Compare commits

..

12 Commits

Author SHA1 Message Date
ahmed fd840b72cb updating secret to s1 to match convention 2023-09-11 16:07:19 -04:00
ahmed 953e7f43bd update gitlabv2 to tri-state 2023-09-11 14:36:48 -04:00
Zubair Khan 3f84a6700e add tri state verification to slack (not slack webhook) (#1731)
* add client, add known false positive check

* fix idiosyncracies

* cleanup

* cleanup comment

* add unexpected error test case

---------

Co-authored-by: Dustin Decker <dustin@trufflesec.com>
2023-09-11 08:38:43 -07:00
ahrav fdeccf06a0 cache dupes w/ different decoders (#1754)
* only cache dupes that have different decoders.

* add test.

* remove file.

* update comment.
2023-09-11 08:18:48 -07:00
ahrav 70cdff915b add test for custom providers. (#1759) 2023-09-11 08:18:34 -07:00
Cody Rose 2c32b14df1 always close aws response body (#1758) 2023-09-11 10:42:14 -04:00
ahrav e53a72abd2 [chore] - Sentry detector update (#1746)
* add test case for account deactivated.

* Handle empty case.
2023-09-11 07:26:09 -07:00
Cody Rose 62ce9bac8b Retry AWS verification 403s (#1757)
This PR introduces retries on 403s in the AWS detector in attempt to work around erroneous SignatureDoesNotMatch errors. As part of the work, the detector has been refactored somewhat, resulting in two minor semantic changes:

Errors crafting the verification HTTP request no longer result in the candidate secret being discarded.
The known-words-based false positive check now runs (and potentially discards candidate secrets) even if verification is disabled. This unifies its behavior with the hash-based false positive check.
2023-09-08 15:32:53 -04:00
trufflesteeeve e68ace5338 Always attempt to return a git link (#1756) 2023-09-08 15:17:29 -04:00
Marwan Sulaiman 3aa5369608 Add Tailscale detector (#1719)
* Add tailscale detector

* PR feedback: match on first element
2023-09-07 19:11:17 -07:00
ah̳̕mͭͭͨͩ̐e̘ͬ́͋ͬ̊̓͂d 8d66fde6de updating sendgrid detector to use tri-state verification (#1735)
* updating sendgrid detector to use tri-state verification
2023-09-07 14:21:03 -04:00
ahrav 2a9f34962d Add optional param to Chunks (#1747)
* Add interface for targeted chunking.

* use optional args.

* update Chunks method signature.

* update tests.

* fix test.

* update QueryCriteria type.
2023-09-07 09:03:37 -07:00
43 changed files with 715 additions and 818 deletions
+10 -20
View File
@@ -105,11 +105,11 @@ var (
filesystemScanIncludePaths = filesystemScan.Flag("include-paths", "Path to file with newline separated regexes for files to include in scan.").Short('i').String()
filesystemScanExcludePaths = filesystemScan.Flag("exclude-paths", "Path to file with newline separated regexes for files to exclude in scan.").Short('x').String()
s3Scan = cli.Command("s3", "Find credentials in S3 buckets.")
s3ScanKey = s3Scan.Flag("key", "S3 key used to authenticate. Can be provided with environment variable AWS_ACCESS_KEY_ID.").Envar("AWS_ACCESS_KEY_ID").String()
s3ScanRoleArns = s3Scan.Flag("role-arn", "Specify the ARN of an IAM role to assume for scanning. You can repeat this flag.").Strings()
s3ScanSecret = s3Scan.Flag("secret", "S3 secret used to authenticate. Can be provided with environment variable AWS_SECRET_ACCESS_KEY.").Envar("AWS_SECRET_ACCESS_KEY").String()
s3ScanSessionToken = s3Scan.Flag("session-token", "S3 session token used to authenticate temporary credentials. Can be provided with environment variable AWS_SESSION_TOKEN.").Envar("AWS_SESSION_TOKEN").String()
s3Scan = cli.Command("s3", "Find credentials in S3 buckets.")
s3ScanKey = s3Scan.Flag("key", "S3 key used to authenticate. Can be provided with environment variable AWS_ACCESS_KEY_ID.").Envar("AWS_ACCESS_KEY_ID").String()
s3ScanRoleArns = s3Scan.Flag("role-arn", "Specify the ARN of an IAM role to assume for scanning. You can repeat this flag.").Strings()
s3ScanSecret = s3Scan.Flag("secret", "S3 secret used to authenticate. Can be provided with environment variable AWS_SECRET_ACCESS_KEY.").Envar("AWS_SECRET_ACCESS_KEY").String()
s3ScanSessionToken = s3Scan.Flag("session-token", "S3 session token used to authenticate temporary credentials. Can be provided with environment variable AWS_SESSION_TOKEN.").Envar("AWS_SESSION_TOKEN").String()
s3ScanCloudEnv = s3Scan.Flag("cloud-environment", "Use IAM credentials in cloud environment.").Bool()
s3ScanBuckets = s3Scan.Flag("bucket", "Name of S3 bucket to scan. You can repeat this flag.").Strings()
s3ScanMaxObjectSize = s3Scan.Flag("max-object-size", "Maximum size of objects to scan. Objects larger than this will be skipped. (Byte units eg. 512B, 2KB, 4MB)").Default("250MB").Bytes()
@@ -136,12 +136,6 @@ var (
circleCiScan = cli.Command("circleci", "Scan CircleCI")
circleCiScanToken = circleCiScan.Flag("token", "CircleCI token. Can also be provided with environment variable").Envar("CIRCLECI_TOKEN").Required().String()
trelloScan = cli.Command("trello", "Scan Trello")
trelloApiKey = trelloScan.Flag("apiKey", "Trello API Key. Can also be provided with environment variable").Envar("TRELLO_API_KEY").Required().String()
trelloToken = trelloScan.Flag("token", "Trello token. Can also be provided with environment variable").Envar("TRELLO_TOKEN").Required().String()
trelloBoardIDs = trelloScan.Flag("boardIDs", "Trello board IDs to scan. Can also be provided with environment variable").Envar("TRELLO_BOARD_IDS").Required().Strings()
dockerScan = cli.Command("docker", "Scan Docker Image")
dockerScanImages = dockerScan.Flag("image", "Docker image to scan. Use the file:// prefix to point to a local tarball, otherwise a image registry is assumed.").Required().Strings()
)
@@ -360,6 +354,10 @@ func run(state overseer.State) {
printer = new(output.PlainPrinter)
}
if !*jsonLegacy && !*jsonOut {
fmt.Fprintf(os.Stderr, "🐷🔑🐷 TruffleHog. Unearth your secrets. 🐷🔑🐷\n\n")
}
e, err := engine.Start(ctx,
engine.WithConcurrency(uint8(*concurrency)),
engine.WithDecoders(decoders.DefaultDecoders()...),
@@ -475,7 +473,7 @@ func run(state overseer.State) {
Secret: *s3ScanSecret,
SessionToken: *s3ScanSessionToken,
Buckets: *s3ScanBuckets,
Roles: *s3ScanRoleArns,
Roles: *s3ScanRoleArns,
CloudCred: *s3ScanCloudEnv,
MaxObjectSize: int64(*s3ScanMaxObjectSize),
}
@@ -498,10 +496,6 @@ func run(state overseer.State) {
if err := e.ScanCircleCI(ctx, *circleCiScanToken); err != nil {
logFatal(err, "Failed to scan CircleCI.")
}
case trelloScan.FullCommand():
if err := e.ScanTrello(ctx, *trelloBoardIDs, *trelloApiKey, *trelloToken); err != nil {
logFatal(err, "Failed to scan Trello.")
}
case gcsScan.FullCommand():
cfg := sources.GCSConfig{
ProjectID: *gcsProjectID,
@@ -535,10 +529,6 @@ func run(state overseer.State) {
}
}
if !*jsonLegacy && !*jsonOut {
fmt.Fprintf(os.Stderr, "🐷🔑🐷 TruffleHog. Unearth your secrets. 🐷🔑🐷\n\n")
}
// Wait for all workers to finish.
if err = e.Finish(ctx); err != nil {
logFatal(err, "engine failed to finish execution")
+130 -110
View File
@@ -118,118 +118,21 @@ func (s scanner) FromData(ctx context.Context, verify bool, data []byte) (result
}
if verify {
// REQUEST VALUES.
method := "GET"
service := "sts"
host := "sts.amazonaws.com"
region := "us-east-1"
endpoint := "https://sts.amazonaws.com"
now := time.Now().UTC()
datestamp := now.Format("20060102")
amzDate := now.Format("20060102T150405Z0700")
req, err := http.NewRequestWithContext(ctx, method, endpoint, nil)
if err != nil {
continue
}
req.Header.Set("Accept", "application/json")
// TASK 1: CREATE A CANONICAL REQUEST.
// http://docs.aws.amazon.com/general/latest/gr/sigv4-create-canonical-request.html
canonicalURI := "/"
canonicalHeaders := "host:" + host + "\n"
signedHeaders := "host"
algorithm := "AWS4-HMAC-SHA256"
credentialScope := fmt.Sprintf("%s/%s/%s/aws4_request", datestamp, region, service)
params := req.URL.Query()
params.Add("Action", "GetCallerIdentity")
params.Add("Version", "2011-06-15")
params.Add("X-Amz-Algorithm", algorithm)
params.Add("X-Amz-Credential", resIDMatch+"/"+credentialScope)
params.Add("X-Amz-Date", amzDate)
params.Add("X-Amz-Expires", "30")
params.Add("X-Amz-SignedHeaders", signedHeaders)
canonicalQuerystring := params.Encode()
payloadHash := GetHash("") // empty payload
canonicalRequest := method + "\n" + canonicalURI + "\n" + canonicalQuerystring + "\n" + canonicalHeaders + "\n" + signedHeaders + "\n" + payloadHash
// TASK 2: CREATE THE STRING TO SIGN.
stringToSign := algorithm + "\n" + amzDate + "\n" + credentialScope + "\n" + GetHash(canonicalRequest)
// TASK 3: CALCULATE THE SIGNATURE.
// https://docs.aws.amazon.com/general/latest/gr/sigv4-calculate-signature.html
hash := GetHMAC([]byte(fmt.Sprintf("AWS4%s", resSecretMatch)), []byte(datestamp))
hash = GetHMAC(hash, []byte(region))
hash = GetHMAC(hash, []byte(service))
hash = GetHMAC(hash, []byte("aws4_request"))
signature2 := GetHMAC(hash, []byte(stringToSign)) // Get Signature HMAC SHA256
signature := hex.EncodeToString(signature2)
// TASK 4: ADD SIGNING INFORMATION TO THE REQUEST.
params.Add("X-Amz-Signature", signature)
req.Header.Add("Content-type", "application/x-www-form-urlencoded; charset=utf-8")
req.URL.RawQuery = params.Encode()
client := s.verificationClient
if client == nil {
client = defaultVerificationClient
}
res, err := client.Do(req)
if err == nil {
if res.StatusCode >= 200 && res.StatusCode < 300 {
identityInfo := identityRes{}
err := json.NewDecoder(res.Body).Decode(&identityInfo)
if err == nil {
s1.Verified = true
s1.ExtraData = map[string]string{
"account": identityInfo.GetCallerIdentityResponse.GetCallerIdentityResult.Account,
"user_id": identityInfo.GetCallerIdentityResponse.GetCallerIdentityResult.UserID,
"arn": identityInfo.GetCallerIdentityResponse.GetCallerIdentityResult.Arn,
}
} else {
s1.VerificationError = err
}
res.Body.Close()
} else {
// This function will check false positives for common test words, but also it will make sure the key appears "random" enough to be a real key.
if detectors.IsKnownFalsePositive(resSecretMatch, detectors.DefaultFalsePositives, true) {
continue
}
if res.StatusCode == 403 {
var body awsErrorResponseBody
err = json.NewDecoder(res.Body).Decode(&body)
res.Body.Close()
if err == nil {
// All instances of the code I've seen in the wild are PascalCased but this check is
// case-insensitive out of an abundance of caution
if strings.EqualFold(body.Error.Code, "InvalidClientTokenId") {
// determinate failure - nothing to do
} else {
// We see a surprising number of false-negative signature mismatch errors here
// (The official SDK somehow elicits even more than just making the request
// ourselves)
s1.VerificationError = fmt.Errorf("request to %v returned status %d with an unexpected reason (%s: %s)", res.Request.URL, res.StatusCode, body.Error.Code, body.Error.Message)
}
} else {
s1.VerificationError = fmt.Errorf("couldn't parse the sts response body (%v)", err)
}
} else {
s1.VerificationError = fmt.Errorf("request to %v returned unexpected status %d", res.Request.URL, res.StatusCode)
}
}
} else {
s1.VerificationError = err
}
verified, extraData, verificationErr := s.verifyMatch(ctx, resIDMatch, resSecretMatch, true)
s1.Verified = verified
s1.ExtraData = extraData
s1.VerificationError = verificationErr
}
// If the result is unverified and matches something like a git hash, don't include it in the results.
if !s1.Verified && falsePositiveSecretCheck.MatchString(resSecretMatch) {
continue
if !s1.Verified {
// Unverified results that contain common test words are probably not secrets
if detectors.IsKnownFalsePositive(resSecretMatch, detectors.DefaultFalsePositives, true) {
continue
}
// Unverified results that look like hashes are probably not secrets
if falsePositiveSecretCheck.MatchString(resSecretMatch) {
continue
}
}
results = append(results, s1)
@@ -242,6 +145,123 @@ func (s scanner) FromData(ctx context.Context, verify bool, data []byte) (result
return awsCustomCleanResults(results), nil
}
func (s scanner) verifyMatch(ctx context.Context, resIDMatch, resSecretMatch string, retryOn403 bool) (bool, map[string]string, error) {
// REQUEST VALUES.
method := "GET"
service := "sts"
host := "sts.amazonaws.com"
region := "us-east-1"
endpoint := "https://sts.amazonaws.com"
now := time.Now().UTC()
datestamp := now.Format("20060102")
amzDate := now.Format("20060102T150405Z0700")
req, err := http.NewRequestWithContext(ctx, method, endpoint, nil)
if err != nil {
return false, nil, err
}
req.Header.Set("Accept", "application/json")
// TASK 1: CREATE A CANONICAL REQUEST.
// http://docs.aws.amazon.com/general/latest/gr/sigv4-create-canonical-request.html
canonicalURI := "/"
canonicalHeaders := "host:" + host + "\n"
signedHeaders := "host"
algorithm := "AWS4-HMAC-SHA256"
credentialScope := fmt.Sprintf("%s/%s/%s/aws4_request", datestamp, region, service)
params := req.URL.Query()
params.Add("Action", "GetCallerIdentity")
params.Add("Version", "2011-06-15")
params.Add("X-Amz-Algorithm", algorithm)
params.Add("X-Amz-Credential", resIDMatch+"/"+credentialScope)
params.Add("X-Amz-Date", amzDate)
params.Add("X-Amz-Expires", "30")
params.Add("X-Amz-SignedHeaders", signedHeaders)
canonicalQuerystring := params.Encode()
payloadHash := GetHash("") // empty payload
canonicalRequest := method + "\n" + canonicalURI + "\n" + canonicalQuerystring + "\n" + canonicalHeaders + "\n" + signedHeaders + "\n" + payloadHash
// TASK 2: CREATE THE STRING TO SIGN.
stringToSign := algorithm + "\n" + amzDate + "\n" + credentialScope + "\n" + GetHash(canonicalRequest)
// TASK 3: CALCULATE THE SIGNATURE.
// https://docs.aws.amazon.com/general/latest/gr/sigv4-calculate-signature.html
hash := GetHMAC([]byte(fmt.Sprintf("AWS4%s", resSecretMatch)), []byte(datestamp))
hash = GetHMAC(hash, []byte(region))
hash = GetHMAC(hash, []byte(service))
hash = GetHMAC(hash, []byte("aws4_request"))
signature2 := GetHMAC(hash, []byte(stringToSign)) // Get Signature HMAC SHA256
signature := hex.EncodeToString(signature2)
// TASK 4: ADD SIGNING INFORMATION TO THE REQUEST.
params.Add("X-Amz-Signature", signature)
req.Header.Add("Content-type", "application/x-www-form-urlencoded; charset=utf-8")
req.URL.RawQuery = params.Encode()
client := s.verificationClient
if client == nil {
client = defaultVerificationClient
}
res, err := client.Do(req)
if err == nil {
defer res.Body.Close()
if res.StatusCode >= 200 && res.StatusCode < 300 {
identityInfo := identityRes{}
err := json.NewDecoder(res.Body).Decode(&identityInfo)
if err == nil {
extraData := map[string]string{
"account": identityInfo.GetCallerIdentityResponse.GetCallerIdentityResult.Account,
"user_id": identityInfo.GetCallerIdentityResponse.GetCallerIdentityResult.UserID,
"arn": identityInfo.GetCallerIdentityResponse.GetCallerIdentityResult.Arn,
}
return true, extraData, nil
} else {
return false, nil, err
}
} else if res.StatusCode == 403 {
// Experimentation has indicated that if you make two GetCallerIdentity requests within five seconds that
// share a key ID but are signed with different secrets the second one will be rejected with a 403 that
// carries a SignatureDoesNotMatch code in its body. This happens even if the second ID-secret pair is
// valid. Since this is exactly our access pattern, we need to work around it.
//
// Fortunately, experimentation has also revealed a workaround: simply resubmit the second request. The
// response to the resubmission will be as expected. But there's a caveat: You can't have closed the body of
// the response to the original second request, or read to its end, or the resubmission will also yield a
// SignatureDoesNotMatch. For this reason, we have to re-request all 403s. We can't re-request only
// SignatureDoesNotMatch responses, because we can only tell whether a given 403 is a SignatureDoesNotMatch
// after decoding its response body, which requires reading the entire response body, which disables the
// workaround.
//
// We are clearly deep in the guts of AWS implementation details here, so this all might change with no
// notice. If you're here because something in this detector broke, you have my condolences.
if retryOn403 {
return s.verifyMatch(ctx, resIDMatch, resSecretMatch, false)
}
var body awsErrorResponseBody
err = json.NewDecoder(res.Body).Decode(&body)
if err == nil {
// All instances of the code I've seen in the wild are PascalCased but this check is
// case-insensitive out of an abundance of caution
if strings.EqualFold(body.Error.Code, "InvalidClientTokenId") {
return false, nil, nil
} else {
return false, nil, fmt.Errorf("request to %v returned status %d with an unexpected reason (%s: %s)", res.Request.URL, res.StatusCode, body.Error.Code, body.Error.Message)
}
} else {
return false, nil, fmt.Errorf("couldn't parse the sts response body (%v)", err)
}
} else {
return false, nil, fmt.Errorf("request to %v returned unexpected status %d", res.Request.URL, res.StatusCode)
}
} else {
return false, nil, err
}
}
func awsCustomCleanResults(results []detectors.Result) []detectors.Result {
if len(results) == 0 {
return results
+22
View File
@@ -248,6 +248,28 @@ func TestAWS_FromChunk(t *testing.T) {
wantErr: false,
wantVerificationError: true,
},
{
name: "verified secret checked directly after unverified secret with same key id",
s: scanner{},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("%s\n%s\n%s", inactiveSecret, id, secret)),
verify: true,
},
want: []detectors.Result{
{
DetectorType: detectorspb.DetectorType_AWS,
Verified: true,
Redacted: "AKIASP2TPHJSQH3FJRUX",
ExtraData: map[string]string{
"account": "171436882533",
"arn": "arn:aws:iam::171436882533:user/canarytokens.com@@4dxkh0pdeop3bzu9zx5wob793",
"user_id": "AIDASP2TPHJSUFRSTTZX4",
},
},
},
wantErr: false,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
+7 -7
View File
@@ -53,7 +53,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
resMatch = keyString[len(keyString)-1]
}
secret := detectors.Result{
s1 := detectors.Result{
DetectorType: detectorspb.DetectorType_Gitlab,
Raw: []byte(match[1]),
}
@@ -84,26 +84,26 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
// 401 is bad key
switch res.StatusCode {
case http.StatusOK:
secret.Verified = true
s1.Verified = true
case http.StatusForbidden:
// Good key but not the right scope
secret.Verified = true
s1.Verified = true
case http.StatusUnauthorized:
// Nothing to do; zero values are the ones we want
default:
secret.VerificationError = fmt.Errorf("unexpected HTTP response status %d", res.StatusCode)
s1.VerificationError = fmt.Errorf("unexpected HTTP response status %d", res.StatusCode)
}
} else {
secret.VerificationError = err
s1.VerificationError = err
}
}
}
if !secret.Verified && detectors.IsKnownFalsePositive(string(secret.Raw), detectors.DefaultFalsePositives, true) {
if !s1.Verified && detectors.IsKnownFalsePositive(string(s1.Raw), detectors.DefaultFalsePositives, true) {
continue
}
results = append(results, secret)
results = append(results, s1)
}
return results, nil
+3 -2
View File
@@ -6,11 +6,12 @@ package gitlab
import (
"context"
"fmt"
"github.com/google/go-cmp/cmp"
"github.com/google/go-cmp/cmp/cmpopts"
"testing"
"time"
"github.com/google/go-cmp/cmp"
"github.com/google/go-cmp/cmp/cmpopts"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+25 -8
View File
@@ -11,7 +11,10 @@ import (
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
)
type Scanner struct{ detectors.EndpointSetter }
type Scanner struct {
client *http.Client
detectors.EndpointSetter
}
// Ensure the Scanner satisfies the interfaces at compile time.
var _ detectors.Detector = (*Scanner)(nil)
@@ -22,7 +25,8 @@ func (Scanner) Version() int { return 2 }
func (Scanner) DefaultEndpoint() string { return "https://gitlab.com" }
var (
keyPat = regexp.MustCompile(`\b(glpat-[a-zA-Z0-9\-=_]{20,22})\b`)
defaultClient = common.SaneHttpClient()
keyPat = regexp.MustCompile(`\b(glpat-[a-zA-Z0-9\-=_]{20,22})\b`)
)
// Keywords are used for efficiently pre-filtering chunks.
@@ -42,7 +46,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
continue
}
secret := detectors.Result{
s1 := detectors.Result{
DetectorType: detectorspb.DetectorType_Gitlab,
Raw: []byte(match[1]),
}
@@ -52,7 +56,10 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
// they all grant access to different parts of the API. I couldn't find an endpoint that every
// one of these scopes has access to, so we just check an example endpoint for each scope. If any
// of them contain data, we know we have a valid key, but if they all fail, we don't
client := common.SaneHttpClient()
client := s.client
if client == nil {
client = defaultClient
}
for _, baseURL := range s.Endpoints(s.DefaultEndpoint()) {
// test `read_user` scope
req, err := http.NewRequestWithContext(ctx, "GET", baseURL+"/api/v4/user", nil)
@@ -67,19 +74,29 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
// 200 means good key and has `read_user` scope
// 403 means good key but not the right scope
// 401 is bad key
if res.StatusCode == http.StatusOK || res.StatusCode == http.StatusForbidden {
secret.Verified = true
switch res.StatusCode {
case http.StatusOK:
s1.Verified = true
case http.StatusForbidden:
// Good key but not the right scope
s1.Verified = true
case http.StatusUnauthorized:
// Nothing to do; zero values are the ones we want
default:
s1.VerificationError = fmt.Errorf("unexpected HTTP response status %d", res.StatusCode)
}
} else {
s1.VerificationError = err
}
}
}
if !secret.Verified && detectors.IsKnownFalsePositive(string(secret.Raw), detectors.DefaultFalsePositives, true) {
if !s1.Verified && detectors.IsKnownFalsePositive(string(s1.Raw), detectors.DefaultFalsePositives, true) {
continue
}
results = append(results, secret)
results = append(results, s1)
}
return
+73 -14
View File
@@ -9,7 +9,8 @@ import (
"testing"
"time"
"github.com/kylelemons/godebug/pretty"
"github.com/google/go-cmp/cmp"
"github.com/google/go-cmp/cmp/cmpopts"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
@@ -18,7 +19,7 @@ import (
func TestGitlab_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")
testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors4")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
@@ -31,18 +32,19 @@ func TestGitlab_FromChunk(t *testing.T) {
verify bool
}
tests := []struct {
name string
s Scanner
args args
want []detectors.Result
wantErr bool
name string
s Scanner
args args
want []detectors.Result
wantErr bool
wantVerificationErr bool
}{
{
name: "found, verified",
s: Scanner{},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a aws secret %s within", secret)),
data: []byte(fmt.Sprintf("You can find a gitlab secret %s within", secret)),
verify: true,
},
want: []detectors.Result{
@@ -58,10 +60,65 @@ func TestGitlab_FromChunk(t *testing.T) {
s: Scanner{},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a aws secret %s within", secretInactive)),
data: []byte(fmt.Sprintf("You can find a gitlab secret %s within", secretInactive)),
verify: true,
},
want: nil,
want: []detectors.Result{
{
DetectorType: detectorspb.DetectorType_Gitlab,
Verified: false,
},
},
wantErr: false,
},
{
name: "found, would be verified but for timeout",
s: Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a gitlab super secret %s within", secret)),
verify: true,
},
want: []detectors.Result{
{
DetectorType: detectorspb.DetectorType_Gitlab,
Verified: false,
},
},
wantErr: false,
wantVerificationErr: true,
},
{
name: "found and valid but unexpected api response",
s: Scanner{client: common.ConstantResponseHttpClient(500, "")},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a gitlab super secret %s within", secret)),
verify: true,
},
want: []detectors.Result{
{
DetectorType: detectorspb.DetectorType_Gitlab,
Verified: false,
},
},
wantErr: false,
wantVerificationErr: true,
},
{
name: "found, good key but wrong scope",
s: Scanner{client: common.ConstantResponseHttpClient(403, "")},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a gitlab super secret %s within", secret)),
verify: true,
},
want: []detectors.Result{
{
DetectorType: detectorspb.DetectorType_Gitlab,
Verified: true,
},
},
wantErr: false,
},
{
@@ -78,8 +135,7 @@ func TestGitlab_FromChunk(t *testing.T) {
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
s := Scanner{}
got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
if (err != nil) != tt.wantErr {
t.Errorf("Gitlab.FromData() error = %v, wantErr %v", err, tt.wantErr)
return
@@ -88,9 +144,12 @@ func TestGitlab_FromChunk(t *testing.T) {
if len(got[i].Raw) == 0 {
t.Fatal("no raw secret present")
}
got[i].Raw = nil
if (got[i].VerificationError != nil) != tt.wantVerificationErr {
t.Fatalf(" wantVerificationError = %v, verification error = %v,", tt.wantVerificationErr, got[i].VerificationError)
}
}
if diff := pretty.Compare(got, tt.want); diff != "" {
opts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "VerificationError")
if diff := cmp.Diff(got, tt.want, opts); diff != "" {
t.Errorf("Gitlab.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
}
})
+15 -6
View File
@@ -13,13 +13,15 @@ import (
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
)
type Scanner struct{}
type Scanner struct {
client *http.Client
}
// Ensure the Scanner satisfies the interface at compile time.
var _ detectors.Detector = (*Scanner)(nil)
var (
client = common.SaneHttpClient()
defaultClient = common.SaneHttpClient()
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"sendgrid"}) + `(SG\.[\w\-_]{20,24}\.[\w\-_]{39,50})\b`)
)
@@ -42,7 +44,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
}
res := strings.TrimSpace(match[1])
s := detectors.Result{
s1 := detectors.Result{
DetectorType: detectorspb.DetectorType_SendGrid,
Raw: []byte(res),
}
@@ -52,6 +54,11 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
// 403 will be issued if the scope is wrong but the key is correct
baseURL := "https://api.sendgrid.com/v3/templates"
client := s.client
if client == nil {
client = defaultClient
}
// test `read_user` scope
req, err := http.NewRequestWithContext(ctx, "GET", baseURL, nil)
if err != nil {
@@ -67,16 +74,18 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
// 403 means good key but not the right scope
// 401 is bad key
if res.StatusCode == http.StatusOK || res.StatusCode == http.StatusForbidden {
s.Verified = true
s1.Verified = true
} else if res.StatusCode != http.StatusUnauthorized {
s1.VerificationError = fmt.Errorf("unexpected HTTP response status %d", res.StatusCode)
}
}
}
if !s.Verified && detectors.IsKnownFalsePositive(string(s.Raw), detectors.DefaultFalsePositives, true) {
if !s1.Verified && detectors.IsKnownFalsePositive(string(s1.Raw), detectors.DefaultFalsePositives, true) {
continue
}
results = append(results, s)
results = append(results, s1)
}
return
+53 -17
View File
@@ -9,7 +9,8 @@ import (
"testing"
"time"
"github.com/kylelemons/godebug/pretty"
"github.com/google/go-cmp/cmp"
"github.com/google/go-cmp/cmp/cmpopts"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
@@ -18,7 +19,7 @@ import (
func TestSendgrid_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")
testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
@@ -30,11 +31,12 @@ func TestSendgrid_FromChunk(t *testing.T) {
verify bool
}
tests := []struct {
name string
s Scanner
args args
want []detectors.Result
wantErr bool
name string
s Scanner
args args
want []detectors.Result
wantErr bool
wantVerificationErr bool
}{
{
name: "found, verified",
@@ -50,7 +52,8 @@ func TestSendgrid_FromChunk(t *testing.T) {
Verified: true,
},
},
wantErr: false,
wantErr: false,
wantVerificationErr: false,
},
{
name: "found, unverified",
@@ -66,7 +69,8 @@ func TestSendgrid_FromChunk(t *testing.T) {
Verified: false,
},
},
wantErr: false,
wantErr: false,
wantVerificationErr: false,
},
{
name: "not found",
@@ -76,22 +80,54 @@ func TestSendgrid_FromChunk(t *testing.T) {
data: []byte("You cannot find the secret within"),
verify: true,
},
want: nil,
wantErr: false,
want: nil,
wantErr: false,
wantVerificationErr: false,
},
{
name: "found, would be verified if not for timeout",
s: Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a sendgrid secret %s within", secret)),
verify: true,
},
want: []detectors.Result{
{
DetectorType: detectorspb.DetectorType_SendGrid,
Verified: false,
},
},
wantErr: false,
wantVerificationErr: true,
},
{
name: "found, verified but unexpected api surface",
s: Scanner{client: common.ConstantResponseHttpClient(404, "")},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a sendgrid secret %s within", secret)),
verify: true,
},
want: []detectors.Result{
{
DetectorType: detectorspb.DetectorType_SendGrid,
Verified: false,
},
},
wantErr: false,
wantVerificationErr: true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
s := Scanner{}
got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
if (err != nil) != tt.wantErr {
t.Errorf("Sendgrid.FromData() error = %v, wantErr %v", err, tt.wantErr)
return
}
for i := range got {
got[i].Raw = nil
}
if diff := pretty.Compare(got, tt.want); diff != "" {
ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "VerificationError")
if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {
t.Errorf("Sendgrid.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
}
})
+3
View File
@@ -122,6 +122,9 @@ func verifyToken(ctx context.Context, client *http.Client, token string) (bool,
if err = json.Unmarshal(bytes, &resp); err != nil {
return false, err
}
if len(resp) == 0 {
return false, fmt.Errorf("unexpected response body: %s", string(bytes))
}
return isVerified, err
}
+39 -1
View File
@@ -123,6 +123,40 @@ func TestSentryToken_FromChunk(t *testing.T) {
},
wantErr: false,
},
{
name: "found, account deactivated",
s: Scanner{client: common.ConstantResponseHttpClient(200, reponseAccountDeactivated)},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a sentry super secret %s within", secret)),
verify: true,
},
want: []detectors.Result{
{
DetectorType: detectorspb.DetectorType_SentryToken,
Verified: false,
},
},
wantErr: false,
wantVerificationErr: true,
},
{
name: "found, account deactivated",
s: Scanner{client: common.ConstantResponseHttpClient(200, responseEnmpty)},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a sentry super secret %s within", secret)),
verify: true,
},
want: []detectors.Result{
{
DetectorType: detectorspb.DetectorType_SentryToken,
Verified: false,
},
},
wantErr: false,
wantVerificationErr: true,
},
{
name: "not found",
s: Scanner{},
@@ -159,7 +193,8 @@ func TestSentryToken_FromChunk(t *testing.T) {
}
}
const responseBody403 = `
const (
responseBody403 = `
[
{
"organization": {
@@ -174,6 +209,9 @@ const responseBody403 = `
}
]
`
reponseAccountDeactivated = `{"detail": "Authentication credentials were not provided"}`
responseEnmpty = `[]`
)
func BenchmarkFromData(benchmark *testing.B) {
ctx := context.Background()
+31 -8
View File
@@ -13,13 +13,16 @@ import (
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
)
type Scanner struct{}
type Scanner struct {
client *http.Client
}
// Check that the Slack scanner implements the SecretScanner interface at compile time.
var _ detectors.Detector = Scanner{}
var (
tokenPats = map[string]*regexp.Regexp{
defaultClient = common.SaneHttpClient()
tokenPats = map[string]*regexp.Regexp{
"Slack Bot Token": regexp.MustCompile(`xoxb\-[0-9]{10,13}\-[0-9]{10,13}[a-zA-Z0-9\-]*`),
"Slack User Token": regexp.MustCompile(`xoxp\-[0-9]{10,13}\-[0-9]{10,13}[a-zA-Z0-9\-]*`),
"Slack Workspace Access Token": regexp.MustCompile(`xoxa\-[0-9]{10,13}\-[0-9]{10,13}[a-zA-Z0-9\-]*`),
@@ -53,34 +56,54 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
tokens := tokenPat.FindAllString(dataStr, -1)
for _, token := range tokens {
s := detectors.Result{
s1 := detectors.Result{
DetectorType: detectorspb.DetectorType_Slack,
Raw: []byte(token),
}
if verify {
client := common.SaneHttpClient()
client := s.client
if s.client == nil {
client = defaultClient
}
req, err := http.NewRequestWithContext(ctx, "POST", verifyURL, nil)
if err != nil {
continue
}
req.Header.Add("Content-Type", "application/json; charset=utf-8")
req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", token))
res, err := client.Do(req)
if err == nil {
defer res.Body.Close()
var authResponse authRes
if err := json.NewDecoder(res.Body).Decode(&authResponse); err != nil {
continue
s1.VerificationError = fmt.Errorf("failed to decode auth response: %w", err)
}
s.Verified = authResponse.Ok
if authResponse.Ok {
s1.Verified = true
// Slack API returns 200 even if the token is invalid. We need to check the error field.
} else if authResponse.Error == "invalid_auth" {
// The secret is determinately not verified (nothing to do)
} else {
s1.VerificationError = fmt.Errorf("unexpected error auth response %+v", authResponse.Error)
}
} else {
s1.VerificationError = err
}
}
results = append(results, s)
if !s1.Verified && detectors.IsKnownFalsePositive(string(s1.Raw), detectors.DefaultFalsePositives, true) {
continue
}
results = append(results, s1)
}
}
return
return results, nil
}
func (s Scanner) Type() detectorspb.DetectorType {
+73 -18
View File
@@ -9,13 +9,15 @@ import (
"testing"
"time"
"github.com/kylelemons/godebug/pretty"
"github.com/google/go-cmp/cmp"
"github.com/google/go-cmp/cmp/cmpopts"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
)
func TestScanner_FromChunk(t *testing.T) {
func TestSlack_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")
@@ -24,17 +26,28 @@ func TestScanner_FromChunk(t *testing.T) {
}
secret := testSecrets.MustGetField("SLACK")
secretInactive := testSecrets.MustGetField("SLACK_INACTIVE")
type args struct {
ctx context.Context
data []byte
verify bool
}
tests := []struct {
name string
data []byte
verify bool
wantResults []detectors.Result
wantErr bool
name string
s Scanner
args args
wantResults []detectors.Result
wantErr bool
wantVerificationErr bool
}{
{
name: "found, verified",
data: []byte(fmt.Sprintf("You can find a slack secret %s within", secret)),
verify: true,
name: "found, verified",
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a slack secret %s within", secret)),
verify: true,
},
wantResults: []detectors.Result{
{
DetectorType: detectorspb.DetectorType_Slack,
@@ -44,9 +57,12 @@ func TestScanner_FromChunk(t *testing.T) {
wantErr: false,
},
{
name: "found but unverified",
data: []byte(fmt.Sprintf("You can find a slack secret %s within", secretInactive)),
verify: true,
name: "found but unverified",
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a slack secret %s within", secretInactive)),
verify: true,
},
wantResults: []detectors.Result{
{
DetectorType: detectorspb.DetectorType_Slack,
@@ -55,23 +71,62 @@ func TestScanner_FromChunk(t *testing.T) {
},
wantErr: false,
},
{
name: "found, would be verified if not for timeout",
s: Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a slack secret %s within", secret)),
verify: true,
},
wantResults: []detectors.Result{
{
DetectorType: detectorspb.DetectorType_Slack,
Verified: false,
},
},
wantErr: false,
wantVerificationErr: true,
},
{
name: "unexpected auth response",
s: Scanner{client: common.ConstantResponseHttpClient(200, `{"ok": false, "error": "unexpected_error"}`)},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a slack secret %s within", secret)),
verify: true,
},
wantResults: []detectors.Result{
{
DetectorType: detectorspb.DetectorType_Slack,
Verified: false,
},
},
wantErr: false,
wantVerificationErr: true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
s := Scanner{}
got, err := s.FromData(context.Background(), tt.verify, tt.data)
got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
if (err != nil) != tt.wantErr {
t.Errorf("Scanner.FromData) error = %v, wantErr %v", err, tt.wantErr)
t.Errorf("Slack.FromData() error = %v, wantErr %v", err, tt.wantErr)
return
}
for i := range got {
if len(got[i].Raw) == 0 {
t.Fatal("no raw secret present")
}
got[i].Raw = nil
if (got[i].VerificationError != nil) != tt.wantVerificationErr {
t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError)
}
}
if diff := pretty.Compare(got, tt.wantResults); diff != "" {
t.Errorf("Scanner.FromData) %s diff: (-got +want)\n%s", tt.name, diff)
ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "VerificationError")
if diff := cmp.Diff(got, tt.wantResults, ignoreOpts); diff != "" {
t.Errorf("Slack.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
}
})
}
+85
View File
@@ -0,0 +1,85 @@
package tailscale
import (
"context"
"fmt"
"net/http"
"net/url"
"regexp"
"strings"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
)
type Scanner struct{}
// Ensure the Scanner satisfies the interface at compile time.
var _ detectors.Detector = (*Scanner)(nil)
var (
client = common.SaneHttpClient()
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
keyPat = regexp.MustCompile(`\btskey-[a-z]+-[0-9A-Za-z_]+-[0-9A-Za-z_]+\b`)
)
// Keywords are used for efficiently pre-filtering chunks.
// Use identifiers in the secret preferably, or the provider name.
func (s Scanner) Keywords() []string {
return []string{"tskey-", "tskey-api-", "tskey-oauth-"}
}
// FromData will find and optionally verify Tailscaleapi secrets in a given set of bytes.
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
dataStr := string(data)
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
for _, match := range matches {
resMatch := strings.TrimSpace(match[0])
s1 := detectors.Result{
DetectorType: detectorspb.DetectorType_Tailscale,
Raw: []byte(resMatch),
}
if verify {
const u = "https://api.tailscale.com/api/v2/secret-scanning/verify"
vals := url.Values{"key": []string{resMatch}}
req, err := http.NewRequestWithContext(ctx, http.MethodPost, u, strings.NewReader(vals.Encode()))
if err != nil {
continue
}
req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
res, err := client.Do(req)
if err == nil {
defer res.Body.Close()
switch res.StatusCode {
case http.StatusNoContent:
s1.Verified = true
case http.StatusUnauthorized:
// The secret is determinately not verified (nothing to do)
default:
s1.VerificationError = fmt.Errorf("unexpected HTTP response status %d", res.StatusCode)
}
} else {
s1.VerificationError = err
}
}
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
if !s1.Verified && detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
continue
}
results = append(results, s1)
}
return results, nil
}
func (s Scanner) Type() detectorspb.DetectorType {
return detectorspb.DetectorType_Tailscale
}
@@ -1,4 +1,7 @@
package gitlab
//go:build detectors
// +build detectors
package tailscale
import (
"context"
@@ -7,20 +10,23 @@ import (
"time"
"github.com/kylelemons/godebug/pretty"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
)
func TestGitlabv2_FromChunk(t *testing.T) {
func TestTailscaleapi_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors4")
if err != nil {
if err != nil && false {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
secret := testSecrets.MustGetField("GITLABV2")
secretInactive := testSecrets.MustGetField("GITLABV2_INACTIVE")
secret := testSecrets.MustGetField("TAILSCALEAPI")
inactiveSecret := testSecrets.MustGetField("TAILSCALEAPI_INACTIVE")
type args struct {
ctx context.Context
data []byte
@@ -34,16 +40,16 @@ func TestGitlabv2_FromChunk(t *testing.T) {
wantErr bool
}{
{
name: "found",
name: "found, verified",
s: Scanner{},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a gitlab secret %s within", secret)),
data: []byte(fmt.Sprintf("You can find a tailscaleapi secret %s within", secret)),
verify: true,
},
want: []detectors.Result{
{
DetectorType: detectorspb.DetectorType_Gitlab,
DetectorType: detectorspb.DetectorType_Tailscale,
Verified: true,
},
},
@@ -54,12 +60,12 @@ func TestGitlabv2_FromChunk(t *testing.T) {
s: Scanner{},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a gitlab secret %s within", secretInactive)),
data: []byte(fmt.Sprintf("You can find a tailscaleapi secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
verify: true,
},
want: []detectors.Result{
{
DetectorType: detectorspb.DetectorType_Gitlab,
DetectorType: detectorspb.DetectorType_Tailscale,
Verified: false,
},
},
@@ -82,28 +88,27 @@ func TestGitlabv2_FromChunk(t *testing.T) {
s := Scanner{}
got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
if (err != nil) != tt.wantErr {
t.Errorf("Gitlab.FromData() error = %v, wantErr %v", err, tt.wantErr)
t.Errorf("Tailscaleapi.FromData() error = %v, wantErr %v", err, tt.wantErr)
return
}
for i := range got {
if len(got[i].Raw) == 0 {
t.Fatal("no raw secret present")
t.Fatalf("no raw secret present: \n %+v", got[i])
}
got[i].Raw = nil
}
if diff := pretty.Compare(got, tt.want); diff != "" {
t.Errorf("Gitlab.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
t.Errorf("Tailscaleapi.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
}
})
}
}
func BenchmarkFromData2(benchmark *testing.B) {
func BenchmarkFromData(benchmark *testing.B) {
ctx := context.Background()
s := Scanner{}
for name, data := range detectors.MustGetBenchmarkData() {
benchmark.Run(name, func(b *testing.B) {
b.ResetTimer()
for n := 0; n < b.N; n++ {
_, err := s.FromData(ctx, false, data)
if err != nil {
+2
View File
@@ -6,6 +6,7 @@ import (
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/salesforce"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/snowflake"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/sourcegraph"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/tailscale"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/trufflehogenterprise"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
@@ -1542,6 +1543,7 @@ func DefaultDetectors() []detectors.Detector {
trufflehogenterprise.Scanner{},
salesforce.Scanner{},
sourcegraph.Scanner{},
tailscale.Scanner{},
}
}
+10 -3
View File
@@ -565,11 +565,18 @@ func (e *Engine) notifyResults(ctx context.Context) {
}
atomic.AddUint32(&e.numFoundResults, 1)
// Dedupe results by comparing the detector type, raw result, and source metadata.
// We want to avoid duplicate results with different decoder types, but we also
// want to include duplicate results with the same decoder type.
// Duplicate results with the same decoder type SHOULD have their own entry in the
// results list, this would happen if the same secret is found multiple times.
key := fmt.Sprintf("%s%s%s%+v", r.DetectorType.String(), r.Raw, r.RawV2, r.SourceMetadata)
if _, ok := e.dedupeCache.Get(key); ok {
continue
if val, ok := e.dedupeCache.Get(key); ok {
if res, ok := val.(detectorspb.DecoderType); ok && res != r.DecoderType {
continue
}
}
e.dedupeCache.Add(key, struct{}{})
e.dedupeCache.Add(key, r.DecoderType)
if r.Verified {
atomic.AddUint64(&e.metrics.VerifiedSecretsFound, 1)
+34
View File
@@ -2,8 +2,13 @@ package engine
import (
"fmt"
"path/filepath"
"testing"
"time"
"github.com/stretchr/testify/assert"
"github.com/trufflesecurity/trufflehog/v3/pkg/context"
"github.com/trufflesecurity/trufflehog/v3/pkg/decoders"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
@@ -171,3 +176,32 @@ func BenchmarkSupportsLineNumbersLoop(b *testing.B) {
_ = SupportsLineNumbers(sourceType)
}
}
// TestEngine_DuplicatSecrets is a test that detects ALL duplicate secrets with the same decoder.
func TestEngine_DuplicatSecrets(t *testing.T) {
ctx := context.Background()
absPath, err := filepath.Abs("./testdata")
assert.Nil(t, err)
ctx, cancel := context.WithTimeout(ctx, 10*time.Second)
defer cancel()
e, err := Start(ctx,
WithConcurrency(1),
WithDecoders(decoders.DefaultDecoders()...),
WithDetectors(true, DefaultDetectors()...),
WithPrinter(new(discardPrinter)),
)
assert.Nil(t, err)
cfg := sources.FilesystemConfig{Paths: []string{absPath}}
if err := e.ScanFileSystem(ctx, cfg); err != nil {
return
}
// Wait for all the chunks to be processed.
assert.Nil(t, e.Finish(ctx))
want := uint64(5)
assert.Equal(t, want, e.GetMetrics().UnverifiedSecretsFound)
}
+6
View File
@@ -0,0 +1,6 @@
1 aws AKIAWARWQKZNHMZBLY4I
2 secret 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
sentry 27ac84f4bcdb4fca9701f4d6f6f58cd7d96b69c9d9754d40800645a51d668f90
sentry 27ac84f4bcdb4fca9701f4d6f6f58cd7d96b69c9d9754d40800645a51d668f90
sentry 27ac84f4bcdb4fca9701f4d6f6f58cd7d96b69c9d9754d40800645a51d668f90
sentry 27ac84f4bcdb4fca9701f4d6f6f58cd7d96b69c9d9754d40800645a51d668f90
-50
View File
@@ -1,50 +0,0 @@
package engine
import (
"runtime"
"google.golang.org/protobuf/proto"
"google.golang.org/protobuf/types/known/anypb"
"github.com/trufflesecurity/trufflehog/v3/pkg/context"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
"github.com/trufflesecurity/trufflehog/v3/pkg/sources/trello"
)
// ScanTrello scans Trello boards for secrets.
func (e *Engine) ScanTrello(ctx context.Context, apiKey string) error {
trelloSource := trello.Source{}
boardIDs, err := trelloSource.GetBoardIDs(ctx, apiKey)
if err != nil {
ctx.Logger().Error(err, "failed to get Trello board IDs")
return err
}
connection := &sourcespb.Trello{
Auth: &sourcespb.Trello_TrelloAuth{
ApiKey: apiKey,
},
Boards: boardIDs,
}
var conn anypb.Any
err = anypb.MarshalFrom(&conn, connection, proto.MarshalOptions{})
if err != nil {
ctx.Logger().Error(err, "failed to marshal Trello connection")
return err
}
handle, err := e.sourceManager.Enroll(ctx, "trufflehog - Trello", new(trello.Source).Type(),
func(ctx context.Context, jobID, sourceID int64) (sources.Source, error) {
if err := trelloSource.Init(ctx, "trufflehog - Trello", jobID, sourceID, true, &conn, runtime.NumCPU()); err != nil {
return nil, err
}
return &trelloSource, nil
})
if err != nil {
return err
}
_, err = e.sourceManager.ScheduleRun(ctx, handle)
return err
}
+11 -10
View File
@@ -119,7 +119,18 @@ func GenerateLink(repo, commit, file string, line int64) string {
case providerBitbucket:
return repo[:len(repo)-4] + "/commits/" + commit
case providerAzure:
baseLink := repo + "/commit/" + commit + "/" + file
if line > 0 {
baseLink += "?line=" + strconv.FormatInt(line, 10)
}
return baseLink
case providerGithub, providerGitlab:
// If the provider name isn't one of the cloud defaults, it is probably an on-prem github or gitlab.
// So do the same thing.
fallthrough
default:
var baseLink string
if file == "" {
baseLink = repo[:len(repo)-4] + "/commit/" + commit
@@ -130,15 +141,5 @@ func GenerateLink(repo, commit, file string, line int64) string {
}
}
return baseLink
case providerAzure:
baseLink := repo + "/commit/" + commit + "/" + file
if line > 0 {
baseLink += "?line=" + strconv.FormatInt(line, 10)
}
return baseLink
default:
return ""
}
}
+18
View File
@@ -160,6 +160,24 @@ func TestGenerateLink(t *testing.T) {
},
want: "https://dev.azure.com/org/project/_git/repo/commit/abcdef/main.go?line=20",
},
{
name: "Unknown provider on-prem instance",
args: args{
repo: "https://onprem.customdomain.com/org/repo.git",
commit: "xyz123",
file: "main.go",
line: int64(30),
},
want: "https://onprem.customdomain.com/org/repo/blob/xyz123/main.go#L30",
},
{
name: "Unknown provider on-prem instance - no file",
args: args{
repo: "https://onprem.customdomain.com/org/repo.git",
commit: "xyz123",
},
want: "https://onprem.customdomain.com/org/repo/commit/xyz123",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
+6 -2
View File
@@ -1000,6 +1000,7 @@ const (
DetectorType_HuggingFace DetectorType = 926
DetectorType_Snowflake DetectorType = 927
DetectorType_Sourcegraph DetectorType = 928
DetectorType_Tailscale DetectorType = 929
)
// Enum value maps for DetectorType.
@@ -1930,6 +1931,7 @@ var (
926: "HuggingFace",
927: "Snowflake",
928: "Sourcegraph",
929: "Tailscale",
}
DetectorType_value = map[string]int32{
"Alibaba": 0,
@@ -2857,6 +2859,7 @@ var (
"HuggingFace": 926,
"Snowflake": 927,
"Sourcegraph": 928,
"Tailscale": 929,
}
)
@@ -3235,7 +3238,7 @@ var file_detectors_proto_rawDesc = []byte{
0x44, 0x65, 0x63, 0x6f, 0x64, 0x65, 0x72, 0x54, 0x79, 0x70, 0x65, 0x12, 0x0b, 0x0a, 0x07, 0x55,
0x4e, 0x4b, 0x4e, 0x4f, 0x57, 0x4e, 0x10, 0x00, 0x12, 0x09, 0x0a, 0x05, 0x50, 0x4c, 0x41, 0x49,
0x4e, 0x10, 0x01, 0x12, 0x0a, 0x0a, 0x06, 0x42, 0x41, 0x53, 0x45, 0x36, 0x34, 0x10, 0x02, 0x12,
0x09, 0x0a, 0x05, 0x55, 0x54, 0x46, 0x31, 0x36, 0x10, 0x03, 0x2a, 0x9e, 0x74, 0x0a, 0x0c, 0x44,
0x09, 0x0a, 0x05, 0x55, 0x54, 0x46, 0x31, 0x36, 0x10, 0x03, 0x2a, 0xae, 0x74, 0x0a, 0x0c, 0x44,
0x65, 0x74, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x54, 0x79, 0x70, 0x65, 0x12, 0x0b, 0x0a, 0x07, 0x41,
0x6c, 0x69, 0x62, 0x61, 0x62, 0x61, 0x10, 0x00, 0x12, 0x08, 0x0a, 0x04, 0x41, 0x4d, 0x51, 0x50,
0x10, 0x01, 0x12, 0x07, 0x0a, 0x03, 0x41, 0x57, 0x53, 0x10, 0x02, 0x12, 0x09, 0x0a, 0x05, 0x41,
@@ -4165,7 +4168,8 @@ var file_detectors_proto_rawDesc = []byte{
0x6f, 0x72, 0x63, 0x65, 0x10, 0x9d, 0x07, 0x12, 0x10, 0x0a, 0x0b, 0x48, 0x75, 0x67, 0x67, 0x69,
0x6e, 0x67, 0x46, 0x61, 0x63, 0x65, 0x10, 0x9e, 0x07, 0x12, 0x0e, 0x0a, 0x09, 0x53, 0x6e, 0x6f,
0x77, 0x66, 0x6c, 0x61, 0x6b, 0x65, 0x10, 0x9f, 0x07, 0x12, 0x10, 0x0a, 0x0b, 0x53, 0x6f, 0x75,
0x72, 0x63, 0x65, 0x67, 0x72, 0x61, 0x70, 0x68, 0x10, 0xa0, 0x07, 0x42, 0x3d, 0x5a, 0x3b, 0x67,
0x72, 0x63, 0x65, 0x67, 0x72, 0x61, 0x70, 0x68, 0x10, 0xa0, 0x07, 0x12, 0x0e, 0x0a, 0x09, 0x54,
0x61, 0x69, 0x6c, 0x73, 0x63, 0x61, 0x6c, 0x65, 0x10, 0xa1, 0x07, 0x42, 0x3d, 0x5a, 0x3b, 0x67,
0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x74, 0x72, 0x75, 0x66, 0x66, 0x6c,
0x65, 0x73, 0x65, 0x63, 0x75, 0x72, 0x69, 0x74, 0x79, 0x2f, 0x74, 0x72, 0x75, 0x66, 0x66, 0x6c,
0x65, 0x68, 0x6f, 0x67, 0x2f, 0x76, 0x33, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x70, 0x62, 0x2f, 0x64,
+1 -82
View File
@@ -60,7 +60,6 @@ const (
SourceType_SOURCE_TYPE_SHAREPOINT SourceType = 29
SourceType_SOURCE_TYPE_GCS_UNAUTHED SourceType = 30
SourceType_SOURCE_TYPE_AZURE_REPOS SourceType = 31
SourceType_SOURCE_TYPE_TRELLO SourceType = 32
)
// Enum value maps for SourceType.
@@ -98,7 +97,6 @@ var (
29: "SOURCE_TYPE_SHAREPOINT",
30: "SOURCE_TYPE_GCS_UNAUTHED",
31: "SOURCE_TYPE_AZURE_REPOS",
32: "SOURCE_TYPE_AZURE_TRELLO",
}
SourceType_value = map[string]int32{
"SOURCE_TYPE_AZURE_STORAGE": 0,
@@ -133,7 +131,6 @@ var (
"SOURCE_TYPE_SHAREPOINT": 29,
"SOURCE_TYPE_GCS_UNAUTHED": 30,
"SOURCE_TYPE_AZURE_REPOS": 31,
"SOURCE_TYPE_AZURE_TRELLO": 32,
}
)
@@ -526,80 +523,6 @@ func (*Bitbucket_Oauth) isBitbucket_Credential() {}
func (*Bitbucket_BasicAuth) isBitbucket_Credential() {}
type Trello struct {
state protoimpl.MessageState
sizeCache protoimpl.SizeCache
unknownFields protoimpl.UnknownFields
Auth *Trello_TrelloAuth `protobuf:"bytes,1,opt,name=auth,proto3,oneof"`
Boards []string `protobuf:"bytes,2,rep,name=boards,proto3" json:"boards,omitempty"`
}
type Trello_TrelloAuth struct {
ApiKey string `protobuf:"bytes,1,opt,name=api_key,json=apiKey,proto3,oneof"`
Token string `protobuf:"bytes,2,opt,name=token,proto3,oneof"`
}
func (x *Trello) Reset() {
*x = Trello{}
if protoimpl.UnsafeEnabled {
mi := &file_sources_proto_msgTypes[4]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
}
func (x *Trello) String() string {
return protoimpl.X.MessageStringOf(x)
}
func (*Trello) ProtoMessage() {}
func (x *Trello) ProtoReflect() protoreflect.Message {
mi := &file_sources_proto_msgTypes[4]
if protoimpl.UnsafeEnabled && x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
ms.StoreMessageInfo(mi)
}
return ms
}
return mi.MessageOf(x)
}
// Deprecated: Use Trello.ProtoReflect.Descriptor instead.
func (*Trello) Descriptor() ([]byte, []int) {
return file_sources_proto_rawDescGZIP(), []int{4} // Assuming Trello is the 5th message in source.proto file
}
func (x *Trello) GetAuth() *Trello_TrelloAuth {
if x != nil {
return x.Auth
}
return nil
}
func (t *Trello_TrelloAuth) GetApiKey() string {
if t != nil && t.ApiKey != nil {
return t.ApiKey
}
return ""
}
func (t *Trello_TrelloAuth) GetToken() string {
if t != nil && t.Token != nil {
return t.Token
}
return ""
}
func (x *Trello) GetBoards() []string {
if x != nil {
return x.Boards
}
return nil
}
type CircleCI struct {
state protoimpl.MessageState
sizeCache protoimpl.SizeCache
@@ -3770,7 +3693,6 @@ var file_sources_proto_goTypes = []interface{}{
(*SlackRealtime)(nil), // 28: sources.SlackRealtime
(*Sharepoint)(nil), // 29: sources.Sharepoint
(*AzureRepos)(nil), // 30: sources.AzureRepos
(*Trello)(nil), // 45: sources.Trello
(*durationpb.Duration)(nil), // 31: google.protobuf.Duration
(*anypb.Any)(nil), // 32: google.protobuf.Any
(*credentialspb.BasicAuth)(nil), // 33: credentials.BasicAuth
@@ -4304,16 +4226,13 @@ func file_sources_proto_init() {
(*AzureRepos_Token)(nil),
(*AzureRepos_Oauth)(nil),
}
file_sources_proto_msgTypes[29].OneofWrappers = []interface{}{
(*Trello_TrelloAuth)(nil),
}
type x struct{}
out := protoimpl.TypeBuilder{
File: protoimpl.DescBuilder{
GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
RawDescriptor: file_sources_proto_rawDesc,
NumEnums: 2,
NumMessages: 30,
NumMessages: 29,
NumExtensions: 0,
NumServices: 0,
},
+1 -1
View File
@@ -75,7 +75,7 @@ func (s *Source) Init(_ context.Context, name string, jobId, sourceId int64, ver
}
// Chunks emits chunks of bytes over a channel.
func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) error {
func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk, _ ...sources.ChunkingTarget) error {
projects, err := s.projects(ctx)
if err != nil {
return fmt.Errorf("error getting projects: %w", err)
+1 -1
View File
@@ -85,7 +85,7 @@ type layerInfo struct {
}
// Chunks emits data over a channel that is decoded and scanned for secrets.
func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) error {
func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk, _ ...sources.ChunkingTarget) error {
ctx = context.WithValues(ctx, "source_type", s.Type(), "source_name", s.name)
workers := new(errgroup.Group)
+1 -1
View File
@@ -77,7 +77,7 @@ func (s *Source) WithFilter(filter *common.Filter) {
}
// Chunks emits chunks of bytes over a channel.
func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) error {
func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk, _ ...sources.ChunkingTarget) error {
for i, path := range s.paths {
logger := ctx.Logger().WithValues("path", path)
if common.IsDone(ctx) {
+1 -1
View File
@@ -248,7 +248,7 @@ func (s *Source) enumerate(ctx context.Context) error {
}
// Chunks emits chunks of bytes over a channel.
func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) error {
func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk, _ ...sources.ChunkingTarget) error {
persistableCache := s.setupCache(ctx)
objectCh, err := s.gcsManager.ListObjects(ctx)
+1 -1
View File
@@ -153,7 +153,7 @@ func (s *Source) Init(aCtx context.Context, name string, jobId, sourceId int64,
}
// Chunks emits chunks of bytes over a channel.
func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) error {
func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk, _ ...sources.ChunkingTarget) error {
if err := s.scanRepos(ctx, chunksChan); err != nil {
return err
}
+1 -1
View File
@@ -413,7 +413,7 @@ func (s *Source) visibilityOf(ctx context.Context, repoURL string) (visibility s
}
// Chunks emits chunks of bytes over a channel.
func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) error {
func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk, _ ...sources.ChunkingTarget) error {
apiEndpoint := s.conn.Endpoint
if len(apiEndpoint) == 0 || endsWithGithub.MatchString(apiEndpoint) {
apiEndpoint = "https://api.github.com"
+1 -1
View File
@@ -139,7 +139,7 @@ func (s *Source) Init(_ context.Context, name string, jobId, sourceId int64, ver
}
// Chunks emits chunks of bytes over a channel.
func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) error {
func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk, _ ...sources.ChunkingTarget) error {
// Start client.
apiClient, err := s.newClient()
if err != nil {
+1 -1
View File
@@ -232,7 +232,7 @@ func (s *Source) scanBuckets(ctx context.Context, client *s3.S3, role string, bu
}
// Chunks emits chunks of bytes over a channel.
func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) error {
func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk, _ ...sources.ChunkingTarget) error {
visitor := func(c context.Context, defaultRegionClient *s3.S3, roleArn string, buckets []string) {
s.scanBuckets(c, defaultRegionClient, roleArn, buckets, chunksChan)
}
+11 -8
View File
@@ -6,10 +6,11 @@ import (
"testing"
"github.com/stretchr/testify/assert"
"google.golang.org/protobuf/types/known/anypb"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/context"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
"google.golang.org/protobuf/types/known/anypb"
)
// DummySource implements Source and is used for testing a SourceManager.
@@ -31,7 +32,7 @@ func (d *DummySource) GetProgress() *Progress { return nil }
// Interface to easily test different chunking methods.
type chunker interface {
Chunks(context.Context, chan *Chunk) error
Chunks(context.Context, chan *Chunk, ...ChunkingTarget) error
ChunkUnit(ctx context.Context, unit SourceUnit, reporter ChunkReporter) error
Enumerate(ctx context.Context, reporter UnitReporter) error
}
@@ -42,7 +43,7 @@ type counterChunker struct {
count int
}
func (c *counterChunker) Chunks(ctx context.Context, ch chan *Chunk) error {
func (c *counterChunker) Chunks(ctx context.Context, ch chan *Chunk, _ ...ChunkingTarget) error {
for i := 0; i < c.count; i++ {
select {
case ch <- &Chunk{Data: []byte{c.chunkCounter}}:
@@ -75,9 +76,9 @@ func (c *counterChunker) ChunkUnit(ctx context.Context, unit SourceUnit, reporte
// Chunk method that always returns an error.
type errorChunker struct{ error }
func (c errorChunker) Chunks(context.Context, chan *Chunk) error { return c }
func (c errorChunker) Enumerate(context.Context, UnitReporter) error { return c }
func (c errorChunker) ChunkUnit(context.Context, SourceUnit, ChunkReporter) error { return c }
func (c errorChunker) Chunks(context.Context, chan *Chunk, ...ChunkingTarget) error { return c }
func (c errorChunker) Enumerate(context.Context, UnitReporter) error { return c }
func (c errorChunker) ChunkUnit(context.Context, SourceUnit, ChunkReporter) error { return c }
// enrollDummy is a helper function to enroll a DummySource with a SourceManager.
func enrollDummy(mgr *SourceManager, chunkMethod chunker) (handle, error) {
@@ -176,7 +177,7 @@ type unitChunk struct {
type unitChunker struct{ steps []unitChunk }
func (c *unitChunker) Chunks(ctx context.Context, ch chan *Chunk) error {
func (c *unitChunker) Chunks(ctx context.Context, ch chan *Chunk, _ ...ChunkingTarget) error {
for _, step := range c.steps {
if step.err != "" {
continue
@@ -294,7 +295,9 @@ type callbackChunker struct {
cb func(context.Context, chan *Chunk) error
}
func (c callbackChunker) Chunks(ctx context.Context, ch chan *Chunk) error { return c.cb(ctx, ch) }
func (c callbackChunker) Chunks(ctx context.Context, ch chan *Chunk, _ ...ChunkingTarget) error {
return c.cb(ctx, ch)
}
func (c callbackChunker) Enumerate(context.Context, UnitReporter) error { return nil }
func (c callbackChunker) ChunkUnit(context.Context, SourceUnit, ChunkReporter) error { return nil }
+16 -2
View File
@@ -30,6 +30,16 @@ type Chunk struct {
Verify bool
}
// ChunkingTarget specifies criteria for a targeted chunking process.
// Instead of collecting data indiscriminately, this struct allows the caller
// to specify particular subsets of data they're interested in. This becomes
// especially useful when one needs to verify or recheck specific data points
// without processing the entire dataset.
type ChunkingTarget struct {
// QueryCriteria represents specific parameters or conditions to target the chunking process.
QueryCriteria source_metadatapb.MetaData
}
// Source defines the interface required to implement a source chunker.
type Source interface {
// Type returns the source type, used for matching against configuration and jobs.
@@ -40,8 +50,12 @@ type Source interface {
JobID() int64
// Init initializes the source.
Init(aCtx context.Context, name string, jobId, sourceId int64, verify bool, connection *anypb.Any, concurrency int) error
// Chunks emits data over a channel that is decoded and scanned for secrets.
Chunks(ctx context.Context, chunksChan chan *Chunk) error
// Chunks emits data over a channel which is then decoded and scanned for secrets.
// By default, data is obtained indiscriminately. However, by providing one or more
// ChunkingTarget parameters, the caller can direct the function to retrieve
// specific chunks of data. This targeted approach allows for efficient and
// intentional data processing, beneficial when verifying or rechecking specific data points.
Chunks(ctx context.Context, chunksChan chan *Chunk, targets ...ChunkingTarget) error
// GetProgress is the completion progress (percentage) for Scanned Source.
GetProgress() *Progress
}
+1 -1
View File
@@ -182,7 +182,7 @@ func (s *Source) verifyConnectionConfig() error {
}
// Chunks emits chunks of bytes over a channel.
func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) error {
func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk, _ ...sources.ChunkingTarget) error {
switch {
case s.conn.TlsCert != nilString || s.conn.TlsKey != nilString:
cert, err := tls.X509KeyPair([]byte(s.conn.TlsCert), []byte(s.conn.TlsKey))
-239
View File
@@ -1,239 +0,0 @@
package trello
import (
"encoding/json"
"fmt"
"net/http"
"sync/atomic"
"github.com/go-errors/errors"
"google.golang.org/protobuf/proto"
"google.golang.org/protobuf/types/known/anypb"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/context"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/source_metadatapb"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
)
const baseURL = "https://api.trello.com/1/"
type Source struct {
name string
apiKey string
token string
boardIDs []string
sourceId int64
jobId int64
verify bool
jobPool *errgroup.Group
sources.Progress
client *http.Client
sources.CommonSourceUnitUnmarshaller
}
// Type returns the type of the source.
func (s *Source) Type() sourcespb.SourceType {
return sourcespb.SourceType_SOURCE_TYPE_TRELLO
}
// Init returns an initialized Source.
func (s *Source) Init(_ context.Context, name string, jobId, sourceId int64, verify bool, connection *anypb.Any, concurrency int) error {
s.name = name
s.sourceId = sourceId
s.jobId = jobId
s.verify = verify
s.jobPool = &errgroup.Group{}
s.jobPool.SetLimit(concurrency)
s.client = common.RetryableHttpClientTimeout(3)
var conn sourcespb.Trello
if err := anypb.UnmarshalTo(connection, &conn, proto.UnmarshalOptions{}); err != nil {
return errors.WrapPrefix(err, "error unmarshalling connection", 0)
}
s.apiKey = conn.GetAuth().GetApiKey()
s.token = conn.GetAuth().GetToken()
s.boardIDs = conn.GetBoards()
return nil
}
// Chunks emits chunks of bytes over a channel.
func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) error {
scanErrs := sources.NewScanErrors()
for _, boardID := range s.boardIDs {
board, err := s.getBoard(ctx, boardID)
if err != nil {
scanErrs.Add(err)
return nil
}
if err = s.chunkBoard(ctx, board, chunksChan); err != nil {
scanErrs.Add(err)
return nil
}
}
_ = s.jobPool.Wait()
if scanErrs.Count() > 0 {
ctx.Logger().V(2).Info("encountered errors while scanning", "count", scanErrs.Count(), "errors", scanErrs)
}
return nil
}
type board struct {
Name string `json:"name"`
Desc string `json:"desc"`
ID string `json:"id"`
URL string `json:"url"`
}
type card struct {
Name string `json:"name"`
Desc string `json:"desc"`
ID string `json:"id"`
URL string `json:"url"`
}
type action struct {
Type string `json:"type"`
Data struct {
Text string `json:"text"`
} `json:"data"`
}
// Add these two methods to Source
func (s *Source) getCards(_ context.Context, boardID string) ([]card, error) {
reqURL := fmt.Sprintf("%sboards/%s/cards?key=%s&token=%s", baseURL, boardID, s.apiKey, s.token)
req, err := http.NewRequest("GET", reqURL, nil)
if err != nil {
return nil, err
}
req.Header.Set("Accept", "application/json")
res, err := s.client.Do(req)
if err != nil {
return nil, err
}
defer res.Body.Close()
var cards []card
if err := json.NewDecoder(res.Body).Decode(&cards); err != nil {
return nil, err
}
return cards, nil
}
func (s *Source) getComments(_ context.Context, cardID string) ([]action, error) {
reqURL := fmt.Sprintf("%scards/%s/actions?key=%s&token=%s", baseURL, cardID, s.apiKey, s.token)
req, err := http.NewRequest("GET", reqURL, nil)
if err != nil {
return nil, err
}
req.Header.Set("Accept", "application/json")
res, err := s.client.Do(req)
if err != nil {
return nil, err
}
defer res.Body.Close()
var actions []action
if err := json.NewDecoder(res.Body).Decode(&actions); err != nil {
return nil, err
}
return actions, nil
}
func (s *Source) getBoard(_ context.Context, boardID string) (*board, error) {
reqURL := fmt.Sprintf("%sboards/%s?key=%s&token=%s", baseURL, boardID, s.apiKey, s.token)
req, err := http.NewRequest("GET", reqURL, nil)
if err != nil {
return nil, err
}
req.Header.Set("Accept", "application/json")
res, err := s.client.Do(req)
if err != nil {
return nil, err
}
defer res.Body.Close()
var b board
if err := json.NewDecoder(res.Body).Decode(&b); err != nil {
return nil, err
}
return &b, nil
}
func (s *Source) chunkBoard(ctx context.Context, board *board, chunksChan chan *sources.Chunk) error {
err := s.chunkItem(ctx, chunksChan, board.ID, board.Desc, board.URL, source_metadatapb.ItemType_ITEM_TYPE_BOARD, board.Name)
if err != nil {
return err
}
cards, err := s.getCards(ctx, board.ID)
if err != nil {
return err
}
for _, card := range cards {
err := s.chunkItem(ctx, chunksChan, card.ID, card.Desc, card.URL, source_metadatapb.ItemType_ITEM_TYPE_CARD, card.Name)
if err != nil {
return err
}
comments, err := s.getComments(ctx, card.ID)
if err != nil {
return err
}
for _, comment := range comments {
if comment.Type != "commentCard" {
continue
}
err := s.chunkItem(ctx, chunksChan, card.ID, comment.Data.Text, card.URL, source_metadatapb.ItemType_ITEM_TYPE_COMMENT, "")
if err != nil {
return err
}
}
}
return nil
}
func (s *Source) chunkItem(ctx context.Context, chunksChan chan *sources.Chunk, id, text, url string, itemType source_metadatapb.ItemType, name string) error {
data := []byte(text)
chunk := &sources.Chunk{
SourceType: s.Type(),
SourceName: s.name,
SourceID: s.sourceId,
JobID: s.jobId,
Data: data,
SourceMetadata: &source_metadatapb.MetaData{
Data: &source_metadatapb.MetaData_Trello{
Trello: &source_metadatapb.Trello{
Id: id,
Url: url,
Type: itemType,
Name: name
},
},
},
Verify: s.verify,
}
atomic.AddInt64(&s.scannedBytes, int64(len(data)))
if err := common.CancellableWrite(ctx, chunksChan, chunk); err != nil {
return err
}
return nil
}
-94
View File
@@ -1,94 +0,0 @@
package trello
import (
"context"
"fmt"
"testing"
"time"
"github.com/kylelemons/godebug/pretty"
"google.golang.org/protobuf/types/known/anypb"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/context"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/source_metadatapb"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
)
func TestSource_Scan(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*3)
defer cancel()
secret, err := common.GetTestSecret(ctx)
if err != nil {
t.Fatal(fmt.Errorf("failed to access secret: %v", err))
}
apiKey := secret.MustGetField("TRELLO_API_KEY")
token := secret.MustGetField("TRELLO_TOKEN")
type init struct {
name string
verify bool
connection *sourcespb.Trello
}
tests := []struct {
name string
init init
wantSourceMetadata *source_metadatapb.MetaData
wantErr bool
}{
{
name: "get a chunk",
init: init{
name: "example board",
connection: &sourcespb.Trello{
Auth: &sourcespb.Trello_TrelloAuth{
ApiKey: apiKey,
Token: token,
},
Boards: []string{"boardID1"},
},
verify: true,
},
wantSourceMetadata: &source_metadatapb.MetaData{
Data: &source_metadatapb.MetaData_Trello{
Trello: &source_metadatapb.Trello{
BoardId: "boardID1",
BoardName: "Example Board",
CardId: "cardID1",
},
},
},
wantErr: false,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
s := Source{}
conn, err := anypb.New(tt.init.connection)
if err != nil {
t.Fatal(err)
}
if err := s.Init(ctx, tt.init.name, 0, 0, tt.init.verify, conn, 5); (err != nil) != tt.wantErr {
t.Errorf("Source.Init() error = %v, wantErr %v", err, tt.wantErr)
return
}
chunksCh := make(chan *sources.Chunk, 1)
go func() {
if err := s.Chunks(ctx, chunksCh); (err != nil) != tt.wantErr {
t.Errorf("Source.Chunks() error = %v, wantErr %v", err, tt.wantErr)
return
}
}()
gotChunk := <-chunksCh
if diff := pretty.Compare(gotChunk.SourceMetadata, tt.wantSourceMetadata); diff != "" {
t.Errorf("Source.Chunks() %s diff: (-got +want)\n%s", tt.name, diff)
}
})
}
}
@@ -68,7 +68,6 @@ func New(c common.Common) *SourceSelect {
OssItem("Filesystem", "Scan your filesystem by selecting what directories to scan."),
OssItem("AWS S3", "Scan Amazon S3 buckets."),
OssItem("CircleCI", "Scan CircleCI, a CI/CD platform."),
OssItem("Trello", "Scan Trello, a ticket system."),
OssItem("Syslog", "Scan syslog, event data logs."),
OssItem("Docker", "Scan a Docker instance, a containerized application."),
OssItem("GCS (Google Cloud Storage)", "Scan a Google Cloud Storage instance."),
-3
View File
@@ -5,7 +5,6 @@ import (
tea "github.com/charmbracelet/bubbletea"
"github.com/trufflesecurity/trufflehog/v3/pkg/tui/sources/circleci"
"github.com/trufflesecurity/trufflehog/v3/pkg/tui/sources/trello"
"github.com/trufflesecurity/trufflehog/v3/pkg/tui/sources/docker"
"github.com/trufflesecurity/trufflehog/v3/pkg/tui/sources/filesystem"
"github.com/trufflesecurity/trufflehog/v3/pkg/tui/sources/gcs"
@@ -53,8 +52,6 @@ func GetSourceFields(sourceName string) CmdModel {
return syslog.GetFields()
case "circleci":
return circleci.GetFields()
case "trello":
return trello.GetFields()
case "docker":
return docker.GetFields()
}
-64
View File
@@ -1,64 +0,0 @@
package trello
import (
"strings"
"github.com/trufflesecurity/trufflehog/v3/pkg/tui/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/tui/components/textinputs"
)
type trelloCmdModel struct {
textinputs.Model
}
func GetFields() trelloCmdModel {
apiKey := textinputs.InputConfig{
Label: "API Key",
Key: "api_key",
Required: true,
Placeholder: "Your Trello API key",
}
token := textinputs.InputConfig{
Label: "Token",
Key: "token",
Required: true,
Placeholder: "Your Trello token",
}
boardIds := textinputs.InputConfig{
Label: "Board IDs",
Key: "board_ids",
Required: true,
Placeholder: "Comma-separated list of Trello board ID(s)",
}
return trelloCmdModel{textinputs.New([]textinputs.InputConfig{apiKey, token, boardIds})}
}
func (m trelloCmdModel) Cmd() string {
var command []string
command = append(command, "trufflehog", "trello")
inputs := m.GetInputs()
if inputs["api_key"] != "" {
command = append(command, "--api-key="+inputs["api_key"])
}
if inputs["token"] != "" {
command = append(command, "--token="+inputs["token"])
}
if inputs["board_ids"] != "" {
command = append(command, "--board-ids="+inputs["board_ids"])
}
return strings.Join(command, " ")
}
func (m trelloCmdModel) Summary() string {
inputs := m.GetInputs()
labels := m.GetLabels()
keys := []string{"api_key", "token", "board_ids"}
return common.SummarizeSource(keys, inputs, labels)
}
+1
View File
@@ -937,6 +937,7 @@ enum DetectorType {
HuggingFace = 926;
Snowflake = 927;
Sourcegraph = 928;
Tailscale = 929;
}
message Result {
-12
View File
@@ -186,17 +186,6 @@ message Test {
string file = 1;
}
message Trello {
string board_id = 1;
string card_id = 2;
string list_id = 3;
string action_id = 4;
string user_id = 5;
string username = 6;
string timestamp = 7;
string link = 8;
}
message Jenkins {
string project_name = 1;
int64 build_number = 2;
@@ -305,6 +294,5 @@ message MetaData {
SharePoint sharepoint = 25;
GoogleDrive googleDrive = 26;
AzureRepos azureRepos = 27;
Trello trello = 28;
}
}
-12
View File
@@ -44,7 +44,6 @@ enum SourceType {
SOURCE_TYPE_SHAREPOINT = 29;
SOURCE_TYPE_GCS_UNAUTHED = 30;
SOURCE_TYPE_AZURE_REPOS = 31;
SOURCE_TYPE_TRELLO = 32;
}
message LocalSource {
@@ -266,17 +265,6 @@ message Jenkins {
}
}
message Trello {
message TrelloAuth {
string api_key = 1;
string token = 2;
}
TrelloAuth auth = 1;
repeated string boards = 2;
}
message Teams {
string endpoint = 1 [(validate.rules).string.uri_ref = true];
oneof credential {