Compare commits

..

5 Commits

Author SHA1 Message Date
Ahrav Dutta d58ff5b6fd remove mutex usage around failFast. 2023-09-08 11:43:17 -07:00
Ahrav Dutta c9c0a7f9bf replace channel w/ bool. 2023-09-07 19:52:19 -07:00
Ahrav Dutta 64880385c4 refctor. 2023-09-07 19:42:34 -07:00
Ahrav Dutta 6195932842 update comment. 2023-09-07 19:14:08 -07:00
Ahrav Dutta 641a0df72d Allow failing fast. 2023-09-07 19:07:03 -07:00
14 changed files with 269 additions and 434 deletions
+11 -6
View File
@@ -53,6 +53,7 @@ var (
printAvgDetectorTime = cli.Flag("print-avg-detector-time", "Print the average time spent on each detector.").Bool()
noUpdate = cli.Flag("no-update", "Don't check for updates.").Bool()
fail = cli.Flag("fail", "Exit with code 183 if results are found.").Bool()
failFast = cli.Flag("fail-fast", "Exit with code 1 as soon as a single result is found.").Bool()
verifiers = cli.Flag("verifier", "Set custom verification endpoints.").StringMap()
archiveMaxSize = cli.Flag("archive-max-size", "Maximum size of archive to scan. (Byte units eg. 512B, 2KB, 4MB)").Bytes()
archiveMaxDepth = cli.Flag("archive-max-depth", "Maximum depth of archive to scan.").Int()
@@ -105,11 +106,11 @@ var (
filesystemScanIncludePaths = filesystemScan.Flag("include-paths", "Path to file with newline separated regexes for files to include in scan.").Short('i').String()
filesystemScanExcludePaths = filesystemScan.Flag("exclude-paths", "Path to file with newline separated regexes for files to exclude in scan.").Short('x').String()
s3Scan = cli.Command("s3", "Find credentials in S3 buckets.")
s3ScanKey = s3Scan.Flag("key", "S3 key used to authenticate. Can be provided with environment variable AWS_ACCESS_KEY_ID.").Envar("AWS_ACCESS_KEY_ID").String()
s3ScanRoleArns = s3Scan.Flag("role-arn", "Specify the ARN of an IAM role to assume for scanning. You can repeat this flag.").Strings()
s3ScanSecret = s3Scan.Flag("secret", "S3 secret used to authenticate. Can be provided with environment variable AWS_SECRET_ACCESS_KEY.").Envar("AWS_SECRET_ACCESS_KEY").String()
s3ScanSessionToken = s3Scan.Flag("session-token", "S3 session token used to authenticate temporary credentials. Can be provided with environment variable AWS_SESSION_TOKEN.").Envar("AWS_SESSION_TOKEN").String()
s3Scan = cli.Command("s3", "Find credentials in S3 buckets.")
s3ScanKey = s3Scan.Flag("key", "S3 key used to authenticate. Can be provided with environment variable AWS_ACCESS_KEY_ID.").Envar("AWS_ACCESS_KEY_ID").String()
s3ScanRoleArns = s3Scan.Flag("role-arn", "Specify the ARN of an IAM role to assume for scanning. You can repeat this flag.").Strings()
s3ScanSecret = s3Scan.Flag("secret", "S3 secret used to authenticate. Can be provided with environment variable AWS_SECRET_ACCESS_KEY.").Envar("AWS_SECRET_ACCESS_KEY").String()
s3ScanSessionToken = s3Scan.Flag("session-token", "S3 session token used to authenticate temporary credentials. Can be provided with environment variable AWS_SESSION_TOKEN.").Envar("AWS_SESSION_TOKEN").String()
s3ScanCloudEnv = s3Scan.Flag("cloud-environment", "Use IAM credentials in cloud environment.").Bool()
s3ScanBuckets = s3Scan.Flag("bucket", "Name of S3 bucket to scan. You can repeat this flag.").Strings()
s3ScanMaxObjectSize = s3Scan.Flag("max-object-size", "Maximum size of objects to scan. Objects larger than this will be skipped. (Byte units eg. 512B, 2KB, 4MB)").Default("250MB").Bytes()
@@ -366,6 +367,7 @@ func run(state overseer.State) {
engine.WithOnlyVerified(*onlyVerified),
engine.WithPrintAvgDetectorTime(*printAvgDetectorTime),
engine.WithPrinter(printer),
engine.WithFailFast(*failFast),
)
if err != nil {
logFatal(err, "error initializing engine")
@@ -469,7 +471,7 @@ func run(state overseer.State) {
Secret: *s3ScanSecret,
SessionToken: *s3ScanSessionToken,
Buckets: *s3ScanBuckets,
Roles: *s3ScanRoleArns,
Roles: *s3ScanRoleArns,
CloudCred: *s3ScanCloudEnv,
MaxObjectSize: int64(*s3ScanMaxObjectSize),
}
@@ -533,6 +535,9 @@ func run(state overseer.State) {
if err = e.Finish(ctx); err != nil {
logFatal(err, "engine failed to finish execution")
}
if e.IsTerminated() {
os.Exit(1)
}
metrics := e.GetMetrics()
// Print results.
+110 -132
View File
@@ -118,21 +118,118 @@ func (s scanner) FromData(ctx context.Context, verify bool, data []byte) (result
}
if verify {
verified, extraData, verificationErr := s.verifyMatch(ctx, resIDMatch, resSecretMatch, true)
s1.Verified = verified
s1.ExtraData = extraData
s1.VerificationError = verificationErr
// REQUEST VALUES.
method := "GET"
service := "sts"
host := "sts.amazonaws.com"
region := "us-east-1"
endpoint := "https://sts.amazonaws.com"
now := time.Now().UTC()
datestamp := now.Format("20060102")
amzDate := now.Format("20060102T150405Z0700")
req, err := http.NewRequestWithContext(ctx, method, endpoint, nil)
if err != nil {
continue
}
req.Header.Set("Accept", "application/json")
// TASK 1: CREATE A CANONICAL REQUEST.
// http://docs.aws.amazon.com/general/latest/gr/sigv4-create-canonical-request.html
canonicalURI := "/"
canonicalHeaders := "host:" + host + "\n"
signedHeaders := "host"
algorithm := "AWS4-HMAC-SHA256"
credentialScope := fmt.Sprintf("%s/%s/%s/aws4_request", datestamp, region, service)
params := req.URL.Query()
params.Add("Action", "GetCallerIdentity")
params.Add("Version", "2011-06-15")
params.Add("X-Amz-Algorithm", algorithm)
params.Add("X-Amz-Credential", resIDMatch+"/"+credentialScope)
params.Add("X-Amz-Date", amzDate)
params.Add("X-Amz-Expires", "30")
params.Add("X-Amz-SignedHeaders", signedHeaders)
canonicalQuerystring := params.Encode()
payloadHash := GetHash("") // empty payload
canonicalRequest := method + "\n" + canonicalURI + "\n" + canonicalQuerystring + "\n" + canonicalHeaders + "\n" + signedHeaders + "\n" + payloadHash
// TASK 2: CREATE THE STRING TO SIGN.
stringToSign := algorithm + "\n" + amzDate + "\n" + credentialScope + "\n" + GetHash(canonicalRequest)
// TASK 3: CALCULATE THE SIGNATURE.
// https://docs.aws.amazon.com/general/latest/gr/sigv4-calculate-signature.html
hash := GetHMAC([]byte(fmt.Sprintf("AWS4%s", resSecretMatch)), []byte(datestamp))
hash = GetHMAC(hash, []byte(region))
hash = GetHMAC(hash, []byte(service))
hash = GetHMAC(hash, []byte("aws4_request"))
signature2 := GetHMAC(hash, []byte(stringToSign)) // Get Signature HMAC SHA256
signature := hex.EncodeToString(signature2)
// TASK 4: ADD SIGNING INFORMATION TO THE REQUEST.
params.Add("X-Amz-Signature", signature)
req.Header.Add("Content-type", "application/x-www-form-urlencoded; charset=utf-8")
req.URL.RawQuery = params.Encode()
client := s.verificationClient
if client == nil {
client = defaultVerificationClient
}
res, err := client.Do(req)
if err == nil {
if res.StatusCode >= 200 && res.StatusCode < 300 {
identityInfo := identityRes{}
err := json.NewDecoder(res.Body).Decode(&identityInfo)
if err == nil {
s1.Verified = true
s1.ExtraData = map[string]string{
"account": identityInfo.GetCallerIdentityResponse.GetCallerIdentityResult.Account,
"user_id": identityInfo.GetCallerIdentityResponse.GetCallerIdentityResult.UserID,
"arn": identityInfo.GetCallerIdentityResponse.GetCallerIdentityResult.Arn,
}
} else {
s1.VerificationError = err
}
res.Body.Close()
} else {
// This function will check false positives for common test words, but also it will make sure the key appears "random" enough to be a real key.
if detectors.IsKnownFalsePositive(resSecretMatch, detectors.DefaultFalsePositives, true) {
continue
}
if res.StatusCode == 403 {
var body awsErrorResponseBody
err = json.NewDecoder(res.Body).Decode(&body)
res.Body.Close()
if err == nil {
// All instances of the code I've seen in the wild are PascalCased but this check is
// case-insensitive out of an abundance of caution
if strings.EqualFold(body.Error.Code, "InvalidClientTokenId") {
// determinate failure - nothing to do
} else {
// We see a surprising number of false-negative signature mismatch errors here
// (The official SDK somehow elicits even more than just making the request
// ourselves)
s1.VerificationError = fmt.Errorf("request to %v returned status %d with an unexpected reason (%s: %s)", res.Request.URL, res.StatusCode, body.Error.Code, body.Error.Message)
}
} else {
s1.VerificationError = fmt.Errorf("couldn't parse the sts response body (%v)", err)
}
} else {
s1.VerificationError = fmt.Errorf("request to %v returned unexpected status %d", res.Request.URL, res.StatusCode)
}
}
} else {
s1.VerificationError = err
}
}
if !s1.Verified {
// Unverified results that contain common test words are probably not secrets
if detectors.IsKnownFalsePositive(resSecretMatch, detectors.DefaultFalsePositives, true) {
continue
}
// Unverified results that look like hashes are probably not secrets
if falsePositiveSecretCheck.MatchString(resSecretMatch) {
continue
}
// If the result is unverified and matches something like a git hash, don't include it in the results.
if !s1.Verified && falsePositiveSecretCheck.MatchString(resSecretMatch) {
continue
}
results = append(results, s1)
@@ -145,125 +242,6 @@ func (s scanner) FromData(ctx context.Context, verify bool, data []byte) (result
return awsCustomCleanResults(results), nil
}
func (s scanner) verifyMatch(ctx context.Context, resIDMatch, resSecretMatch string, retryOn403 bool) (bool, map[string]string, error) {
// REQUEST VALUES.
method := "GET"
service := "sts"
host := "sts.amazonaws.com"
region := "us-east-1"
endpoint := "https://sts.amazonaws.com"
now := time.Now().UTC()
datestamp := now.Format("20060102")
amzDate := now.Format("20060102T150405Z0700")
req, err := http.NewRequestWithContext(ctx, method, endpoint, nil)
if err != nil {
return false, nil, err
}
req.Header.Set("Accept", "application/json")
// TASK 1: CREATE A CANONICAL REQUEST.
// http://docs.aws.amazon.com/general/latest/gr/sigv4-create-canonical-request.html
canonicalURI := "/"
canonicalHeaders := "host:" + host + "\n"
signedHeaders := "host"
algorithm := "AWS4-HMAC-SHA256"
credentialScope := fmt.Sprintf("%s/%s/%s/aws4_request", datestamp, region, service)
params := req.URL.Query()
params.Add("Action", "GetCallerIdentity")
params.Add("Version", "2011-06-15")
params.Add("X-Amz-Algorithm", algorithm)
params.Add("X-Amz-Credential", resIDMatch+"/"+credentialScope)
params.Add("X-Amz-Date", amzDate)
params.Add("X-Amz-Expires", "30")
params.Add("X-Amz-SignedHeaders", signedHeaders)
canonicalQuerystring := params.Encode()
payloadHash := GetHash("") // empty payload
canonicalRequest := method + "\n" + canonicalURI + "\n" + canonicalQuerystring + "\n" + canonicalHeaders + "\n" + signedHeaders + "\n" + payloadHash
// TASK 2: CREATE THE STRING TO SIGN.
stringToSign := algorithm + "\n" + amzDate + "\n" + credentialScope + "\n" + GetHash(canonicalRequest)
// TASK 3: CALCULATE THE SIGNATURE.
// https://docs.aws.amazon.com/general/latest/gr/sigv4-calculate-signature.html
hash := GetHMAC([]byte(fmt.Sprintf("AWS4%s", resSecretMatch)), []byte(datestamp))
hash = GetHMAC(hash, []byte(region))
hash = GetHMAC(hash, []byte(service))
hash = GetHMAC(hash, []byte("aws4_request"))
signature2 := GetHMAC(hash, []byte(stringToSign)) // Get Signature HMAC SHA256
signature := hex.EncodeToString(signature2)
// TASK 4: ADD SIGNING INFORMATION TO THE REQUEST.
params.Add("X-Amz-Signature", signature)
req.Header.Add("Content-type", "application/x-www-form-urlencoded; charset=utf-8")
req.URL.RawQuery = params.Encode()
client := s.verificationClient
if client == nil {
client = defaultVerificationClient
}
res, err := client.Do(req)
if err == nil {
if res.StatusCode >= 200 && res.StatusCode < 300 {
identityInfo := identityRes{}
err := json.NewDecoder(res.Body).Decode(&identityInfo)
res.Body.Close()
if err == nil {
extraData := map[string]string{
"account": identityInfo.GetCallerIdentityResponse.GetCallerIdentityResult.Account,
"user_id": identityInfo.GetCallerIdentityResponse.GetCallerIdentityResult.UserID,
"arn": identityInfo.GetCallerIdentityResponse.GetCallerIdentityResult.Arn,
}
return true, extraData, nil
} else {
return false, nil, err
}
} else if res.StatusCode == 403 {
// Experimentation has indicated that if you make two GetCallerIdentity requests within five seconds that
// share a key ID but are signed with different secrets the second one will be rejected with a 403 that
// carries a SignatureDoesNotMatch code in its body. This happens even if the second ID-secret pair is
// valid. Since this is exactly our access pattern, we need to work around it.
//
// Fortunately, experimentation has also revealed a workaround: simply resubmit the second request. The
// response to the resubmission will be as expected. But there's a caveat: You can't have closed the body of
// the response to the original second request, or read to its end, or the resubmission will also yield a
// SignatureDoesNotMatch. For this reason, we have to re-request all 403s. We can't re-request only
// SignatureDoesNotMatch responses, because we can only tell whether a given 403 is a SignatureDoesNotMatch
// after decoding its response body, which requires reading the entire response body, which disables the
// workaround.
//
// We are clearly deep in the guts of AWS implementation details here, so this all might change with no
// notice. If you're here because something in this detector broke, you have my condolences.
if retryOn403 {
return s.verifyMatch(ctx, resIDMatch, resSecretMatch, false)
}
var body awsErrorResponseBody
err = json.NewDecoder(res.Body).Decode(&body)
res.Body.Close()
if err == nil {
// All instances of the code I've seen in the wild are PascalCased but this check is
// case-insensitive out of an abundance of caution
if strings.EqualFold(body.Error.Code, "InvalidClientTokenId") {
return false, nil, nil
} else {
return false, nil, fmt.Errorf("request to %v returned status %d with an unexpected reason (%s: %s)", res.Request.URL, res.StatusCode, body.Error.Code, body.Error.Message)
}
} else {
return false, nil, fmt.Errorf("couldn't parse the sts response body (%v)", err)
}
} else {
return false, nil, fmt.Errorf("request to %v returned unexpected status %d", res.Request.URL, res.StatusCode)
}
} else {
return false, nil, err
}
}
func awsCustomCleanResults(results []detectors.Result) []detectors.Result {
if len(results) == 0 {
return results
-22
View File
@@ -248,28 +248,6 @@ func TestAWS_FromChunk(t *testing.T) {
wantErr: false,
wantVerificationError: true,
},
{
name: "verified secret checked directly after unverified secret with same key id",
s: scanner{},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("%s\n%s\n%s", inactiveSecret, id, secret)),
verify: true,
},
want: []detectors.Result{
{
DetectorType: detectorspb.DetectorType_AWS,
Verified: true,
Redacted: "AKIASP2TPHJSQH3FJRUX",
ExtraData: map[string]string{
"account": "171436882533",
"arn": "arn:aws:iam::171436882533:user/canarytokens.com@@4dxkh0pdeop3bzu9zx5wob793",
"user_id": "AIDASP2TPHJSUFRSTTZX4",
},
},
},
wantErr: false,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
-85
View File
@@ -1,85 +0,0 @@
package tailscale
import (
"context"
"fmt"
"net/http"
"net/url"
"regexp"
"strings"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
)
type Scanner struct{}
// Ensure the Scanner satisfies the interface at compile time.
var _ detectors.Detector = (*Scanner)(nil)
var (
client = common.SaneHttpClient()
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
keyPat = regexp.MustCompile(`\btskey-[a-z]+-[0-9A-Za-z_]+-[0-9A-Za-z_]+\b`)
)
// Keywords are used for efficiently pre-filtering chunks.
// Use identifiers in the secret preferably, or the provider name.
func (s Scanner) Keywords() []string {
return []string{"tskey-", "tskey-api-", "tskey-oauth-"}
}
// FromData will find and optionally verify Tailscaleapi secrets in a given set of bytes.
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
dataStr := string(data)
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
for _, match := range matches {
resMatch := strings.TrimSpace(match[0])
s1 := detectors.Result{
DetectorType: detectorspb.DetectorType_Tailscale,
Raw: []byte(resMatch),
}
if verify {
const u = "https://api.tailscale.com/api/v2/secret-scanning/verify"
vals := url.Values{"key": []string{resMatch}}
req, err := http.NewRequestWithContext(ctx, http.MethodPost, u, strings.NewReader(vals.Encode()))
if err != nil {
continue
}
req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
res, err := client.Do(req)
if err == nil {
defer res.Body.Close()
switch res.StatusCode {
case http.StatusNoContent:
s1.Verified = true
case http.StatusUnauthorized:
// The secret is determinately not verified (nothing to do)
default:
s1.VerificationError = fmt.Errorf("unexpected HTTP response status %d", res.StatusCode)
}
} else {
s1.VerificationError = err
}
}
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
if !s1.Verified && detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
continue
}
results = append(results, s1)
}
return results, nil
}
func (s Scanner) Type() detectorspb.DetectorType {
return detectorspb.DetectorType_Tailscale
}
-120
View File
@@ -1,120 +0,0 @@
//go:build detectors
// +build detectors
package tailscale
import (
"context"
"fmt"
"testing"
"time"
"github.com/kylelemons/godebug/pretty"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
)
func TestTailscaleapi_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors4")
if err != nil && false {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
secret := testSecrets.MustGetField("TAILSCALEAPI")
inactiveSecret := testSecrets.MustGetField("TAILSCALEAPI_INACTIVE")
type args struct {
ctx context.Context
data []byte
verify bool
}
tests := []struct {
name string
s Scanner
args args
want []detectors.Result
wantErr bool
}{
{
name: "found, verified",
s: Scanner{},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a tailscaleapi secret %s within", secret)),
verify: true,
},
want: []detectors.Result{
{
DetectorType: detectorspb.DetectorType_Tailscale,
Verified: true,
},
},
wantErr: false,
},
{
name: "found, unverified",
s: Scanner{},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a tailscaleapi secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
verify: true,
},
want: []detectors.Result{
{
DetectorType: detectorspb.DetectorType_Tailscale,
Verified: false,
},
},
wantErr: false,
},
{
name: "not found",
s: Scanner{},
args: args{
ctx: context.Background(),
data: []byte("You cannot find the secret within"),
verify: true,
},
want: nil,
wantErr: false,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
s := Scanner{}
got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
if (err != nil) != tt.wantErr {
t.Errorf("Tailscaleapi.FromData() error = %v, wantErr %v", err, tt.wantErr)
return
}
for i := range got {
if len(got[i].Raw) == 0 {
t.Fatalf("no raw secret present: \n %+v", got[i])
}
got[i].Raw = nil
}
if diff := pretty.Compare(got, tt.want); diff != "" {
t.Errorf("Tailscaleapi.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
}
})
}
}
func BenchmarkFromData(benchmark *testing.B) {
ctx := context.Background()
s := Scanner{}
for name, data := range detectors.MustGetBenchmarkData() {
benchmark.Run(name, func(b *testing.B) {
for n := 0; n < b.N; n++ {
_, err := s.FromData(ctx, false, data)
if err != nil {
b.Fatal(err)
}
}
})
}
}
-2
View File
@@ -6,7 +6,6 @@ import (
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/salesforce"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/snowflake"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/sourcegraph"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/tailscale"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/trufflehogenterprise"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
@@ -1543,7 +1542,6 @@ func DefaultDetectors() []detectors.Detector {
trufflehogenterprise.Scanner{},
salesforce.Scanner{},
sourcegraph.Scanner{},
tailscale.Scanner{},
}
}
+132 -33
View File
@@ -51,6 +51,53 @@ type Printer interface {
Print(ctx context.Context, r *detectors.ResultWithMetadata) error
}
// terminationSignal provides a mechanism to safely signal a termination
// event across different goroutines. It uses a combination of a condition variable
// and a mutex to enable safe signaling and waiting, ensuring that the signaling
// is consistent and that there are no race conditions when checking or updating
// the status of the termination.
type terminationSignal struct {
// isTerminated indicates the signal state (true if the engine is terminated).
isTerminated bool
// isFailFast indicates if the engine is in fail-fast mode.
isFailFast bool
// mutex is used to protect access to isFailFast and to synchronize
// the condition variable.
mutex sync.Mutex
// failFastCondition is a condition variable that's signaled when
// a fail-fast event occurs. It's used in Finish to wait for or
// react to this signal.
failFastCondition *sync.Cond
}
// newTerminationSignal initializes a new terminationSignal.
// The failFast argument determines the initial state.
func newTerminationSignal(failFast bool) *terminationSignal {
ts := &terminationSignal{isFailFast: failFast}
// Initialize the condition variable with the mutex.
ts.failFastCondition = sync.NewCond(&ts.mutex)
return ts
}
// signalFinish signals that the engine has finished its work.
// It uses the condition variable to notify any waiting goroutines
// (e.g., the Finish method).
func (t *terminationSignal) signalFinish() {
t.mutex.Lock()
defer t.mutex.Unlock()
t.isTerminated = true
t.failFastCondition.Signal() // Signal the condition to unblock Finish.
}
func (t *terminationSignal) wait() {
t.mutex.Lock()
defer t.mutex.Unlock()
for !t.isTerminated {
t.failFastCondition.Wait()
}
}
type Engine struct {
// CLI flags.
concurrency uint8
@@ -74,6 +121,11 @@ type Engine struct {
workersWg sync.WaitGroup
wgDetectorWorkers sync.WaitGroup
WgNotifier sync.WaitGroup
// termSignal is used to signal the engine to terminate.
// It is used to implement the fail-fast feature,
// where the engine will terminate scanning as soon as
// the first potential secret is detected.
termSignal *terminationSignal
// Runtime information.
metrics runtimeMetrics
@@ -169,6 +221,19 @@ func WithPrinter(printer Printer) EngineOption {
}
}
// WithFailFast returns an EngineOption that configures the engine's termination behavior.
// If 'failFast' is enabled, the engine will terminate scanning upon detecting the first potential secret.
// This mode is valuable in contexts like CI/CD pipelines or preliminary scans where
// immediate feedback is more important than a comprehensive review. In the non-failFast mode,
// the engine continues its operation without waiting for a result.
// The boolean parameter 'failFast' dictates the termination behavior.
func WithFailFast(failFast bool) EngineOption {
return func(e *Engine) {
// Configure the termination signal with the desired mode (failFast or not).
e.termSignal = newTerminationSignal(failFast)
}
}
func filterDetectors(filterFunc func(detectors.Detector) bool, input []detectors.Detector) []detectors.Detector {
var output []detectors.Detector
for _, detector := range input {
@@ -276,6 +341,7 @@ func Start(ctx context.Context, options ...EngineOption) (*Engine, error) {
dedupeCache: cache,
printer: new(output.PlainPrinter), // default printer
metrics: runtimeMetrics{Metrics: Metrics{scanStartTime: time.Now()}},
termSignal: &terminationSignal{},
}
for _, option := range options {
@@ -367,22 +433,15 @@ func Start(ctx context.Context, options ...EngineOption) (*Engine, error) {
}()
}
// We want 1/4th of the notifier workers as the number of scanner workers.
const notifierWorkerRatio = 4
maxNotifierWorkers := 1
if numWorkers := e.concurrency / notifierWorkerRatio; numWorkers > 0 {
maxNotifierWorkers = int(numWorkers)
}
ctx.Logger().V(2).Info("starting notifier workers", "count", maxNotifierWorkers)
for worker := 0; worker < maxNotifierWorkers; worker++ {
e.WgNotifier.Add(1)
go func() {
ctx := context.WithValue(ctx, "notifier_worker_id", common.RandomID(5))
defer common.Recover(ctx)
defer e.WgNotifier.Done()
e.notifyResults(ctx)
}()
}
// We want only one notifier worker.
ctx.Logger().V(2).Info("starting notifier worker")
e.WgNotifier.Add(1)
go func() {
ctx := context.WithValue(ctx, "notifier_worker_id", common.RandomID(5))
defer common.Recover(ctx)
defer e.WgNotifier.Done()
e.notifyResults(ctx)
}()
return e, nil
}
@@ -392,6 +451,13 @@ func Start(ctx context.Context, options ...EngineOption) (*Engine, error) {
// more sources may be scanned by the engine.
func (e *Engine) Finish(ctx context.Context) error {
defer common.RecoverWithExit(ctx)
// Handle the fail-fast termination signal.
if e.termSignal.isFailFast {
e.termSignal.wait()
return fmt.Errorf("terminated early due to fail-fast flag after printing the first result")
}
// Wait for the sources to finish putting chunks onto the chunks channel.
err := e.sourceManager.Wait()
@@ -407,6 +473,15 @@ func (e *Engine) Finish(ctx context.Context) error {
return err
}
// IsTerminated checks if the engine received a termination signal.
// This can be used, for example, to determine if the engine
// terminated early due to the fail-fast feature being activated.
func (e *Engine) IsTerminated() bool {
e.termSignal.mutex.Lock()
defer e.termSignal.mutex.Unlock()
return e.termSignal.isTerminated
}
func (e *Engine) ChunksChan() <-chan *sources.Chunk {
return e.sourceManager.Chunks()
}
@@ -559,27 +634,51 @@ func (e *Engine) processResult(data detectableChunk, res detectors.Result) {
}
func (e *Engine) notifyResults(ctx context.Context) {
if e.termSignal.isFailFast {
e.processResultsWithFailFast(ctx)
} else {
e.processRegularResults(ctx)
}
}
func (e *Engine) processResultsWithFailFast(ctx context.Context) {
for r := range e.ResultsChan() {
if e.onlyVerified && !r.Verified {
continue
}
atomic.AddUint32(&e.numFoundResults, 1)
e.processSingleResult(ctx, &r)
key := fmt.Sprintf("%s%s%s%+v", r.DetectorType.String(), r.Raw, r.RawV2, r.SourceMetadata)
if _, ok := e.dedupeCache.Get(key); ok {
continue
}
e.dedupeCache.Add(key, struct{}{})
// If fail-fast is enabled, signal finish after processing the first result.
e.termSignal.signalFinish()
return
}
}
if r.Verified {
atomic.AddUint64(&e.metrics.VerifiedSecretsFound, 1)
} else {
atomic.AddUint64(&e.metrics.UnverifiedSecretsFound, 1)
}
func (e *Engine) processRegularResults(ctx context.Context) {
for r := range e.ResultsChan() {
e.processSingleResult(ctx, &r)
}
}
if err := e.printer.Print(ctx, &r); err != nil {
ctx.Logger().Error(err, "error printing result")
}
func (e *Engine) processSingleResult(ctx context.Context, r *detectors.ResultWithMetadata) {
if e.onlyVerified && !r.Verified {
return
}
atomic.AddUint32(&e.numFoundResults, 1)
key := fmt.Sprintf("%s%s%s%+v", r.DetectorType.String(), r.Raw, r.RawV2, r.SourceMetadata)
if _, ok := e.dedupeCache.Get(key); ok {
return
}
e.dedupeCache.Add(key, struct{}{})
if r.Verified {
atomic.AddUint64(&e.metrics.VerifiedSecretsFound, 1)
} else {
atomic.AddUint64(&e.metrics.UnverifiedSecretsFound, 1)
}
if err := e.printer.Print(ctx, r); err != nil {
ctx.Logger().Error(err, "error printing result")
}
}
+10 -11
View File
@@ -119,18 +119,7 @@ func GenerateLink(repo, commit, file string, line int64) string {
case providerBitbucket:
return repo[:len(repo)-4] + "/commits/" + commit
case providerAzure:
baseLink := repo + "/commit/" + commit + "/" + file
if line > 0 {
baseLink += "?line=" + strconv.FormatInt(line, 10)
}
return baseLink
case providerGithub, providerGitlab:
// If the provider name isn't one of the cloud defaults, it is probably an on-prem github or gitlab.
// So do the same thing.
fallthrough
default:
var baseLink string
if file == "" {
baseLink = repo[:len(repo)-4] + "/commit/" + commit
@@ -141,5 +130,15 @@ func GenerateLink(repo, commit, file string, line int64) string {
}
}
return baseLink
case providerAzure:
baseLink := repo + "/commit/" + commit + "/" + file
if line > 0 {
baseLink += "?line=" + strconv.FormatInt(line, 10)
}
return baseLink
default:
return ""
}
}
+1 -4
View File
@@ -4,7 +4,6 @@ import (
"crypto/sha256"
"encoding/hex"
"fmt"
"sync"
"github.com/trufflesecurity/trufflehog/v3/pkg/context"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
@@ -14,7 +13,7 @@ import (
var dedupeCache = make(map[string]struct{})
// GitHubActionsPrinter is a printer that prints results in GitHub Actions format.
type GitHubActionsPrinter struct{ mu sync.Mutex }
type GitHubActionsPrinter struct{}
func (p *GitHubActionsPrinter) Print(_ context.Context, r *detectors.ResultWithMetadata) error {
out := gitHubActionsOutputFormat{
@@ -57,8 +56,6 @@ func (p *GitHubActionsPrinter) Print(_ context.Context, r *detectors.ResultWithM
}
dedupeCache[key] = struct{}{}
p.mu.Lock()
defer p.mu.Unlock()
message := fmt.Sprintf("Found %s %s result 🐷🔑\n", verifiedStatus, out.DetectorType)
if r.Result.DecoderType != detectorspb.DecoderType_PLAIN {
message = fmt.Sprintf("Found %s %s result with %s encoding 🐷🔑\n", verifiedStatus, out.DetectorType, out.DecoderType)
+1 -4
View File
@@ -3,7 +3,6 @@ package output
import (
"encoding/json"
"fmt"
"sync"
"github.com/trufflesecurity/trufflehog/v3/pkg/context"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
@@ -13,7 +12,7 @@ import (
)
// JSONPrinter is a printer that prints results in JSON format.
type JSONPrinter struct{ mu sync.Mutex }
type JSONPrinter struct{}
func (p *JSONPrinter) Print(_ context.Context, r *detectors.ResultWithMetadata) error {
v := &struct {
@@ -62,8 +61,6 @@ func (p *JSONPrinter) Print(_ context.Context, r *detectors.ResultWithMetadata)
return fmt.Errorf("could not marshal result: %w", err)
}
p.mu.Lock()
fmt.Println(string(out))
p.mu.Unlock()
return nil
}
+1 -4
View File
@@ -7,7 +7,6 @@ import (
"net/url"
"os"
"strings"
"sync"
gogit "github.com/go-git/go-git/v5"
"github.com/go-git/go-git/v5/plumbing"
@@ -21,7 +20,7 @@ import (
)
// LegacyJSONPrinter is a printer that prints results in legacy JSON format for backwards compatibility.
type LegacyJSONPrinter struct{ mu sync.Mutex }
type LegacyJSONPrinter struct{}
func (p *LegacyJSONPrinter) Print(ctx context.Context, r *detectors.ResultWithMetadata) error {
var repo string
@@ -54,9 +53,7 @@ func (p *LegacyJSONPrinter) Print(ctx context.Context, r *detectors.ResultWithMe
return fmt.Errorf("could not marshal result: %w", err)
}
p.mu.Lock()
fmt.Println(string(out))
p.mu.Unlock()
return nil
}
+1 -4
View File
@@ -5,7 +5,6 @@ import (
"fmt"
"sort"
"strings"
"sync"
"github.com/fatih/color"
"golang.org/x/text/cases"
@@ -23,7 +22,7 @@ var (
)
// PlainPrinter is a printer that prints results in plain text format.
type PlainPrinter struct{ mu sync.Mutex }
type PlainPrinter struct{}
func (p *PlainPrinter) Print(_ context.Context, r *detectors.ResultWithMetadata) error {
out := outputFormat{
@@ -40,8 +39,6 @@ func (p *PlainPrinter) Print(_ context.Context, r *detectors.ResultWithMetadata)
}
printer := greenPrinter
p.mu.Lock()
defer p.mu.Unlock()
if out.Verified {
yellowPrinter.Print("Found verified result 🐷🔑\n")
+2 -6
View File
@@ -1000,7 +1000,6 @@ const (
DetectorType_HuggingFace DetectorType = 926
DetectorType_Snowflake DetectorType = 927
DetectorType_Sourcegraph DetectorType = 928
DetectorType_Tailscale DetectorType = 929
)
// Enum value maps for DetectorType.
@@ -1931,7 +1930,6 @@ var (
926: "HuggingFace",
927: "Snowflake",
928: "Sourcegraph",
929: "Tailscale",
}
DetectorType_value = map[string]int32{
"Alibaba": 0,
@@ -2859,7 +2857,6 @@ var (
"HuggingFace": 926,
"Snowflake": 927,
"Sourcegraph": 928,
"Tailscale": 929,
}
)
@@ -3238,7 +3235,7 @@ var file_detectors_proto_rawDesc = []byte{
0x44, 0x65, 0x63, 0x6f, 0x64, 0x65, 0x72, 0x54, 0x79, 0x70, 0x65, 0x12, 0x0b, 0x0a, 0x07, 0x55,
0x4e, 0x4b, 0x4e, 0x4f, 0x57, 0x4e, 0x10, 0x00, 0x12, 0x09, 0x0a, 0x05, 0x50, 0x4c, 0x41, 0x49,
0x4e, 0x10, 0x01, 0x12, 0x0a, 0x0a, 0x06, 0x42, 0x41, 0x53, 0x45, 0x36, 0x34, 0x10, 0x02, 0x12,
0x09, 0x0a, 0x05, 0x55, 0x54, 0x46, 0x31, 0x36, 0x10, 0x03, 0x2a, 0xae, 0x74, 0x0a, 0x0c, 0x44,
0x09, 0x0a, 0x05, 0x55, 0x54, 0x46, 0x31, 0x36, 0x10, 0x03, 0x2a, 0x9e, 0x74, 0x0a, 0x0c, 0x44,
0x65, 0x74, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x54, 0x79, 0x70, 0x65, 0x12, 0x0b, 0x0a, 0x07, 0x41,
0x6c, 0x69, 0x62, 0x61, 0x62, 0x61, 0x10, 0x00, 0x12, 0x08, 0x0a, 0x04, 0x41, 0x4d, 0x51, 0x50,
0x10, 0x01, 0x12, 0x07, 0x0a, 0x03, 0x41, 0x57, 0x53, 0x10, 0x02, 0x12, 0x09, 0x0a, 0x05, 0x41,
@@ -4168,8 +4165,7 @@ var file_detectors_proto_rawDesc = []byte{
0x6f, 0x72, 0x63, 0x65, 0x10, 0x9d, 0x07, 0x12, 0x10, 0x0a, 0x0b, 0x48, 0x75, 0x67, 0x67, 0x69,
0x6e, 0x67, 0x46, 0x61, 0x63, 0x65, 0x10, 0x9e, 0x07, 0x12, 0x0e, 0x0a, 0x09, 0x53, 0x6e, 0x6f,
0x77, 0x66, 0x6c, 0x61, 0x6b, 0x65, 0x10, 0x9f, 0x07, 0x12, 0x10, 0x0a, 0x0b, 0x53, 0x6f, 0x75,
0x72, 0x63, 0x65, 0x67, 0x72, 0x61, 0x70, 0x68, 0x10, 0xa0, 0x07, 0x12, 0x0e, 0x0a, 0x09, 0x54,
0x61, 0x69, 0x6c, 0x73, 0x63, 0x61, 0x6c, 0x65, 0x10, 0xa1, 0x07, 0x42, 0x3d, 0x5a, 0x3b, 0x67,
0x72, 0x63, 0x65, 0x67, 0x72, 0x61, 0x70, 0x68, 0x10, 0xa0, 0x07, 0x42, 0x3d, 0x5a, 0x3b, 0x67,
0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x74, 0x72, 0x75, 0x66, 0x66, 0x6c,
0x65, 0x73, 0x65, 0x63, 0x75, 0x72, 0x69, 0x74, 0x79, 0x2f, 0x74, 0x72, 0x75, 0x66, 0x66, 0x6c,
0x65, 0x68, 0x6f, 0x67, 0x2f, 0x76, 0x33, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x70, 0x62, 0x2f, 0x64,
-1
View File
@@ -937,7 +937,6 @@ enum DetectorType {
HuggingFace = 926;
Snowflake = 927;
Sourcegraph = 928;
Tailscale = 929;
}
message Result {