Compare commits
6 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| b4e6b6e32f | |||
| c8b9735834 | |||
| 769d708b84 | |||
| 572cb0e5dc | |||
| 183037ab34 | |||
| 8b7c50825e |
@@ -39,7 +39,7 @@ require (
|
||||
github.com/jpillora/overseer v1.1.6
|
||||
github.com/kylelemons/godebug v1.1.0
|
||||
github.com/lib/pq v1.10.9
|
||||
github.com/mattn/go-sqlite3 v1.14.16
|
||||
github.com/mattn/go-sqlite3 v1.14.17
|
||||
github.com/mholt/archiver/v4 v4.0.0-alpha.8
|
||||
github.com/patrickmn/go-cache v2.1.0+incompatible
|
||||
github.com/paulbellamy/ratecounter v0.2.0
|
||||
@@ -47,7 +47,7 @@ require (
|
||||
github.com/pkg/errors v0.9.1
|
||||
github.com/rabbitmq/amqp091-go v1.8.1
|
||||
github.com/sergi/go-diff v1.3.1
|
||||
github.com/stretchr/testify v1.8.3
|
||||
github.com/stretchr/testify v1.8.4
|
||||
github.com/tailscale/depaware v0.0.0-20210622194025-720c4b409502
|
||||
github.com/xanzy/go-gitlab v0.83.0
|
||||
go.mongodb.org/mongo-driver v1.11.6
|
||||
|
||||
@@ -323,8 +323,8 @@ github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovk
|
||||
github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
|
||||
github.com/mattn/go-isatty v0.0.17 h1:BTarxUcIeDqL27Mc+vyvdWYSL28zpIhv3RoTdsLMPng=
|
||||
github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
|
||||
github.com/mattn/go-sqlite3 v1.14.16 h1:yOQRA0RpS5PFz/oikGwBEqvAWhWg5ufRz4ETLjwpU1Y=
|
||||
github.com/mattn/go-sqlite3 v1.14.16/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg=
|
||||
github.com/mattn/go-sqlite3 v1.14.17 h1:mCRHCLDUBXgpKAqIKsaAaAsrAlbkeomtRFKXh2L6YIM=
|
||||
github.com/mattn/go-sqlite3 v1.14.17/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg=
|
||||
github.com/mholt/archiver/v4 v4.0.0-alpha.8 h1:tRGQuDVPh66WCOelqe6LIGh0gwmfwxUrSSDunscGsRM=
|
||||
github.com/mholt/archiver/v4 v4.0.0-alpha.8/go.mod h1:5f7FUYGXdJWUjESffJaYR4R60VhnHxb2X3T1teMyv5A=
|
||||
github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y=
|
||||
@@ -399,8 +399,8 @@ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/
|
||||
github.com/stretchr/testify v1.7.2/go.mod h1:R6va5+xMeoiuVRoj+gSkQ7d3FALtqAAGI1FQKckRals=
|
||||
github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
|
||||
github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
|
||||
github.com/stretchr/testify v1.8.3 h1:RP3t2pwF7cMEbC1dqtB6poj3niw/9gnV4Cjg5oW5gtY=
|
||||
github.com/stretchr/testify v1.8.3/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
|
||||
github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
|
||||
github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
|
||||
github.com/tailscale/depaware v0.0.0-20210622194025-720c4b409502 h1:34icjjmqJ2HPjrSuJYEkdZ+0ItmGQAQ75cRHIiftIyE=
|
||||
github.com/tailscale/depaware v0.0.0-20210622194025-720c4b409502/go.mod h1:p9lPsd+cx33L3H9nNoecRRxPssFKUwwI50I3pZ0yT+8=
|
||||
github.com/therootcompany/xz v1.0.1 h1:CmOtsn1CbtmyYiusbfmhmkpAAETj0wBIH6kCYaX+xzw=
|
||||
|
||||
@@ -163,9 +163,10 @@ func (c *customRegexWebhook) createResults(ctx context.Context, match map[string
|
||||
// TODO: Read response body.
|
||||
res.Body.Close()
|
||||
if res.StatusCode == http.StatusOK {
|
||||
result.Verified = true
|
||||
result.Verified = detectors.Verified
|
||||
break
|
||||
}
|
||||
result.Verified = detectors.Unverified
|
||||
}
|
||||
|
||||
select {
|
||||
|
||||
+22
-21
@@ -172,37 +172,38 @@ func (s scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
req.URL.RawQuery = params.Encode()
|
||||
|
||||
res, err := client.Do(req)
|
||||
if err == nil {
|
||||
|
||||
if res.StatusCode >= 200 && res.StatusCode < 300 {
|
||||
identityInfo := identityRes{}
|
||||
err := json.NewDecoder(res.Body).Decode(&identityInfo)
|
||||
if err == nil {
|
||||
s1.Verified = true
|
||||
s1.ExtraData = map[string]string{
|
||||
"account": identityInfo.GetCallerIdentityResponse.GetCallerIdentityResult.Account,
|
||||
"user_id": identityInfo.GetCallerIdentityResponse.GetCallerIdentityResult.UserID,
|
||||
"arn": identityInfo.GetCallerIdentityResponse.GetCallerIdentityResult.Arn,
|
||||
}
|
||||
}
|
||||
res.Body.Close()
|
||||
} else {
|
||||
// This function will check false positives for common test words, but also it will make sure the key appears "random" enough to be a real key.
|
||||
if detectors.IsKnownFalsePositive(resSecretMatch, detectors.DefaultFalsePositives, true) {
|
||||
continue
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
s1.Verified = detectors.Unverified
|
||||
if res.StatusCode >= 200 && res.StatusCode < 300 {
|
||||
identityInfo := identityRes{}
|
||||
err := json.NewDecoder(res.Body).Decode(&identityInfo)
|
||||
if err == nil {
|
||||
s1.Verified = detectors.Verified
|
||||
s1.ExtraData = map[string]string{
|
||||
"account": identityInfo.GetCallerIdentityResponse.GetCallerIdentityResult.Account,
|
||||
"user_id": identityInfo.GetCallerIdentityResponse.GetCallerIdentityResult.UserID,
|
||||
"arn": identityInfo.GetCallerIdentityResponse.GetCallerIdentityResult.Arn,
|
||||
}
|
||||
}
|
||||
res.Body.Close()
|
||||
} else {
|
||||
// This function will check false positives for common test words, but also it will make sure the key appears "random" enough to be a real key.
|
||||
if detectors.IsKnownFalsePositive(resSecretMatch, detectors.DefaultFalsePositives, true) {
|
||||
continue
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// If the result is unverified and matches something like a git hash, don't include it in the results.
|
||||
if !s1.Verified && falsePositiveSecretCheck.MatchString(resSecretMatch) {
|
||||
if s1.Verified == detectors.Unverified && falsePositiveSecretCheck.MatchString(resSecretMatch) {
|
||||
continue
|
||||
}
|
||||
|
||||
results = append(results, s1)
|
||||
// If we've found a verified match with this ID, we don't need to look for any more. So move on to the next ID.
|
||||
if s1.Verified {
|
||||
if s1.Verified == detectors.Verified {
|
||||
break
|
||||
}
|
||||
}
|
||||
@@ -219,7 +220,7 @@ func awsCustomCleanResults(results []detectors.Result) []detectors.Result {
|
||||
idResults := map[string]detectors.Result{}
|
||||
for _, result := range results {
|
||||
// Always accept the verified result as the result for the given ID.
|
||||
if result.Verified {
|
||||
if result.Verified == detectors.Verified {
|
||||
idResults[result.Redacted] = result
|
||||
continue
|
||||
}
|
||||
|
||||
@@ -14,6 +14,13 @@ import (
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
|
||||
)
|
||||
|
||||
const (
|
||||
// Convenience consts for implementing detectors.
|
||||
Unknown = detectorspb.VerifiedResult_VERIFIED_UNKNOWN
|
||||
Verified = detectorspb.VerifiedResult_VERIFIED
|
||||
Unverified = detectorspb.VerifiedResult_UNVERIFIED
|
||||
)
|
||||
|
||||
// Detector defines an interface for scanning for and verifying secrets.
|
||||
type Detector interface {
|
||||
// FromData will scan bytes for results, and optionally verify them.
|
||||
@@ -45,7 +52,7 @@ type Result struct {
|
||||
DetectorName string
|
||||
// DecoderType is the type of Decoder.
|
||||
DecoderType detectorspb.DecoderType
|
||||
Verified bool
|
||||
Verified detectorspb.VerifiedResult
|
||||
// Raw contains the raw secret identifier data. Prefer IDs over secrets since it is used for deduping after hashing.
|
||||
Raw []byte
|
||||
// RawV2 contains the raw secret identifier that is a combination of both the ID and the secret.
|
||||
@@ -94,7 +101,7 @@ func CleanResults(results []Result) []Result {
|
||||
var cleaned = make(map[string]Result, 0)
|
||||
|
||||
for _, s := range results {
|
||||
if s.Verified {
|
||||
if s.Verified == Verified {
|
||||
cleaned[s.Redacted] = s
|
||||
}
|
||||
}
|
||||
|
||||
@@ -21,7 +21,7 @@ var (
|
||||
client = common.SaneHttpClient()
|
||||
|
||||
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
|
||||
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"float"}) + `\b([a-zA-Z0-9-._+=/]{59,60})\b`)
|
||||
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"float"}) + `\b([a-f0-9]{16}[A-Za-z0-9+/]{42,43}=)`)
|
||||
)
|
||||
|
||||
// Keywords are used for efficiently pre-filtering chunks.
|
||||
|
||||
@@ -15,7 +15,7 @@ func PrintGitHubActionsOutput(r *detectors.ResultWithMetadata) error {
|
||||
out := gitHubActionsOutputFormat{
|
||||
DetectorType: r.Result.DetectorType.String(),
|
||||
DecoderType: r.Result.DecoderType.String(),
|
||||
Verified: r.Result.Verified,
|
||||
Verified: r.Result.Verified.String(),
|
||||
}
|
||||
|
||||
meta, err := structToMap(r.SourceMetadata.Data)
|
||||
@@ -38,9 +38,11 @@ func PrintGitHubActionsOutput(r *detectors.ResultWithMetadata) error {
|
||||
}
|
||||
}
|
||||
|
||||
verifiedStatus := "unverified"
|
||||
if out.Verified {
|
||||
verifiedStatus := "unknown"
|
||||
if out.Verified == detectors.Verified.String() {
|
||||
verifiedStatus = "verified"
|
||||
} else if out.Verified == detectors.Unverified.String() {
|
||||
verifiedStatus = "unverified"
|
||||
}
|
||||
|
||||
key := fmt.Sprintf("%s:%s:%s:%s:%d", out.DecoderType, out.DetectorType, verifiedStatus, out.Filename, out.StartLine)
|
||||
@@ -66,7 +68,7 @@ func PrintGitHubActionsOutput(r *detectors.ResultWithMetadata) error {
|
||||
type gitHubActionsOutputFormat struct {
|
||||
DetectorType,
|
||||
DecoderType string
|
||||
Verified bool
|
||||
Verified string
|
||||
StartLine int64
|
||||
Filename string
|
||||
}
|
||||
|
||||
+2
-2
@@ -26,7 +26,7 @@ func PrintJSON(r *detectors.ResultWithMetadata) error {
|
||||
DetectorName string
|
||||
// DecoderName is the string name of the DecoderType.
|
||||
DecoderName string
|
||||
Verified bool
|
||||
Verified string
|
||||
// Raw contains the raw secret data.
|
||||
Raw string
|
||||
// RawV2 contains the raw secret identifier that is a combination of both the ID and the secret.
|
||||
@@ -45,7 +45,7 @@ func PrintJSON(r *detectors.ResultWithMetadata) error {
|
||||
DetectorType: r.DetectorType,
|
||||
DetectorName: r.DetectorType.String(),
|
||||
DecoderName: r.DecoderType.String(),
|
||||
Verified: r.Verified,
|
||||
Verified: r.Verified.String(),
|
||||
Raw: string(r.Raw),
|
||||
RawV2: string(r.RawV2),
|
||||
Redacted: r.Redacted,
|
||||
|
||||
+7
-4
@@ -18,13 +18,14 @@ var (
|
||||
yellowPrinter = color.New(color.FgYellow)
|
||||
greenPrinter = color.New(color.FgHiGreen)
|
||||
whitePrinter = color.New(color.FgWhite)
|
||||
redPrinter = color.New(color.FgRed)
|
||||
)
|
||||
|
||||
func PrintPlainOutput(r *detectors.ResultWithMetadata) error {
|
||||
out := outputFormat{
|
||||
DetectorType: r.Result.DetectorType.String(),
|
||||
DecoderType: r.Result.DecoderType.String(),
|
||||
Verified: r.Result.Verified,
|
||||
Verified: r.Result.Verified.String(),
|
||||
MetaData: r.SourceMetadata,
|
||||
Raw: strings.TrimSpace(string(r.Result.Raw)),
|
||||
}
|
||||
@@ -36,11 +37,13 @@ func PrintPlainOutput(r *detectors.ResultWithMetadata) error {
|
||||
|
||||
printer := greenPrinter
|
||||
|
||||
if out.Verified {
|
||||
if out.Verified == detectors.Verified.String() {
|
||||
yellowPrinter.Print("Found verified result 🐷🔑\n")
|
||||
} else {
|
||||
} else if out.Verified == detectors.Unverified.String() {
|
||||
printer = whitePrinter
|
||||
whitePrinter.Print("Found unverified result 🐷🔑❓\n")
|
||||
} else {
|
||||
redPrinter.Print("Found unknown result 🐷🔑❓\n")
|
||||
}
|
||||
printer.Printf("Detector Type: %s\n", out.DetectorType)
|
||||
printer.Printf("Decoder Type: %s\n", out.DecoderType)
|
||||
@@ -98,7 +101,7 @@ func structToMap(obj interface{}) (m map[string]map[string]interface{}, err erro
|
||||
type outputFormat struct {
|
||||
DetectorType,
|
||||
DecoderType string
|
||||
Verified bool
|
||||
Verified string
|
||||
Raw string
|
||||
*source_metadatapb.MetaData
|
||||
}
|
||||
|
||||
+1052
-994
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,40 @@
|
||||
package git
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os/exec"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/go-errors/errors"
|
||||
)
|
||||
|
||||
// GitCmdCheck checks if git is installed and meets 2.20.0<=x<3.0.0 version requirements.
|
||||
func GitCmdCheck() error {
|
||||
if errors.Is(exec.Command("git").Run(), exec.ErrNotFound) {
|
||||
return fmt.Errorf("'git' command not found in $PATH. Make sure git is installed and included in $PATH")
|
||||
}
|
||||
|
||||
// Check the version is greater than or equal to 2.20.0
|
||||
out, err := exec.Command("git", "--version").Output()
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to check git version: %w", err)
|
||||
}
|
||||
|
||||
// Extract the version string using a regex to find the version numbers
|
||||
r := regexp.MustCompile(`\d+\.\d+\.\d+`)
|
||||
versionStr := r.FindString(string(out))
|
||||
versionParts := strings.Split(versionStr, ".")
|
||||
|
||||
// Parse version numbers
|
||||
major, _ := strconv.Atoi(versionParts[0])
|
||||
minor, _ := strconv.Atoi(versionParts[1])
|
||||
|
||||
// Compare with version 2.20.0<=x<3.0.0
|
||||
if major == 2 && minor >= 20 {
|
||||
return nil
|
||||
} else {
|
||||
return fmt.Errorf("git version is %s, but must be greater than or equal to 2.20.0, and less than 3.0.0", versionStr)
|
||||
}
|
||||
}
|
||||
@@ -337,14 +337,6 @@ func CloneRepoUsingSSH(ctx context.Context, gitUrl string, args ...string) (stri
|
||||
return CloneRepo(ctx, userInfo, gitUrl, args...)
|
||||
}
|
||||
|
||||
// GitCmdCheck checks if git is installed.
|
||||
func GitCmdCheck() error {
|
||||
if errors.Is(exec.Command("git").Run(), exec.ErrNotFound) {
|
||||
return fmt.Errorf("'git' command not found in $PATH. Make sure git is installed and included in $PATH")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Git) ScanCommits(ctx context.Context, repo *git.Repository, path string, scanOptions *ScanOptions, chunksChan chan *sources.Chunk) error {
|
||||
if err := GitCmdCheck(); err != nil {
|
||||
return err
|
||||
|
||||
@@ -929,10 +929,16 @@ enum DetectorType {
|
||||
Prefect = 918;
|
||||
}
|
||||
|
||||
enum VerifiedResult {
|
||||
VERIFIED_UNKNOWN = 0;
|
||||
UNVERIFIED = 1;
|
||||
VERIFIED = 2;
|
||||
}
|
||||
|
||||
message Result {
|
||||
int64 source_id = 2;
|
||||
string redacted = 3;
|
||||
bool verified = 4;
|
||||
VerifiedResult verified = 4;
|
||||
string hash = 5;
|
||||
map<string, string> extra_data = 6;
|
||||
StructuredData structured_data = 7;
|
||||
|
||||
Reference in New Issue
Block a user