Compare commits

...

6 Commits

Author SHA1 Message Date
Miccah Castorina b4e6b6e32f [draft] Switch to using tristate verification state 2023-06-07 17:35:06 -05:00
dependabot[bot] c8b9735834 Bump github.com/stretchr/testify from 1.8.3 to 1.8.4 (#1377)
Bumps [github.com/stretchr/testify](https://github.com/stretchr/testify) from 1.8.3 to 1.8.4.
- [Release notes](https://github.com/stretchr/testify/releases)
- [Commits](https://github.com/stretchr/testify/compare/v1.8.3...v1.8.4)

---
updated-dependencies:
- dependency-name: github.com/stretchr/testify
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2023-06-06 10:41:01 -07:00
dependabot[bot] 769d708b84 Bump github.com/mattn/go-sqlite3 from 1.14.16 to 1.14.17 (#1380)
Bumps [github.com/mattn/go-sqlite3](https://github.com/mattn/go-sqlite3) from 1.14.16 to 1.14.17.
- [Release notes](https://github.com/mattn/go-sqlite3/releases)
- [Commits](https://github.com/mattn/go-sqlite3/compare/v1.14.16...v1.14.17)

---
updated-dependencies:
- dependency-name: github.com/mattn/go-sqlite3
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2023-06-06 10:40:34 -07:00
Dustin Decker 572cb0e5dc Loosen up version check for git 2023-06-01 12:17:48 -07:00
Dustin Decker 183037ab34 Check that git meets version requirements (#1373) 2023-06-01 09:41:06 -07:00
ahrav 8b7c50825e update detector regex. (#1368) 2023-06-01 08:16:18 -07:00
13 changed files with 1154 additions and 1044 deletions
+2 -2
View File
@@ -39,7 +39,7 @@ require (
github.com/jpillora/overseer v1.1.6
github.com/kylelemons/godebug v1.1.0
github.com/lib/pq v1.10.9
github.com/mattn/go-sqlite3 v1.14.16
github.com/mattn/go-sqlite3 v1.14.17
github.com/mholt/archiver/v4 v4.0.0-alpha.8
github.com/patrickmn/go-cache v2.1.0+incompatible
github.com/paulbellamy/ratecounter v0.2.0
@@ -47,7 +47,7 @@ require (
github.com/pkg/errors v0.9.1
github.com/rabbitmq/amqp091-go v1.8.1
github.com/sergi/go-diff v1.3.1
github.com/stretchr/testify v1.8.3
github.com/stretchr/testify v1.8.4
github.com/tailscale/depaware v0.0.0-20210622194025-720c4b409502
github.com/xanzy/go-gitlab v0.83.0
go.mongodb.org/mongo-driver v1.11.6
+4 -4
View File
@@ -323,8 +323,8 @@ github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovk
github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
github.com/mattn/go-isatty v0.0.17 h1:BTarxUcIeDqL27Mc+vyvdWYSL28zpIhv3RoTdsLMPng=
github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
github.com/mattn/go-sqlite3 v1.14.16 h1:yOQRA0RpS5PFz/oikGwBEqvAWhWg5ufRz4ETLjwpU1Y=
github.com/mattn/go-sqlite3 v1.14.16/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg=
github.com/mattn/go-sqlite3 v1.14.17 h1:mCRHCLDUBXgpKAqIKsaAaAsrAlbkeomtRFKXh2L6YIM=
github.com/mattn/go-sqlite3 v1.14.17/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg=
github.com/mholt/archiver/v4 v4.0.0-alpha.8 h1:tRGQuDVPh66WCOelqe6LIGh0gwmfwxUrSSDunscGsRM=
github.com/mholt/archiver/v4 v4.0.0-alpha.8/go.mod h1:5f7FUYGXdJWUjESffJaYR4R60VhnHxb2X3T1teMyv5A=
github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y=
@@ -399,8 +399,8 @@ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/
github.com/stretchr/testify v1.7.2/go.mod h1:R6va5+xMeoiuVRoj+gSkQ7d3FALtqAAGI1FQKckRals=
github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
github.com/stretchr/testify v1.8.3 h1:RP3t2pwF7cMEbC1dqtB6poj3niw/9gnV4Cjg5oW5gtY=
github.com/stretchr/testify v1.8.3/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
github.com/tailscale/depaware v0.0.0-20210622194025-720c4b409502 h1:34icjjmqJ2HPjrSuJYEkdZ+0ItmGQAQ75cRHIiftIyE=
github.com/tailscale/depaware v0.0.0-20210622194025-720c4b409502/go.mod h1:p9lPsd+cx33L3H9nNoecRRxPssFKUwwI50I3pZ0yT+8=
github.com/therootcompany/xz v1.0.1 h1:CmOtsn1CbtmyYiusbfmhmkpAAETj0wBIH6kCYaX+xzw=
+2 -1
View File
@@ -163,9 +163,10 @@ func (c *customRegexWebhook) createResults(ctx context.Context, match map[string
// TODO: Read response body.
res.Body.Close()
if res.StatusCode == http.StatusOK {
result.Verified = true
result.Verified = detectors.Verified
break
}
result.Verified = detectors.Unverified
}
select {
+22 -21
View File
@@ -172,37 +172,38 @@ func (s scanner) FromData(ctx context.Context, verify bool, data []byte) (result
req.URL.RawQuery = params.Encode()
res, err := client.Do(req)
if err == nil {
if res.StatusCode >= 200 && res.StatusCode < 300 {
identityInfo := identityRes{}
err := json.NewDecoder(res.Body).Decode(&identityInfo)
if err == nil {
s1.Verified = true
s1.ExtraData = map[string]string{
"account": identityInfo.GetCallerIdentityResponse.GetCallerIdentityResult.Account,
"user_id": identityInfo.GetCallerIdentityResponse.GetCallerIdentityResult.UserID,
"arn": identityInfo.GetCallerIdentityResponse.GetCallerIdentityResult.Arn,
}
}
res.Body.Close()
} else {
// This function will check false positives for common test words, but also it will make sure the key appears "random" enough to be a real key.
if detectors.IsKnownFalsePositive(resSecretMatch, detectors.DefaultFalsePositives, true) {
continue
if err != nil {
continue
}
s1.Verified = detectors.Unverified
if res.StatusCode >= 200 && res.StatusCode < 300 {
identityInfo := identityRes{}
err := json.NewDecoder(res.Body).Decode(&identityInfo)
if err == nil {
s1.Verified = detectors.Verified
s1.ExtraData = map[string]string{
"account": identityInfo.GetCallerIdentityResponse.GetCallerIdentityResult.Account,
"user_id": identityInfo.GetCallerIdentityResponse.GetCallerIdentityResult.UserID,
"arn": identityInfo.GetCallerIdentityResponse.GetCallerIdentityResult.Arn,
}
}
res.Body.Close()
} else {
// This function will check false positives for common test words, but also it will make sure the key appears "random" enough to be a real key.
if detectors.IsKnownFalsePositive(resSecretMatch, detectors.DefaultFalsePositives, true) {
continue
}
}
}
// If the result is unverified and matches something like a git hash, don't include it in the results.
if !s1.Verified && falsePositiveSecretCheck.MatchString(resSecretMatch) {
if s1.Verified == detectors.Unverified && falsePositiveSecretCheck.MatchString(resSecretMatch) {
continue
}
results = append(results, s1)
// If we've found a verified match with this ID, we don't need to look for any more. So move on to the next ID.
if s1.Verified {
if s1.Verified == detectors.Verified {
break
}
}
@@ -219,7 +220,7 @@ func awsCustomCleanResults(results []detectors.Result) []detectors.Result {
idResults := map[string]detectors.Result{}
for _, result := range results {
// Always accept the verified result as the result for the given ID.
if result.Verified {
if result.Verified == detectors.Verified {
idResults[result.Redacted] = result
continue
}
+9 -2
View File
@@ -14,6 +14,13 @@ import (
"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
)
const (
// Convenience consts for implementing detectors.
Unknown = detectorspb.VerifiedResult_VERIFIED_UNKNOWN
Verified = detectorspb.VerifiedResult_VERIFIED
Unverified = detectorspb.VerifiedResult_UNVERIFIED
)
// Detector defines an interface for scanning for and verifying secrets.
type Detector interface {
// FromData will scan bytes for results, and optionally verify them.
@@ -45,7 +52,7 @@ type Result struct {
DetectorName string
// DecoderType is the type of Decoder.
DecoderType detectorspb.DecoderType
Verified bool
Verified detectorspb.VerifiedResult
// Raw contains the raw secret identifier data. Prefer IDs over secrets since it is used for deduping after hashing.
Raw []byte
// RawV2 contains the raw secret identifier that is a combination of both the ID and the secret.
@@ -94,7 +101,7 @@ func CleanResults(results []Result) []Result {
var cleaned = make(map[string]Result, 0)
for _, s := range results {
if s.Verified {
if s.Verified == Verified {
cleaned[s.Redacted] = s
}
}
+1 -1
View File
@@ -21,7 +21,7 @@ var (
client = common.SaneHttpClient()
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"float"}) + `\b([a-zA-Z0-9-._+=/]{59,60})\b`)
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"float"}) + `\b([a-f0-9]{16}[A-Za-z0-9+/]{42,43}=)`)
)
// Keywords are used for efficiently pre-filtering chunks.
+6 -4
View File
@@ -15,7 +15,7 @@ func PrintGitHubActionsOutput(r *detectors.ResultWithMetadata) error {
out := gitHubActionsOutputFormat{
DetectorType: r.Result.DetectorType.String(),
DecoderType: r.Result.DecoderType.String(),
Verified: r.Result.Verified,
Verified: r.Result.Verified.String(),
}
meta, err := structToMap(r.SourceMetadata.Data)
@@ -38,9 +38,11 @@ func PrintGitHubActionsOutput(r *detectors.ResultWithMetadata) error {
}
}
verifiedStatus := "unverified"
if out.Verified {
verifiedStatus := "unknown"
if out.Verified == detectors.Verified.String() {
verifiedStatus = "verified"
} else if out.Verified == detectors.Unverified.String() {
verifiedStatus = "unverified"
}
key := fmt.Sprintf("%s:%s:%s:%s:%d", out.DecoderType, out.DetectorType, verifiedStatus, out.Filename, out.StartLine)
@@ -66,7 +68,7 @@ func PrintGitHubActionsOutput(r *detectors.ResultWithMetadata) error {
type gitHubActionsOutputFormat struct {
DetectorType,
DecoderType string
Verified bool
Verified string
StartLine int64
Filename string
}
+2 -2
View File
@@ -26,7 +26,7 @@ func PrintJSON(r *detectors.ResultWithMetadata) error {
DetectorName string
// DecoderName is the string name of the DecoderType.
DecoderName string
Verified bool
Verified string
// Raw contains the raw secret data.
Raw string
// RawV2 contains the raw secret identifier that is a combination of both the ID and the secret.
@@ -45,7 +45,7 @@ func PrintJSON(r *detectors.ResultWithMetadata) error {
DetectorType: r.DetectorType,
DetectorName: r.DetectorType.String(),
DecoderName: r.DecoderType.String(),
Verified: r.Verified,
Verified: r.Verified.String(),
Raw: string(r.Raw),
RawV2: string(r.RawV2),
Redacted: r.Redacted,
+7 -4
View File
@@ -18,13 +18,14 @@ var (
yellowPrinter = color.New(color.FgYellow)
greenPrinter = color.New(color.FgHiGreen)
whitePrinter = color.New(color.FgWhite)
redPrinter = color.New(color.FgRed)
)
func PrintPlainOutput(r *detectors.ResultWithMetadata) error {
out := outputFormat{
DetectorType: r.Result.DetectorType.String(),
DecoderType: r.Result.DecoderType.String(),
Verified: r.Result.Verified,
Verified: r.Result.Verified.String(),
MetaData: r.SourceMetadata,
Raw: strings.TrimSpace(string(r.Result.Raw)),
}
@@ -36,11 +37,13 @@ func PrintPlainOutput(r *detectors.ResultWithMetadata) error {
printer := greenPrinter
if out.Verified {
if out.Verified == detectors.Verified.String() {
yellowPrinter.Print("Found verified result 🐷🔑\n")
} else {
} else if out.Verified == detectors.Unverified.String() {
printer = whitePrinter
whitePrinter.Print("Found unverified result 🐷🔑❓\n")
} else {
redPrinter.Print("Found unknown result 🐷🔑❓\n")
}
printer.Printf("Detector Type: %s\n", out.DetectorType)
printer.Printf("Decoder Type: %s\n", out.DecoderType)
@@ -98,7 +101,7 @@ func structToMap(obj interface{}) (m map[string]map[string]interface{}, err erro
type outputFormat struct {
DetectorType,
DecoderType string
Verified bool
Verified string
Raw string
*source_metadatapb.MetaData
}
File diff suppressed because it is too large Load Diff
+40
View File
@@ -0,0 +1,40 @@
package git
import (
"fmt"
"os/exec"
"regexp"
"strconv"
"strings"
"github.com/go-errors/errors"
)
// GitCmdCheck checks if git is installed and meets 2.20.0<=x<3.0.0 version requirements.
func GitCmdCheck() error {
if errors.Is(exec.Command("git").Run(), exec.ErrNotFound) {
return fmt.Errorf("'git' command not found in $PATH. Make sure git is installed and included in $PATH")
}
// Check the version is greater than or equal to 2.20.0
out, err := exec.Command("git", "--version").Output()
if err != nil {
return fmt.Errorf("failed to check git version: %w", err)
}
// Extract the version string using a regex to find the version numbers
r := regexp.MustCompile(`\d+\.\d+\.\d+`)
versionStr := r.FindString(string(out))
versionParts := strings.Split(versionStr, ".")
// Parse version numbers
major, _ := strconv.Atoi(versionParts[0])
minor, _ := strconv.Atoi(versionParts[1])
// Compare with version 2.20.0<=x<3.0.0
if major == 2 && minor >= 20 {
return nil
} else {
return fmt.Errorf("git version is %s, but must be greater than or equal to 2.20.0, and less than 3.0.0", versionStr)
}
}
-8
View File
@@ -337,14 +337,6 @@ func CloneRepoUsingSSH(ctx context.Context, gitUrl string, args ...string) (stri
return CloneRepo(ctx, userInfo, gitUrl, args...)
}
// GitCmdCheck checks if git is installed.
func GitCmdCheck() error {
if errors.Is(exec.Command("git").Run(), exec.ErrNotFound) {
return fmt.Errorf("'git' command not found in $PATH. Make sure git is installed and included in $PATH")
}
return nil
}
func (s *Git) ScanCommits(ctx context.Context, repo *git.Repository, path string, scanOptions *ScanOptions, chunksChan chan *sources.Chunk) error {
if err := GitCmdCheck(); err != nil {
return err
+7 -1
View File
@@ -929,10 +929,16 @@ enum DetectorType {
Prefect = 918;
}
enum VerifiedResult {
VERIFIED_UNKNOWN = 0;
UNVERIFIED = 1;
VERIFIED = 2;
}
message Result {
int64 source_id = 2;
string redacted = 3;
bool verified = 4;
VerifiedResult verified = 4;
string hash = 5;
map<string, string> extra_data = 6;
StructuredData structured_data = 7;