Compare commits

..

1 Commits

Author SHA1 Message Date
Ahrav Dutta a39e354c5f PR 2 of 40 2023-08-25 21:39:54 -07:00
43 changed files with 486 additions and 894 deletions
+1 -1
View File
@@ -32,7 +32,7 @@ Have questions? Feedback? Jump in slack or discord and hang out with us
Join our [Slack Community](https://join.slack.com/t/trufflehog-community/shared_invite/zt-pw2qbi43-Aa86hkiimstfdKH9UCpPzQ)
Join the [Secret Scanning Discord](https://discord.gg/8Hzbrnkr7E)
Join the [Secret Scanning Discord](https://discord.gg/sydS6AHTUP)
# :tv: Demo
+2 -40
View File
@@ -26,10 +26,9 @@ type Context interface {
Logger() logr.Logger
}
// CancelFunc and CancelCauseFunc are type aliases to allow use as if they are
// the same types as the standard library variants.
// CancelFunc is a type alias to context.CancelFunc to allow use as if they are
// the same types.
type CancelFunc = context.CancelFunc
type CancelCauseFunc = context.CancelCauseFunc
// logCtx implements Context.
type logCtx struct {
@@ -69,16 +68,6 @@ func WithCancel(parent Context) (Context, context.CancelFunc) {
return lCtx, cancel
}
// WithCancelCause returns context.WithCancelCause with the log object propagated.
func WithCancelCause(parent Context) (Context, context.CancelCauseFunc) {
ctx, cancel := context.WithCancelCause(parent)
lCtx := logCtx{
log: parent.Logger(),
Context: ctx,
}
return lCtx, cancel
}
// WithDeadline returns context.WithDeadline with the log object propagated and
// the deadline added to the structured log values.
func WithDeadline(parent Context, d time.Time) (Context, context.CancelFunc) {
@@ -90,17 +79,6 @@ func WithDeadline(parent Context, d time.Time) (Context, context.CancelFunc) {
return lCtx, cancel
}
// WithDeadlineCause returns context.WithDeadlineCause with the log object
// propagated and the deadline added to the structured log values.
func WithDeadlineCause(parent Context, d time.Time, cause error) (Context, context.CancelFunc) {
ctx, cancel := context.WithDeadlineCause(parent, d, cause)
lCtx := logCtx{
log: parent.Logger().WithValues("deadline", d),
Context: ctx,
}
return lCtx, cancel
}
// WithTimeout returns context.WithTimeout with the log object propagated and
// the timeout added to the structured log values.
func WithTimeout(parent Context, timeout time.Duration) (Context, context.CancelFunc) {
@@ -112,22 +90,6 @@ func WithTimeout(parent Context, timeout time.Duration) (Context, context.Cancel
return lCtx, cancel
}
// WithTimeoutCause returns context.WithTimeoutCause with the log object
// propagated and the timeout added to the structured log values.
func WithTimeoutCause(parent Context, timeout time.Duration, cause error) (Context, context.CancelFunc) {
ctx, cancel := context.WithTimeoutCause(parent, timeout, cause)
lCtx := logCtx{
log: parent.Logger().WithValues("timeout", timeout),
Context: ctx,
}
return lCtx, cancel
}
// Cause returns the context.Cause of the context.
func Cause(ctx context.Context) error {
return context.Cause(ctx)
}
// WithValue returns context.WithValue with the log object propagated and
// the value added to the structured log values (if the key is a string).
func WithValue(parent Context, key, val any) Context {
-8
View File
@@ -3,7 +3,6 @@ package context
import (
"bytes"
"context"
"fmt"
"strings"
"testing"
"time"
@@ -178,10 +177,3 @@ func TestRace(t *testing.T) {
cancel()
_ = ctx.Err()
}
func TestCause(t *testing.T) {
ctx, cancel := WithCancelCause(Background())
err := fmt.Errorf("oh no")
cancel(err)
assert.Equal(t, err, Cause(ctx))
}
+15 -20
View File
@@ -1,11 +1,11 @@
package appcues
import (
"bytes"
"context"
"fmt"
"net/http"
"regexp"
"strings"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
@@ -20,64 +20,59 @@ var _ detectors.Detector = (*Scanner)(nil)
var (
client = common.SaneHttpClient()
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"appcues"}) + `\b([a-z0-9-]{36})\b`)
userPat = regexp.MustCompile(detectors.PrefixRegex([]string{"appcues"}) + `\b([a-z0-9-]{39})\b`)
idPat = regexp.MustCompile(detectors.PrefixRegex([]string{"appcues"}) + `\b([0-9]{5})\b`)
)
// Keywords are used for efficiently pre-filtering chunks.
// Use identifiers in the secret preferably, or the provider name.
func (s Scanner) Keywords() []string {
return []string{"appcues"}
func (s Scanner) Keywords() [][]byte {
return [][]byte{[]byte("appcues")}
}
// FromData will find and optionally verify Appcues secrets in a given set of bytes.
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
dataStr := string(data)
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
userMatches := userPat.FindAllStringSubmatch(dataStr, -1)
idMatches := idPat.FindAllStringSubmatch(dataStr, -1)
keyMatches := keyPat.FindAllSubmatch(data, -1)
userMatches := userPat.FindAllSubmatch(data, -1)
idMatches := idPat.FindAllSubmatch(data, -1)
for _, match := range matches {
for _, match := range keyMatches {
if len(match) != 2 {
continue
}
resMatch := strings.TrimSpace(match[1])
resMatch := bytes.TrimSpace(match[1])
for _, userMatch := range userMatches {
if len(userMatch) != 2 {
continue
}
resUserMatch := strings.TrimSpace(userMatch[1])
resUserMatch := bytes.TrimSpace(userMatch[1])
for _, idMatch := range idMatches {
if len(idMatch) != 2 {
continue
}
resIdMatch := strings.TrimSpace(idMatch[1])
resIdMatch := bytes.TrimSpace(idMatch[1])
s1 := detectors.Result{
DetectorType: detectorspb.DetectorType_Appcues,
Raw: []byte(resMatch),
RawV2: []byte(resMatch + resUserMatch),
Raw: resMatch,
RawV2: append(resMatch, resUserMatch...),
}
if verify {
req, err := http.NewRequestWithContext(ctx, "GET", fmt.Sprintf("https://api.appcues.com/v2/accounts/%s/flows", resIdMatch), nil)
req, err := http.NewRequestWithContext(ctx, "GET", fmt.Sprintf("https://api.appcues.com/v2/accounts/%s/flows", string(resIdMatch)), nil)
if err != nil {
continue
}
req.SetBasicAuth(resUserMatch, resMatch)
req.SetBasicAuth(string(resUserMatch), string(resMatch))
res, err := client.Do(req)
if err == nil {
defer res.Body.Close()
if res.StatusCode >= 200 && res.StatusCode < 300 {
s1.Verified = true
} else {
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
continue
}
+8 -17
View File
@@ -6,7 +6,6 @@ import (
"fmt"
"net/http"
"regexp"
"strings"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
@@ -15,42 +14,35 @@ import (
type Scanner struct{}
// Ensure the Scanner satisfies the interface at compile time.
var _ detectors.Detector = (*Scanner)(nil)
var (
client = common.SaneHttpClient()
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"appfollow"}) + `\b([0-9A-Za-z]{20})\b`)
)
// Keywords are used for efficiently pre-filtering chunks.
// Use identifiers in the secret preferably, or the provider name.
func (s Scanner) Keywords() []string {
return []string{"appfollow"}
func (s Scanner) Keywords() [][]byte {
return [][]byte{[]byte("appfollow")}
}
// FromData will find and optionally verify Appfollow secrets in a given set of bytes.
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
dataStr := string(data)
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
matches := keyPat.FindAllSubmatch(data, -1)
for _, match := range matches {
if len(match) != 2 {
continue
}
resMatch := strings.TrimSpace(match[1])
resMatch := match[1]
s1 := detectors.Result{
DetectorType: detectorspb.DetectorType_Appfollow,
Raw: []byte(resMatch),
Raw: resMatch,
}
if verify {
data := fmt.Sprintf("%s:", resMatch)
sEnc := b64.StdEncoding.EncodeToString([]byte(data))
data := append(resMatch, ':')
sEnc := b64.StdEncoding.EncodeToString(data)
req, err := http.NewRequestWithContext(ctx, "GET", "https://api.appfollow.io/test", nil)
if err != nil {
continue
@@ -62,7 +54,6 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
if res.StatusCode >= 200 && res.StatusCode < 300 {
s1.Verified = true
} else {
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
continue
}
+14 -27
View File
@@ -1,11 +1,11 @@
package appointedd
import (
"bytes"
"context"
"io"
"net/http"
"regexp"
"strings"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
@@ -14,66 +14,53 @@ import (
type Scanner struct{}
// Ensure the Scanner satisfies the interface at compile time
var _ detectors.Detector = (*Scanner)(nil)
var (
client = common.SaneHttpClient()
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"appointedd"}) + `\b([a-zA-Z0-9=+]{88})`)
)
// Keywords are used for efficiently pre-filtering chunks.
// Use identifiers in the secret preferably, or the provider name.
func (s Scanner) Keywords() []string {
return []string{"appointedd"}
func (s Scanner) Keywords() [][]byte {
return [][]byte{[]byte("appointedd")}
}
// FromData will find and optionally verify appointedd secrets in a given set of bytes.
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
dataStr := string(data)
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
matches := keyPat.FindAllSubmatch(data, -1)
for _, match := range matches {
if len(match) != 2 {
continue
}
resMatch := strings.TrimSpace(match[1])
resMatch := bytes.TrimSpace(match[1])
s1 := detectors.Result{
DetectorType: detectorspb.DetectorType_Appointedd,
Raw: []byte(resMatch),
Raw: resMatch,
}
if verify {
req, err := http.NewRequestWithContext(ctx, "GET", "https://api.appointedd.com/v1/availability/slots", nil)
if err != nil {
continue
}
req.Header.Add("X-API-KEY", resMatch)
req.Header.Add("X-API-KEY", string(resMatch))
res, err := client.Do(req)
if err == nil {
defer res.Body.Close()
bodyBytes, err := io.ReadAll(res.Body)
if err != nil {
continue
}
body := string(bodyBytes)
if strings.Contains(body, "total") {
s1.Verified = true
} else {
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
continue
if bodyBytes, err := io.ReadAll(res.Body); err == nil {
if bytes.Contains(bodyBytes, []byte("total")) {
s1.Verified = true
} else {
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
continue
}
}
}
}
}
results = append(results, s1)
}
return results, nil
}
+8 -17
View File
@@ -1,11 +1,11 @@
package appsynergy
import (
"bytes"
"context"
"fmt"
"net/http"
"regexp"
"strings"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
@@ -14,42 +14,34 @@ import (
type Scanner struct{}
// Ensure the Scanner satisfies the interface at compile time.
var _ detectors.Detector = (*Scanner)(nil)
var (
client = common.SaneHttpClient()
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"appsynergy"}) + `\b([a-z0-9]{64})\b`)
)
// Keywords are used for efficiently pre-filtering chunks.
// Use identifiers in the secret preferably, or the provider name.
func (s Scanner) Keywords() []string {
return []string{"appsynergy"}
func (s Scanner) Keywords() [][]byte {
return [][]byte{[]byte("appsynergy")}
}
// FromData will find and optionally verify AppSynergy secrets in a given set of bytes.
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
dataStr := string(data)
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
matches := keyPat.FindAllSubmatch(data, -1)
for _, match := range matches {
if len(match) != 2 {
continue
}
resMatch := strings.TrimSpace(match[1])
resMatch := bytes.TrimSpace(match[1])
s1 := detectors.Result{
DetectorType: detectorspb.DetectorType_AppSynergy,
Raw: []byte(resMatch),
Raw: resMatch,
}
if verify {
payload := strings.NewReader(`{"html":"<html><body><h1>Hello World</h1></body></html>","filename":"HelloWorld.pdf"}`)
req, err := http.NewRequestWithContext(ctx, "POST", fmt.Sprintf("https://www.appsynergy.com/api?action=HTML2PDF&apiKey=%s", resMatch), payload)
payload := bytes.NewBuffer([]byte(`{"html":"<html><body><h1>Hello World</h1></body></html>","filename":"HelloWorld.pdf"}`))
req, err := http.NewRequestWithContext(ctx, "POST", fmt.Sprintf("https://www.appsynergy.com/api?action=HTML2PDF&apiKey=%s", string(resMatch)), payload)
if err != nil {
continue
}
@@ -59,7 +51,6 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
if res.StatusCode >= 200 && res.StatusCode < 300 {
s1.Verified = true
} else {
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
continue
}
+14 -22
View File
@@ -1,12 +1,13 @@
package apptivo
import (
"bytes"
"context"
"fmt"
"io"
"net/http"
"regexp"
"strings"
"io/ioutil"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
@@ -15,60 +16,52 @@ import (
type Scanner struct{}
// Ensure the Scanner satisfies the interface at compile time.
var _ detectors.Detector = (*Scanner)(nil)
var (
client = common.SaneHttpClient()
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"apptivo"}) + `\b([a-z0-9-]{36})\b`)
idPat = regexp.MustCompile(detectors.PrefixRegex([]string{"apptivo"}) + `\b([a-zA-Z0-9-]{32})\b`)
)
// Keywords are used for efficiently pre-filtering chunks.
// Use identifiers in the secret preferably, or the provider name.
func (s Scanner) Keywords() []string {
return []string{"apptivo"}
func (s Scanner) Keywords() [][]byte {
return [][]byte{[]byte("apptivo")}
}
// FromData will find and optionally verify Apptivo secrets in a given set of bytes.
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
dataStr := string(data)
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
idMatches := idPat.FindAllStringSubmatch(dataStr, -1)
matches := keyPat.FindAllSubmatch(data, -1)
idMatches := idPat.FindAllSubmatch(data, -1)
for _, match := range matches {
if len(match) != 2 {
continue
}
resMatch := strings.TrimSpace(match[1])
resMatch := bytes.TrimSpace(match[1])
for _, idMatch := range idMatches {
if len(idMatch) != 2 {
continue
}
resIdMatch := strings.TrimSpace(idMatch[1])
resIdMatch := bytes.TrimSpace(idMatch[1])
s1 := detectors.Result{
DetectorType: detectorspb.DetectorType_Apptivo,
Raw: []byte(resMatch),
RawV2: []byte(resMatch + resIdMatch),
Raw: resMatch,
RawV2: append(resMatch, resIdMatch...),
}
if verify {
req, err := http.NewRequestWithContext(ctx, "GET", fmt.Sprintf("https://api.apptivo.com/app/dao/v6/leads?a=getConfigData&apiKey=%s&accessKey=%s", resMatch, resIdMatch), nil)
req, err := http.NewRequestWithContext(ctx, "GET", fmt.Sprintf("https://api.apptivo.com/app/dao/v6/leads?a=getConfigData&apiKey=%s&accessKey=%s", string(resMatch), string(resIdMatch)), nil)
if err != nil {
continue
}
res, err := client.Do(req)
if err == nil {
bodyBytes, err := io.ReadAll(res.Body)
bodyBytes, err := ioutil.ReadAll(res.Body)
if err != nil {
continue
}
bodyString := string(bodyBytes)
validResponse := strings.Contains(bodyString, `displayName`)
validResponse := bytes.Contains(bodyBytes, []byte(`displayName`))
defer res.Body.Close()
if res.StatusCode >= 200 && res.StatusCode < 300 {
if validResponse {
@@ -77,7 +70,6 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
s1.Verified = false
}
} else {
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
continue
}
+14 -18
View File
@@ -1,10 +1,10 @@
package artifactory
import (
"bytes"
"context"
"net/http"
"regexp"
"strings"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
@@ -19,58 +19,54 @@ var _ detectors.Detector = (*Scanner)(nil)
var (
client = common.SaneHttpClient()
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
// Keep the regexp patterns unchanged
keyPat = regexp.MustCompile(`\b([a-zA-Z0-9]{73})`)
URLPat = regexp.MustCompile(`\b([A-Za-z0-9](?:[A-Za-z0-9\-]{0,61}[A-Za-z0-9])\.jfrog\.io)`)
)
// Keywords are used for efficiently pre-filtering chunks.
// Use identifiers in the secret preferably, or the provider name.
func (s Scanner) Keywords() []string {
return []string{"artifactory"}
func (s Scanner) Keywords() [][]byte {
return [][]byte{[]byte("artifactory")}
}
// FromData will find and optionally verify Artifactory secrets in a given set of bytes.
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
dataStr := string(data)
URLmatches := URLPat.FindAllStringSubmatch(dataStr, -1)
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
URLmatches := URLPat.FindAllSubmatch(data, -1)
matches := keyPat.FindAllSubmatch(data, -1)
var resURLMatch []byte
resURLMatch := ""
for _, URLmatch := range URLmatches {
if len(URLmatch) != 2 {
continue
}
resURLMatch = strings.TrimSpace(URLmatch[1])
resURLMatch = bytes.TrimSpace(URLmatch[1])
}
for _, match := range matches {
if len(match) != 2 {
continue
}
resMatch := strings.TrimSpace(match[1])
resMatch := bytes.TrimSpace(match[1])
s1 := detectors.Result{
DetectorType: detectorspb.DetectorType_ArtifactoryAccessToken,
Raw: []byte(resMatch),
RawV2: []byte(resMatch + resURLMatch),
Raw: resMatch,
RawV2: append(resMatch, resURLMatch...),
}
if verify {
req, err := http.NewRequestWithContext(ctx, "GET", "https://"+resURLMatch+"/artifactory/api/storageinfo", nil)
req, err := http.NewRequestWithContext(ctx, "GET", "https://"+string(resURLMatch)+"/artifactory/api/storageinfo", nil)
if err != nil {
continue
}
req.Header.Add("X-JFrog-Art-Api", resMatch)
req.Header.Add("X-JFrog-Art-Api", string(resMatch))
res, err := client.Do(req)
if err == nil {
defer res.Body.Close()
if res.StatusCode >= 200 && res.StatusCode < 300 {
s1.Verified = true
} else {
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
continue
}
+11 -20
View File
@@ -1,10 +1,10 @@
package artsy
import (
"bytes"
"context"
"net/http"
"regexp"
"strings"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
@@ -13,51 +13,44 @@ import (
type Scanner struct{}
// Ensure the Scanner satisfies the interface at compile time.
var _ detectors.Detector = (*Scanner)(nil)
var (
client = common.SaneHttpClient()
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"artsy"}) + `\b([0-9a-zA-Z]{32})\b`)
idPat = regexp.MustCompile(detectors.PrefixRegex([]string{"artsy"}) + `\b([0-9a-zA-Z]{20})\b`)
)
// Keywords are used for efficiently pre-filtering chunks.
// Use identifiers in the secret preferably, or the provider name.
func (s Scanner) Keywords() []string {
return []string{"artsy"}
func (s Scanner) Keywords() [][]byte {
return [][]byte{[]byte("artsy")}
}
// FromData will find and optionally verify Artsy secrets in a given set of bytes.
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
dataStr := string(data)
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
idmatches := idPat.FindAllStringSubmatch(dataStr, -1)
matches := keyPat.FindAllSubmatch(data, -1)
idmatches := idPat.FindAllSubmatch(data, -1)
for _, match := range matches {
if len(match) != 2 {
continue
}
resMatch := strings.TrimSpace(match[1])
resMatch := bytes.TrimSpace(match[1])
for _, idmatch := range idmatches {
if len(idmatch) != 2 {
continue
}
resIdMatch := strings.TrimSpace(idmatch[1])
resIdMatch := bytes.TrimSpace(idmatch[1])
s1 := detectors.Result{
DetectorType: detectorspb.DetectorType_Artsy,
Raw: []byte(resMatch),
RawV2: []byte(resMatch + resIdMatch),
Raw: resMatch,
RawV2: append(resMatch, resIdMatch...),
}
if verify {
req, err := http.NewRequestWithContext(ctx, "POST", "https://api.artsy.net/api/tokens/xapp_token?client_id="+resIdMatch+"&client_secret="+resMatch, nil)
req, err := http.NewRequestWithContext(ctx, "POST", "https://api.artsy.net/api/tokens/xapp_token?client_id="+string(resIdMatch)+"&client_secret="+string(resMatch), nil)
if err != nil {
continue
}
@@ -67,8 +60,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
if res.StatusCode >= 200 && res.StatusCode < 300 {
s1.Verified = true
} else {
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
if detectors.IsKnownFalsePositive(bytes.Join([][]byte{resMatch, resIdMatch}, nil), detectors.DefaultFalsePositives, true) {
continue
}
}
@@ -77,7 +69,6 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
results = append(results, s1)
}
}
return results, nil
+7 -15
View File
@@ -5,7 +5,6 @@ import (
"fmt"
"net/http"
"regexp"
"strings"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
@@ -20,31 +19,25 @@ var _ detectors.Detector = (*Scanner)(nil)
var (
client = common.SaneHttpClient()
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"asana"}) + `\b([a-z\/:0-9]{51})\b`)
)
// Keywords are used for efficiently pre-filtering chunks.
// Use identifiers in the secret preferably, or the provider name.
func (s Scanner) Keywords() []string {
return []string{"asana"}
func (s Scanner) Keywords() [][]byte {
return [][]byte{[]byte("asana")}
}
// FromData will find and optionally verify AsanaOauth secrets in a given set of bytes.
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
dataStr := string(data)
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
matches := keyPat.FindAllSubmatch(data, -1)
for _, match := range matches {
if len(match) != 2 {
continue
}
resMatch := strings.TrimSpace(match[1])
resMatch := match[1]
s1 := detectors.Result{
DetectorType: detectorspb.DetectorType_AsanaOauth,
Raw: []byte(resMatch),
Raw: resMatch,
}
if verify {
@@ -52,14 +45,13 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
if err != nil {
continue
}
req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", resMatch))
req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", string(resMatch)))
res, err := client.Do(req)
if err == nil {
if err == nil && res.Body != nil {
defer res.Body.Close()
if res.StatusCode >= 200 && res.StatusCode < 300 {
s1.Verified = true
} else {
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
continue
}
@@ -1,11 +1,11 @@
package asanapersonalaccesstoken
import (
"bytes"
"context"
"fmt"
"net/http"
"regexp"
"strings"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
@@ -14,7 +14,6 @@ import (
type Scanner struct{}
// Ensure the Scanner satisfies the interface at compile time.
var _ detectors.Detector = (*Scanner)(nil)
var (
@@ -23,27 +22,22 @@ var (
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"asana"}) + `\b([0-9]{1,}\/[0-9]{16,}:[A-Za-z0-9]{32,})\b`)
)
// Keywords are used for efficiently pre-filtering chunks.
// Use identifiers in the secret preferably, or the provider name.
func (s Scanner) Keywords() []string {
return []string{"asana"}
func (s Scanner) Keywords() [][]byte {
return [][]byte{[]byte("asana")}
}
// FromData will find and optionally verify AsanaPersonalAccessToken secrets in a given set of bytes.
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
dataStr := string(data)
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
matches := keyPat.FindAllSubmatch(data, -1)
for _, match := range matches {
if len(match) != 2 {
continue
}
resMatch := strings.TrimSpace(match[1])
resMatch := bytes.TrimSpace(match[1])
s1 := detectors.Result{
DetectorType: detectorspb.DetectorType_AsanaPersonalAccessToken,
Raw: []byte(resMatch),
Raw: resMatch,
}
if verify {
@@ -51,14 +45,13 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
if err != nil {
continue
}
req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", resMatch))
req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", string(resMatch)))
res, err := client.Do(req)
if err == nil {
defer res.Body.Close()
if res.StatusCode >= 200 && res.StatusCode < 300 {
s1.Verified = true
} else {
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
continue
}
+8 -12
View File
@@ -1,10 +1,10 @@
package assemblyai
import (
"bytes"
"context"
"net/http"
"regexp"
"strings"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
@@ -19,31 +19,28 @@ var _ detectors.Detector = (*Scanner)(nil)
var (
client = common.SaneHttpClient()
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
// Keep the regexp patterns unchanged
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"assemblyai"}) + `\b([0-9a-z]{32})\b`)
)
// Keywords are used for efficiently pre-filtering chunks.
// Use identifiers in the secret preferably, or the provider name.
func (s Scanner) Keywords() []string {
return []string{"assemblyai"}
func (s Scanner) Keywords() [][]byte {
return [][]byte{[]byte("assemblyai")}
}
// FromData will find and optionally verify Assemblyai secrets in a given set of bytes.
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
dataStr := string(data)
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
matches := keyPat.FindAllSubmatch(data, -1)
for _, match := range matches {
if len(match) != 2 {
continue
}
resMatch := strings.TrimSpace(match[1])
resMatch := bytes.TrimSpace(match[1])
s1 := detectors.Result{
DetectorType: detectorspb.DetectorType_AssemblyAI,
Raw: []byte(resMatch),
Raw: resMatch,
}
if verify {
@@ -52,14 +49,13 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
continue
}
req.Header.Add("Content-Type", "application/json")
req.Header.Add("Authorization", resMatch)
req.Header.Add("Authorization", string(resMatch))
res, err := client.Do(req)
if err == nil {
defer res.Body.Close()
if res.StatusCode >= 200 && res.StatusCode < 300 {
s1.Verified = true
} else {
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
continue
}
+9 -14
View File
@@ -1,10 +1,10 @@
package atera
import (
"bytes"
"context"
"net/http"
"regexp"
"strings"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
@@ -18,32 +18,27 @@ var _ detectors.Detector = (*Scanner)(nil)
var (
client = common.SaneHttpClient()
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"atera"}) + `\b([[0-9a-z]{32})\b`)
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"atera"}) + `\b([0-9a-z]{32})\b`)
)
// Keywords are used for efficiently pre-filtering chunks.
// Use identifiers in the secret preferably, or the provider name.
func (s Scanner) Keywords() []string {
return []string{"atera"}
func (s Scanner) Keywords() [][]byte {
return [][]byte{[]byte("atera")}
}
// FromData will find and optionally verify Atera secrets in a given set of bytes.
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
dataStr := string(data)
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
matches := keyPat.FindAllSubmatch(data, -1)
for _, match := range matches {
if len(match) != 2 {
continue
}
resMatch := strings.TrimSpace(match[1])
resMatch := bytes.TrimSpace(match[1])
s1 := detectors.Result{
DetectorType: detectorspb.DetectorType_Atera,
Raw: []byte(resMatch),
Raw: resMatch,
}
if verify {
@@ -52,14 +47,14 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
continue
}
req.Header.Add("Accept", "application/json")
req.Header.Add("X-API-KEY", resMatch)
req.Header.Add("X-API-KEY", string(resMatch))
res, err := client.Do(req)
if err == nil {
defer res.Body.Close()
if res.StatusCode >= 200 && res.StatusCode < 300 {
s1.Verified = true
} else {
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
// Consider adding the []byte conversion here if the input also changed to a []byte.
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
continue
}
+8 -14
View File
@@ -1,12 +1,12 @@
package audd
import (
"bytes"
"context"
"fmt"
"io"
"net/http"
"regexp"
"strings"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
@@ -26,30 +26,26 @@ var (
)
// Keywords are used for efficiently pre-filtering chunks.
// Use identifiers in the secret preferably, or the provider name.
func (s Scanner) Keywords() []string {
return []string{"audd"}
func (s Scanner) Keywords() [][]byte {
return [][]byte{[]byte("audd")}
}
// FromData will find and optionally verify Audd secrets in a given set of bytes.
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
dataStr := string(data)
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
matches := keyPat.FindAllSubmatch(data, -1)
for _, match := range matches {
if len(match) != 2 {
continue
}
resMatch := strings.TrimSpace(match[1])
resMatch := bytes.TrimSpace(match[1])
s1 := detectors.Result{
DetectorType: detectorspb.DetectorType_Audd,
Raw: []byte(resMatch),
Raw: resMatch,
}
if verify {
req, err := http.NewRequestWithContext(ctx, "GET", fmt.Sprintf("https://api.audd.io/setCallbackUrl/?api_token=%s&url=https://yourwebsite.com/callbacks_handler/", resMatch), nil)
req, err := http.NewRequestWithContext(ctx, "GET", fmt.Sprintf("https://api.audd.io/setCallbackUrl/?api_token=%s&url=https://yourwebsite.com/callbacks_handler/", string(resMatch)), nil)
if err != nil {
continue
}
@@ -57,9 +53,8 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
if err == nil {
bodyBytes, err := io.ReadAll(res.Body)
if err == nil {
bodyString := string(bodyBytes)
validResponse := strings.Contains(bodyString, `"status":"success"`)
defer res.Body.Close()
validResponse := bytes.Contains(bodyBytes, []byte(`"status":"success"`))
if res.StatusCode >= 200 && res.StatusCode < 300 {
if validResponse {
s1.Verified = true
@@ -67,7 +62,6 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
s1.Verified = false
}
} else {
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
continue
}
@@ -5,7 +5,6 @@ import (
"fmt"
"net/http"
"regexp"
"strings"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
@@ -14,36 +13,28 @@ import (
type Scanner struct{}
// Ensure the Scanner satisfies the interface at compile time.
var _ detectors.Detector = (*Scanner)(nil)
var (
client = common.SaneHttpClient()
// long jwt token but note this is default 8640000 seconds = 24 hours but could be set to maximum 2592000 seconds = 720 hours = 30 days
// at https://manage.auth0.com/dashboard/us/dev-63memjo3/apis/management/explorer
managementApiTokenPat = regexp.MustCompile(detectors.PrefixRegex([]string{"auth0"}) + `\b(ey[a-zA-Z0-9._-]+)\b`)
domainPat = regexp.MustCompile(`([a-zA-Z0-9\-]{2,16}\.[a-zA-Z0-9_-]{2,3}\.auth0\.com)`) // could be part of url
domainPat = regexp.MustCompile(`([a-zA-Z0-9\-]{2,16}\.[a-zA-Z0-9_-]{2,3}\.auth0\.com)`)
)
// Keywords are used for efficiently pre-filtering chunks.
// Use identifiers in the secret preferably, or the provider name.
func (s Scanner) Keywords() []string {
return []string{"auth0"}
func (s Scanner) Keywords() [][]byte {
return [][]byte{[]byte("auth0")}
}
// FromData will find and optionally verify Auth0ManagementApiToken secrets in a given set of bytes.
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
dataStr := string(data)
managementApiTokenMatches := managementApiTokenPat.FindAllStringSubmatch(dataStr, -1)
domainMatches := domainPat.FindAllStringSubmatch(dataStr, -1)
managementApiTokenMatches := managementApiTokenPat.FindAllSubmatch(data, -1)
domainMatches := domainPat.FindAllSubmatch(data, -1)
for _, managementApiTokenMatch := range managementApiTokenMatches {
if len(managementApiTokenMatch) != 2 {
continue
}
managementApiTokenRes := strings.TrimSpace(managementApiTokenMatch[1])
managementApiTokenRes := managementApiTokenMatch[1]
if len(managementApiTokenRes) < 2000 || len(managementApiTokenRes) > 5000 {
continue
}
@@ -52,25 +43,22 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
if len(domainMatch) != 2 {
continue
}
domainRes := strings.TrimSpace(domainMatch[1])
domainRes := domainMatch[1]
s1 := detectors.Result{
DetectorType: detectorspb.DetectorType_Auth0ManagementApiToken,
Redacted: domainRes,
Raw: []byte(managementApiTokenRes),
RawV2: []byte(managementApiTokenRes + domainRes),
Redacted: string(domainRes),
Raw: managementApiTokenRes,
RawV2: append(managementApiTokenRes, domainRes...),
}
if verify {
/*
curl -H "Authorization: Bearer $token" https://domain/api/v2/users
*/
req, err := http.NewRequestWithContext(ctx, "GET", "https://"+domainRes+"/api/v2/users", nil)
req, err := http.NewRequestWithContext(ctx, "GET", "https://"+string(domainRes)+"/api/v2/users", nil)
if err != nil {
continue
}
req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", managementApiTokenRes))
req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", string(managementApiTokenRes)))
res, err := client.Do(req)
if err == nil {
defer res.Body.Close()
+17 -31
View File
@@ -1,6 +1,7 @@
package auth0oauth
import (
"bytes"
"context"
"io"
"net/http"
@@ -15,7 +16,6 @@ import (
type Scanner struct{}
// Ensure the Scanner satisfies the interface at compile time.
var _ detectors.Detector = (*Scanner)(nil)
var (
@@ -23,64 +23,54 @@ var (
clientIdPat = regexp.MustCompile(detectors.PrefixRegex([]string{"auth0"}) + `\b([a-zA-Z0-9_-]{32,60})\b`)
clientSecretPat = regexp.MustCompile(`\b([a-zA-Z0-9_-]{64,})\b`)
domainPat = regexp.MustCompile(`\b([a-zA-Z0-9][a-zA-Z0-9._-]*auth0\.com)\b`) // could be part of url
domainPat = regexp.MustCompile(`\b([a-zA-Z0-9][a-zA-Z0-9._-]*auth0\.com)\b`)
)
// Keywords are used for efficiently pre-filtering chunks.
// Use identifiers in the secret preferably, or the provider name.
func (s Scanner) Keywords() []string {
return []string{"auth0"}
func (s Scanner) Keywords() [][]byte {
return [][]byte{[]byte("auth0")}
}
// FromData will find and optionally verify Auth0oauth secrets in a given set of bytes.
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
dataStr := string(data)
clientIdMatches := clientIdPat.FindAllStringSubmatch(dataStr, -1)
clientSecretMatches := clientSecretPat.FindAllStringSubmatch(dataStr, -1)
domainMatches := domainPat.FindAllStringSubmatch(dataStr, -1)
clientIdMatches := clientIdPat.FindAllSubmatch(data, -1)
clientSecretMatches := clientSecretPat.FindAllSubmatch(data, -1)
domainMatches := domainPat.FindAllSubmatch(data, -1)
for _, clientIdMatch := range clientIdMatches {
if len(clientIdMatch) != 2 {
continue
}
clientIdRes := strings.TrimSpace(clientIdMatch[1])
clientIdRes := bytes.TrimSpace(clientIdMatch[1])
for _, clientSecretMatch := range clientSecretMatches {
if len(clientSecretMatch) != 2 {
continue
}
clientSecretRes := strings.TrimSpace(clientSecretMatch[1])
clientSecretRes := bytes.TrimSpace(clientSecretMatch[1])
for _, domainMatch := range domainMatches {
if len(domainMatch) != 2 {
continue
}
domainRes := strings.TrimSpace(domainMatch[1])
domainRes := bytes.TrimSpace(domainMatch[1])
s1 := detectors.Result{
DetectorType: detectorspb.DetectorType_Auth0oauth,
Redacted: clientIdRes,
Raw: []byte(clientSecretRes),
RawV2: []byte(clientIdRes + clientSecretRes),
Redacted: string(clientIdRes),
Raw: clientSecretRes,
RawV2: append(clientIdRes, clientSecretRes...),
}
if verify {
/*
curl --request POST \
--url 'https://YOUR_DOMAIN/oauth/token' \
--header 'content-type: application/x-www-form-urlencoded' \
--data 'grant_type=authorization_code&client_id=W44JmL3qD6LxHeEJyKe9lMuhcwvPOaOq&client_secret=YOUR_CLIENT_SECRET&code=AUTHORIZATION_CODE&redirect_uri=undefined'
*/
data := url.Values{}
data.Set("grant_type", "authorization_code")
data.Set("client_id", clientIdRes)
data.Set("client_secret", clientSecretRes)
data.Set("client_id", string(clientIdRes))
data.Set("client_secret", string(clientSecretRes))
data.Set("code", "AUTHORIZATION_CODE")
data.Set("redirect_uri", "undefined")
req, err := http.NewRequestWithContext(ctx, http.MethodPost, "https://"+domainRes+"/oauth/token", strings.NewReader(data.Encode())) // URL-encoded payload
req, err := http.NewRequestWithContext(ctx, http.MethodPost, "https://"+string(domainRes)+"/oauth/token", strings.NewReader(data.Encode()))
if err != nil {
continue
}
@@ -94,11 +84,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
}
body := string(bodyBytes)
// if client_id and client_secret is valid -> 403 {"error":"invalid_grant","error_description":"Invalid authorization code"}
// if invalid -> 401 {"error":"access_denied","error_description":"Unauthorized"}
// ingenious!
if !strings.Contains(body, "access_denied") {
if !bytes.Contains(bodyBytes, []byte("access_denied")) {
s1.Verified = true
} else {
if detectors.IsKnownFalsePositive(clientIdRes, detectors.DefaultFalsePositives, true) {
+11 -20
View File
@@ -1,11 +1,11 @@
package autodesk
import (
"bytes"
"context"
"fmt"
"net/http"
"regexp"
"strings"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
@@ -14,49 +14,41 @@ import (
type Scanner struct{}
// Ensure the Scanner satisfies the interface at compile time.
var _ detectors.Detector = (*Scanner)(nil)
var (
client = common.SaneHttpClient()
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
client = common.SaneHttpClient()
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"autodesk"}) + `\b([0-9A-Za-z]{32})\b`)
secretPat = regexp.MustCompile(detectors.PrefixRegex([]string{"autodesk"}) + `\b([0-9A-Za-z]{16})\b`)
)
// Keywords are used for efficiently pre-filtering chunks.
// Use identifiers in the secret preferably, or the provider name.
func (s Scanner) Keywords() []string {
return []string{"autodesk"}
func (s Scanner) Keywords() [][]byte {
return [][]byte{[]byte("autodesk")}
}
// FromData will find and optionally verify Autodesk secrets in a given set of bytes.
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
dataStr := string(data)
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
secretMatches := secretPat.FindAllStringSubmatch(dataStr, -1)
matches := keyPat.FindAllSubmatch(data, -1)
secretMatches := secretPat.FindAllSubmatch(data, -1)
for _, match := range matches {
if len(match) != 2 {
continue
}
resMatch := strings.TrimSpace(match[1])
resMatch := bytes.TrimSpace(match[1])
for _, secretMatch := range secretMatches {
if len(secretMatch) != 2 {
continue
}
resSecret := strings.TrimSpace(secretMatch[1])
resSecret := bytes.TrimSpace(secretMatch[1])
s1 := detectors.Result{
DetectorType: detectorspb.DetectorType_Autodesk,
Raw: []byte(resMatch),
RawV2: []byte(resMatch + resSecret),
Raw: resMatch,
RawV2: append(resMatch, resSecret...),
}
if verify {
payload := strings.NewReader(fmt.Sprintf(`grant_type=client_credentials&client_id=%s&client_secret=%s`, resMatch, resSecret))
payload := bytes.NewReader([]byte(fmt.Sprintf(`grant_type=client_credentials&client_id=%s&client_secret=%s`, string(resMatch), string(resSecret))))
req, err := http.NewRequestWithContext(ctx, "POST", "https://developer.api.autodesk.com/authentication/v1/authenticate", payload)
if err != nil {
continue
@@ -68,7 +60,6 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
if res.StatusCode >= 200 && res.StatusCode < 300 {
s1.Verified = true
} else {
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
continue
}
+7 -15
View File
@@ -1,13 +1,13 @@
package autoklose
import (
"bytes"
"context"
"encoding/json"
"fmt"
"io"
"net/http"
"regexp"
"strings"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
@@ -16,41 +16,34 @@ import (
type Scanner struct{}
// Ensure the Scanner satisfies the interface at compile time.
var _ detectors.Detector = (*Scanner)(nil)
var (
client = common.SaneHttpClient()
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"autoklose"}) + `\b([a-zA-Z0-9-]{32})\b`)
)
// Keywords are used for efficiently pre-filtering chunks.
// Use identifiers in the secret preferably, or the provider name.
func (s Scanner) Keywords() []string {
return []string{"autoklose"}
func (s Scanner) Keywords() [][]byte {
return [][]byte{[]byte("autoklose")}
}
// FromData will find and optionally verify Autoklose secrets in a given set of bytes.
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
dataStr := string(data)
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
matches := keyPat.FindAllSubmatch(data, -1)
for _, match := range matches {
if len(match) != 2 {
continue
}
resMatch := strings.TrimSpace(match[1])
resMatch := bytes.TrimSpace(match[1])
s1 := detectors.Result{
DetectorType: detectorspb.DetectorType_Autoklose,
Raw: []byte(resMatch),
Raw: resMatch,
}
if verify {
req, err := http.NewRequestWithContext(ctx, "GET", fmt.Sprintf("https://api.autoklose.com/api/campaigns/?api_token=%s", resMatch), nil)
req, err := http.NewRequestWithContext(ctx, "GET", fmt.Sprintf("https://api.autoklose.com/api/campaigns/?api_token=%s", string(resMatch)), nil)
if err != nil {
continue
}
@@ -68,7 +61,6 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
s1.Verified = false
}
} else {
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
continue
}
+7 -15
View File
@@ -1,10 +1,10 @@
package autopilot
import (
"bytes"
"context"
"net/http"
"regexp"
"strings"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
@@ -13,37 +13,30 @@ import (
type Scanner struct{}
// Ensure the Scanner satisfies the interface at compile time.
var _ detectors.Detector = (*Scanner)(nil)
var (
client = common.SaneHttpClient()
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"autopilot"}) + `\b([0-9a-f]{32})\b`)
)
// Keywords are used for efficiently pre-filtering chunks.
// Use identifiers in the secret preferably, or the provider name.
func (s Scanner) Keywords() []string {
return []string{"autopilot"}
func (s Scanner) Keywords() [][]byte {
return [][]byte{[]byte("autopilot")}
}
// FromData will find and optionally verify AutoPilot secrets in a given set of bytes.
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
dataStr := string(data)
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
matches := keyPat.FindAllSubmatch(data, -1)
for _, match := range matches {
if len(match) != 2 {
continue
}
resMatch := strings.TrimSpace(match[1])
resMatch := bytes.TrimSpace(match[1])
s1 := detectors.Result{
DetectorType: detectorspb.DetectorType_AutoPilot,
Raw: []byte(resMatch),
Raw: resMatch,
}
if verify {
@@ -51,14 +44,13 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
if err != nil {
continue
}
req.Header.Add("autopilotapikey", resMatch)
req.Header.Add("autopilotapikey", string(resMatch))
res, err := client.Do(req)
if err == nil {
defer res.Body.Close()
if res.StatusCode >= 200 && res.StatusCode < 300 {
s1.Verified = true
} else {
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
continue
}
@@ -1,11 +1,11 @@
package avazapersonalaccesstoken
import (
"bytes"
"context"
"fmt"
"net/http"
"regexp"
"strings"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
@@ -14,53 +14,43 @@ import (
type Scanner struct{}
// Ensure the Scanner satisfies the interface at compile time.
var _ detectors.Detector = (*Scanner)(nil)
var (
client = common.SaneHttpClient()
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
// The number prefix increments for every Personal Access Token created.
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"avaza"}) + `\b([0-9]+-[0-9a-f]{40})\b`)
)
// Keywords are used for efficiently pre-filtering chunks.
// Use identifiers in the secret preferably, or the provider name.
func (s Scanner) Keywords() []string {
return []string{"avaza"}
func (s Scanner) Keywords() [][]byte {
return [][]byte{[]byte("avaza")}
}
// FromData will find and optionally verify AvazaPersonalAccessToken secrets in a given set of bytes.
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
dataStr := string(data)
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
matches := keyPat.FindAllSubmatch(data, -1)
for _, match := range matches {
if len(match) != 2 {
continue
}
resMatch := strings.TrimSpace(match[1])
resMatch := bytes.TrimSpace(match[1])
s1 := detectors.Result{
DetectorType: detectorspb.DetectorType_AvazaPersonalAccessToken,
Raw: []byte(resMatch),
Raw: resMatch,
}
if verify {
req, err := http.NewRequest("GET", "https://api.avaza.com/api/Account", nil)
req, err := http.NewRequestWithContext(ctx, "GET", "https://api.avaza.com/api/Account", nil)
if err != nil {
continue
}
req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", resMatch))
req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", string(resMatch)))
res, err := client.Do(req)
if err == nil {
defer res.Body.Close()
if res.StatusCode >= 200 && res.StatusCode < 300 {
s1.Verified = true
} else {
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
continue
}
+7 -15
View File
@@ -1,12 +1,11 @@
package aviationstack
import (
"bytes"
"context"
"fmt"
"net/http"
"regexp"
"strings"
"time"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
@@ -21,37 +20,31 @@ var _ detectors.Detector = (*Scanner)(nil)
var (
client = common.SaneHttpClient()
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"aviationstack"}) + `\b([a-z0-9]{32})\b`)
)
// Keywords are used for efficiently pre-filtering chunks.
// Use identifiers in the secret preferably, or the provider name.
func (s Scanner) Keywords() []string {
return []string{"aviationstack"}
func (s Scanner) Keywords() [][]byte {
return [][]byte{[]byte("aviationstack")}
}
// FromData will find and optionally verify AviationStack secrets in a given set of bytes.
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
dataStr := string(data)
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
matches := keyPat.FindAllSubmatch(data, -1)
for _, match := range matches {
if len(match) != 2 {
continue
}
resMatch := strings.TrimSpace(match[1])
resMatch := bytes.TrimSpace(match[1])
s1 := detectors.Result{
DetectorType: detectorspb.DetectorType_AviationStack,
Raw: []byte(resMatch),
Raw: resMatch,
}
if verify {
timeout := 10 * time.Second
client.Timeout = timeout
req, err := http.NewRequestWithContext(ctx, "GET", fmt.Sprintf("https://api.aviationstack.com/v1/flights?access_key=%s", resMatch), nil)
req, err := http.NewRequestWithContext(ctx, "GET", fmt.Sprintf("https://api.aviationstack.com/v1/flights?access_key=%s", string(resMatch)), nil)
if err != nil {
continue
}
@@ -61,7 +54,6 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
if res.StatusCode >= 200 && res.StatusCode < 300 {
s1.Verified = true
} else {
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
continue
}
+173 -23
View File
@@ -1,6 +1,7 @@
package aws
import (
"bytes"
"context"
"crypto/hmac"
"crypto/sha256"
@@ -19,7 +20,8 @@ import (
type scanner struct {
verificationClient *http.Client
skipIDs map[string]struct{}
skipIDs map[string]struct{}
}
func New(opts ...func(*scanner)) *scanner {
@@ -61,14 +63,13 @@ var (
falsePositiveSecretCheck = regexp.MustCompile(`[a-f0-9]{40}`)
)
// Keywords are used for efficiently pre-filtering chunks.
// Use identifiers in the secret preferably, or the provider name.
func (s scanner) Keywords() []string {
return []string{
"AKIA",
"ABIA",
"ACCA",
"ASIA",
// Returning the keywords as byte arrays
func (s scanner) Keywords() [][]byte {
return [][]byte{
[]byte("AKIA"),
[]byte("ABIA"),
[]byte("ACCA"),
[]byte("ASIA"),
}
}
@@ -87,19 +88,17 @@ func GetHMAC(key []byte, data []byte) []byte {
// FromData will find and optionally verify AWS secrets in a given set of bytes.
func (s scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
dataStr := string(data)
idMatches := idPat.FindAllStringSubmatch(dataStr, -1)
secretMatches := secretPat.FindAllStringSubmatch(dataStr, -1)
idMatches := idPat.FindAllSubmatch(data, -1)
secretMatches := secretPat.FindAllSubmatch(data, -1)
for _, idMatch := range idMatches {
if len(idMatch) != 2 {
continue
}
resIDMatch := strings.TrimSpace(idMatch[1])
resIDMatch := bytes.TrimSpace(idMatch[1])
if s.skipIDs != nil {
if _, ok := s.skipIDs[resIDMatch]; ok {
if _, ok := s.skipIDs[string(resIDMatch)]; ok {
continue
}
}
@@ -108,13 +107,13 @@ func (s scanner) FromData(ctx context.Context, verify bool, data []byte) (result
if len(secretMatch) != 2 {
continue
}
resSecretMatch := strings.TrimSpace(secretMatch[1])
resSecretMatch := bytes.TrimSpace(secretMatch[1])
s1 := detectors.Result{
DetectorType: detectorspb.DetectorType_AWS,
Raw: []byte(resIDMatch),
Redacted: resIDMatch,
RawV2: []byte(resIDMatch + resSecretMatch),
Raw: resIDMatch,
Redacted: string(resIDMatch),
RawV2: append(append([]byte(nil), resIDMatch...), resSecretMatch...),
}
if verify {
@@ -146,17 +145,17 @@ func (s scanner) FromData(ctx context.Context, verify bool, data []byte) (result
params.Add("Action", "GetCallerIdentity")
params.Add("Version", "2011-06-15")
params.Add("X-Amz-Algorithm", algorithm)
params.Add("X-Amz-Credential", resIDMatch+"/"+credentialScope)
params.Add("X-Amz-Credential", string(resIDMatch)+"/"+credentialScope)
params.Add("X-Amz-Date", amzDate)
params.Add("X-Amz-Expires", "30")
params.Add("X-Amz-SignedHeaders", signedHeaders)
canonicalQuerystring := params.Encode()
payloadHash := GetHash("") // empty payload
canonicalRequest := method + "\n" + canonicalURI + "\n" + canonicalQuerystring + "\n" + canonicalHeaders + "\n" + signedHeaders + "\n" + payloadHash
canonicalRequest := string(method) + "\n" + string(canonicalURI) + "\n" + string(canonicalQuerystring) + "\n" + string(canonicalHeaders) + "\n" + string(signedHeaders) + "\n" + string(payloadHash)
// TASK 2: CREATE THE STRING TO SIGN.
stringToSign := algorithm + "\n" + amzDate + "\n" + credentialScope + "\n" + GetHash(canonicalRequest)
stringToSign := string(algorithm) + "\n" + string(amzDate) + "\n" + string(credentialScope) + "\n" + string(GetHash(canonicalRequest))
// TASK 3: CALCULATE THE SIGNATURE.
// https://docs.aws.amazon.com/general/latest/gr/sigv4-calculate-signature.html
@@ -228,7 +227,7 @@ func (s scanner) FromData(ctx context.Context, verify bool, data []byte) (result
}
// If the result is unverified and matches something like a git hash, don't include it in the results.
if !s1.Verified && falsePositiveSecretCheck.MatchString(resSecretMatch) {
if !s1.Verified && falsePositiveSecretCheck.Match(resSecretMatch) {
continue
}
@@ -242,6 +241,157 @@ func (s scanner) FromData(ctx context.Context, verify bool, data []byte) (result
return awsCustomCleanResults(results), nil
}
// func (s scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
// idMatches := idPat.FindAllSubmatch(data, -1)
// secretMatches := secretPat.FindAllSubmatch(data, -1)
//
// for _, idMatch := range idMatches {
// if len(idMatch) != 2 {
// continue
// }
// resIDMatch := bytes.TrimSpace(idMatch[1])
//
// if s.skipIDs != nil {
// if _, ok := s.skipIDs[string(resIDMatch)]; ok {
// continue
// }
// }
//
// for _, secretMatch := range secretMatches {
// if len(secretMatch) != 2 {
// continue
// }
// resSecretMatch := bytes.TrimSpace(secretMatch[1])
//
// s1 := detectors.Result{
// DetectorType: detectorspb.DetectorType_AWS,
// Raw: resIDMatch,
// Redacted: string(resIDMatch),
// RawV2: append(append([]byte(nil), resIDMatch...), resSecretMatch...),
// }
//
// if verify {
// // Each string operation is converted into byte operations
//
// method := []byte("GET")
// service := []byte("sts")
// host := []byte("sts.amazonaws.com")
// region := []byte("us-east-1")
// endpoint := []byte("https://sts.amazonaws.com")
// datestamp := []byte(time.Now().UTC().Format("20060102"))
// amzDate := []byte(time.Now().UTC().Format("20060102T150405Z0700"))
//
// req, err := http.NewRequestWithContext(ctx, string(method), string(endpoint), nil)
// if err != nil {
// continue
// }
// req.Header.Set("Accept", "application/json")
//
// // Byte operations for the subsequent string operations
//
// canonicalURI := []byte("/")
// canonicalHeaders := []byte("host:" + string(host) + "\n")
// signedHeaders := []byte("host")
// algorithm := []byte("AWS4-HMAC-SHA256")
// credentialScope := []byte(fmt.Sprintf("%s/%s/%s/aws4_request", datestamp, region, service))
//
// params := req.URL.Query()
// params.Add("Action", "GetCallerIdentity")
// params.Add("Version", "2011-06-15")
// params.Add("X-Amz-Algorithm", string(algorithm))
// params.Add("X-Amz-Credential", string(resIDMatch)+"/"+string(credentialScope))
// params.Add("X-Amz-Date", string(amzDate))
// params.Add("X-Amz-Expires", "30")
// params.Add("X-Amz-SignedHeaders", string(signedHeaders))
//
// canonicalQuerystring := params.Encode()
// payloadHash := GetHash([]byte("")) // empty payload
// canonicalRequest := string(method) + "\n" + string(canonicalURI) + "\n" + string(canonicalQuerystring) + "\n" + string(canonicalHeaders) + "\n" + string(signedHeaders) + "\n" + string(payloadHash)
//
// // TASK 2: CREATE THE STRING TO SIGN.
// stringToSign := string(algorithm) + "\n" + string(amzDate) + "\n" + string(credentialScope) + "\n" + string(GetHash([]byte(canonicalRequest)))
//
// hash := GetHMAC([]byte(fmt.Sprintf("AWS4%s", string(resSecretMatch))), datestamp)
// hash = GetHMAC(hash, region)
// hash = GetHMAC(hash, service)
// hash = GetHMAC(hash, []byte("aws4_request"))
//
// signature2 := GetHMAC(hash, []byte(stringToSign)) // Get Signature HMAC SHA256
// signature := hex.EncodeToString(signature2)
//
// params.Add("X-Amz-Signature", signature)
// req.Header.Add("Content-type", "application/x-www-form-urlencoded; charset=utf-8")
// req.URL.RawQuery = params.Encode()
//
// client := s.verificationClient
// if client == nil {
// client = defaultVerificationClient
// }
// res, err := client.Do(req)
// if err == nil {
//
// if res.StatusCode >= 200 && res.StatusCode < 300 {
// identityInfo := identityRes{}
// err := json.NewDecoder(res.Body).Decode(&identityInfo)
// if err == nil {
// s1.Verified = true
// s1.ExtraData = map[string]string{
// "account": identityInfo.GetCallerIdentityResponse.GetCallerIdentityResult.Account,
// "user_id": identityInfo.GetCallerIdentityResponse.GetCallerIdentityResult.UserID,
// "arn": identityInfo.GetCallerIdentityResponse.GetCallerIdentityResult.Arn,
// }
// } else {
// s1.VerificationError = err
// }
// res.Body.Close()
// } else {
// // This function will check false positives for common test words, but also it will make sure the key appears "random" enough to be a real key.
// if detectors.IsKnownFalsePositive(resSecretMatch, detectors.DefaultFalsePositives, true) {
// continue
// }
//
// if res.StatusCode == 403 {
// var body awsErrorResponseBody
// err = json.NewDecoder(res.Body).Decode(&body)
// res.Body.Close()
// if err == nil {
// // All instances of the code I've seen in the wild are PascalCased but this check is
// // case-insensitive out of an abundance of caution
// if strings.EqualFold(body.Error.Code, "InvalidClientTokenId") {
// // determinate failure - nothing to do
// } else {
// // We see a surprising number of false-negative signature mismatch errors here
// // (The official SDK somehow elicits even more than just making the request
// // ourselves)
// s1.VerificationError = fmt.Errorf("request to %v returned status %d with an unexpected reason (%s: %s)", res.Request.URL, res.StatusCode, body.Error.Code, body.Error.Message)
// }
// } else {
// s1.VerificationError = fmt.Errorf("couldn't parse the sts response body (%v)", err)
// }
// } else {
// s1.VerificationError = fmt.Errorf("request to %v returned unexpected status %d", res.Request.URL, res.StatusCode)
// }
// }
// } else {
// s1.VerificationError = err
// }
// }
//
// // If the result is unverified and matches something like a git hash, don't include it in the results.
// if !s1.Verified && falsePositiveSecretCheck.Match(resSecretMatch) {
// continue
// }
//
// results = append(results, s1)
// // If we've found a verified match with this ID, we don't need to look for any more. So move on to the next ID.
// if s1.Verified {
// break
// }
// }
// }
// return awsCustomCleanResults(results), nil
// }
func awsCustomCleanResults(results []detectors.Result) []detectors.Result {
if len(results) == 0 {
return results
+2
View File
@@ -43,6 +43,8 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
instanceMatches := instancePat.FindAllStringSubmatch(dataStr, -1)
tokenMatches := accessTokenPat.FindAllStringSubmatch(dataStr, -1)
fmt.Printf("instanceMatches: %v\n", instanceMatches)
for _, instance := range instanceMatches {
if len(instance) != 1 {
continue
+1 -9
View File
@@ -2,10 +2,6 @@ package engine
import (
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/envoyapikey"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/huggingface"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/salesforce"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/snowflake"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/trufflehogenterprise"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/abbysale"
@@ -291,7 +287,6 @@ import (
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/getresponse"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/getsandbox"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/github"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/github_oauth2"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/github_old"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/githubapp"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/gitlab"
@@ -561,6 +556,7 @@ import (
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/satismeterwritekey"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/saucelabs"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/scalewaykey"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/github_oauth2"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/scalr"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/scrapeowl"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/scraperapi"
@@ -1536,10 +1532,6 @@ func DefaultDetectors() []detectors.Detector {
couchbase.Scanner{},
envoyapikey.Scanner{},
github_oauth2.Scanner{},
snowflake.Scanner{},
huggingface.Scanner{},
trufflehogenterprise.Scanner{},
salesforce.Scanner{},
}
}
+31 -84
View File
@@ -12,8 +12,6 @@ import (
"strings"
"time"
v1 "github.com/google/go-containerregistry/pkg/v1"
"github.com/google/go-containerregistry/pkg/v1/tarball"
"github.com/h2non/filetype"
"github.com/mholt/archiver/v4"
@@ -31,40 +29,17 @@ var (
maxDepth = 5
maxSize = 250 * 1024 * 1024 // 20MB
maxTimeout = time.Duration(30) * time.Second
defaultBufferSize = 512
)
// Ensure the Archive satisfies the interfaces at compile time.
var _ SpecializedHandler = (*Archive)(nil)
// tempEnv contains the temporary file and directory information for the archive.
type tempEnv struct {
tempFile *os.File
tempFileName string
extractPath string
}
// dockerImageDetails contains the details of the Docker image and the temporary environment.
type dockerImageDetails struct {
image v1.Image
temp tempEnv
}
// Archive is a handler for extracting and decompressing archives.
type Archive struct {
size int
currentDepth int
}
// DockerTarReader is a wrapper for io.Reader that also stores the temp filepath and image of the tar file.
// Temporary file path information needed for inner-workings of go-containerregistry Image methods.
type DockerTarReader struct {
io.Reader
img v1.Image
tmpEnv tempEnv
}
// New sets a default maximum size and current size counter.
func (a *Archive) New() {
a.size = 0
@@ -87,7 +62,7 @@ func SetArchiveMaxTimeout(timeout time.Duration) {
// FromFile extracts the files from an archive.
func (a *Archive) FromFile(originalCtx context.Context, data io.Reader) chan []byte {
archiveChan := make(chan []byte, defaultBufferSize)
archiveChan := make(chan []byte, 512)
go func() {
ctx, cancel := context.WithTimeout(originalCtx, maxTimeout)
logger := logContext.AddLogger(ctx).Logger()
@@ -208,35 +183,35 @@ func (a *Archive) ReadToMax(ctx context.Context, reader io.Reader) (data []byte,
logger.Error(err, "Panic occurred when reading archive")
}
}()
if common.IsDone(ctx) {
return nil, ctx.Err()
fileContent := bytes.Buffer{}
logger.V(5).Info("Remaining buffer capacity", "bytes", maxSize-a.size)
for i := 0; i <= maxSize/512; i++ {
if common.IsDone(ctx) {
return nil, ctx.Err()
}
fileChunk := make([]byte, 512)
bRead, err := reader.Read(fileChunk)
if err != nil && !errors.Is(err, io.EOF) {
return []byte{}, err
}
a.size += bRead
if len(fileChunk) > 0 {
fileContent.Write(fileChunk[0:bRead])
}
if bRead < 512 {
return fileContent.Bytes(), nil
}
if a.size >= maxSize && bRead == 512 {
logger.V(2).Info("Max archive size reached.")
return fileContent.Bytes(), nil
}
}
var fileContent bytes.Buffer
// Create a limited reader to ensure we don't read more than the max size.
lr := io.LimitReader(reader, int64(maxSize))
// Using io.CopyBuffer for performance advantages. Though buf is mandatory
// for the method, due to the internal implementation of io.CopyBuffer, when
// *bytes.Buffer implements io.WriterTo or io.ReaderFrom, the provided buf
// is simply ignored. Thus, we can pass nil for the buf parameter.
_, err = io.CopyBuffer(&fileContent, lr, nil)
if err != nil && !errors.Is(err, io.EOF) {
return nil, err
}
if fileContent.Len() == maxSize {
logger.V(2).Info("Max archive size reached.")
}
return fileContent.Bytes(), nil
}
const (
arMimeType = "application/x-unix-archive"
rpmMimeType = "application/x-rpm"
tarMimeType = "application/x-tar"
)
// Define a map of mime types to corresponding command-line tools
@@ -290,21 +265,6 @@ func (a *Archive) HandleSpecialized(ctx logContext.Context, reader io.Reader) (i
return nil, false, err
}
reader, err = a.extractRpmContent(ctx, reader)
case tarMimeType:
//check if tar is a docker image
isImg, details, err := a.isDockerImage(ctx, reader)
if err != nil {
return nil, false, err
}
if !isImg {
return nil, false, nil
}
// Build dockerReader containing docker image
reader = DockerTarReader{
Reader: reader,
img: details.image,
tmpEnv: details.temp,
}
default:
return reader, false, nil
}
@@ -329,8 +289,8 @@ func (a *Archive) extractDebContent(ctx logContext.Context, file io.Reader) (io.
if err != nil {
return nil, err
}
defer os.RemoveAll(tmpEnv.extractPath)
defer os.Remove(tmpEnv.tempFileName)
defer os.RemoveAll(tmpEnv.extractPath)
cmd := exec.Command("ar", "x", tmpEnv.tempFile.Name())
cmd.Dir = tmpEnv.extractPath
@@ -367,8 +327,8 @@ func (a *Archive) extractRpmContent(ctx logContext.Context, file io.Reader) (io.
if err != nil {
return nil, err
}
defer os.RemoveAll(tmpEnv.extractPath)
defer os.Remove(tmpEnv.tempFileName)
defer os.RemoveAll(tmpEnv.extractPath)
// Use rpm2cpio to convert the RPM file to a cpio archive and then extract it using cpio command.
cmd := exec.Command("sh", "-c", "rpm2cpio "+tmpEnv.tempFile.Name()+" | cpio -id")
@@ -467,6 +427,12 @@ func (a *Archive) handleExtractedFiles(ctx logContext.Context, env tempEnv, hand
return dataArchiveName, nil
}
type tempEnv struct {
tempFile *os.File
tempFileName string
extractPath string
}
// createTempEnv creates a temporary file and a temporary directory for extracting archives.
// The caller is responsible for removing these temporary resources
// (both the file and directory) when they are no longer needed.
@@ -510,22 +476,3 @@ func openDataArchive(extractPath string, dataArchiveName string) (io.ReadCloser,
}
return dataFile, nil
}
// isDockerImage checks if a reader object is a docker image.
// Returns true if the tar file is a docker image, dockerImageDetails, and error.
// Caller is responsible for removing temporary files and directories IF the file is a docker image.
// The docker scanner requires a file path to the image, so we must leave the temporary file on disk.
func (a *Archive) isDockerImage(ctx context.Context, file io.Reader) (isImg bool, details dockerImageDetails, err error) {
tmpEnv, err := a.createTempEnv(ctx, file)
if err != nil {
return false, dockerImageDetails{}, err
}
img, err := tarball.ImageFromPath(tmpEnv.tempFileName, nil)
if err != nil {
os.Remove(tmpEnv.tempFileName)
os.RemoveAll(tmpEnv.extractPath)
return false, dockerImageDetails{}, err
}
return true, dockerImageDetails{image: img, temp: tmpEnv}, nil
}
-102
View File
@@ -1,102 +0,0 @@
//go:build integration
// +build integration
package handlers
import (
"context"
"os"
"testing"
diskbufferreader "github.com/bill-rich/disk-buffer-reader"
"github.com/google/go-containerregistry/pkg/name"
"github.com/google/go-containerregistry/pkg/v1/remote"
"github.com/google/go-containerregistry/pkg/v1/tarball"
"github.com/stretchr/testify/assert"
logContext "github.com/trufflesecurity/trufflehog/v3/pkg/context"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/source_metadatapb"
"github.com/trufflesecurity/trufflehog/v3/pkg/sanitizer"
"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
)
func TestFoundKeyInDockerTar(t *testing.T) {
// URI of the Docker image to clone
// imageURI := "ghcr.io/joeleonjr/getting-started-app-with-canary-token:main"
imageURI := "trufflesecurity/secrets"
var imageName name.Reference
imageName, err := name.NewTag(imageURI)
if err != nil {
t.Errorf("%s: error creating image name: %s", imageURI, err)
}
// Pull the image
img, err := remote.Image(imageName)
if err != nil {
t.Errorf("%s: error pulling image: %s", imageURI, err)
}
tempFile, err := os.CreateTemp("", "archive_test_docker_img.tar")
if err != nil {
t.Errorf("%s: error creating temporary file: %s", imageURI, err)
return
}
defer os.Remove(tempFile.Name()) // Clean up the temporary file
// Save the image as a tar file
err = tarball.WriteToFile(tempFile.Name(), imageName, img)
if err != nil {
t.Errorf("%s: error saving image as tar file: %s", imageURI, err)
return
}
ctx := logContext.AddLogger(context.Background())
inputFile, err := os.Open(tempFile.Name())
if err != nil {
t.Errorf("%s: error opening tar file: %s", imageURI, err)
return
}
defer inputFile.Close()
reReader, err := diskbufferreader.New(inputFile)
if err != nil {
t.Errorf("%s: error creating re-readable reader: %s", imageURI, err)
return
}
defer reReader.Close()
chunkSkel := &sources.Chunk{
SourceType: 1,
SourceName: "filesystem",
SourceID: 1,
SourceMetadata: &source_metadatapb.MetaData{
Data: &source_metadatapb.MetaData_Filesystem{
Filesystem: &source_metadatapb.Filesystem{
File: sanitizer.UTF8(tempFile.Name()),
},
},
},
Verify: true,
}
chunksChan := make(chan *sources.Chunk, 1)
go func() {
defer close(chunksChan)
HandleFile(ctx, reReader, chunkSkel, chunksChan)
}()
secret := "AKIAXYZDQCEN4B6JSJQI"
// Read from the channel and validate the secrets.
foundSecret := ""
for chunkCh := range chunksChan {
foundSecret += string(chunkCh.Data)
}
assert.Contains(t, foundSecret, secret)
}
+3 -57
View File
@@ -1,7 +1,6 @@
package handlers
import (
"bytes"
"context"
"io"
"net/http"
@@ -108,8 +107,7 @@ func TestHandleFile(t *testing.T) {
ch := make(chan *sources.Chunk, 2)
// Context cancels the operation.
ctx := logContext.AddLogger(context.Background())
canceledCtx, cancel := logContext.WithCancel(ctx)
canceledCtx, cancel := context.WithCancel(context.Background())
cancel()
assert.False(t, HandleFile(canceledCtx, strings.NewReader("file"), &sources.Chunk{}, ch))
@@ -124,62 +122,10 @@ func TestHandleFile(t *testing.T) {
assert.NoError(t, err)
assert.Equal(t, 0, len(ch))
assert.True(t, HandleFile(ctx, reader, &sources.Chunk{}, ch))
assert.True(t, HandleFile(context.Background(), reader, &sources.Chunk{}, ch))
assert.Equal(t, 1, len(ch))
}
func TestReadToMax(t *testing.T) {
tests := []struct {
name string
input []byte
expected []byte
}{
{
name: "read full content within maxSize",
input: []byte("abcdefg"),
expected: []byte("abcdefg"),
},
{
name: "read content larger than maxSize",
input: make([]byte, maxSize+10), // this creates a byte slice 10 bytes larger than maxSize
expected: make([]byte, maxSize),
},
{
name: "empty input",
input: []byte(""),
expected: []byte(""),
},
}
a := &Archive{}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
reader := bytes.NewReader(tt.input)
output, err := a.ReadToMax(context.Background(), reader)
assert.Nil(t, err)
assert.Equal(t, tt.expected, output)
})
}
}
func BenchmarkReadToMax(b *testing.B) {
data := bytes.Repeat([]byte("a"), 1024*1000) // 1MB of data.
reader := bytes.NewReader(data)
a := &Archive{}
b.ResetTimer()
for i := 0; i < b.N; i++ {
b.StartTimer()
_, _ = a.ReadToMax(context.Background(), reader)
b.StopTimer()
_, _ = reader.Seek(0, 0) // Reset the reader position.
a.size = 0 // Reset archive size.
}
}
func TestExtractDebContent(t *testing.T) {
// Open the sample .deb file from the testdata folder.
file, err := os.Open("testdata/test.deb")
@@ -203,7 +149,7 @@ func TestExtractTarContent(t *testing.T) {
assert.Nil(t, err)
defer file.Close()
ctx := logContext.AddLogger(context.Background())
ctx := context.Background()
chunkCh := make(chan *sources.Chunk)
go func() {
+2 -10
View File
@@ -3,13 +3,11 @@ package handlers
import (
"context"
"io"
"os"
diskbufferreader "github.com/bill-rich/disk-buffer-reader"
logContext "github.com/trufflesecurity/trufflehog/v3/pkg/context"
"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
"github.com/trufflesecurity/trufflehog/v3/pkg/sources/docker"
)
func DefaultHandlers() []Handler {
@@ -39,7 +37,7 @@ type Handler interface {
// packages them in the provided chunk skeleton, and sends them to chunksChan.
// The function returns true if processing was successful and false otherwise.
// Context is used for cancellation, and the caller is responsible for canceling it if needed.
func HandleFile(ctx logContext.Context, file io.Reader, chunkSkel *sources.Chunk, chunksChan chan *sources.Chunk) bool {
func HandleFile(ctx context.Context, file io.Reader, chunkSkel *sources.Chunk, chunksChan chan *sources.Chunk) bool {
aCtx := logContext.AddLogger(ctx)
for _, h := range DefaultHandlers() {
h.New()
@@ -58,15 +56,9 @@ func HandleFile(ctx logContext.Context, file io.Reader, chunkSkel *sources.Chunk
if specialHandler, ok := h.(SpecializedHandler); ok {
file, isSpecial, err := specialHandler.HandleSpecialized(aCtx, reReader)
if isSpecial {
if dockerTarReader, ok := file.(DockerTarReader); ok {
// Clean up temporary files when done.
defer os.RemoveAll(dockerTarReader.tmpEnv.extractPath)
defer os.Remove(dockerTarReader.tmpEnv.tempFileName)
err = docker.ScanDockerImg(ctx, dockerTarReader.img, chunksChan, chunkSkel)
return err == nil
}
return handleChunks(aCtx, h.FromFile(ctx, file), chunkSkel, chunksChan)
}
if err != nil {
aCtx.Logger().Error(err, "error handling file")
}
+1 -1
View File
@@ -133,7 +133,7 @@ func BranchHeads(repo *gogit.Repository) (map[string]*object.Commit, error) {
}
headCommit, err := repo.CommitObject(*headHash)
if err != nil {
logger.Error(err, "unable to get commit", "head_hash", headHash.String())
logger.Error(err, "unable to get commit", "commit", headCommit.String())
return nil
}
branches[branchName] = headCommit
+1 -1
View File
@@ -20,7 +20,7 @@ const (
// Chunker takes a chunk and splits it into chunks of ChunkSize.
func Chunker(originalChunk *Chunk) chan *Chunk {
chunkChan := make(chan *Chunk, 1)
chunkChan := make(chan *Chunk)
go func() {
defer close(chunkChan)
if len(originalChunk.Data) <= TotalChunkSize {
-1
View File
@@ -238,7 +238,6 @@ func (s *Source) chunkAction(ctx context.Context, proj project, bld build, act a
SourceType: s.Type(),
SourceName: s.name,
SourceID: s.SourceID(),
JobID: s.JobID(),
Data: removeCircleSha1Line(data.Bytes()),
SourceMetadata: &source_metadatapb.MetaData{
Data: &source_metadatapb.MetaData_Circleci{
+65 -90
View File
@@ -78,7 +78,7 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) err
return err
}
ctx.Logger().WithValues("source_type", s.Type(), "source_name", s.name)
logger := ctx.Logger().WithValues("source_type", s.Type(), "source_name", s.name)
for _, image := range s.conn.GetImages() {
var img v1.Image
@@ -115,117 +115,92 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) err
}
}
chunkSkel := &sources.Chunk{
SourceType: s.Type(),
SourceName: s.name,
SourceID: s.SourceID(),
JobID: s.JobID(),
SourceMetadata: &source_metadatapb.MetaData{
Data: &source_metadatapb.MetaData_Docker{
Docker: &source_metadatapb.Docker{
Image: base,
Tag: tag,
},
},
},
Verify: s.verify,
}
err = ScanDockerImg(ctx, img, chunksChan, chunkSkel)
logger = logger.WithValues("image", base, "tag", tag)
logger.V(2).Info("scanning image")
layers, err := img.Layers()
if err != nil {
return err
}
}
for _, layer := range layers {
return nil
}
func ScanDockerImg(ctx context.Context, img v1.Image, chunksChan chan *sources.Chunk, chunkSkel *sources.Chunk) error {
base, tag := chunkSkel.BaseAndTagForDockerImg()
logger := ctx.Logger().WithValues("image", base, "tag", tag)
logger.V(2).Info("scanning image")
layers, err := img.Layers()
if err != nil {
return err
}
for _, layer := range layers {
digest, err := layer.Digest()
if err != nil {
return err
}
logger = logger.WithValues("layer", digest.String())
logger.V(2).Info("scanning layer")
rc, err := layer.Compressed()
if err != nil {
return err
}
defer rc.Close()
gzipReader, err := gzip.NewReader(rc)
if err != nil {
return err
}
defer gzipReader.Close()
tarReader := tar.NewReader(gzipReader)
for {
header, err := tarReader.Next()
if errors.Is(err, io.EOF) {
break // End of archive
}
digest, err := layer.Digest()
if err != nil {
return err
}
// Skip files larger than FilesizeLimitBytes
if header.Size > FilesizeLimitBytes {
continue
}
logger = logger.WithValues("layer", digest.String())
logger.V(2).Info("scanning layer")
file := bytes.NewBuffer(make([]byte, 0, header.Size))
_, err = io.Copy(file, tarReader)
rc, err := layer.Compressed()
if err != nil {
return err
}
chunk := &sources.Chunk{
// This must be TYPE_DOCKER or SetResultLineNumber could panic
SourceType: sourcespb.SourceType_SOURCE_TYPE_DOCKER,
SourceName: chunkSkel.SourceName,
SourceID: chunkSkel.SourceID,
Data: file.Bytes(),
SourceMetadata: &source_metadatapb.MetaData{
Data: &source_metadatapb.MetaData_Docker{
Docker: &source_metadatapb.Docker{
File: "/" + header.Name, // Prepend /, because the file is relative to the root of the image
Image: base,
Tag: tag,
Layer: digest.String(),
defer rc.Close()
gzipReader, err := gzip.NewReader(rc)
if err != nil {
return err
}
defer gzipReader.Close()
tarReader := tar.NewReader(gzipReader)
for {
header, err := tarReader.Next()
if errors.Is(err, io.EOF) {
break // End of archive
}
if err != nil {
return err
}
// Skip files larger than FilesizeLimitBytes
if header.Size > FilesizeLimitBytes {
continue
}
file := bytes.NewBuffer(make([]byte, 0, header.Size))
_, err = io.Copy(file, tarReader)
if err != nil {
return err
}
chunk := &sources.Chunk{
SourceType: s.Type(),
SourceName: s.name,
SourceID: s.SourceID(),
Data: file.Bytes(),
SourceMetadata: &source_metadatapb.MetaData{
Data: &source_metadatapb.MetaData_Docker{
Docker: &source_metadatapb.Docker{
File: "/" + header.Name, // Prepend /, because the file is relative to the root of the image
Image: base,
Tag: tag,
Layer: digest.String(),
},
},
},
},
Verify: chunkSkel.Verify,
Verify: s.verify,
}
select {
case chunksChan <- chunk:
case <-ctx.Done():
return ctx.Err()
}
}
select {
case chunksChan <- chunk:
case <-ctx.Done():
return ctx.Err()
}
dockerLayersScanned.WithLabelValues(s.name).Inc()
}
dockerLayersScanned.WithLabelValues(chunkSkel.SourceName).Inc()
dockerImagesScanned.WithLabelValues(s.name).Inc()
}
dockerImagesScanned.WithLabelValues(chunkSkel.SourceName).Inc()
return nil
}
-2
View File
@@ -161,7 +161,6 @@ func (s *Source) scanFile(ctx context.Context, path string, chunksChan chan *sou
SourceType: s.Type(),
SourceName: s.name,
SourceID: s.SourceID(),
JobID: s.JobID(),
SourceMetadata: &source_metadatapb.MetaData{
Data: &source_metadatapb.MetaData_Filesystem{
Filesystem: &source_metadatapb.Filesystem{
@@ -192,7 +191,6 @@ func (s *Source) scanFile(ctx context.Context, path string, chunksChan chan *sou
SourceType: s.Type(),
SourceName: s.name,
SourceID: s.SourceID(),
JobID: s.JobID(),
Data: data.Bytes(),
SourceMetadata: &source_metadatapb.MetaData{
Data: &source_metadatapb.MetaData_Filesystem{
-1
View File
@@ -324,7 +324,6 @@ func (s *Source) processObject(ctx context.Context, o object) error {
chunkSkel := &sources.Chunk{
SourceName: s.name,
SourceType: s.Type(),
JobID: s.JobID(),
SourceID: s.sourceId,
Verify: s.verify,
SourceMetadata: &source_metadatapb.MetaData{
-7
View File
@@ -446,7 +446,6 @@ func (s *Git) ScanCommits(ctx context.Context, repo *git.Repository, path string
chunkSkel := &sources.Chunk{
SourceName: s.sourceName,
SourceID: s.sourceID,
JobID: s.jobID,
SourceType: s.sourceType,
SourceMetadata: metadata,
Verify: s.verify,
@@ -465,7 +464,6 @@ func (s *Git) ScanCommits(ctx context.Context, repo *git.Repository, path string
chunksChan <- &sources.Chunk{
SourceName: s.sourceName,
SourceID: s.sourceID,
JobID: s.jobID,
SourceType: s.sourceType,
SourceMetadata: metadata,
Data: diff.Content.Bytes(),
@@ -492,7 +490,6 @@ func (s *Git) gitChunk(ctx context.Context, diff gitparse.Diff, fileName, email,
chunksChan <- &sources.Chunk{
SourceName: s.sourceName,
SourceID: s.sourceID,
JobID: s.jobID,
SourceType: s.sourceType,
SourceMetadata: metadata,
Data: append([]byte{}, newChunkBuffer.Bytes()...),
@@ -507,7 +504,6 @@ func (s *Git) gitChunk(ctx context.Context, diff gitparse.Diff, fileName, email,
chunksChan <- &sources.Chunk{
SourceName: s.sourceName,
SourceID: s.sourceID,
JobID: s.jobID,
SourceType: s.sourceType,
SourceMetadata: metadata,
Data: line,
@@ -527,7 +523,6 @@ func (s *Git) gitChunk(ctx context.Context, diff gitparse.Diff, fileName, email,
chunksChan <- &sources.Chunk{
SourceName: s.sourceName,
SourceID: s.sourceID,
JobID: s.jobID,
SourceType: s.sourceType,
SourceMetadata: metadata,
Data: append([]byte{}, newChunkBuffer.Bytes()...),
@@ -593,7 +588,6 @@ func (s *Git) ScanStaged(ctx context.Context, repo *git.Repository, path string,
chunkSkel := &sources.Chunk{
SourceName: s.sourceName,
SourceID: s.sourceID,
JobID: s.jobID,
SourceType: s.sourceType,
SourceMetadata: metadata,
Verify: s.verify,
@@ -608,7 +602,6 @@ func (s *Git) ScanStaged(ctx context.Context, repo *git.Repository, path string,
chunksChan <- &sources.Chunk{
SourceName: s.sourceName,
SourceID: s.sourceID,
JobID: s.jobID,
SourceType: s.sourceType,
SourceMetadata: metadata,
Data: diff.Content.Bytes(),
+2 -7
View File
@@ -149,16 +149,14 @@ func (s *Source) newFilteredRepoCache(c cache.Cache, include, exclude []string)
for _, ig := range include {
g, err := glob.Compile(ig)
if err != nil {
s.log.V(1).Info("invalid include glob", "include_value", ig, "err", err)
continue
s.log.V(1).Info("invalid include glob", "glob", g, "err", err)
}
includeGlobs = append(includeGlobs, g)
}
for _, eg := range exclude {
g, err := glob.Compile(eg)
if err != nil {
s.log.V(1).Info("invalid exclude glob", "exclude_value", eg, "err", err)
continue
s.log.V(1).Info("invalid exclude glob", "glob", g, "err", err)
}
excludeGlobs = append(excludeGlobs, g)
}
@@ -1225,7 +1223,6 @@ func (s *Source) chunkIssueComments(ctx context.Context, repo, repoPath string,
chunk := &sources.Chunk{
SourceName: s.name,
SourceID: s.SourceID(),
JobID: s.JobID(),
SourceType: s.Type(),
SourceMetadata: &source_metadatapb.MetaData{
Data: &source_metadatapb.MetaData_Github{
@@ -1259,7 +1256,6 @@ func (s *Source) chunkPullRequestComments(ctx context.Context, repo string, comm
SourceName: s.name,
SourceID: s.SourceID(),
SourceType: s.Type(),
JobID: s.JobID(),
SourceMetadata: &source_metadatapb.MetaData{
Data: &source_metadatapb.MetaData_Github{
Github: &source_metadatapb.Github{
@@ -1291,7 +1287,6 @@ func (s *Source) chunkGistComments(ctx context.Context, gistUrl string, comments
SourceName: s.name,
SourceID: s.SourceID(),
SourceType: s.Type(),
JobID: s.JobID(),
SourceMetadata: &source_metadatapb.MetaData{
Data: &source_metadatapb.MetaData_Github{
Github: &source_metadatapb.Github{
+4 -4
View File
@@ -71,11 +71,11 @@ func (r *JobProgressRef) Done() <-chan struct{} {
// CancelRun requests that the job this is referencing is cancelled and stops
// running. This method will have no effect if the job does not allow
// cancellation.
func (r *JobProgressRef) CancelRun(cause error) {
func (r *JobProgressRef) CancelRun() {
if r.jobProgress == nil || r.jobProgress.jobCancel == nil {
return
}
r.jobProgress.jobCancel(cause)
r.jobProgress.jobCancel()
}
// Fatal is a wrapper around error to differentiate non-fatal errors from fatal
@@ -108,7 +108,7 @@ type JobProgress struct {
ctx context.Context
cancel context.CancelFunc
// Requests to cancel the job.
jobCancel context.CancelCauseFunc
jobCancel context.CancelFunc
// Metrics.
metrics JobProgressMetrics
metricsLock sync.Mutex
@@ -150,7 +150,7 @@ func WithHooks(hooks ...JobProgressHook) func(*JobProgress) {
}
// WithCancel allows cancelling the job by the JobProgressRef.
func WithCancel(cancel context.CancelCauseFunc) func(*JobProgress) {
func WithCancel(cancel context.CancelFunc) func(*JobProgress) {
return func(jp *JobProgress) { jp.jobCancel = cancel }
}
-1
View File
@@ -376,7 +376,6 @@ func (s *Source) pageChunker(ctx context.Context, client *s3.S3, chunksChan chan
SourceType: s.Type(),
SourceName: s.name,
SourceID: s.SourceID(),
JobID: s.JobID(),
SourceMetadata: &source_metadatapb.MetaData{
Data: &source_metadatapb.MetaData_S3{
S3: &source_metadatapb.S3{
+4 -28
View File
@@ -36,10 +36,7 @@ type SourceManager struct {
handles map[handle]sourceInfo
handlesLock sync.Mutex
// Pool limiting the amount of concurrent sources running.
pool errgroup.Group
poolLimit int
currentRunningCount int32
// Max number of units to scan concurrently per source.
pool errgroup.Group
concurrentUnits int
// Run the sources using source unit enumeration / chunking if available.
useSourceUnits bool
@@ -71,10 +68,7 @@ func WithReportHook(hook JobProgressHook) func(*SourceManager) {
// WithConcurrentSources limits the concurrent number of sources a manager can run.
func WithConcurrentSources(concurrency int) func(*SourceManager) {
return func(mgr *SourceManager) {
mgr.pool.SetLimit(concurrency)
mgr.poolLimit = concurrency
}
return func(mgr *SourceManager) { mgr.pool.SetLimit(concurrency) }
}
// WithBufferedOutput sets the size of the buffer used for the Chunks() channel.
@@ -170,17 +164,15 @@ func (s *SourceManager) asyncRun(ctx context.Context, handle handle) (JobProgres
return JobProgressRef{SourceID: int64(handle), SourceName: sourceName}, err
}
// Create a JobProgress object for tracking progress.
ctx, cancel := context.WithCancelCause(ctx)
ctx, cancel := context.WithCancel(ctx)
progress := NewJobProgress(jobID, int64(handle), sourceName, WithHooks(s.hooks...), WithCancel(cancel))
s.pool.Go(func() error {
atomic.AddInt32(&s.currentRunningCount, 1)
defer atomic.AddInt32(&s.currentRunningCount, -1)
ctx := context.WithValues(ctx,
"job_id", jobID,
"source_manager_worker_id", common.RandomID(5),
)
defer common.Recover(ctx)
defer cancel(nil)
defer cancel()
return s.run(ctx, handle, jobID, progress)
})
return progress.Ref(), nil
@@ -215,16 +207,6 @@ func (s *SourceManager) ScanChunk(chunk *Chunk) {
s.outputChunks <- chunk
}
// AvailableCapacity returns the number of concurrent jobs the manager can
// accommodate at this time. If there is no limit, -1 is returned.
func (s *SourceManager) AvailableCapacity() int {
if s.poolLimit == 0 {
return -1
}
runCount := atomic.LoadInt32(&s.currentRunningCount)
return s.poolLimit - int(runCount)
}
// preflightChecks is a helper method to check the Manager or the context isn't
// done and that the handle is valid.
func (s *SourceManager) preflightChecks(ctx context.Context, handle handle) error {
@@ -247,12 +229,6 @@ func (s *SourceManager) run(ctx context.Context, handle handle, jobID int64, rep
report.Start(time.Now())
defer func() { report.End(time.Now()) }()
defer func() {
if err := context.Cause(ctx); err != nil {
report.ReportError(Fatal{err})
}
}()
// Initialize the source.
sourceInfo, ok := s.getSourceInfo(handle)
if !ok {
+1 -24
View File
@@ -312,31 +312,8 @@ func TestSourceManagerCancelRun(t *testing.T) {
ref, err := mgr.ScheduleRun(context.Background(), handle)
assert.NoError(t, err)
cancelErr := fmt.Errorf("abort! abort!")
ref.CancelRun(cancelErr)
ref.CancelRun()
<-ref.Done()
assert.Error(t, ref.Snapshot().FatalError())
assert.True(t, errors.Is(ref.Snapshot().FatalError(), returnedErr))
assert.True(t, errors.Is(ref.Snapshot().FatalErrors(), cancelErr))
}
func TestSourceManagerAvailableCapacity(t *testing.T) {
mgr := NewManager(WithConcurrentSources(1337))
start, end := make(chan struct{}), make(chan struct{})
handle, err := enrollDummy(mgr, callbackChunker{func(context.Context, chan *Chunk) error {
start <- struct{}{} // Send start signal.
<-end // Wait for end signal.
return nil
}})
assert.NoError(t, err)
assert.Equal(t, 1337, mgr.AvailableCapacity())
ref, err := mgr.ScheduleRun(context.Background(), handle)
assert.NoError(t, err)
<-start // Wait for start signal.
assert.Equal(t, 1336, mgr.AvailableCapacity())
end <- struct{}{} // Send end signal.
<-ref.Done() // Wait for the job to finish.
assert.Equal(t, 1337, mgr.AvailableCapacity())
}
-42
View File
@@ -30,48 +30,6 @@ type Chunk struct {
Verify bool
}
// BaseAndTagForDockerImg returns a base and tag value for a
// docker image regardless of the source type. If the source type
// is not a docker image, the base will be a filename/path from the source metadata.
// The default is the entire source metadata string. Update for new source types.
func (c *Chunk) BaseAndTagForDockerImg() (base string, tag string) {
switch c.SourceType {
case sourcespb.SourceType_SOURCE_TYPE_DOCKER:
base = c.SourceMetadata.GetDocker().GetImage()
tag = c.SourceMetadata.GetDocker().GetTag()
case sourcespb.SourceType_SOURCE_TYPE_FILESYSTEM:
base = c.SourceMetadata.GetFilesystem().GetFile()
case sourcespb.SourceType_SOURCE_TYPE_S3:
bucket := c.SourceMetadata.GetS3().GetBucket()
file := c.SourceMetadata.GetS3().GetFile()
base = bucket + "/" + file
case sourcespb.SourceType_SOURCE_TYPE_GCS:
bucket := c.SourceMetadata.GetGcs().GetBucket()
file := c.SourceMetadata.GetGcs().GetFilename()
base = bucket + "/" + file
case sourcespb.SourceType_SOURCE_TYPE_SYSLOG:
host := c.SourceMetadata.GetSyslog().GetHostname()
app := c.SourceMetadata.GetSyslog().GetAppname()
base = host + "/" + app
case sourcespb.SourceType_SOURCE_TYPE_GIT:
repo := c.SourceMetadata.GetGit().GetRepository()
file := c.SourceMetadata.GetGit().GetFile()
base = file + " (repo: " + repo + ")"
case sourcespb.SourceType_SOURCE_TYPE_GITLAB:
repo := c.SourceMetadata.GetGitlab().GetRepository()
file := c.SourceMetadata.GetGitlab().GetFile()
base = file + " (repo: " + repo + ")"
case sourcespb.SourceType_SOURCE_TYPE_GITHUB:
repo := c.SourceMetadata.GetGithub().GetRepository()
file := c.SourceMetadata.GetGithub().GetFile()
base = file + " (repo: " + repo + ")"
default:
base = c.SourceMetadata.String()
tag = ""
}
return base, tag
}
// Source defines the interface required to implement a source chunker.
type Source interface {
// Type returns the source type, used for matching against configuration and jobs.
-2
View File
@@ -272,7 +272,6 @@ func (s *Source) monitorConnection(ctx context.Context, conn net.Conn, chunksCha
SourceName: s.syslog.sourceName,
SourceID: s.syslog.sourceID,
SourceType: s.syslog.sourceType,
JobID: s.JobID(),
SourceMetadata: metadata,
Data: input,
Verify: s.verify,
@@ -314,7 +313,6 @@ func (s *Source) acceptUDPConnections(ctx context.Context, netListener net.Packe
chunksChan <- &sources.Chunk{
SourceName: s.syslog.sourceName,
SourceID: s.syslog.sourceID,
JobID: s.JobID(),
SourceType: s.syslog.sourceType,
SourceMetadata: metadata,
Data: input,