Compare commits

..

28 Commits

Author SHA1 Message Date
joeleonjr f623c0686f Merge branch 'main' into docker-tar-identification-from-filesystem-2 2023-08-30 15:03:48 -04:00
Miccah 522b2fab29 Add a cancel cause to job cancellation (#1728) 2023-08-30 12:00:44 -07:00
Miccah c77c1172c8 Add the 'Cause' family of functions to the context wrapper library (#1725)
Go 1.20 introduced `WithCancelCause`, `WithTimeoutCause`, and
`WithDeadlineCause` to allow adding a reason to context cancellations.
Adding it to our wrapper will allow us to use these features.
2023-08-30 07:57:45 -07:00
ahrav d6afca682b remove fmt.Print (#1727) 2023-08-30 07:17:38 -07:00
ahrav 4dc5eb7912 Optimize read to max (#1714)
* optimize the ReadToMax.

* add comment.

* remove dumb comment.

* update comment.

* fix test.

* lint.

* address comments.

* use limit reader.

* update equality check.

* update test.'

* use custom limit reader.

* address comments.

* revert fun.
2023-08-29 17:31:40 -07:00
Miccah 7ba880f47a Add AvailableCapacity method to SourceManager (#1665) 2023-08-29 12:36:44 -07:00
ahrav 2b1b1b5ad0 Add jobID to chunk. (#1721) 2023-08-29 12:02:30 -07:00
ahrav c51e8f8af5 buffer channel. (#1718) 2023-08-28 18:08:31 -07:00
Zubair Khan 66271aed7b add detectors that were missed (#1716) 2023-08-28 18:02:11 -07:00
Zachary Rice d1b1ac408e Expired invite link fix (#1713) 2023-08-28 14:21:13 -05:00
ahrav 0932ea224b [chore] - Prevent nil deref panic (#1709) 2023-08-26 20:39:50 -07:00
joeleonjr b8468af367 updating secret string for docker detection 2023-08-24 13:59:51 -04:00
joeleonjr b1a1fa393f Update archive_integration_test.go 2023-08-24 13:54:07 -04:00
joeleonjr 7077f69552 Merge branch 'main' into docker-tar-identification-from-filesystem-2 2023-08-24 13:50:33 -04:00
joeleonjr b38bb0d110 Merge branch 'main' into docker-tar-identification-from-filesystem-2 2023-08-24 08:46:44 -04:00
Joe Leon 51aadf96e1 added ahrav's comments 2023-08-24 08:46:30 -04:00
joeleonjr f7cf80ee85 Merge branch 'main' into docker-tar-identification-from-filesystem-2 2023-08-23 15:49:10 -04:00
Joe Leon e523b69a4c reverted size 2023-08-23 15:47:27 -04:00
Joe Leon f4119799d9 cleaned up code 2023-08-23 15:46:02 -04:00
Joe Leon 6b8ce2fba2 saving test case progress 2023-08-23 15:43:37 -04:00
Joe Leon 95f65281ea moved docker base/img hint to sources.go 2023-08-23 11:46:04 -04:00
Joe Leon 5f4f513014 refactored to call docker chunk scanner directly 2023-08-23 10:49:58 -04:00
Joe Leon 2d46cf1f74 temp increase file size in maxreader, handler improvements 2023-08-21 12:15:34 -04:00
Joe Leon aa461ae806 attempting to fix archive test 2023-08-18 21:12:54 -04:00
joeleonjr a55237376a Merge branch 'main' into docker-tar-identification-from-filesystem-2 2023-08-18 21:10:54 -04:00
Joe Leon 5b9a862ffa adjusted contexts in archive tests 2023-08-18 21:01:37 -04:00
Joe Leon e01f6d0257 switched back log context naming 2023-08-18 16:48:07 -04:00
Joe Leon af83970f77 reworked tarred docker img scanning 2023-08-18 13:07:14 -04:00
43 changed files with 889 additions and 332 deletions
+1 -1
View File
@@ -32,7 +32,7 @@ Have questions? Feedback? Jump in slack or discord and hang out with us
Join our [Slack Community](https://join.slack.com/t/trufflehog-community/shared_invite/zt-pw2qbi43-Aa86hkiimstfdKH9UCpPzQ)
Join the [Secret Scanning Discord](https://discord.gg/sydS6AHTUP)
Join the [Secret Scanning Discord](https://discord.gg/8Hzbrnkr7E)
# :tv: Demo
+40 -2
View File
@@ -26,9 +26,10 @@ type Context interface {
Logger() logr.Logger
}
// CancelFunc is a type alias to context.CancelFunc to allow use as if they are
// the same types.
// CancelFunc and CancelCauseFunc are type aliases to allow use as if they are
// the same types as the standard library variants.
type CancelFunc = context.CancelFunc
type CancelCauseFunc = context.CancelCauseFunc
// logCtx implements Context.
type logCtx struct {
@@ -68,6 +69,16 @@ func WithCancel(parent Context) (Context, context.CancelFunc) {
return lCtx, cancel
}
// WithCancelCause returns context.WithCancelCause with the log object propagated.
func WithCancelCause(parent Context) (Context, context.CancelCauseFunc) {
ctx, cancel := context.WithCancelCause(parent)
lCtx := logCtx{
log: parent.Logger(),
Context: ctx,
}
return lCtx, cancel
}
// WithDeadline returns context.WithDeadline with the log object propagated and
// the deadline added to the structured log values.
func WithDeadline(parent Context, d time.Time) (Context, context.CancelFunc) {
@@ -79,6 +90,17 @@ func WithDeadline(parent Context, d time.Time) (Context, context.CancelFunc) {
return lCtx, cancel
}
// WithDeadlineCause returns context.WithDeadlineCause with the log object
// propagated and the deadline added to the structured log values.
func WithDeadlineCause(parent Context, d time.Time, cause error) (Context, context.CancelFunc) {
ctx, cancel := context.WithDeadlineCause(parent, d, cause)
lCtx := logCtx{
log: parent.Logger().WithValues("deadline", d),
Context: ctx,
}
return lCtx, cancel
}
// WithTimeout returns context.WithTimeout with the log object propagated and
// the timeout added to the structured log values.
func WithTimeout(parent Context, timeout time.Duration) (Context, context.CancelFunc) {
@@ -90,6 +112,22 @@ func WithTimeout(parent Context, timeout time.Duration) (Context, context.Cancel
return lCtx, cancel
}
// WithTimeoutCause returns context.WithTimeoutCause with the log object
// propagated and the timeout added to the structured log values.
func WithTimeoutCause(parent Context, timeout time.Duration, cause error) (Context, context.CancelFunc) {
ctx, cancel := context.WithTimeoutCause(parent, timeout, cause)
lCtx := logCtx{
log: parent.Logger().WithValues("timeout", timeout),
Context: ctx,
}
return lCtx, cancel
}
// Cause returns the context.Cause of the context.
func Cause(ctx context.Context) error {
return context.Cause(ctx)
}
// WithValue returns context.WithValue with the log object propagated and
// the value added to the structured log values (if the key is a string).
func WithValue(parent Context, key, val any) Context {
+8
View File
@@ -3,6 +3,7 @@ package context
import (
"bytes"
"context"
"fmt"
"strings"
"testing"
"time"
@@ -177,3 +178,10 @@ func TestRace(t *testing.T) {
cancel()
_ = ctx.Err()
}
func TestCause(t *testing.T) {
ctx, cancel := WithCancelCause(Background())
err := fmt.Errorf("oh no")
cancel(err)
assert.Equal(t, err, Cause(ctx))
}
+16 -7
View File
@@ -1,10 +1,10 @@
package axonaut
import (
"bytes"
"context"
"net/http"
"regexp"
"strings"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
@@ -13,29 +13,37 @@ import (
type Scanner struct{}
// Ensure the Scanner satisfies the interface at compile time.
var _ detectors.Detector = (*Scanner)(nil)
var (
client = common.SaneHttpClient()
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"axonaut"}) + `\b([a-z0-9]{32})\b`)
)
func (s Scanner) Keywords() [][]byte {
return [][]byte{[]byte("axonaut")}
// Keywords are used for efficiently pre-filtering chunks.
// Use identifiers in the secret preferably, or the provider name.
func (s Scanner) Keywords() []string {
return []string{"axonaut"}
}
// FromData will find and optionally verify Axonaut secrets in a given set of bytes.
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
matches := keyPat.FindAllSubmatch(data, -1)
dataStr := string(data)
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
for _, match := range matches {
if len(match) != 2 {
continue
}
resMatch := bytes.TrimSpace(match[1])
resMatch := strings.TrimSpace(match[1])
s1 := detectors.Result{
DetectorType: detectorspb.DetectorType_Axonaut,
Raw: resMatch,
Raw: []byte(resMatch),
}
if verify {
@@ -43,13 +51,14 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
if err != nil {
continue
}
req.Header.Add("userApiKey", string(resMatch))
req.Header.Add("userApiKey", resMatch)
res, err := client.Do(req)
if err == nil {
defer res.Body.Close()
if res.StatusCode >= 200 && res.StatusCode < 300 {
s1.Verified = true
} else {
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
continue
}
+15 -14
View File
@@ -1,10 +1,10 @@
package aylien
import (
"bytes"
"context"
"net/http"
"regexp"
"strings"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
@@ -26,48 +26,49 @@ var (
// Keywords are used for efficiently pre-filtering chunks.
// Use identifiers in the secret preferably, or the provider name.
func (s Scanner) Keywords() [][]byte {
return [][]byte{[]byte("aylien")}
func (s Scanner) Keywords() []string {
return []string{"aylien"}
}
// FromData will find and optionally verify Aylien secrets in a given set of bytes.
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
matches := keyPat.FindAllSubmatch(data, -1)
idMatches := idPat.FindAllSubmatch(data, -1)
dataStr := string(data)
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
idMatches := idPat.FindAllStringSubmatch(dataStr, -1)
for _, match := range matches {
if len(match) != 2 {
continue
}
resMatch := bytes.TrimSpace(match[1])
resMatch := strings.TrimSpace(match[1])
for _, idMatch := range idMatches {
if len(idMatch) != 2 {
continue
}
resIdMatch := bytes.TrimSpace(idMatch[1])
newSecret := append(resMatch, resIdMatch...)
resIdMatch := strings.TrimSpace(idMatch[1])
s1 := detectors.Result{
DetectorType: detectorspb.DetectorType_Aylien,
Raw: resMatch,
RawV2: newSecret,
Raw: []byte(resMatch),
RawV2: []byte(resMatch + resIdMatch),
}
if verify {
req, err := http.NewRequestWithContext(ctx, "GET", "https://api.aylien.com/news/stories", nil)
if err != nil {
continue
}
req.Header.Add("X-AYLIEN-NewsAPI-Application-ID", string(resMatch))
req.Header.Add("X-AYLIEN-NewsAPI-Application-Key", string(resIdMatch))
req.Header.Add("X-AYLIEN-NewsAPI-Application-ID", resIdMatch)
req.Header.Add("X-AYLIEN-NewsAPI-Application-Key", resMatch)
res, err := client.Do(req)
if err == nil {
defer res.Body.Close()
if res.StatusCode >= 200 && res.StatusCode < 300 {
s1.Verified = true
} else {
if detectors.IsKnownFalsePositive(newSecret, detectors.DefaultFalsePositives, true) {
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
continue
}
}
+19 -7
View File
@@ -1,11 +1,11 @@
package ayrshare
import (
"bytes"
"context"
"fmt"
"net/http"
"regexp"
"strings"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
@@ -14,28 +14,37 @@ import (
type Scanner struct{}
// Ensure the Scanner satisfies the interface at compile time.
var _ detectors.Detector = (*Scanner)(nil)
var (
client = common.SaneHttpClient()
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"ayrshare"}) + `\b([A-Z]{7}-[A-Z0-9]{7}-[A-Z0-9]{7}-[A-Z0-9]{7})\b`)
)
func (s Scanner) Keywords() [][]byte {
return [][]byte{[]byte("ayrshare")}
// Keywords are used for efficiently pre-filtering chunks.
// Use identifiers in the secret preferably, or the provider name.
func (s Scanner) Keywords() []string {
return []string{"ayrshare"}
}
// FromData will find and optionally verify Ayrshare secrets in a given set of bytes.
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
matches := keyPat.FindAllSubmatch(data, -1)
dataStr := string(data)
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
for _, match := range matches {
if len(match) != 2 {
continue
}
resMatch := bytes.TrimSpace(match[1])
resMatch := strings.TrimSpace(match[1])
s1 := detectors.Result{
DetectorType: detectorspb.DetectorType_Ayrshare,
Raw: resMatch,
Raw: []byte(resMatch),
}
if verify {
@@ -43,21 +52,24 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
if err != nil {
continue
}
req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", string(resMatch)))
req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", resMatch))
res, err := client.Do(req)
if err == nil {
defer res.Body.Close()
if res.StatusCode >= 200 && res.StatusCode < 300 {
s1.Verified = true
} else {
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
continue
}
}
}
}
results = append(results, s1)
}
return results, nil
}
+15 -12
View File
@@ -7,6 +7,7 @@ import (
"strings"
"github.com/Azure/go-autorest/autorest/azure/auth"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
)
@@ -30,33 +31,35 @@ var (
tenantIDPat = mustFmtPat("tenant_id", idPatFmt)
// TODO: support old patterns
secretPatFmt = `(?i)(%s).{0,20}([a-z0-9_.\-~]{34})`
secretPatFmt = `(?i)(%s).{0,20}([a-z0-9_\.\-~]{34})`
clientSecretPat = mustFmtPat("client_secret", secretPatFmt)
)
// Keywords are used for efficiently pre-filtering chunks.
func (s Scanner) Keywords() [][]byte {
return [][]byte{[]byte("azure")}
// Use identifiers in the secret preferably, or the provider name.
func (s Scanner) Keywords() []string {
return []string{"azure"}
}
// FromData will find and optionally verify Azure secrets in a given set of bytes.
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
clientSecretMatches := clientSecretPat.FindAllSubmatch(data, -1)
dataStr := string(data)
clientSecretMatches := clientSecretPat.FindAllStringSubmatch(dataStr, -1)
for _, clientSecret := range clientSecretMatches {
tenantIDMatches := tenantIDPat.FindAllSubmatch(data, -1)
tenantIDMatches := tenantIDPat.FindAllStringSubmatch(dataStr, -1)
for _, tenantID := range tenantIDMatches {
clientIDMatches := clientIDPat.FindAllSubmatch(data, -1)
clientIDMatches := clientIDPat.FindAllStringSubmatch(dataStr, -1)
for _, clientID := range clientIDMatches {
s := detectors.Result{
DetectorType: detectorspb.DetectorType_Azure,
Raw: clientSecret[2],
RawV2: append(clientID[2], append(clientSecret[2], tenantID[2]...)...),
Redacted: string(clientID[2]),
Raw: []byte(clientSecret[2]),
RawV2: []byte(clientID[2] + clientSecret[2] + tenantID[2]),
Redacted: clientID[2],
}
if verify {
cred := auth.NewClientCredentialsConfig(string(clientID[2]), string(clientSecret[2]), string(tenantID[2]))
cred := auth.NewClientCredentialsConfig(clientID[2], clientSecret[2], tenantID[2])
token, err := cred.ServicePrincipalToken()
if err != nil {
continue
@@ -68,10 +71,10 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
}
if !s.Verified {
if detectors.IsKnownFalsePositive([]byte(s.Redacted), detectors.DefaultFalsePositives, true) {
if detectors.IsKnownFalsePositive(s.Redacted, detectors.DefaultFalsePositives, true) {
continue
}
if detectors.IsKnownFalsePositive(s.Raw, detectors.DefaultFalsePositives, true) {
if detectors.IsKnownFalsePositive(string(s.Raw), detectors.DefaultFalsePositives, true) {
continue
}
}
+10 -7
View File
@@ -1,11 +1,11 @@
package bannerbear
import (
"bytes"
"context"
"fmt"
"net/http"
"regexp"
"strings"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
@@ -20,29 +20,31 @@ var _ detectors.Detector = (*Scanner)(nil)
var (
client = common.SaneHttpClient()
// Keep the regexp patterns unchanged
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"bannerbear"}) + `\b([0-9a-zA-Z]{22}tt)\b`)
)
// Keywords are used for efficiently pre-filtering chunks.
func (s Scanner) Keywords() [][]byte {
return [][]byte{[]byte("bannerbear")}
// Use identifiers in the secret preferably, or the provider name.
func (s Scanner) Keywords() []string {
return []string{"bannerbear"}
}
// FromData will find and optionally verify Bannerbear secrets in a given set of bytes.
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
dataStr := string(data)
matches := keyPat.FindAllSubmatch(data, -1)
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
for _, match := range matches {
if len(match) != 2 {
continue
}
resMatch := bytes.TrimSpace(match[1])
resMatch := strings.TrimSpace(match[1])
s1 := detectors.Result{
DetectorType: detectorspb.DetectorType_Bannerbear,
Raw: resMatch,
Raw: []byte(resMatch),
}
if verify {
@@ -57,6 +59,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
if res.StatusCode >= 200 && res.StatusCode < 300 {
s1.Verified = true
} else {
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
continue
}
+15 -7
View File
@@ -1,11 +1,11 @@
package baremetrics
import (
"bytes"
"context"
"fmt"
"net/http"
"regexp"
"strings"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
@@ -14,30 +14,37 @@ import (
type Scanner struct{}
// Ensure the Scanner satisfies the interface at compile time.
var _ detectors.Detector = (*Scanner)(nil)
var (
client = common.SaneHttpClient()
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"baremetrics"}) + `\b([a-zA-Z0-9_]{25})\b`)
)
func (s Scanner) Keywords() [][]byte {
return [][]byte{[]byte("baremetrics")}
// Keywords are used for efficiently pre-filtering chunks.
// Use identifiers in the secret preferably, or the provider name.
func (s Scanner) Keywords() []string {
return []string{"baremetrics"}
}
// FromData will find and optionally verify Baremetrics secrets in a given set of bytes.
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
matches := keyPat.FindAllSubmatch(data, -1)
dataStr := string(data)
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
for _, match := range matches {
if len(match) != 2 {
continue
}
resMatch := bytes.TrimSpace(match[1])
resMatch := strings.TrimSpace(match[1])
s1 := detectors.Result{
DetectorType: detectorspb.DetectorType_Baremetrics,
Raw: resMatch,
Raw: []byte(resMatch),
}
if verify {
@@ -45,13 +52,14 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
if err != nil {
continue
}
req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", string(resMatch)))
req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", resMatch))
res, err := client.Do(req)
if err == nil {
defer res.Body.Close()
if res.StatusCode >= 200 && res.StatusCode < 300 {
s1.Verified = true
} else {
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
continue
}
+23 -8
View File
@@ -1,11 +1,12 @@
package baseapiio
import (
"bytes"
"context"
"fmt"
"io"
"net/http"
"regexp"
"strings"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
@@ -14,30 +15,37 @@ import (
type Scanner struct{}
// Ensure the Scanner satisfies the interface at compile time.
var _ detectors.Detector = (*Scanner)(nil)
var (
client = common.SaneHttpClient()
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"baseapi", "base-api"}) + `\b([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})\b`)
)
func (s Scanner) Keywords() [][]byte {
return [][]byte{[]byte("baseapi"), []byte("base-api")}
// Keywords are used for efficiently pre-filtering chunks.
// Use identifiers in the secret preferably, or the provider name.
func (s Scanner) Keywords() []string {
return []string{"baseapi", "base-api"}
}
// FromData will find and optionally verify BaseApiIO secrets in a given set of bytes.
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
matches := keyPat.FindAllSubmatch(data, -1)
dataStr := string(data)
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
for _, match := range matches {
if len(match) != 2 {
continue
}
resMatch := bytes.TrimSpace(match[1])
resMatch := strings.TrimSpace(match[1])
s1 := detectors.Result{
DetectorType: detectorspb.DetectorType_BaseApiIO,
Raw: resMatch,
Raw: []byte(resMatch),
}
if verify {
@@ -45,17 +53,24 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
if err != nil {
continue
}
req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", string(resMatch)))
req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", resMatch))
res, err := client.Do(req)
if err == nil {
defer res.Body.Close()
if bytes.Contains(resMatch, []byte("items")) {
bodyBytes, err := io.ReadAll(res.Body)
if err != nil {
continue
}
body := string(bodyBytes)
if strings.Contains(body, "items") {
s1.Verified = true
} else {
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
continue
}
}
}
}
+12 -8
View File
@@ -1,10 +1,10 @@
package beamer
import (
"bytes"
"context"
"net/http"
"regexp"
"strings"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
@@ -19,27 +19,31 @@ var _ detectors.Detector = (*Scanner)(nil)
var (
client = common.SaneHttpClient()
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"beamer"}) + `\b([a-zA-Z0-9_+/]{45}=)`)
)
// Keywords are used for efficiently pre-filtering chunks.
func (s Scanner) Keywords() [][]byte {
return [][]byte{[]byte("beamer")}
// Use identifiers in the secret preferably, or the provider name.
func (s Scanner) Keywords() []string {
return []string{"beamer"}
}
// FromData will find and optionally verify Beamer secrets in a given set of bytes.
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
matches := keyPat.FindAllSubmatch(data, -1)
dataStr := string(data)
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
for _, match := range matches {
if len(match) != 2 {
continue
}
resMatch := bytes.TrimSpace(match[1])
resMatch := strings.TrimSpace(match[1])
s1 := detectors.Result{
DetectorType: detectorspb.DetectorType_Beamer,
Raw: resMatch,
Raw: []byte(resMatch),
}
if verify {
@@ -47,14 +51,14 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
if err != nil {
continue
}
req.Header.Add("Beamer-Api-Key", string(resMatch))
req.Header.Add("Beamer-Api-Key", resMatch)
res, err := client.Do(req)
if err == nil {
defer res.Body.Close()
if res.StatusCode >= 200 && res.StatusCode < 300 {
s1.Verified = true
} else {
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
continue
}
+16 -10
View File
@@ -1,12 +1,12 @@
package beebole
import (
"bytes"
"context"
b64 "encoding/base64"
"fmt"
"net/http"
"regexp"
"strings"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
@@ -15,50 +15,56 @@ import (
type Scanner struct{}
// Ensure the Scanner satisfies the interface at compile time.
var _ detectors.Detector = (*Scanner)(nil)
var (
client = common.SaneHttpClient()
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"beebole"}) + `\b([0-9a-z]{40})\b`)
)
func (s Scanner) Keywords() [][]byte {
return [][]byte{[]byte("beebole")}
// Keywords are used for efficiently pre-filtering chunks.
// Use identifiers in the secret preferably, or the provider name.
func (s Scanner) Keywords() []string {
return []string{"beebole"}
}
// FromData will find and optionally verify Beebole secrets in a given set of bytes.
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
matches := keyPat.FindAllSubmatch(data, -1)
dataStr := string(data)
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
for _, match := range matches {
if len(match) != 2 {
continue
}
resMatch := bytes.TrimSpace(match[1])
resMatch := strings.TrimSpace(match[1])
s1 := detectors.Result{
DetectorType: detectorspb.DetectorType_Beebole,
Raw: resMatch,
Raw: []byte(resMatch),
}
if verify {
data := fmt.Sprintf("%s:X", string(resMatch))
data := fmt.Sprintf("%s:X", resMatch)
sEnc := b64.StdEncoding.EncodeToString([]byte(data))
payload := bytes.NewReader([]byte(`{"service": "custom_field.list"}`))
payload := strings.NewReader(`{"service": "custom_field.list"}`)
req, err := http.NewRequestWithContext(ctx, "POST", "https://beebole-apps.com/api/v2", payload)
if err != nil {
continue
}
req.Header.Add("Content-Type", "application/json")
req.Header.Add("Authorization", fmt.Sprintf("Basic %s", sEnc))
res, err := client.Do(req)
if err == nil {
defer res.Body.Close()
if res.StatusCode >= 200 && res.StatusCode < 300 {
s1.Verified = true
} else {
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
continue
}
+14 -6
View File
@@ -1,12 +1,12 @@
package besnappy
import (
b "bytes"
"context"
b64 "encoding/base64"
"fmt"
"net/http"
"regexp"
"strings"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
@@ -15,30 +15,37 @@ import (
type Scanner struct{}
// Ensure the Scanner satisfies the interface at compile time
var _ detectors.Detector = (*Scanner)(nil)
var (
client = common.SaneHttpClient()
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"besnappy"}) + `\b([a-f0-9]{64})\b`)
)
func (s Scanner) Keywords() [][]byte {
return [][]byte{[]byte("besnappy")}
// Keywords are used for efficiently pre-filtering chunks.
// Use identifiers in the secret preferably, or the provider name.
func (s Scanner) Keywords() []string {
return []string{"besnappy"}
}
// FromData will find and optionally verify Besnappy secrets in a given set of bytes.
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
dataStr := string(data)
matches := keyPat.FindAllSubmatch(data, -1)
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
for _, match := range matches {
if len(match) != 2 {
continue
}
resMatch := b.TrimSpace(match[1])
resMatch := strings.TrimSpace(match[1])
s1 := detectors.Result{
DetectorType: detectorspb.DetectorType_Besnappy,
Raw: resMatch,
Raw: []byte(resMatch),
}
if verify {
data := fmt.Sprintf("%s:x", resMatch)
@@ -54,6 +61,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
if res.StatusCode >= 200 && res.StatusCode < 300 {
s1.Verified = true
} else {
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
continue
}
+29 -22
View File
@@ -1,11 +1,11 @@
package besttime
import (
"bytes"
"context"
"io"
"net/http"
"regexp"
"strings"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
@@ -14,54 +14,61 @@ import (
type Scanner struct{}
// Ensure the Scanner satisfies the interface at compile time.
var _ detectors.Detector = (*Scanner)(nil)
var (
client = common.SaneHttpClient()
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"besttime"}) + `\b([0-9A-Za-z_]{36})\b`)
)
func (s Scanner) Keywords() [][]byte {
return [][]byte{[]byte("besttime")}
// Keywords are used for efficiently pre-filtering chunks.
// Use identifiers in the secret preferably, or the provider name.
func (s Scanner) Keywords() []string {
return []string{"besttime"}
}
// FromData will find and optionally verify Besttime secrets in a given set of bytes.
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
matches := keyPat.FindAllSubmatch(data, -1)
dataStr := string(data)
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
for _, match := range matches {
if len(match) != 2 {
continue
}
resMatch := bytes.TrimSpace(match[1])
resMatch := strings.TrimSpace(match[1])
s1 := detectors.Result{
DetectorType: detectorspb.DetectorType_Besttime,
Raw: resMatch,
Raw: []byte(resMatch),
}
if verify {
req, err := http.NewRequestWithContext(ctx, "GET", "https://besttime.app/api/v1/keys/"+string(resMatch), nil)
req, err := http.NewRequestWithContext(ctx, "GET", "https://besttime.app/api/v1/keys/"+resMatch, nil)
if err != nil {
continue
}
res, err := client.Do(req)
if err != nil {
continue
}
defer res.Body.Close()
bodyBytes, err := io.ReadAll(res.Body)
if err != nil {
continue
}
if bytes.Contains(bodyBytes, []byte(`"status": "OK"`)) {
s1.Verified = true
} else {
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
if err == nil {
defer res.Body.Close()
bodyBytes, err := io.ReadAll(res.Body)
if err != nil {
continue
}
body := string(bodyBytes)
if strings.Contains(body, `"status": "OK"`) {
s1.Verified = true
} else {
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
continue
}
}
}
}
+18 -12
View File
@@ -1,11 +1,11 @@
package billomat
import (
"bytes"
"context"
"fmt"
"net/http"
"regexp"
"strings"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
@@ -20,49 +20,55 @@ var _ detectors.Detector = (*Scanner)(nil)
var (
client = common.SaneHttpClient()
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"billomat"}) + `\b([0-9a-z]{32})\b`)
idPat = regexp.MustCompile(detectors.PrefixRegex([]string{"billomat"}) + `\b([0-9a-z]{1,})\b`)
)
func (s Scanner) Keywords() [][]byte {
return [][]byte{[]byte("billomat")}
// Keywords are used for efficiently pre-filtering chunks.
// Use identifiers in the secret preferably, or the provider name.
func (s Scanner) Keywords() []string {
return []string{"billomat"}
}
// FromData will find and optionally verify Billomat secrets in a given set of bytes.
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
matches := keyPat.FindAllSubmatch(data, -1)
idMatches := idPat.FindAllSubmatch(data, -1)
dataStr := string(data)
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
idMatches := idPat.FindAllStringSubmatch(dataStr, -1)
for _, match := range matches {
if len(match) != 2 {
continue
}
resMatch := bytes.TrimSpace(match[1])
resMatch := strings.TrimSpace(match[1])
for _, idMatch := range idMatches {
if len(idMatch) != 2 {
continue
}
resId := bytes.TrimSpace(idMatch[1])
resId := strings.TrimSpace(idMatch[1])
s1 := detectors.Result{
DetectorType: detectorspb.DetectorType_Billomat,
Raw: resMatch,
RawV2: append(resMatch, resId...),
Raw: []byte(resMatch),
RawV2: []byte(resMatch + resId),
}
if verify {
req, err := http.NewRequestWithContext(ctx, "GET", fmt.Sprintf("https://%s.billomat.net/api/v2/clients/myself", string(resId)), nil)
req, err := http.NewRequestWithContext(ctx, "GET", fmt.Sprintf("https://%s.billomat.net/api/v2/clients/myself", resId), nil)
if err != nil {
continue
}
req.Header.Add("Content-Type", "application/json")
req.Header.Add("X-BillomatApiKey", string(resMatch))
req.Header.Add("X-BillomatApiKey", resMatch)
res, err := client.Do(req)
if err == nil {
defer res.Body.Close()
if res.StatusCode >= 200 && res.StatusCode < 300 {
s1.Verified = true
} else {
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
continue
}
+17 -7
View File
@@ -6,6 +6,7 @@ import (
"fmt"
"net/http"
"regexp"
"strings"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
@@ -14,34 +15,42 @@ import (
type Scanner struct{}
// Ensure the Scanner satisfies the interface at compile time.
var _ detectors.Detector = (*Scanner)(nil)
var (
client = common.SaneHttpClient()
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"bitbar"}) + `\b([0-9a-z]{32})\b`)
)
func (s Scanner) Keywords() [][]byte {
return [][]byte{[]byte("bitbar")}
// Keywords are used for efficiently pre-filtering chunks.
// Use identifiers in the secret preferably, or the provider name.
func (s Scanner) Keywords() []string {
return []string{"bitbar"}
}
// FromData will find and optionally verify Bitbar secrets in a given set of bytes.
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
matches := keyPat.FindAllSubmatch(data, -1)
dataStr := string(data)
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
for _, match := range matches {
if len(match) != 2 {
continue
}
resMatch := match[1]
resMatch := strings.TrimSpace(match[1])
s1 := detectors.Result{
DetectorType: detectorspb.DetectorType_Bitbar,
Raw: resMatch,
Raw: []byte(resMatch),
}
if verify {
data := []byte(fmt.Sprintf("%s:", resMatch))
baseToken := b64.StdEncoding.EncodeToString(data)
data := fmt.Sprintf("%s:", resMatch)
baseToken := b64.StdEncoding.EncodeToString([]byte(data))
req, err := http.NewRequestWithContext(ctx, "GET", "https://cloud.bitbar.com/api/me", nil)
if err != nil {
continue
@@ -54,6 +63,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
if res.StatusCode >= 200 && res.StatusCode < 300 {
s1.Verified = true
} else {
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
continue
}
+14 -7
View File
@@ -1,10 +1,10 @@
package bitcoinaverage
import (
"bytes"
"context"
"net/http"
"regexp"
"strings"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
@@ -19,38 +19,45 @@ var _ detectors.Detector = (*Scanner)(nil)
var (
client = common.SaneHttpClient()
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"bitcoinaverage"}) + `\b([a-zA-Z0-9]{43})\b`)
)
func (s Scanner) Keywords() [][]byte {
return [][]byte{[]byte("bitcoinaverage")}
// Keywords are used for efficiently pre-filtering chunks.
// Use identifiers in the secret preferably, or the provider name.
func (s Scanner) Keywords() []string {
return []string{"bitcoinaverage"}
}
// FromData will find and optionally verify BitcoinAverage secrets in a given set of bytes.
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
matches := keyPat.FindAllSubmatch(data, -1)
dataStr := string(data)
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
for _, match := range matches {
if len(match) != 2 {
continue
}
resMatch := bytes.TrimSpace(match[1])
resMatch := strings.TrimSpace(match[1])
s1 := detectors.Result{
DetectorType: detectorspb.DetectorType_BitcoinAverage,
Raw: resMatch,
Raw: []byte(resMatch),
}
if verify {
req, err := http.NewRequestWithContext(ctx, "GET", "https://apiv2.bitcoinaverage.com/websocket/get_ticket", nil)
if err != nil {
continue
}
req.Header.Add("x-ba-key", string(resMatch))
req.Header.Add("x-ba-key", resMatch)
res, err := client.Do(req)
if err == nil {
defer res.Body.Close()
if res.StatusCode >= 200 && res.StatusCode < 300 {
s1.Verified = true
} else {
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
continue
}
+38 -19
View File
@@ -1,80 +1,99 @@
package bitfinex
import (
"bytes"
"context"
"flag"
"net/http"
"regexp"
"strings"
"github.com/bitfinexcom/bitfinex-api-go/v2/rest"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
)
type Scanner struct{}
// Ensure the Scanner satisfies the interface at compile time.
var _ detectors.Detector = (*Scanner)(nil)
var (
client = common.SaneHttpClient()
client = common.SaneHttpClient()
// related resource https://medium.com/@Bitfinex/api-development-update-april-65fe52f84124
apiKeyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"bitfinex"}) + `\b([A-Za-z0-9_-]{43})\b`)
apiSecretPat = regexp.MustCompile(detectors.PrefixRegex([]string{"bitfinex"}) + `\b([A-Za-z0-9_-]{43})\b`)
)
func (s Scanner) Keywords() [][]byte {
return [][]byte{[]byte("bitfinex")}
var (
api = flag.String("api", "https://api-pub.bitfinex.com/v2/", "v2 REST API URL")
)
// Keywords are used for efficiently pre-filtering chunks.
// Use identifiers in the secret preferably, or the provider name.
func (s Scanner) Keywords() []string {
return []string{"bitfinex"}
}
// FromData will find and optionally verify Bitfinex secrets in a given set of bytes.
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
apiKeyMatches := apiKeyPat.FindAllSubmatch(data, -1)
apiSecretMatches := apiSecretPat.FindAllSubmatch(data, -1)
dataStr := string(data)
apiKeyMatches := apiKeyPat.FindAllStringSubmatch(dataStr, -1)
apiSecretMatches := apiSecretPat.FindAllStringSubmatch(dataStr, -1)
for _, apiKeyMatch := range apiKeyMatches {
if len(apiKeyMatch) != 2 {
continue
}
apiKeyRes := bytes.TrimSpace(apiKeyMatch[1])
apiKeyRes := strings.TrimSpace(apiKeyMatch[1])
s1 := detectors.Result{
DetectorType: detectorspb.DetectorType_Bitfinex,
Raw: apiKeyRes,
Raw: []byte(apiKeyRes),
}
for _, apiSecretMatch := range apiSecretMatches {
if len(apiSecretMatch) != 2 {
continue
}
apiSecretRes := bytes.TrimSpace(apiSecretMatch[1])
apiSecretRes := strings.TrimSpace(apiSecretMatch[1])
if bytes.Equal(apiKeyRes, apiSecretRes) {
if apiKeyRes == apiSecretRes {
continue
}
if verify {
key := string(apiKeyRes)
secret := string(apiSecretRes)
http.DefaultClient = client
c := rest.NewClient().Credentials(key, secret)
// thankfully official golang examples exist but you just need to dig their many repos https://github.com/bitfinexcom/bitfinex-api-go/blob/master/examples/v2/rest-orders/main.go
key := apiKeyRes
secret := apiSecretRes
http.DefaultClient = client // filed https://github.com/bitfinexcom/bitfinex-api-go/issues/238 to improve this
c := rest.NewClientWithURL(*api).Credentials(key, secret)
isValid := true
isValid := true // assume valid
_, err = c.Orders.AllHistory()
if err != nil {
if bytes.HasPrefix([]byte(err.Error()), []byte("POST https://")) {
if strings.HasPrefix(err.Error(), "POST https://") { // eg POST https://api-pub.bitfinex.com/v2/auth/r/orders/hist: 500 apikey: digest invalid (10100)
isValid = false
} else if detectors.IsKnownFalsePositive(apiKeyRes, detectors.DefaultFalsePositives, true) {
continue
} else {
if detectors.IsKnownFalsePositive(apiKeyRes, detectors.DefaultFalsePositives, true) {
continue
}
}
}
s1.Verified = isValid
// If there is a valid one, we need to stop iterating now and return the valid result
if isValid {
break
}
}
}
// By appending resutls in the outer loop we can reduce false positives if there are multiple
// combinations of secrets and IDs found.
if len(apiSecretMatches) > 0 {
results = append(results, s1)
}
@@ -1,11 +1,11 @@
package bitlyaccesstoken
import (
"bytes"
"context"
"fmt"
"net/http"
"regexp"
"strings"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
@@ -14,30 +14,37 @@ import (
type Scanner struct{}
// Ensure the Scanner satisfies the interface at compile time.
var _ detectors.Detector = (*Scanner)(nil)
var (
client = common.SaneHttpClient()
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"bitly"}) + `\b([a-zA-Z-0-9]{40})\b`)
)
func (s Scanner) Keywords() [][]byte {
return [][]byte{[]byte("bitly")}
// Keywords are used for efficiently pre-filtering chunks.
// Use identifiers in the secret preferably, or the provider name.
func (s Scanner) Keywords() []string {
return []string{"bitly"}
}
// FromData will find and optionally verify BitLyAccessToken secrets in a given set of bytes.
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
matches := keyPat.FindAllSubmatch(data, -1)
dataStr := string(data)
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
for _, match := range matches {
if len(match) != 2 {
continue
}
resMatch := bytes.TrimSpace(match[1])
resMatch := strings.TrimSpace(match[1])
s1 := detectors.Result{
DetectorType: detectorspb.DetectorType_BitLyAccessToken,
Raw: resMatch,
Raw: []byte(resMatch),
}
if verify {
@@ -45,13 +52,14 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
if err != nil {
continue
}
req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", string(resMatch)))
req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", resMatch))
res, err := client.Do(req)
if err == nil {
defer res.Body.Close()
if res.StatusCode >= 200 && res.StatusCode < 300 {
s1.Verified = true
} else {
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
continue
}
+21 -15
View File
@@ -1,7 +1,6 @@
package bitmex
import (
"bytes"
"context"
"crypto/hmac"
"crypto/sha256"
@@ -10,6 +9,7 @@ import (
"net/url"
"regexp"
"strconv"
"strings"
"time"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
@@ -19,40 +19,46 @@ import (
type Scanner struct{}
// Ensure the Scanner satisfies the interface at compile time.
var _ detectors.Detector = (*Scanner)(nil)
var (
client = common.SaneHttpClient()
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"bitmex"}) + string([]byte{32, 13, 10}) + `[0-9a-zA-Z\-\_]{24}` + string([]byte{32, 13, 10}))
secretPat = regexp.MustCompile(detectors.PrefixRegex([]string{"bitmex"}) + string([]byte{32, 13, 10}) + `[0-9a-zA-Z\-\_]{48}` + string([]byte{32, 13, 10}))
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"bitmex"}) + `([ \r\n]{1}[0-9a-zA-Z\-\_]{24}[ \r\n]{1})`)
secretPat = regexp.MustCompile(detectors.PrefixRegex([]string{"bitmex"}) + `([ \r\n]{1}[0-9a-zA-Z\-\_]{48}[ \r\n]{1})`)
)
func (s Scanner) Keywords() [][]byte {
return [][]byte{[]byte("bitmex")}
// Keywords are used for efficiently pre-filtering chunks.
// Use identifiers in the secret preferably, or the provider name.
func (s Scanner) Keywords() []string {
return []string{"bitmex"}
}
// FromData will find and optionally verify Bitmex secrets in a given set of bytes.
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
dataStr := string(data)
matches := keyPat.FindAllSubmatch(data, -1)
secretMatches := secretPat.FindAllSubmatch(data, -1)
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
secretMatches := secretPat.FindAllStringSubmatch(dataStr, -1)
for _, match := range matches {
if len(match) != 2 {
continue
}
resMatch := bytes.TrimSpace(match[1])
resMatch := strings.TrimSpace(match[1])
for _, secretMatch := range secretMatches {
if len(secretMatch) != 2 {
continue
}
resSecretMatch := bytes.TrimSpace(secretMatch[1])
resSecretMatch := strings.TrimSpace(secretMatch[1])
s1 := detectors.Result{
DetectorType: detectorspb.DetectorType_Bitmex,
Raw: resSecretMatch,
RawV2: append(resMatch, resSecretMatch...),
Raw: []byte(resSecretMatch),
RawV2: []byte(resMatch + resSecretMatch),
}
if verify {
@@ -62,15 +68,14 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
path := "/api/v1/user"
payload := url.Values{}
signature := getBitmexSignature(timestamp, string(resSecretMatch), action, path, payload.Encode())
req, err := http.NewRequestWithContext(ctx, action, "https://www.bitmex.com"+path, bytes.NewReader([]byte(payload.Encode())))
signature := getBitmexSignature(timestamp, resSecretMatch, action, path, payload.Encode())
req, err := http.NewRequestWithContext(ctx, action, "https://www.bitmex.com"+path, strings.NewReader(payload.Encode()))
if err != nil {
continue
}
req.Header.Add("api-expires", timestamp)
req.Header.Add("api-key", string(resMatch))
req.Header.Add("api-key", resMatch)
req.Header.Add("api-signature", signature)
res, err := client.Do(req)
if err == nil {
@@ -78,6 +83,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
if res.StatusCode >= 200 && res.StatusCode < 300 {
s1.Verified = true
} else {
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
continue
}
+15 -7
View File
@@ -1,11 +1,11 @@
package blablabus
import (
"bytes"
"context"
"fmt"
"net/http"
"regexp"
"strings"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
@@ -14,30 +14,37 @@ import (
type Scanner struct{}
// Ensure the Scanner satisfies the interface at compile time.
var _ detectors.Detector = (*Scanner)(nil)
var (
client = common.SaneHttpClient()
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"blablabus"}) + `\b([0-9A-Za-z]{22})\b`)
)
func (s Scanner) Keywords() [][]byte {
return [][]byte{[]byte("blablabus")}
// Keywords are used for efficiently pre-filtering chunks.
// Use identifiers in the secret preferably, or the provider name.
func (s Scanner) Keywords() []string {
return []string{"blablabus"}
}
// FromData will find and optionally verify Blablabus secrets in a given set of bytes.
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
dataStr := string(data)
matches := keyPat.FindAllSubmatch(data, -1)
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
for _, match := range matches {
if len(match) != 2 {
continue
}
resMatch := bytes.TrimSpace(match[1])
resMatch := strings.TrimSpace(match[1])
s1 := detectors.Result{
DetectorType: detectorspb.DetectorType_Blablabus,
Raw: resMatch,
Raw: []byte(resMatch),
}
if verify {
@@ -46,13 +53,14 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
continue
}
req.Header.Add("Content-Type", "application/json")
req.Header.Add("Authorization", fmt.Sprintf("Token %s", string(resMatch)))
req.Header.Add("Authorization", fmt.Sprintf("Token %s", resMatch))
res, err := client.Do(req)
if err == nil {
defer res.Body.Close()
if res.StatusCode >= 200 && res.StatusCode < 300 {
s1.Verified = true
} else {
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
continue
}
+14 -7
View File
@@ -1,11 +1,11 @@
package blazemeter
import (
"bytes"
"context"
"fmt"
"net/http"
"regexp"
"strings"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
@@ -20,25 +20,31 @@ var _ detectors.Detector = (*Scanner)(nil)
var (
client = common.SaneHttpClient()
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"blazemeter", "runscope"}) + `\b([0-9a-z]{8}-[0-9a-z]{4}-[0-9a-z]{4}-[0-9a-z]{4}-[0-9a-z]{12})\b`)
)
func (s Scanner) Keywords() [][]byte {
return [][]byte{[]byte("blazemeter"), []byte("runscope")}
// Keywords are used for efficiently pre-filtering chunks.
// Use identifiers in the secret preferably, or the provider name.
func (s Scanner) Keywords() []string {
return []string{"blazemeter", "runscope"}
}
// FromData will find and optionally verify Blazemeter secrets in a given set of bytes.
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
matches := keyPat.FindAllSubmatch(data, -1)
dataStr := string(data)
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
for _, match := range matches {
if len(match) != 2 {
continue
}
resMatch := bytes.TrimSpace(match[1])
resMatch := strings.TrimSpace(match[1])
s1 := detectors.Result{
DetectorType: detectorspb.DetectorType_Blazemeter,
Raw: resMatch,
Raw: []byte(resMatch),
}
if verify {
@@ -47,13 +53,14 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
continue
}
req.Header.Add("Content-Type", "application/json")
req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", string(resMatch)))
req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", resMatch))
res, err := client.Do(req)
if err == nil {
defer res.Body.Close()
if res.StatusCode >= 200 && res.StatusCode < 300 {
s1.Verified = true
} else {
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
continue
}
+15 -7
View File
@@ -1,10 +1,10 @@
package blitapp
import (
"bytes"
"context"
"net/http"
"regexp"
"strings"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
@@ -13,30 +13,37 @@ import (
type Scanner struct{}
// Ensure the Scanner satisfies the interface at compile time.
var _ detectors.Detector = (*Scanner)(nil)
var (
client = common.SaneHttpClient()
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"blitapp"}) + `\b([a-zA-Z0-9_-]{39})\b`)
)
func (s Scanner) Keywords() [][]byte {
return [][]byte{[]byte("blitapp")}
// Keywords are used for efficiently pre-filtering chunks.
// Use identifiers in the secret preferably, or the provider name.
func (s Scanner) Keywords() []string {
return []string{"blitapp"}
}
// FromData will find and optionally verify BlitApp secrets in a given set of bytes.
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
matches := keyPat.FindAllSubmatch(data, -1)
dataStr := string(data)
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
for _, match := range matches {
if len(match) != 2 {
continue
}
resMatch := bytes.TrimSpace(match[1])
resMatch := strings.TrimSpace(match[1])
s1 := detectors.Result{
DetectorType: detectorspb.DetectorType_BlitApp,
Raw: resMatch,
Raw: []byte(resMatch),
}
if verify {
@@ -44,13 +51,14 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
if err != nil {
continue
}
req.Header.Add("API-Key", string(resMatch))
req.Header.Add("API-Key", resMatch)
res, err := client.Do(req)
if err == nil {
defer res.Body.Close()
if res.StatusCode >= 200 && res.StatusCode < 300 {
s1.Verified = true
} else {
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
continue
}
-2
View File
@@ -43,8 +43,6 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
instanceMatches := instancePat.FindAllStringSubmatch(dataStr, -1)
tokenMatches := accessTokenPat.FindAllStringSubmatch(dataStr, -1)
fmt.Printf("instanceMatches: %v\n", instanceMatches)
for _, instance := range instanceMatches {
if len(instance) != 1 {
continue
+9 -1
View File
@@ -2,6 +2,10 @@ package engine
import (
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/envoyapikey"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/huggingface"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/salesforce"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/snowflake"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/trufflehogenterprise"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/abbysale"
@@ -287,6 +291,7 @@ import (
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/getresponse"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/getsandbox"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/github"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/github_oauth2"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/github_old"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/githubapp"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/gitlab"
@@ -556,7 +561,6 @@ import (
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/satismeterwritekey"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/saucelabs"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/scalewaykey"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/github_oauth2"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/scalr"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/scrapeowl"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/scraperapi"
@@ -1532,6 +1536,10 @@ func DefaultDetectors() []detectors.Detector {
couchbase.Scanner{},
envoyapikey.Scanner{},
github_oauth2.Scanner{},
snowflake.Scanner{},
huggingface.Scanner{},
trufflehogenterprise.Scanner{},
salesforce.Scanner{},
}
}
+84 -31
View File
@@ -12,6 +12,8 @@ import (
"strings"
"time"
v1 "github.com/google/go-containerregistry/pkg/v1"
"github.com/google/go-containerregistry/pkg/v1/tarball"
"github.com/h2non/filetype"
"github.com/mholt/archiver/v4"
@@ -29,17 +31,40 @@ var (
maxDepth = 5
maxSize = 250 * 1024 * 1024 // 20MB
maxTimeout = time.Duration(30) * time.Second
defaultBufferSize = 512
)
// Ensure the Archive satisfies the interfaces at compile time.
var _ SpecializedHandler = (*Archive)(nil)
// tempEnv contains the temporary file and directory information for the archive.
type tempEnv struct {
tempFile *os.File
tempFileName string
extractPath string
}
// dockerImageDetails contains the details of the Docker image and the temporary environment.
type dockerImageDetails struct {
image v1.Image
temp tempEnv
}
// Archive is a handler for extracting and decompressing archives.
type Archive struct {
size int
currentDepth int
}
// DockerTarReader is a wrapper for io.Reader that also stores the temp filepath and image of the tar file.
// Temporary file path information needed for inner-workings of go-containerregistry Image methods.
type DockerTarReader struct {
io.Reader
img v1.Image
tmpEnv tempEnv
}
// New sets a default maximum size and current size counter.
func (a *Archive) New() {
a.size = 0
@@ -62,7 +87,7 @@ func SetArchiveMaxTimeout(timeout time.Duration) {
// FromFile extracts the files from an archive.
func (a *Archive) FromFile(originalCtx context.Context, data io.Reader) chan []byte {
archiveChan := make(chan []byte, 512)
archiveChan := make(chan []byte, defaultBufferSize)
go func() {
ctx, cancel := context.WithTimeout(originalCtx, maxTimeout)
logger := logContext.AddLogger(ctx).Logger()
@@ -183,35 +208,35 @@ func (a *Archive) ReadToMax(ctx context.Context, reader io.Reader) (data []byte,
logger.Error(err, "Panic occurred when reading archive")
}
}()
fileContent := bytes.Buffer{}
logger.V(5).Info("Remaining buffer capacity", "bytes", maxSize-a.size)
for i := 0; i <= maxSize/512; i++ {
if common.IsDone(ctx) {
return nil, ctx.Err()
}
fileChunk := make([]byte, 512)
bRead, err := reader.Read(fileChunk)
if err != nil && !errors.Is(err, io.EOF) {
return []byte{}, err
}
a.size += bRead
if len(fileChunk) > 0 {
fileContent.Write(fileChunk[0:bRead])
}
if bRead < 512 {
return fileContent.Bytes(), nil
}
if a.size >= maxSize && bRead == 512 {
logger.V(2).Info("Max archive size reached.")
return fileContent.Bytes(), nil
}
if common.IsDone(ctx) {
return nil, ctx.Err()
}
var fileContent bytes.Buffer
// Create a limited reader to ensure we don't read more than the max size.
lr := io.LimitReader(reader, int64(maxSize))
// Using io.CopyBuffer for performance advantages. Though buf is mandatory
// for the method, due to the internal implementation of io.CopyBuffer, when
// *bytes.Buffer implements io.WriterTo or io.ReaderFrom, the provided buf
// is simply ignored. Thus, we can pass nil for the buf parameter.
_, err = io.CopyBuffer(&fileContent, lr, nil)
if err != nil && !errors.Is(err, io.EOF) {
return nil, err
}
if fileContent.Len() == maxSize {
logger.V(2).Info("Max archive size reached.")
}
return fileContent.Bytes(), nil
}
const (
arMimeType = "application/x-unix-archive"
rpmMimeType = "application/x-rpm"
tarMimeType = "application/x-tar"
)
// Define a map of mime types to corresponding command-line tools
@@ -265,6 +290,21 @@ func (a *Archive) HandleSpecialized(ctx logContext.Context, reader io.Reader) (i
return nil, false, err
}
reader, err = a.extractRpmContent(ctx, reader)
case tarMimeType:
//check if tar is a docker image
isImg, details, err := a.isDockerImage(ctx, reader)
if err != nil {
return nil, false, err
}
if !isImg {
return nil, false, nil
}
// Build dockerReader containing docker image
reader = DockerTarReader{
Reader: reader,
img: details.image,
tmpEnv: details.temp,
}
default:
return reader, false, nil
}
@@ -289,8 +329,8 @@ func (a *Archive) extractDebContent(ctx logContext.Context, file io.Reader) (io.
if err != nil {
return nil, err
}
defer os.Remove(tmpEnv.tempFileName)
defer os.RemoveAll(tmpEnv.extractPath)
defer os.Remove(tmpEnv.tempFileName)
cmd := exec.Command("ar", "x", tmpEnv.tempFile.Name())
cmd.Dir = tmpEnv.extractPath
@@ -327,8 +367,8 @@ func (a *Archive) extractRpmContent(ctx logContext.Context, file io.Reader) (io.
if err != nil {
return nil, err
}
defer os.Remove(tmpEnv.tempFileName)
defer os.RemoveAll(tmpEnv.extractPath)
defer os.Remove(tmpEnv.tempFileName)
// Use rpm2cpio to convert the RPM file to a cpio archive and then extract it using cpio command.
cmd := exec.Command("sh", "-c", "rpm2cpio "+tmpEnv.tempFile.Name()+" | cpio -id")
@@ -427,12 +467,6 @@ func (a *Archive) handleExtractedFiles(ctx logContext.Context, env tempEnv, hand
return dataArchiveName, nil
}
type tempEnv struct {
tempFile *os.File
tempFileName string
extractPath string
}
// createTempEnv creates a temporary file and a temporary directory for extracting archives.
// The caller is responsible for removing these temporary resources
// (both the file and directory) when they are no longer needed.
@@ -476,3 +510,22 @@ func openDataArchive(extractPath string, dataArchiveName string) (io.ReadCloser,
}
return dataFile, nil
}
// isDockerImage checks if a reader object is a docker image.
// Returns true if the tar file is a docker image, dockerImageDetails, and error.
// Caller is responsible for removing temporary files and directories IF the file is a docker image.
// The docker scanner requires a file path to the image, so we must leave the temporary file on disk.
func (a *Archive) isDockerImage(ctx context.Context, file io.Reader) (isImg bool, details dockerImageDetails, err error) {
tmpEnv, err := a.createTempEnv(ctx, file)
if err != nil {
return false, dockerImageDetails{}, err
}
img, err := tarball.ImageFromPath(tmpEnv.tempFileName, nil)
if err != nil {
os.Remove(tmpEnv.tempFileName)
os.RemoveAll(tmpEnv.extractPath)
return false, dockerImageDetails{}, err
}
return true, dockerImageDetails{image: img, temp: tmpEnv}, nil
}
+102
View File
@@ -0,0 +1,102 @@
//go:build integration
// +build integration
package handlers
import (
"context"
"os"
"testing"
diskbufferreader "github.com/bill-rich/disk-buffer-reader"
"github.com/google/go-containerregistry/pkg/name"
"github.com/google/go-containerregistry/pkg/v1/remote"
"github.com/google/go-containerregistry/pkg/v1/tarball"
"github.com/stretchr/testify/assert"
logContext "github.com/trufflesecurity/trufflehog/v3/pkg/context"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/source_metadatapb"
"github.com/trufflesecurity/trufflehog/v3/pkg/sanitizer"
"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
)
func TestFoundKeyInDockerTar(t *testing.T) {
// URI of the Docker image to clone
// imageURI := "ghcr.io/joeleonjr/getting-started-app-with-canary-token:main"
imageURI := "trufflesecurity/secrets"
var imageName name.Reference
imageName, err := name.NewTag(imageURI)
if err != nil {
t.Errorf("%s: error creating image name: %s", imageURI, err)
}
// Pull the image
img, err := remote.Image(imageName)
if err != nil {
t.Errorf("%s: error pulling image: %s", imageURI, err)
}
tempFile, err := os.CreateTemp("", "archive_test_docker_img.tar")
if err != nil {
t.Errorf("%s: error creating temporary file: %s", imageURI, err)
return
}
defer os.Remove(tempFile.Name()) // Clean up the temporary file
// Save the image as a tar file
err = tarball.WriteToFile(tempFile.Name(), imageName, img)
if err != nil {
t.Errorf("%s: error saving image as tar file: %s", imageURI, err)
return
}
ctx := logContext.AddLogger(context.Background())
inputFile, err := os.Open(tempFile.Name())
if err != nil {
t.Errorf("%s: error opening tar file: %s", imageURI, err)
return
}
defer inputFile.Close()
reReader, err := diskbufferreader.New(inputFile)
if err != nil {
t.Errorf("%s: error creating re-readable reader: %s", imageURI, err)
return
}
defer reReader.Close()
chunkSkel := &sources.Chunk{
SourceType: 1,
SourceName: "filesystem",
SourceID: 1,
SourceMetadata: &source_metadatapb.MetaData{
Data: &source_metadatapb.MetaData_Filesystem{
Filesystem: &source_metadatapb.Filesystem{
File: sanitizer.UTF8(tempFile.Name()),
},
},
},
Verify: true,
}
chunksChan := make(chan *sources.Chunk, 1)
go func() {
defer close(chunksChan)
HandleFile(ctx, reReader, chunkSkel, chunksChan)
}()
secret := "AKIAXYZDQCEN4B6JSJQI"
// Read from the channel and validate the secrets.
foundSecret := ""
for chunkCh := range chunksChan {
foundSecret += string(chunkCh.Data)
}
assert.Contains(t, foundSecret, secret)
}
+57 -3
View File
@@ -1,6 +1,7 @@
package handlers
import (
"bytes"
"context"
"io"
"net/http"
@@ -107,7 +108,8 @@ func TestHandleFile(t *testing.T) {
ch := make(chan *sources.Chunk, 2)
// Context cancels the operation.
canceledCtx, cancel := context.WithCancel(context.Background())
ctx := logContext.AddLogger(context.Background())
canceledCtx, cancel := logContext.WithCancel(ctx)
cancel()
assert.False(t, HandleFile(canceledCtx, strings.NewReader("file"), &sources.Chunk{}, ch))
@@ -122,10 +124,62 @@ func TestHandleFile(t *testing.T) {
assert.NoError(t, err)
assert.Equal(t, 0, len(ch))
assert.True(t, HandleFile(context.Background(), reader, &sources.Chunk{}, ch))
assert.True(t, HandleFile(ctx, reader, &sources.Chunk{}, ch))
assert.Equal(t, 1, len(ch))
}
func TestReadToMax(t *testing.T) {
tests := []struct {
name string
input []byte
expected []byte
}{
{
name: "read full content within maxSize",
input: []byte("abcdefg"),
expected: []byte("abcdefg"),
},
{
name: "read content larger than maxSize",
input: make([]byte, maxSize+10), // this creates a byte slice 10 bytes larger than maxSize
expected: make([]byte, maxSize),
},
{
name: "empty input",
input: []byte(""),
expected: []byte(""),
},
}
a := &Archive{}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
reader := bytes.NewReader(tt.input)
output, err := a.ReadToMax(context.Background(), reader)
assert.Nil(t, err)
assert.Equal(t, tt.expected, output)
})
}
}
func BenchmarkReadToMax(b *testing.B) {
data := bytes.Repeat([]byte("a"), 1024*1000) // 1MB of data.
reader := bytes.NewReader(data)
a := &Archive{}
b.ResetTimer()
for i := 0; i < b.N; i++ {
b.StartTimer()
_, _ = a.ReadToMax(context.Background(), reader)
b.StopTimer()
_, _ = reader.Seek(0, 0) // Reset the reader position.
a.size = 0 // Reset archive size.
}
}
func TestExtractDebContent(t *testing.T) {
// Open the sample .deb file from the testdata folder.
file, err := os.Open("testdata/test.deb")
@@ -149,7 +203,7 @@ func TestExtractTarContent(t *testing.T) {
assert.Nil(t, err)
defer file.Close()
ctx := context.Background()
ctx := logContext.AddLogger(context.Background())
chunkCh := make(chan *sources.Chunk)
go func() {
+10 -2
View File
@@ -3,11 +3,13 @@ package handlers
import (
"context"
"io"
"os"
diskbufferreader "github.com/bill-rich/disk-buffer-reader"
logContext "github.com/trufflesecurity/trufflehog/v3/pkg/context"
"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
"github.com/trufflesecurity/trufflehog/v3/pkg/sources/docker"
)
func DefaultHandlers() []Handler {
@@ -37,7 +39,7 @@ type Handler interface {
// packages them in the provided chunk skeleton, and sends them to chunksChan.
// The function returns true if processing was successful and false otherwise.
// Context is used for cancellation, and the caller is responsible for canceling it if needed.
func HandleFile(ctx context.Context, file io.Reader, chunkSkel *sources.Chunk, chunksChan chan *sources.Chunk) bool {
func HandleFile(ctx logContext.Context, file io.Reader, chunkSkel *sources.Chunk, chunksChan chan *sources.Chunk) bool {
aCtx := logContext.AddLogger(ctx)
for _, h := range DefaultHandlers() {
h.New()
@@ -56,9 +58,15 @@ func HandleFile(ctx context.Context, file io.Reader, chunkSkel *sources.Chunk, c
if specialHandler, ok := h.(SpecializedHandler); ok {
file, isSpecial, err := specialHandler.HandleSpecialized(aCtx, reReader)
if isSpecial {
if dockerTarReader, ok := file.(DockerTarReader); ok {
// Clean up temporary files when done.
defer os.RemoveAll(dockerTarReader.tmpEnv.extractPath)
defer os.Remove(dockerTarReader.tmpEnv.tempFileName)
err = docker.ScanDockerImg(ctx, dockerTarReader.img, chunksChan, chunkSkel)
return err == nil
}
return handleChunks(aCtx, h.FromFile(ctx, file), chunkSkel, chunksChan)
}
if err != nil {
aCtx.Logger().Error(err, "error handling file")
}
+1 -1
View File
@@ -133,7 +133,7 @@ func BranchHeads(repo *gogit.Repository) (map[string]*object.Commit, error) {
}
headCommit, err := repo.CommitObject(*headHash)
if err != nil {
logger.Error(err, "unable to get commit", "commit", headCommit.String())
logger.Error(err, "unable to get commit", "head_hash", headHash.String())
return nil
}
branches[branchName] = headCommit
+1 -1
View File
@@ -20,7 +20,7 @@ const (
// Chunker takes a chunk and splits it into chunks of ChunkSize.
func Chunker(originalChunk *Chunk) chan *Chunk {
chunkChan := make(chan *Chunk)
chunkChan := make(chan *Chunk, 1)
go func() {
defer close(chunkChan)
if len(originalChunk.Data) <= TotalChunkSize {
+1
View File
@@ -238,6 +238,7 @@ func (s *Source) chunkAction(ctx context.Context, proj project, bld build, act a
SourceType: s.Type(),
SourceName: s.name,
SourceID: s.SourceID(),
JobID: s.JobID(),
Data: removeCircleSha1Line(data.Bytes()),
SourceMetadata: &source_metadatapb.MetaData{
Data: &source_metadatapb.MetaData_Circleci{
+106 -81
View File
@@ -78,7 +78,7 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) err
return err
}
logger := ctx.Logger().WithValues("source_type", s.Type(), "source_name", s.name)
ctx.Logger().WithValues("source_type", s.Type(), "source_name", s.name)
for _, image := range s.conn.GetImages() {
var img v1.Image
@@ -115,95 +115,120 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) err
}
}
logger = logger.WithValues("image", base, "tag", tag)
logger.V(2).Info("scanning image")
layers, err := img.Layers()
chunkSkel := &sources.Chunk{
SourceType: s.Type(),
SourceName: s.name,
SourceID: s.SourceID(),
JobID: s.JobID(),
SourceMetadata: &source_metadatapb.MetaData{
Data: &source_metadatapb.MetaData_Docker{
Docker: &source_metadatapb.Docker{
Image: base,
Tag: tag,
},
},
},
Verify: s.verify,
}
err = ScanDockerImg(ctx, img, chunksChan, chunkSkel)
if err != nil {
return err
}
for _, layer := range layers {
digest, err := layer.Digest()
if err != nil {
return err
}
logger = logger.WithValues("layer", digest.String())
logger.V(2).Info("scanning layer")
rc, err := layer.Compressed()
if err != nil {
return err
}
defer rc.Close()
gzipReader, err := gzip.NewReader(rc)
if err != nil {
return err
}
defer gzipReader.Close()
tarReader := tar.NewReader(gzipReader)
for {
header, err := tarReader.Next()
if errors.Is(err, io.EOF) {
break // End of archive
}
if err != nil {
return err
}
// Skip files larger than FilesizeLimitBytes
if header.Size > FilesizeLimitBytes {
continue
}
file := bytes.NewBuffer(make([]byte, 0, header.Size))
_, err = io.Copy(file, tarReader)
if err != nil {
return err
}
chunk := &sources.Chunk{
SourceType: s.Type(),
SourceName: s.name,
SourceID: s.SourceID(),
Data: file.Bytes(),
SourceMetadata: &source_metadatapb.MetaData{
Data: &source_metadatapb.MetaData_Docker{
Docker: &source_metadatapb.Docker{
File: "/" + header.Name, // Prepend /, because the file is relative to the root of the image
Image: base,
Tag: tag,
Layer: digest.String(),
},
},
},
Verify: s.verify,
}
select {
case chunksChan <- chunk:
case <-ctx.Done():
return ctx.Err()
}
}
dockerLayersScanned.WithLabelValues(s.name).Inc()
}
dockerImagesScanned.WithLabelValues(s.name).Inc()
}
return nil
}
func ScanDockerImg(ctx context.Context, img v1.Image, chunksChan chan *sources.Chunk, chunkSkel *sources.Chunk) error {
base, tag := chunkSkel.BaseAndTagForDockerImg()
logger := ctx.Logger().WithValues("image", base, "tag", tag)
logger.V(2).Info("scanning image")
layers, err := img.Layers()
if err != nil {
return err
}
for _, layer := range layers {
digest, err := layer.Digest()
if err != nil {
return err
}
logger = logger.WithValues("layer", digest.String())
logger.V(2).Info("scanning layer")
rc, err := layer.Compressed()
if err != nil {
return err
}
defer rc.Close()
gzipReader, err := gzip.NewReader(rc)
if err != nil {
return err
}
defer gzipReader.Close()
tarReader := tar.NewReader(gzipReader)
for {
header, err := tarReader.Next()
if errors.Is(err, io.EOF) {
break // End of archive
}
if err != nil {
return err
}
// Skip files larger than FilesizeLimitBytes
if header.Size > FilesizeLimitBytes {
continue
}
file := bytes.NewBuffer(make([]byte, 0, header.Size))
_, err = io.Copy(file, tarReader)
if err != nil {
return err
}
chunk := &sources.Chunk{
// This must be TYPE_DOCKER or SetResultLineNumber could panic
SourceType: sourcespb.SourceType_SOURCE_TYPE_DOCKER,
SourceName: chunkSkel.SourceName,
SourceID: chunkSkel.SourceID,
Data: file.Bytes(),
SourceMetadata: &source_metadatapb.MetaData{
Data: &source_metadatapb.MetaData_Docker{
Docker: &source_metadatapb.Docker{
File: "/" + header.Name, // Prepend /, because the file is relative to the root of the image
Image: base,
Tag: tag,
Layer: digest.String(),
},
},
},
Verify: chunkSkel.Verify,
}
select {
case chunksChan <- chunk:
case <-ctx.Done():
return ctx.Err()
}
}
dockerLayersScanned.WithLabelValues(chunkSkel.SourceName).Inc()
}
dockerImagesScanned.WithLabelValues(chunkSkel.SourceName).Inc()
return nil
}
func baseAndTagFromImage(image string) (base, tag string, hasDigest bool) {
regRepoDelimiter := "/"
tagDelim := ":"
+2
View File
@@ -161,6 +161,7 @@ func (s *Source) scanFile(ctx context.Context, path string, chunksChan chan *sou
SourceType: s.Type(),
SourceName: s.name,
SourceID: s.SourceID(),
JobID: s.JobID(),
SourceMetadata: &source_metadatapb.MetaData{
Data: &source_metadatapb.MetaData_Filesystem{
Filesystem: &source_metadatapb.Filesystem{
@@ -191,6 +192,7 @@ func (s *Source) scanFile(ctx context.Context, path string, chunksChan chan *sou
SourceType: s.Type(),
SourceName: s.name,
SourceID: s.SourceID(),
JobID: s.JobID(),
Data: data.Bytes(),
SourceMetadata: &source_metadatapb.MetaData{
Data: &source_metadatapb.MetaData_Filesystem{
+1
View File
@@ -324,6 +324,7 @@ func (s *Source) processObject(ctx context.Context, o object) error {
chunkSkel := &sources.Chunk{
SourceName: s.name,
SourceType: s.Type(),
JobID: s.JobID(),
SourceID: s.sourceId,
Verify: s.verify,
SourceMetadata: &source_metadatapb.MetaData{
+7
View File
@@ -446,6 +446,7 @@ func (s *Git) ScanCommits(ctx context.Context, repo *git.Repository, path string
chunkSkel := &sources.Chunk{
SourceName: s.sourceName,
SourceID: s.sourceID,
JobID: s.jobID,
SourceType: s.sourceType,
SourceMetadata: metadata,
Verify: s.verify,
@@ -464,6 +465,7 @@ func (s *Git) ScanCommits(ctx context.Context, repo *git.Repository, path string
chunksChan <- &sources.Chunk{
SourceName: s.sourceName,
SourceID: s.sourceID,
JobID: s.jobID,
SourceType: s.sourceType,
SourceMetadata: metadata,
Data: diff.Content.Bytes(),
@@ -490,6 +492,7 @@ func (s *Git) gitChunk(ctx context.Context, diff gitparse.Diff, fileName, email,
chunksChan <- &sources.Chunk{
SourceName: s.sourceName,
SourceID: s.sourceID,
JobID: s.jobID,
SourceType: s.sourceType,
SourceMetadata: metadata,
Data: append([]byte{}, newChunkBuffer.Bytes()...),
@@ -504,6 +507,7 @@ func (s *Git) gitChunk(ctx context.Context, diff gitparse.Diff, fileName, email,
chunksChan <- &sources.Chunk{
SourceName: s.sourceName,
SourceID: s.sourceID,
JobID: s.jobID,
SourceType: s.sourceType,
SourceMetadata: metadata,
Data: line,
@@ -523,6 +527,7 @@ func (s *Git) gitChunk(ctx context.Context, diff gitparse.Diff, fileName, email,
chunksChan <- &sources.Chunk{
SourceName: s.sourceName,
SourceID: s.sourceID,
JobID: s.jobID,
SourceType: s.sourceType,
SourceMetadata: metadata,
Data: append([]byte{}, newChunkBuffer.Bytes()...),
@@ -588,6 +593,7 @@ func (s *Git) ScanStaged(ctx context.Context, repo *git.Repository, path string,
chunkSkel := &sources.Chunk{
SourceName: s.sourceName,
SourceID: s.sourceID,
JobID: s.jobID,
SourceType: s.sourceType,
SourceMetadata: metadata,
Verify: s.verify,
@@ -602,6 +608,7 @@ func (s *Git) ScanStaged(ctx context.Context, repo *git.Repository, path string,
chunksChan <- &sources.Chunk{
SourceName: s.sourceName,
SourceID: s.sourceID,
JobID: s.jobID,
SourceType: s.sourceType,
SourceMetadata: metadata,
Data: diff.Content.Bytes(),
+7 -2
View File
@@ -149,14 +149,16 @@ func (s *Source) newFilteredRepoCache(c cache.Cache, include, exclude []string)
for _, ig := range include {
g, err := glob.Compile(ig)
if err != nil {
s.log.V(1).Info("invalid include glob", "glob", g, "err", err)
s.log.V(1).Info("invalid include glob", "include_value", ig, "err", err)
continue
}
includeGlobs = append(includeGlobs, g)
}
for _, eg := range exclude {
g, err := glob.Compile(eg)
if err != nil {
s.log.V(1).Info("invalid exclude glob", "glob", g, "err", err)
s.log.V(1).Info("invalid exclude glob", "exclude_value", eg, "err", err)
continue
}
excludeGlobs = append(excludeGlobs, g)
}
@@ -1223,6 +1225,7 @@ func (s *Source) chunkIssueComments(ctx context.Context, repo, repoPath string,
chunk := &sources.Chunk{
SourceName: s.name,
SourceID: s.SourceID(),
JobID: s.JobID(),
SourceType: s.Type(),
SourceMetadata: &source_metadatapb.MetaData{
Data: &source_metadatapb.MetaData_Github{
@@ -1256,6 +1259,7 @@ func (s *Source) chunkPullRequestComments(ctx context.Context, repo string, comm
SourceName: s.name,
SourceID: s.SourceID(),
SourceType: s.Type(),
JobID: s.JobID(),
SourceMetadata: &source_metadatapb.MetaData{
Data: &source_metadatapb.MetaData_Github{
Github: &source_metadatapb.Github{
@@ -1287,6 +1291,7 @@ func (s *Source) chunkGistComments(ctx context.Context, gistUrl string, comments
SourceName: s.name,
SourceID: s.SourceID(),
SourceType: s.Type(),
JobID: s.JobID(),
SourceMetadata: &source_metadatapb.MetaData{
Data: &source_metadatapb.MetaData_Github{
Github: &source_metadatapb.Github{
+4 -4
View File
@@ -71,11 +71,11 @@ func (r *JobProgressRef) Done() <-chan struct{} {
// CancelRun requests that the job this is referencing is cancelled and stops
// running. This method will have no effect if the job does not allow
// cancellation.
func (r *JobProgressRef) CancelRun() {
func (r *JobProgressRef) CancelRun(cause error) {
if r.jobProgress == nil || r.jobProgress.jobCancel == nil {
return
}
r.jobProgress.jobCancel()
r.jobProgress.jobCancel(cause)
}
// Fatal is a wrapper around error to differentiate non-fatal errors from fatal
@@ -108,7 +108,7 @@ type JobProgress struct {
ctx context.Context
cancel context.CancelFunc
// Requests to cancel the job.
jobCancel context.CancelFunc
jobCancel context.CancelCauseFunc
// Metrics.
metrics JobProgressMetrics
metricsLock sync.Mutex
@@ -150,7 +150,7 @@ func WithHooks(hooks ...JobProgressHook) func(*JobProgress) {
}
// WithCancel allows cancelling the job by the JobProgressRef.
func WithCancel(cancel context.CancelFunc) func(*JobProgress) {
func WithCancel(cancel context.CancelCauseFunc) func(*JobProgress) {
return func(jp *JobProgress) { jp.jobCancel = cancel }
}
+1
View File
@@ -376,6 +376,7 @@ func (s *Source) pageChunker(ctx context.Context, client *s3.S3, chunksChan chan
SourceType: s.Type(),
SourceName: s.name,
SourceID: s.SourceID(),
JobID: s.JobID(),
SourceMetadata: &source_metadatapb.MetaData{
Data: &source_metadatapb.MetaData_S3{
S3: &source_metadatapb.S3{
+28 -4
View File
@@ -36,7 +36,10 @@ type SourceManager struct {
handles map[handle]sourceInfo
handlesLock sync.Mutex
// Pool limiting the amount of concurrent sources running.
pool errgroup.Group
pool errgroup.Group
poolLimit int
currentRunningCount int32
// Max number of units to scan concurrently per source.
concurrentUnits int
// Run the sources using source unit enumeration / chunking if available.
useSourceUnits bool
@@ -68,7 +71,10 @@ func WithReportHook(hook JobProgressHook) func(*SourceManager) {
// WithConcurrentSources limits the concurrent number of sources a manager can run.
func WithConcurrentSources(concurrency int) func(*SourceManager) {
return func(mgr *SourceManager) { mgr.pool.SetLimit(concurrency) }
return func(mgr *SourceManager) {
mgr.pool.SetLimit(concurrency)
mgr.poolLimit = concurrency
}
}
// WithBufferedOutput sets the size of the buffer used for the Chunks() channel.
@@ -164,15 +170,17 @@ func (s *SourceManager) asyncRun(ctx context.Context, handle handle) (JobProgres
return JobProgressRef{SourceID: int64(handle), SourceName: sourceName}, err
}
// Create a JobProgress object for tracking progress.
ctx, cancel := context.WithCancel(ctx)
ctx, cancel := context.WithCancelCause(ctx)
progress := NewJobProgress(jobID, int64(handle), sourceName, WithHooks(s.hooks...), WithCancel(cancel))
s.pool.Go(func() error {
atomic.AddInt32(&s.currentRunningCount, 1)
defer atomic.AddInt32(&s.currentRunningCount, -1)
ctx := context.WithValues(ctx,
"job_id", jobID,
"source_manager_worker_id", common.RandomID(5),
)
defer common.Recover(ctx)
defer cancel()
defer cancel(nil)
return s.run(ctx, handle, jobID, progress)
})
return progress.Ref(), nil
@@ -207,6 +215,16 @@ func (s *SourceManager) ScanChunk(chunk *Chunk) {
s.outputChunks <- chunk
}
// AvailableCapacity returns the number of concurrent jobs the manager can
// accommodate at this time. If there is no limit, -1 is returned.
func (s *SourceManager) AvailableCapacity() int {
if s.poolLimit == 0 {
return -1
}
runCount := atomic.LoadInt32(&s.currentRunningCount)
return s.poolLimit - int(runCount)
}
// preflightChecks is a helper method to check the Manager or the context isn't
// done and that the handle is valid.
func (s *SourceManager) preflightChecks(ctx context.Context, handle handle) error {
@@ -229,6 +247,12 @@ func (s *SourceManager) run(ctx context.Context, handle handle, jobID int64, rep
report.Start(time.Now())
defer func() { report.End(time.Now()) }()
defer func() {
if err := context.Cause(ctx); err != nil {
report.ReportError(Fatal{err})
}
}()
// Initialize the source.
sourceInfo, ok := s.getSourceInfo(handle)
if !ok {
+24 -1
View File
@@ -312,8 +312,31 @@ func TestSourceManagerCancelRun(t *testing.T) {
ref, err := mgr.ScheduleRun(context.Background(), handle)
assert.NoError(t, err)
ref.CancelRun()
cancelErr := fmt.Errorf("abort! abort!")
ref.CancelRun(cancelErr)
<-ref.Done()
assert.Error(t, ref.Snapshot().FatalError())
assert.True(t, errors.Is(ref.Snapshot().FatalError(), returnedErr))
assert.True(t, errors.Is(ref.Snapshot().FatalErrors(), cancelErr))
}
func TestSourceManagerAvailableCapacity(t *testing.T) {
mgr := NewManager(WithConcurrentSources(1337))
start, end := make(chan struct{}), make(chan struct{})
handle, err := enrollDummy(mgr, callbackChunker{func(context.Context, chan *Chunk) error {
start <- struct{}{} // Send start signal.
<-end // Wait for end signal.
return nil
}})
assert.NoError(t, err)
assert.Equal(t, 1337, mgr.AvailableCapacity())
ref, err := mgr.ScheduleRun(context.Background(), handle)
assert.NoError(t, err)
<-start // Wait for start signal.
assert.Equal(t, 1336, mgr.AvailableCapacity())
end <- struct{}{} // Send end signal.
<-ref.Done() // Wait for the job to finish.
assert.Equal(t, 1337, mgr.AvailableCapacity())
}
+42
View File
@@ -30,6 +30,48 @@ type Chunk struct {
Verify bool
}
// BaseAndTagForDockerImg returns a base and tag value for a
// docker image regardless of the source type. If the source type
// is not a docker image, the base will be a filename/path from the source metadata.
// The default is the entire source metadata string. Update for new source types.
func (c *Chunk) BaseAndTagForDockerImg() (base string, tag string) {
switch c.SourceType {
case sourcespb.SourceType_SOURCE_TYPE_DOCKER:
base = c.SourceMetadata.GetDocker().GetImage()
tag = c.SourceMetadata.GetDocker().GetTag()
case sourcespb.SourceType_SOURCE_TYPE_FILESYSTEM:
base = c.SourceMetadata.GetFilesystem().GetFile()
case sourcespb.SourceType_SOURCE_TYPE_S3:
bucket := c.SourceMetadata.GetS3().GetBucket()
file := c.SourceMetadata.GetS3().GetFile()
base = bucket + "/" + file
case sourcespb.SourceType_SOURCE_TYPE_GCS:
bucket := c.SourceMetadata.GetGcs().GetBucket()
file := c.SourceMetadata.GetGcs().GetFilename()
base = bucket + "/" + file
case sourcespb.SourceType_SOURCE_TYPE_SYSLOG:
host := c.SourceMetadata.GetSyslog().GetHostname()
app := c.SourceMetadata.GetSyslog().GetAppname()
base = host + "/" + app
case sourcespb.SourceType_SOURCE_TYPE_GIT:
repo := c.SourceMetadata.GetGit().GetRepository()
file := c.SourceMetadata.GetGit().GetFile()
base = file + " (repo: " + repo + ")"
case sourcespb.SourceType_SOURCE_TYPE_GITLAB:
repo := c.SourceMetadata.GetGitlab().GetRepository()
file := c.SourceMetadata.GetGitlab().GetFile()
base = file + " (repo: " + repo + ")"
case sourcespb.SourceType_SOURCE_TYPE_GITHUB:
repo := c.SourceMetadata.GetGithub().GetRepository()
file := c.SourceMetadata.GetGithub().GetFile()
base = file + " (repo: " + repo + ")"
default:
base = c.SourceMetadata.String()
tag = ""
}
return base, tag
}
// Source defines the interface required to implement a source chunker.
type Source interface {
// Type returns the source type, used for matching against configuration and jobs.
+2
View File
@@ -272,6 +272,7 @@ func (s *Source) monitorConnection(ctx context.Context, conn net.Conn, chunksCha
SourceName: s.syslog.sourceName,
SourceID: s.syslog.sourceID,
SourceType: s.syslog.sourceType,
JobID: s.JobID(),
SourceMetadata: metadata,
Data: input,
Verify: s.verify,
@@ -313,6 +314,7 @@ func (s *Source) acceptUDPConnections(ctx context.Context, netListener net.Packe
chunksChan <- &sources.Chunk{
SourceName: s.syslog.sourceName,
SourceID: s.syslog.sourceID,
JobID: s.JobID(),
SourceType: s.syslog.sourceType,
SourceMetadata: metadata,
Data: input,