Compare commits
28 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| f623c0686f | |||
| 522b2fab29 | |||
| c77c1172c8 | |||
| d6afca682b | |||
| 4dc5eb7912 | |||
| 7ba880f47a | |||
| 2b1b1b5ad0 | |||
| c51e8f8af5 | |||
| 66271aed7b | |||
| d1b1ac408e | |||
| 0932ea224b | |||
| b8468af367 | |||
| b1a1fa393f | |||
| 7077f69552 | |||
| b38bb0d110 | |||
| 51aadf96e1 | |||
| f7cf80ee85 | |||
| e523b69a4c | |||
| f4119799d9 | |||
| 6b8ce2fba2 | |||
| 95f65281ea | |||
| 5f4f513014 | |||
| 2d46cf1f74 | |||
| aa461ae806 | |||
| a55237376a | |||
| 5b9a862ffa | |||
| e01f6d0257 | |||
| af83970f77 |
@@ -32,7 +32,7 @@ Have questions? Feedback? Jump in slack or discord and hang out with us
|
||||
|
||||
Join our [Slack Community](https://join.slack.com/t/trufflehog-community/shared_invite/zt-pw2qbi43-Aa86hkiimstfdKH9UCpPzQ)
|
||||
|
||||
Join the [Secret Scanning Discord](https://discord.gg/sydS6AHTUP)
|
||||
Join the [Secret Scanning Discord](https://discord.gg/8Hzbrnkr7E)
|
||||
|
||||
# :tv: Demo
|
||||
|
||||
|
||||
+40
-2
@@ -26,9 +26,10 @@ type Context interface {
|
||||
Logger() logr.Logger
|
||||
}
|
||||
|
||||
// CancelFunc is a type alias to context.CancelFunc to allow use as if they are
|
||||
// the same types.
|
||||
// CancelFunc and CancelCauseFunc are type aliases to allow use as if they are
|
||||
// the same types as the standard library variants.
|
||||
type CancelFunc = context.CancelFunc
|
||||
type CancelCauseFunc = context.CancelCauseFunc
|
||||
|
||||
// logCtx implements Context.
|
||||
type logCtx struct {
|
||||
@@ -68,6 +69,16 @@ func WithCancel(parent Context) (Context, context.CancelFunc) {
|
||||
return lCtx, cancel
|
||||
}
|
||||
|
||||
// WithCancelCause returns context.WithCancelCause with the log object propagated.
|
||||
func WithCancelCause(parent Context) (Context, context.CancelCauseFunc) {
|
||||
ctx, cancel := context.WithCancelCause(parent)
|
||||
lCtx := logCtx{
|
||||
log: parent.Logger(),
|
||||
Context: ctx,
|
||||
}
|
||||
return lCtx, cancel
|
||||
}
|
||||
|
||||
// WithDeadline returns context.WithDeadline with the log object propagated and
|
||||
// the deadline added to the structured log values.
|
||||
func WithDeadline(parent Context, d time.Time) (Context, context.CancelFunc) {
|
||||
@@ -79,6 +90,17 @@ func WithDeadline(parent Context, d time.Time) (Context, context.CancelFunc) {
|
||||
return lCtx, cancel
|
||||
}
|
||||
|
||||
// WithDeadlineCause returns context.WithDeadlineCause with the log object
|
||||
// propagated and the deadline added to the structured log values.
|
||||
func WithDeadlineCause(parent Context, d time.Time, cause error) (Context, context.CancelFunc) {
|
||||
ctx, cancel := context.WithDeadlineCause(parent, d, cause)
|
||||
lCtx := logCtx{
|
||||
log: parent.Logger().WithValues("deadline", d),
|
||||
Context: ctx,
|
||||
}
|
||||
return lCtx, cancel
|
||||
}
|
||||
|
||||
// WithTimeout returns context.WithTimeout with the log object propagated and
|
||||
// the timeout added to the structured log values.
|
||||
func WithTimeout(parent Context, timeout time.Duration) (Context, context.CancelFunc) {
|
||||
@@ -90,6 +112,22 @@ func WithTimeout(parent Context, timeout time.Duration) (Context, context.Cancel
|
||||
return lCtx, cancel
|
||||
}
|
||||
|
||||
// WithTimeoutCause returns context.WithTimeoutCause with the log object
|
||||
// propagated and the timeout added to the structured log values.
|
||||
func WithTimeoutCause(parent Context, timeout time.Duration, cause error) (Context, context.CancelFunc) {
|
||||
ctx, cancel := context.WithTimeoutCause(parent, timeout, cause)
|
||||
lCtx := logCtx{
|
||||
log: parent.Logger().WithValues("timeout", timeout),
|
||||
Context: ctx,
|
||||
}
|
||||
return lCtx, cancel
|
||||
}
|
||||
|
||||
// Cause returns the context.Cause of the context.
|
||||
func Cause(ctx context.Context) error {
|
||||
return context.Cause(ctx)
|
||||
}
|
||||
|
||||
// WithValue returns context.WithValue with the log object propagated and
|
||||
// the value added to the structured log values (if the key is a string).
|
||||
func WithValue(parent Context, key, val any) Context {
|
||||
|
||||
@@ -3,6 +3,7 @@ package context
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
@@ -177,3 +178,10 @@ func TestRace(t *testing.T) {
|
||||
cancel()
|
||||
_ = ctx.Err()
|
||||
}
|
||||
|
||||
func TestCause(t *testing.T) {
|
||||
ctx, cancel := WithCancelCause(Background())
|
||||
err := fmt.Errorf("oh no")
|
||||
cancel(err)
|
||||
assert.Equal(t, err, Cause(ctx))
|
||||
}
|
||||
|
||||
@@ -1,10 +1,10 @@
|
||||
package axonaut
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
@@ -13,29 +13,37 @@ import (
|
||||
|
||||
type Scanner struct{}
|
||||
|
||||
// Ensure the Scanner satisfies the interface at compile time.
|
||||
var _ detectors.Detector = (*Scanner)(nil)
|
||||
|
||||
var (
|
||||
client = common.SaneHttpClient()
|
||||
|
||||
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
|
||||
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"axonaut"}) + `\b([a-z0-9]{32})\b`)
|
||||
)
|
||||
|
||||
func (s Scanner) Keywords() [][]byte {
|
||||
return [][]byte{[]byte("axonaut")}
|
||||
// Keywords are used for efficiently pre-filtering chunks.
|
||||
// Use identifiers in the secret preferably, or the provider name.
|
||||
func (s Scanner) Keywords() []string {
|
||||
return []string{"axonaut"}
|
||||
}
|
||||
|
||||
// FromData will find and optionally verify Axonaut secrets in a given set of bytes.
|
||||
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
|
||||
matches := keyPat.FindAllSubmatch(data, -1)
|
||||
dataStr := string(data)
|
||||
|
||||
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
|
||||
|
||||
for _, match := range matches {
|
||||
if len(match) != 2 {
|
||||
continue
|
||||
}
|
||||
resMatch := bytes.TrimSpace(match[1])
|
||||
resMatch := strings.TrimSpace(match[1])
|
||||
|
||||
s1 := detectors.Result{
|
||||
DetectorType: detectorspb.DetectorType_Axonaut,
|
||||
Raw: resMatch,
|
||||
Raw: []byte(resMatch),
|
||||
}
|
||||
|
||||
if verify {
|
||||
@@ -43,13 +51,14 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
req.Header.Add("userApiKey", string(resMatch))
|
||||
req.Header.Add("userApiKey", resMatch)
|
||||
res, err := client.Do(req)
|
||||
if err == nil {
|
||||
defer res.Body.Close()
|
||||
if res.StatusCode >= 200 && res.StatusCode < 300 {
|
||||
s1.Verified = true
|
||||
} else {
|
||||
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
|
||||
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
|
||||
continue
|
||||
}
|
||||
|
||||
@@ -1,10 +1,10 @@
|
||||
package aylien
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
@@ -26,48 +26,49 @@ var (
|
||||
|
||||
// Keywords are used for efficiently pre-filtering chunks.
|
||||
// Use identifiers in the secret preferably, or the provider name.
|
||||
func (s Scanner) Keywords() [][]byte {
|
||||
return [][]byte{[]byte("aylien")}
|
||||
func (s Scanner) Keywords() []string {
|
||||
return []string{"aylien"}
|
||||
}
|
||||
|
||||
// FromData will find and optionally verify Aylien secrets in a given set of bytes.
|
||||
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
|
||||
matches := keyPat.FindAllSubmatch(data, -1)
|
||||
idMatches := idPat.FindAllSubmatch(data, -1)
|
||||
dataStr := string(data)
|
||||
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
|
||||
idMatches := idPat.FindAllStringSubmatch(dataStr, -1)
|
||||
|
||||
for _, match := range matches {
|
||||
if len(match) != 2 {
|
||||
continue
|
||||
}
|
||||
resMatch := bytes.TrimSpace(match[1])
|
||||
resMatch := strings.TrimSpace(match[1])
|
||||
|
||||
for _, idMatch := range idMatches {
|
||||
if len(idMatch) != 2 {
|
||||
continue
|
||||
}
|
||||
resIdMatch := bytes.TrimSpace(idMatch[1])
|
||||
newSecret := append(resMatch, resIdMatch...)
|
||||
|
||||
resIdMatch := strings.TrimSpace(idMatch[1])
|
||||
|
||||
s1 := detectors.Result{
|
||||
DetectorType: detectorspb.DetectorType_Aylien,
|
||||
Raw: resMatch,
|
||||
RawV2: newSecret,
|
||||
Raw: []byte(resMatch),
|
||||
RawV2: []byte(resMatch + resIdMatch),
|
||||
}
|
||||
|
||||
if verify {
|
||||
req, err := http.NewRequestWithContext(ctx, "GET", "https://api.aylien.com/news/stories", nil)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
req.Header.Add("X-AYLIEN-NewsAPI-Application-ID", string(resMatch))
|
||||
req.Header.Add("X-AYLIEN-NewsAPI-Application-Key", string(resIdMatch))
|
||||
req.Header.Add("X-AYLIEN-NewsAPI-Application-ID", resIdMatch)
|
||||
req.Header.Add("X-AYLIEN-NewsAPI-Application-Key", resMatch)
|
||||
res, err := client.Do(req)
|
||||
if err == nil {
|
||||
defer res.Body.Close()
|
||||
if res.StatusCode >= 200 && res.StatusCode < 300 {
|
||||
s1.Verified = true
|
||||
} else {
|
||||
if detectors.IsKnownFalsePositive(newSecret, detectors.DefaultFalsePositives, true) {
|
||||
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
|
||||
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,11 +1,11 @@
|
||||
package ayrshare
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
@@ -14,28 +14,37 @@ import (
|
||||
|
||||
type Scanner struct{}
|
||||
|
||||
// Ensure the Scanner satisfies the interface at compile time.
|
||||
var _ detectors.Detector = (*Scanner)(nil)
|
||||
|
||||
var (
|
||||
client = common.SaneHttpClient()
|
||||
|
||||
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
|
||||
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"ayrshare"}) + `\b([A-Z]{7}-[A-Z0-9]{7}-[A-Z0-9]{7}-[A-Z0-9]{7})\b`)
|
||||
)
|
||||
|
||||
func (s Scanner) Keywords() [][]byte {
|
||||
return [][]byte{[]byte("ayrshare")}
|
||||
// Keywords are used for efficiently pre-filtering chunks.
|
||||
// Use identifiers in the secret preferably, or the provider name.
|
||||
func (s Scanner) Keywords() []string {
|
||||
return []string{"ayrshare"}
|
||||
}
|
||||
|
||||
// FromData will find and optionally verify Ayrshare secrets in a given set of bytes.
|
||||
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
|
||||
matches := keyPat.FindAllSubmatch(data, -1)
|
||||
dataStr := string(data)
|
||||
|
||||
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
|
||||
|
||||
for _, match := range matches {
|
||||
if len(match) != 2 {
|
||||
continue
|
||||
}
|
||||
resMatch := bytes.TrimSpace(match[1])
|
||||
resMatch := strings.TrimSpace(match[1])
|
||||
|
||||
s1 := detectors.Result{
|
||||
DetectorType: detectorspb.DetectorType_Ayrshare,
|
||||
Raw: resMatch,
|
||||
Raw: []byte(resMatch),
|
||||
}
|
||||
|
||||
if verify {
|
||||
@@ -43,21 +52,24 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", string(resMatch)))
|
||||
req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", resMatch))
|
||||
res, err := client.Do(req)
|
||||
if err == nil {
|
||||
defer res.Body.Close()
|
||||
if res.StatusCode >= 200 && res.StatusCode < 300 {
|
||||
s1.Verified = true
|
||||
} else {
|
||||
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
|
||||
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
|
||||
continue
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
results = append(results, s1)
|
||||
}
|
||||
|
||||
return results, nil
|
||||
}
|
||||
|
||||
|
||||
@@ -7,6 +7,7 @@ import (
|
||||
"strings"
|
||||
|
||||
"github.com/Azure/go-autorest/autorest/azure/auth"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
|
||||
)
|
||||
@@ -30,33 +31,35 @@ var (
|
||||
tenantIDPat = mustFmtPat("tenant_id", idPatFmt)
|
||||
|
||||
// TODO: support old patterns
|
||||
secretPatFmt = `(?i)(%s).{0,20}([a-z0-9_.\-~]{34})`
|
||||
secretPatFmt = `(?i)(%s).{0,20}([a-z0-9_\.\-~]{34})`
|
||||
clientSecretPat = mustFmtPat("client_secret", secretPatFmt)
|
||||
)
|
||||
|
||||
// Keywords are used for efficiently pre-filtering chunks.
|
||||
func (s Scanner) Keywords() [][]byte {
|
||||
return [][]byte{[]byte("azure")}
|
||||
// Use identifiers in the secret preferably, or the provider name.
|
||||
func (s Scanner) Keywords() []string {
|
||||
return []string{"azure"}
|
||||
}
|
||||
|
||||
// FromData will find and optionally verify Azure secrets in a given set of bytes.
|
||||
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
|
||||
clientSecretMatches := clientSecretPat.FindAllSubmatch(data, -1)
|
||||
dataStr := string(data)
|
||||
|
||||
clientSecretMatches := clientSecretPat.FindAllStringSubmatch(dataStr, -1)
|
||||
for _, clientSecret := range clientSecretMatches {
|
||||
tenantIDMatches := tenantIDPat.FindAllSubmatch(data, -1)
|
||||
tenantIDMatches := tenantIDPat.FindAllStringSubmatch(dataStr, -1)
|
||||
for _, tenantID := range tenantIDMatches {
|
||||
clientIDMatches := clientIDPat.FindAllSubmatch(data, -1)
|
||||
clientIDMatches := clientIDPat.FindAllStringSubmatch(dataStr, -1)
|
||||
for _, clientID := range clientIDMatches {
|
||||
s := detectors.Result{
|
||||
DetectorType: detectorspb.DetectorType_Azure,
|
||||
Raw: clientSecret[2],
|
||||
RawV2: append(clientID[2], append(clientSecret[2], tenantID[2]...)...),
|
||||
Redacted: string(clientID[2]),
|
||||
Raw: []byte(clientSecret[2]),
|
||||
RawV2: []byte(clientID[2] + clientSecret[2] + tenantID[2]),
|
||||
Redacted: clientID[2],
|
||||
}
|
||||
|
||||
if verify {
|
||||
cred := auth.NewClientCredentialsConfig(string(clientID[2]), string(clientSecret[2]), string(tenantID[2]))
|
||||
cred := auth.NewClientCredentialsConfig(clientID[2], clientSecret[2], tenantID[2])
|
||||
token, err := cred.ServicePrincipalToken()
|
||||
if err != nil {
|
||||
continue
|
||||
@@ -68,10 +71,10 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
}
|
||||
|
||||
if !s.Verified {
|
||||
if detectors.IsKnownFalsePositive([]byte(s.Redacted), detectors.DefaultFalsePositives, true) {
|
||||
if detectors.IsKnownFalsePositive(s.Redacted, detectors.DefaultFalsePositives, true) {
|
||||
continue
|
||||
}
|
||||
if detectors.IsKnownFalsePositive(s.Raw, detectors.DefaultFalsePositives, true) {
|
||||
if detectors.IsKnownFalsePositive(string(s.Raw), detectors.DefaultFalsePositives, true) {
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,11 +1,11 @@
|
||||
package bannerbear
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
@@ -20,29 +20,31 @@ var _ detectors.Detector = (*Scanner)(nil)
|
||||
var (
|
||||
client = common.SaneHttpClient()
|
||||
|
||||
// Keep the regexp patterns unchanged
|
||||
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
|
||||
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"bannerbear"}) + `\b([0-9a-zA-Z]{22}tt)\b`)
|
||||
)
|
||||
|
||||
// Keywords are used for efficiently pre-filtering chunks.
|
||||
func (s Scanner) Keywords() [][]byte {
|
||||
return [][]byte{[]byte("bannerbear")}
|
||||
// Use identifiers in the secret preferably, or the provider name.
|
||||
func (s Scanner) Keywords() []string {
|
||||
return []string{"bannerbear"}
|
||||
}
|
||||
|
||||
// FromData will find and optionally verify Bannerbear secrets in a given set of bytes.
|
||||
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
|
||||
dataStr := string(data)
|
||||
|
||||
matches := keyPat.FindAllSubmatch(data, -1)
|
||||
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
|
||||
|
||||
for _, match := range matches {
|
||||
if len(match) != 2 {
|
||||
continue
|
||||
}
|
||||
resMatch := bytes.TrimSpace(match[1])
|
||||
resMatch := strings.TrimSpace(match[1])
|
||||
|
||||
s1 := detectors.Result{
|
||||
DetectorType: detectorspb.DetectorType_Bannerbear,
|
||||
Raw: resMatch,
|
||||
Raw: []byte(resMatch),
|
||||
}
|
||||
|
||||
if verify {
|
||||
@@ -57,6 +59,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
if res.StatusCode >= 200 && res.StatusCode < 300 {
|
||||
s1.Verified = true
|
||||
} else {
|
||||
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
|
||||
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
|
||||
continue
|
||||
}
|
||||
|
||||
@@ -1,11 +1,11 @@
|
||||
package baremetrics
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
@@ -14,30 +14,37 @@ import (
|
||||
|
||||
type Scanner struct{}
|
||||
|
||||
// Ensure the Scanner satisfies the interface at compile time.
|
||||
var _ detectors.Detector = (*Scanner)(nil)
|
||||
|
||||
var (
|
||||
client = common.SaneHttpClient()
|
||||
|
||||
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
|
||||
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"baremetrics"}) + `\b([a-zA-Z0-9_]{25})\b`)
|
||||
)
|
||||
|
||||
func (s Scanner) Keywords() [][]byte {
|
||||
return [][]byte{[]byte("baremetrics")}
|
||||
// Keywords are used for efficiently pre-filtering chunks.
|
||||
// Use identifiers in the secret preferably, or the provider name.
|
||||
func (s Scanner) Keywords() []string {
|
||||
return []string{"baremetrics"}
|
||||
}
|
||||
|
||||
// FromData will find and optionally verify Baremetrics secrets in a given set of bytes.
|
||||
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
|
||||
matches := keyPat.FindAllSubmatch(data, -1)
|
||||
dataStr := string(data)
|
||||
|
||||
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
|
||||
|
||||
for _, match := range matches {
|
||||
if len(match) != 2 {
|
||||
continue
|
||||
}
|
||||
resMatch := bytes.TrimSpace(match[1])
|
||||
resMatch := strings.TrimSpace(match[1])
|
||||
|
||||
s1 := detectors.Result{
|
||||
DetectorType: detectorspb.DetectorType_Baremetrics,
|
||||
Raw: resMatch,
|
||||
Raw: []byte(resMatch),
|
||||
}
|
||||
|
||||
if verify {
|
||||
@@ -45,13 +52,14 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", string(resMatch)))
|
||||
req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", resMatch))
|
||||
res, err := client.Do(req)
|
||||
if err == nil {
|
||||
defer res.Body.Close()
|
||||
if res.StatusCode >= 200 && res.StatusCode < 300 {
|
||||
s1.Verified = true
|
||||
} else {
|
||||
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
|
||||
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
|
||||
continue
|
||||
}
|
||||
|
||||
@@ -1,11 +1,12 @@
|
||||
package baseapiio
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
@@ -14,30 +15,37 @@ import (
|
||||
|
||||
type Scanner struct{}
|
||||
|
||||
// Ensure the Scanner satisfies the interface at compile time.
|
||||
var _ detectors.Detector = (*Scanner)(nil)
|
||||
|
||||
var (
|
||||
client = common.SaneHttpClient()
|
||||
|
||||
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
|
||||
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"baseapi", "base-api"}) + `\b([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})\b`)
|
||||
)
|
||||
|
||||
func (s Scanner) Keywords() [][]byte {
|
||||
return [][]byte{[]byte("baseapi"), []byte("base-api")}
|
||||
// Keywords are used for efficiently pre-filtering chunks.
|
||||
// Use identifiers in the secret preferably, or the provider name.
|
||||
func (s Scanner) Keywords() []string {
|
||||
return []string{"baseapi", "base-api"}
|
||||
}
|
||||
|
||||
// FromData will find and optionally verify BaseApiIO secrets in a given set of bytes.
|
||||
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
|
||||
matches := keyPat.FindAllSubmatch(data, -1)
|
||||
dataStr := string(data)
|
||||
|
||||
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
|
||||
|
||||
for _, match := range matches {
|
||||
if len(match) != 2 {
|
||||
continue
|
||||
}
|
||||
resMatch := bytes.TrimSpace(match[1])
|
||||
resMatch := strings.TrimSpace(match[1])
|
||||
|
||||
s1 := detectors.Result{
|
||||
DetectorType: detectorspb.DetectorType_BaseApiIO,
|
||||
Raw: resMatch,
|
||||
Raw: []byte(resMatch),
|
||||
}
|
||||
|
||||
if verify {
|
||||
@@ -45,17 +53,24 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", string(resMatch)))
|
||||
req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", resMatch))
|
||||
res, err := client.Do(req)
|
||||
if err == nil {
|
||||
defer res.Body.Close()
|
||||
if bytes.Contains(resMatch, []byte("items")) {
|
||||
bodyBytes, err := io.ReadAll(res.Body)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
body := string(bodyBytes)
|
||||
|
||||
if strings.Contains(body, "items") {
|
||||
s1.Verified = true
|
||||
} else {
|
||||
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -1,10 +1,10 @@
|
||||
package beamer
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
@@ -19,27 +19,31 @@ var _ detectors.Detector = (*Scanner)(nil)
|
||||
var (
|
||||
client = common.SaneHttpClient()
|
||||
|
||||
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
|
||||
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"beamer"}) + `\b([a-zA-Z0-9_+/]{45}=)`)
|
||||
)
|
||||
|
||||
// Keywords are used for efficiently pre-filtering chunks.
|
||||
func (s Scanner) Keywords() [][]byte {
|
||||
return [][]byte{[]byte("beamer")}
|
||||
// Use identifiers in the secret preferably, or the provider name.
|
||||
func (s Scanner) Keywords() []string {
|
||||
return []string{"beamer"}
|
||||
}
|
||||
|
||||
// FromData will find and optionally verify Beamer secrets in a given set of bytes.
|
||||
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
|
||||
matches := keyPat.FindAllSubmatch(data, -1)
|
||||
dataStr := string(data)
|
||||
|
||||
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
|
||||
|
||||
for _, match := range matches {
|
||||
if len(match) != 2 {
|
||||
continue
|
||||
}
|
||||
resMatch := bytes.TrimSpace(match[1])
|
||||
resMatch := strings.TrimSpace(match[1])
|
||||
|
||||
s1 := detectors.Result{
|
||||
DetectorType: detectorspb.DetectorType_Beamer,
|
||||
Raw: resMatch,
|
||||
Raw: []byte(resMatch),
|
||||
}
|
||||
|
||||
if verify {
|
||||
@@ -47,14 +51,14 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
req.Header.Add("Beamer-Api-Key", string(resMatch))
|
||||
req.Header.Add("Beamer-Api-Key", resMatch)
|
||||
res, err := client.Do(req)
|
||||
if err == nil {
|
||||
defer res.Body.Close()
|
||||
if res.StatusCode >= 200 && res.StatusCode < 300 {
|
||||
s1.Verified = true
|
||||
} else {
|
||||
|
||||
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
|
||||
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
|
||||
continue
|
||||
}
|
||||
|
||||
@@ -1,12 +1,12 @@
|
||||
package beebole
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
b64 "encoding/base64"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
@@ -15,50 +15,56 @@ import (
|
||||
|
||||
type Scanner struct{}
|
||||
|
||||
// Ensure the Scanner satisfies the interface at compile time.
|
||||
var _ detectors.Detector = (*Scanner)(nil)
|
||||
|
||||
var (
|
||||
client = common.SaneHttpClient()
|
||||
|
||||
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
|
||||
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"beebole"}) + `\b([0-9a-z]{40})\b`)
|
||||
)
|
||||
|
||||
func (s Scanner) Keywords() [][]byte {
|
||||
return [][]byte{[]byte("beebole")}
|
||||
// Keywords are used for efficiently pre-filtering chunks.
|
||||
// Use identifiers in the secret preferably, or the provider name.
|
||||
func (s Scanner) Keywords() []string {
|
||||
return []string{"beebole"}
|
||||
}
|
||||
|
||||
// FromData will find and optionally verify Beebole secrets in a given set of bytes.
|
||||
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
|
||||
matches := keyPat.FindAllSubmatch(data, -1)
|
||||
dataStr := string(data)
|
||||
|
||||
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
|
||||
|
||||
for _, match := range matches {
|
||||
if len(match) != 2 {
|
||||
continue
|
||||
}
|
||||
resMatch := bytes.TrimSpace(match[1])
|
||||
resMatch := strings.TrimSpace(match[1])
|
||||
|
||||
s1 := detectors.Result{
|
||||
DetectorType: detectorspb.DetectorType_Beebole,
|
||||
Raw: resMatch,
|
||||
Raw: []byte(resMatch),
|
||||
}
|
||||
|
||||
if verify {
|
||||
data := fmt.Sprintf("%s:X", string(resMatch))
|
||||
data := fmt.Sprintf("%s:X", resMatch)
|
||||
sEnc := b64.StdEncoding.EncodeToString([]byte(data))
|
||||
payload := bytes.NewReader([]byte(`{"service": "custom_field.list"}`))
|
||||
payload := strings.NewReader(`{"service": "custom_field.list"}`)
|
||||
req, err := http.NewRequestWithContext(ctx, "POST", "https://beebole-apps.com/api/v2", payload)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
|
||||
req.Header.Add("Content-Type", "application/json")
|
||||
req.Header.Add("Authorization", fmt.Sprintf("Basic %s", sEnc))
|
||||
res, err := client.Do(req)
|
||||
|
||||
if err == nil {
|
||||
defer res.Body.Close()
|
||||
if res.StatusCode >= 200 && res.StatusCode < 300 {
|
||||
s1.Verified = true
|
||||
} else {
|
||||
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
|
||||
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
|
||||
continue
|
||||
}
|
||||
|
||||
@@ -1,12 +1,12 @@
|
||||
package besnappy
|
||||
|
||||
import (
|
||||
b "bytes"
|
||||
"context"
|
||||
b64 "encoding/base64"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
@@ -15,30 +15,37 @@ import (
|
||||
|
||||
type Scanner struct{}
|
||||
|
||||
// Ensure the Scanner satisfies the interface at compile time
|
||||
var _ detectors.Detector = (*Scanner)(nil)
|
||||
|
||||
var (
|
||||
client = common.SaneHttpClient()
|
||||
|
||||
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives
|
||||
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"besnappy"}) + `\b([a-f0-9]{64})\b`)
|
||||
)
|
||||
|
||||
func (s Scanner) Keywords() [][]byte {
|
||||
return [][]byte{[]byte("besnappy")}
|
||||
// Keywords are used for efficiently pre-filtering chunks.
|
||||
// Use identifiers in the secret preferably, or the provider name.
|
||||
func (s Scanner) Keywords() []string {
|
||||
return []string{"besnappy"}
|
||||
}
|
||||
|
||||
// FromData will find and optionally verify Besnappy secrets in a given set of bytes.
|
||||
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
|
||||
dataStr := string(data)
|
||||
|
||||
matches := keyPat.FindAllSubmatch(data, -1)
|
||||
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
|
||||
|
||||
for _, match := range matches {
|
||||
if len(match) != 2 {
|
||||
continue
|
||||
}
|
||||
resMatch := b.TrimSpace(match[1])
|
||||
resMatch := strings.TrimSpace(match[1])
|
||||
|
||||
s1 := detectors.Result{
|
||||
DetectorType: detectorspb.DetectorType_Besnappy,
|
||||
Raw: resMatch,
|
||||
Raw: []byte(resMatch),
|
||||
}
|
||||
if verify {
|
||||
data := fmt.Sprintf("%s:x", resMatch)
|
||||
@@ -54,6 +61,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
if res.StatusCode >= 200 && res.StatusCode < 300 {
|
||||
s1.Verified = true
|
||||
} else {
|
||||
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key
|
||||
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
|
||||
continue
|
||||
}
|
||||
|
||||
@@ -1,11 +1,11 @@
|
||||
package besttime
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"io"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
@@ -14,54 +14,61 @@ import (
|
||||
|
||||
type Scanner struct{}
|
||||
|
||||
// Ensure the Scanner satisfies the interface at compile time.
|
||||
var _ detectors.Detector = (*Scanner)(nil)
|
||||
|
||||
var (
|
||||
client = common.SaneHttpClient()
|
||||
|
||||
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
|
||||
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"besttime"}) + `\b([0-9A-Za-z_]{36})\b`)
|
||||
)
|
||||
|
||||
func (s Scanner) Keywords() [][]byte {
|
||||
return [][]byte{[]byte("besttime")}
|
||||
// Keywords are used for efficiently pre-filtering chunks.
|
||||
// Use identifiers in the secret preferably, or the provider name.
|
||||
func (s Scanner) Keywords() []string {
|
||||
return []string{"besttime"}
|
||||
}
|
||||
|
||||
// FromData will find and optionally verify Besttime secrets in a given set of bytes.
|
||||
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
|
||||
matches := keyPat.FindAllSubmatch(data, -1)
|
||||
dataStr := string(data)
|
||||
|
||||
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
|
||||
|
||||
for _, match := range matches {
|
||||
if len(match) != 2 {
|
||||
continue
|
||||
}
|
||||
|
||||
resMatch := bytes.TrimSpace(match[1])
|
||||
resMatch := strings.TrimSpace(match[1])
|
||||
|
||||
s1 := detectors.Result{
|
||||
DetectorType: detectorspb.DetectorType_Besttime,
|
||||
Raw: resMatch,
|
||||
Raw: []byte(resMatch),
|
||||
}
|
||||
|
||||
if verify {
|
||||
req, err := http.NewRequestWithContext(ctx, "GET", "https://besttime.app/api/v1/keys/"+string(resMatch), nil)
|
||||
req, err := http.NewRequestWithContext(ctx, "GET", "https://besttime.app/api/v1/keys/"+resMatch, nil)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
|
||||
res, err := client.Do(req)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
defer res.Body.Close()
|
||||
bodyBytes, err := io.ReadAll(res.Body)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
|
||||
if bytes.Contains(bodyBytes, []byte(`"status": "OK"`)) {
|
||||
s1.Verified = true
|
||||
} else {
|
||||
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
|
||||
if err == nil {
|
||||
defer res.Body.Close()
|
||||
bodyBytes, err := io.ReadAll(res.Body)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
body := string(bodyBytes)
|
||||
|
||||
if strings.Contains(body, `"status": "OK"`) {
|
||||
s1.Verified = true
|
||||
} else {
|
||||
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -1,11 +1,11 @@
|
||||
package billomat
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
@@ -20,49 +20,55 @@ var _ detectors.Detector = (*Scanner)(nil)
|
||||
var (
|
||||
client = common.SaneHttpClient()
|
||||
|
||||
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
|
||||
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"billomat"}) + `\b([0-9a-z]{32})\b`)
|
||||
idPat = regexp.MustCompile(detectors.PrefixRegex([]string{"billomat"}) + `\b([0-9a-z]{1,})\b`)
|
||||
)
|
||||
|
||||
func (s Scanner) Keywords() [][]byte {
|
||||
return [][]byte{[]byte("billomat")}
|
||||
// Keywords are used for efficiently pre-filtering chunks.
|
||||
// Use identifiers in the secret preferably, or the provider name.
|
||||
func (s Scanner) Keywords() []string {
|
||||
return []string{"billomat"}
|
||||
}
|
||||
|
||||
// FromData will find and optionally verify Billomat secrets in a given set of bytes.
|
||||
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
|
||||
matches := keyPat.FindAllSubmatch(data, -1)
|
||||
idMatches := idPat.FindAllSubmatch(data, -1)
|
||||
dataStr := string(data)
|
||||
|
||||
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
|
||||
idMatches := idPat.FindAllStringSubmatch(dataStr, -1)
|
||||
|
||||
for _, match := range matches {
|
||||
if len(match) != 2 {
|
||||
continue
|
||||
}
|
||||
resMatch := bytes.TrimSpace(match[1])
|
||||
|
||||
resMatch := strings.TrimSpace(match[1])
|
||||
for _, idMatch := range idMatches {
|
||||
if len(idMatch) != 2 {
|
||||
continue
|
||||
}
|
||||
resId := bytes.TrimSpace(idMatch[1])
|
||||
resId := strings.TrimSpace(idMatch[1])
|
||||
|
||||
s1 := detectors.Result{
|
||||
DetectorType: detectorspb.DetectorType_Billomat,
|
||||
Raw: resMatch,
|
||||
RawV2: append(resMatch, resId...),
|
||||
Raw: []byte(resMatch),
|
||||
RawV2: []byte(resMatch + resId),
|
||||
}
|
||||
|
||||
if verify {
|
||||
req, err := http.NewRequestWithContext(ctx, "GET", fmt.Sprintf("https://%s.billomat.net/api/v2/clients/myself", string(resId)), nil)
|
||||
req, err := http.NewRequestWithContext(ctx, "GET", fmt.Sprintf("https://%s.billomat.net/api/v2/clients/myself", resId), nil)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
req.Header.Add("Content-Type", "application/json")
|
||||
req.Header.Add("X-BillomatApiKey", string(resMatch))
|
||||
req.Header.Add("X-BillomatApiKey", resMatch)
|
||||
res, err := client.Do(req)
|
||||
if err == nil {
|
||||
defer res.Body.Close()
|
||||
if res.StatusCode >= 200 && res.StatusCode < 300 {
|
||||
s1.Verified = true
|
||||
} else {
|
||||
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
|
||||
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
|
||||
continue
|
||||
}
|
||||
|
||||
@@ -6,6 +6,7 @@ import (
|
||||
"fmt"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
@@ -14,34 +15,42 @@ import (
|
||||
|
||||
type Scanner struct{}
|
||||
|
||||
// Ensure the Scanner satisfies the interface at compile time.
|
||||
var _ detectors.Detector = (*Scanner)(nil)
|
||||
|
||||
var (
|
||||
client = common.SaneHttpClient()
|
||||
|
||||
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
|
||||
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"bitbar"}) + `\b([0-9a-z]{32})\b`)
|
||||
)
|
||||
|
||||
func (s Scanner) Keywords() [][]byte {
|
||||
return [][]byte{[]byte("bitbar")}
|
||||
// Keywords are used for efficiently pre-filtering chunks.
|
||||
// Use identifiers in the secret preferably, or the provider name.
|
||||
func (s Scanner) Keywords() []string {
|
||||
return []string{"bitbar"}
|
||||
}
|
||||
|
||||
// FromData will find and optionally verify Bitbar secrets in a given set of bytes.
|
||||
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
|
||||
matches := keyPat.FindAllSubmatch(data, -1)
|
||||
dataStr := string(data)
|
||||
|
||||
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
|
||||
|
||||
for _, match := range matches {
|
||||
if len(match) != 2 {
|
||||
continue
|
||||
}
|
||||
resMatch := match[1]
|
||||
resMatch := strings.TrimSpace(match[1])
|
||||
|
||||
s1 := detectors.Result{
|
||||
DetectorType: detectorspb.DetectorType_Bitbar,
|
||||
Raw: resMatch,
|
||||
Raw: []byte(resMatch),
|
||||
}
|
||||
|
||||
if verify {
|
||||
data := []byte(fmt.Sprintf("%s:", resMatch))
|
||||
baseToken := b64.StdEncoding.EncodeToString(data)
|
||||
data := fmt.Sprintf("%s:", resMatch)
|
||||
baseToken := b64.StdEncoding.EncodeToString([]byte(data))
|
||||
req, err := http.NewRequestWithContext(ctx, "GET", "https://cloud.bitbar.com/api/me", nil)
|
||||
if err != nil {
|
||||
continue
|
||||
@@ -54,6 +63,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
if res.StatusCode >= 200 && res.StatusCode < 300 {
|
||||
s1.Verified = true
|
||||
} else {
|
||||
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
|
||||
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
|
||||
continue
|
||||
}
|
||||
|
||||
@@ -1,10 +1,10 @@
|
||||
package bitcoinaverage
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
@@ -19,38 +19,45 @@ var _ detectors.Detector = (*Scanner)(nil)
|
||||
var (
|
||||
client = common.SaneHttpClient()
|
||||
|
||||
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
|
||||
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"bitcoinaverage"}) + `\b([a-zA-Z0-9]{43})\b`)
|
||||
)
|
||||
|
||||
func (s Scanner) Keywords() [][]byte {
|
||||
return [][]byte{[]byte("bitcoinaverage")}
|
||||
// Keywords are used for efficiently pre-filtering chunks.
|
||||
// Use identifiers in the secret preferably, or the provider name.
|
||||
func (s Scanner) Keywords() []string {
|
||||
return []string{"bitcoinaverage"}
|
||||
}
|
||||
|
||||
// FromData will find and optionally verify BitcoinAverage secrets in a given set of bytes.
|
||||
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
|
||||
matches := keyPat.FindAllSubmatch(data, -1)
|
||||
dataStr := string(data)
|
||||
|
||||
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
|
||||
|
||||
for _, match := range matches {
|
||||
if len(match) != 2 {
|
||||
continue
|
||||
}
|
||||
resMatch := bytes.TrimSpace(match[1])
|
||||
resMatch := strings.TrimSpace(match[1])
|
||||
|
||||
s1 := detectors.Result{
|
||||
DetectorType: detectorspb.DetectorType_BitcoinAverage,
|
||||
Raw: resMatch,
|
||||
Raw: []byte(resMatch),
|
||||
}
|
||||
if verify {
|
||||
req, err := http.NewRequestWithContext(ctx, "GET", "https://apiv2.bitcoinaverage.com/websocket/get_ticket", nil)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
req.Header.Add("x-ba-key", string(resMatch))
|
||||
req.Header.Add("x-ba-key", resMatch)
|
||||
res, err := client.Do(req)
|
||||
if err == nil {
|
||||
defer res.Body.Close()
|
||||
if res.StatusCode >= 200 && res.StatusCode < 300 {
|
||||
s1.Verified = true
|
||||
} else {
|
||||
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
|
||||
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
|
||||
continue
|
||||
}
|
||||
|
||||
@@ -1,80 +1,99 @@
|
||||
package bitfinex
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"flag"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"github.com/bitfinexcom/bitfinex-api-go/v2/rest"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
|
||||
)
|
||||
|
||||
type Scanner struct{}
|
||||
|
||||
// Ensure the Scanner satisfies the interface at compile time.
|
||||
var _ detectors.Detector = (*Scanner)(nil)
|
||||
|
||||
var (
|
||||
client = common.SaneHttpClient()
|
||||
client = common.SaneHttpClient()
|
||||
|
||||
// related resource https://medium.com/@Bitfinex/api-development-update-april-65fe52f84124
|
||||
apiKeyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"bitfinex"}) + `\b([A-Za-z0-9_-]{43})\b`)
|
||||
apiSecretPat = regexp.MustCompile(detectors.PrefixRegex([]string{"bitfinex"}) + `\b([A-Za-z0-9_-]{43})\b`)
|
||||
)
|
||||
|
||||
func (s Scanner) Keywords() [][]byte {
|
||||
return [][]byte{[]byte("bitfinex")}
|
||||
var (
|
||||
api = flag.String("api", "https://api-pub.bitfinex.com/v2/", "v2 REST API URL")
|
||||
)
|
||||
|
||||
// Keywords are used for efficiently pre-filtering chunks.
|
||||
// Use identifiers in the secret preferably, or the provider name.
|
||||
func (s Scanner) Keywords() []string {
|
||||
return []string{"bitfinex"}
|
||||
}
|
||||
|
||||
// FromData will find and optionally verify Bitfinex secrets in a given set of bytes.
|
||||
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
|
||||
apiKeyMatches := apiKeyPat.FindAllSubmatch(data, -1)
|
||||
apiSecretMatches := apiSecretPat.FindAllSubmatch(data, -1)
|
||||
dataStr := string(data)
|
||||
|
||||
apiKeyMatches := apiKeyPat.FindAllStringSubmatch(dataStr, -1)
|
||||
apiSecretMatches := apiSecretPat.FindAllStringSubmatch(dataStr, -1)
|
||||
|
||||
for _, apiKeyMatch := range apiKeyMatches {
|
||||
if len(apiKeyMatch) != 2 {
|
||||
continue
|
||||
}
|
||||
apiKeyRes := bytes.TrimSpace(apiKeyMatch[1])
|
||||
apiKeyRes := strings.TrimSpace(apiKeyMatch[1])
|
||||
|
||||
s1 := detectors.Result{
|
||||
DetectorType: detectorspb.DetectorType_Bitfinex,
|
||||
Raw: apiKeyRes,
|
||||
Raw: []byte(apiKeyRes),
|
||||
}
|
||||
|
||||
for _, apiSecretMatch := range apiSecretMatches {
|
||||
if len(apiSecretMatch) != 2 {
|
||||
continue
|
||||
}
|
||||
apiSecretRes := bytes.TrimSpace(apiSecretMatch[1])
|
||||
apiSecretRes := strings.TrimSpace(apiSecretMatch[1])
|
||||
|
||||
if bytes.Equal(apiKeyRes, apiSecretRes) {
|
||||
if apiKeyRes == apiSecretRes {
|
||||
continue
|
||||
}
|
||||
|
||||
if verify {
|
||||
key := string(apiKeyRes)
|
||||
secret := string(apiSecretRes)
|
||||
http.DefaultClient = client
|
||||
c := rest.NewClient().Credentials(key, secret)
|
||||
// thankfully official golang examples exist but you just need to dig their many repos https://github.com/bitfinexcom/bitfinex-api-go/blob/master/examples/v2/rest-orders/main.go
|
||||
key := apiKeyRes
|
||||
secret := apiSecretRes
|
||||
http.DefaultClient = client // filed https://github.com/bitfinexcom/bitfinex-api-go/issues/238 to improve this
|
||||
c := rest.NewClientWithURL(*api).Credentials(key, secret)
|
||||
|
||||
isValid := true
|
||||
isValid := true // assume valid
|
||||
_, err = c.Orders.AllHistory()
|
||||
if err != nil {
|
||||
if bytes.HasPrefix([]byte(err.Error()), []byte("POST https://")) {
|
||||
if strings.HasPrefix(err.Error(), "POST https://") { // eg POST https://api-pub.bitfinex.com/v2/auth/r/orders/hist: 500 apikey: digest invalid (10100)
|
||||
isValid = false
|
||||
} else if detectors.IsKnownFalsePositive(apiKeyRes, detectors.DefaultFalsePositives, true) {
|
||||
continue
|
||||
} else {
|
||||
if detectors.IsKnownFalsePositive(apiKeyRes, detectors.DefaultFalsePositives, true) {
|
||||
continue
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
s1.Verified = isValid
|
||||
// If there is a valid one, we need to stop iterating now and return the valid result
|
||||
if isValid {
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// By appending resutls in the outer loop we can reduce false positives if there are multiple
|
||||
// combinations of secrets and IDs found.
|
||||
if len(apiSecretMatches) > 0 {
|
||||
results = append(results, s1)
|
||||
}
|
||||
|
||||
@@ -1,11 +1,11 @@
|
||||
package bitlyaccesstoken
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
@@ -14,30 +14,37 @@ import (
|
||||
|
||||
type Scanner struct{}
|
||||
|
||||
// Ensure the Scanner satisfies the interface at compile time.
|
||||
var _ detectors.Detector = (*Scanner)(nil)
|
||||
|
||||
var (
|
||||
client = common.SaneHttpClient()
|
||||
|
||||
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
|
||||
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"bitly"}) + `\b([a-zA-Z-0-9]{40})\b`)
|
||||
)
|
||||
|
||||
func (s Scanner) Keywords() [][]byte {
|
||||
return [][]byte{[]byte("bitly")}
|
||||
// Keywords are used for efficiently pre-filtering chunks.
|
||||
// Use identifiers in the secret preferably, or the provider name.
|
||||
func (s Scanner) Keywords() []string {
|
||||
return []string{"bitly"}
|
||||
}
|
||||
|
||||
// FromData will find and optionally verify BitLyAccessToken secrets in a given set of bytes.
|
||||
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
|
||||
matches := keyPat.FindAllSubmatch(data, -1)
|
||||
dataStr := string(data)
|
||||
|
||||
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
|
||||
|
||||
for _, match := range matches {
|
||||
if len(match) != 2 {
|
||||
continue
|
||||
}
|
||||
resMatch := bytes.TrimSpace(match[1])
|
||||
resMatch := strings.TrimSpace(match[1])
|
||||
|
||||
s1 := detectors.Result{
|
||||
DetectorType: detectorspb.DetectorType_BitLyAccessToken,
|
||||
Raw: resMatch,
|
||||
Raw: []byte(resMatch),
|
||||
}
|
||||
|
||||
if verify {
|
||||
@@ -45,13 +52,14 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", string(resMatch)))
|
||||
req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", resMatch))
|
||||
res, err := client.Do(req)
|
||||
if err == nil {
|
||||
defer res.Body.Close()
|
||||
if res.StatusCode >= 200 && res.StatusCode < 300 {
|
||||
s1.Verified = true
|
||||
} else {
|
||||
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
|
||||
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
|
||||
continue
|
||||
}
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
package bitmex
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/hmac"
|
||||
"crypto/sha256"
|
||||
@@ -10,6 +9,7 @@ import (
|
||||
"net/url"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
@@ -19,40 +19,46 @@ import (
|
||||
|
||||
type Scanner struct{}
|
||||
|
||||
// Ensure the Scanner satisfies the interface at compile time.
|
||||
var _ detectors.Detector = (*Scanner)(nil)
|
||||
|
||||
var (
|
||||
client = common.SaneHttpClient()
|
||||
|
||||
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"bitmex"}) + string([]byte{32, 13, 10}) + `[0-9a-zA-Z\-\_]{24}` + string([]byte{32, 13, 10}))
|
||||
secretPat = regexp.MustCompile(detectors.PrefixRegex([]string{"bitmex"}) + string([]byte{32, 13, 10}) + `[0-9a-zA-Z\-\_]{48}` + string([]byte{32, 13, 10}))
|
||||
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
|
||||
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"bitmex"}) + `([ \r\n]{1}[0-9a-zA-Z\-\_]{24}[ \r\n]{1})`)
|
||||
secretPat = regexp.MustCompile(detectors.PrefixRegex([]string{"bitmex"}) + `([ \r\n]{1}[0-9a-zA-Z\-\_]{48}[ \r\n]{1})`)
|
||||
)
|
||||
|
||||
func (s Scanner) Keywords() [][]byte {
|
||||
return [][]byte{[]byte("bitmex")}
|
||||
// Keywords are used for efficiently pre-filtering chunks.
|
||||
// Use identifiers in the secret preferably, or the provider name.
|
||||
func (s Scanner) Keywords() []string {
|
||||
return []string{"bitmex"}
|
||||
}
|
||||
|
||||
// FromData will find and optionally verify Bitmex secrets in a given set of bytes.
|
||||
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
|
||||
dataStr := string(data)
|
||||
|
||||
matches := keyPat.FindAllSubmatch(data, -1)
|
||||
secretMatches := secretPat.FindAllSubmatch(data, -1)
|
||||
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
|
||||
secretMatches := secretPat.FindAllStringSubmatch(dataStr, -1)
|
||||
|
||||
for _, match := range matches {
|
||||
if len(match) != 2 {
|
||||
continue
|
||||
}
|
||||
resMatch := bytes.TrimSpace(match[1])
|
||||
resMatch := strings.TrimSpace(match[1])
|
||||
|
||||
for _, secretMatch := range secretMatches {
|
||||
if len(secretMatch) != 2 {
|
||||
continue
|
||||
}
|
||||
resSecretMatch := bytes.TrimSpace(secretMatch[1])
|
||||
resSecretMatch := strings.TrimSpace(secretMatch[1])
|
||||
|
||||
s1 := detectors.Result{
|
||||
DetectorType: detectorspb.DetectorType_Bitmex,
|
||||
Raw: resSecretMatch,
|
||||
RawV2: append(resMatch, resSecretMatch...),
|
||||
Raw: []byte(resSecretMatch),
|
||||
RawV2: []byte(resMatch + resSecretMatch),
|
||||
}
|
||||
|
||||
if verify {
|
||||
@@ -62,15 +68,14 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
path := "/api/v1/user"
|
||||
payload := url.Values{}
|
||||
|
||||
signature := getBitmexSignature(timestamp, string(resSecretMatch), action, path, payload.Encode())
|
||||
|
||||
req, err := http.NewRequestWithContext(ctx, action, "https://www.bitmex.com"+path, bytes.NewReader([]byte(payload.Encode())))
|
||||
signature := getBitmexSignature(timestamp, resSecretMatch, action, path, payload.Encode())
|
||||
|
||||
req, err := http.NewRequestWithContext(ctx, action, "https://www.bitmex.com"+path, strings.NewReader(payload.Encode()))
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
req.Header.Add("api-expires", timestamp)
|
||||
req.Header.Add("api-key", string(resMatch))
|
||||
req.Header.Add("api-key", resMatch)
|
||||
req.Header.Add("api-signature", signature)
|
||||
res, err := client.Do(req)
|
||||
if err == nil {
|
||||
@@ -78,6 +83,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
if res.StatusCode >= 200 && res.StatusCode < 300 {
|
||||
s1.Verified = true
|
||||
} else {
|
||||
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
|
||||
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
|
||||
continue
|
||||
}
|
||||
|
||||
@@ -1,11 +1,11 @@
|
||||
package blablabus
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
@@ -14,30 +14,37 @@ import (
|
||||
|
||||
type Scanner struct{}
|
||||
|
||||
// Ensure the Scanner satisfies the interface at compile time.
|
||||
var _ detectors.Detector = (*Scanner)(nil)
|
||||
|
||||
var (
|
||||
client = common.SaneHttpClient()
|
||||
|
||||
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
|
||||
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"blablabus"}) + `\b([0-9A-Za-z]{22})\b`)
|
||||
)
|
||||
|
||||
func (s Scanner) Keywords() [][]byte {
|
||||
return [][]byte{[]byte("blablabus")}
|
||||
// Keywords are used for efficiently pre-filtering chunks.
|
||||
// Use identifiers in the secret preferably, or the provider name.
|
||||
func (s Scanner) Keywords() []string {
|
||||
return []string{"blablabus"}
|
||||
}
|
||||
|
||||
// FromData will find and optionally verify Blablabus secrets in a given set of bytes.
|
||||
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
|
||||
dataStr := string(data)
|
||||
|
||||
matches := keyPat.FindAllSubmatch(data, -1)
|
||||
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
|
||||
|
||||
for _, match := range matches {
|
||||
if len(match) != 2 {
|
||||
continue
|
||||
}
|
||||
resMatch := bytes.TrimSpace(match[1])
|
||||
resMatch := strings.TrimSpace(match[1])
|
||||
|
||||
s1 := detectors.Result{
|
||||
DetectorType: detectorspb.DetectorType_Blablabus,
|
||||
Raw: resMatch,
|
||||
Raw: []byte(resMatch),
|
||||
}
|
||||
|
||||
if verify {
|
||||
@@ -46,13 +53,14 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
continue
|
||||
}
|
||||
req.Header.Add("Content-Type", "application/json")
|
||||
req.Header.Add("Authorization", fmt.Sprintf("Token %s", string(resMatch)))
|
||||
req.Header.Add("Authorization", fmt.Sprintf("Token %s", resMatch))
|
||||
res, err := client.Do(req)
|
||||
if err == nil {
|
||||
defer res.Body.Close()
|
||||
if res.StatusCode >= 200 && res.StatusCode < 300 {
|
||||
s1.Verified = true
|
||||
} else {
|
||||
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
|
||||
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
|
||||
continue
|
||||
}
|
||||
|
||||
@@ -1,11 +1,11 @@
|
||||
package blazemeter
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
@@ -20,25 +20,31 @@ var _ detectors.Detector = (*Scanner)(nil)
|
||||
var (
|
||||
client = common.SaneHttpClient()
|
||||
|
||||
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
|
||||
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"blazemeter", "runscope"}) + `\b([0-9a-z]{8}-[0-9a-z]{4}-[0-9a-z]{4}-[0-9a-z]{4}-[0-9a-z]{12})\b`)
|
||||
)
|
||||
|
||||
func (s Scanner) Keywords() [][]byte {
|
||||
return [][]byte{[]byte("blazemeter"), []byte("runscope")}
|
||||
// Keywords are used for efficiently pre-filtering chunks.
|
||||
// Use identifiers in the secret preferably, or the provider name.
|
||||
func (s Scanner) Keywords() []string {
|
||||
return []string{"blazemeter", "runscope"}
|
||||
}
|
||||
|
||||
// FromData will find and optionally verify Blazemeter secrets in a given set of bytes.
|
||||
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
|
||||
matches := keyPat.FindAllSubmatch(data, -1)
|
||||
dataStr := string(data)
|
||||
|
||||
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
|
||||
|
||||
for _, match := range matches {
|
||||
if len(match) != 2 {
|
||||
continue
|
||||
}
|
||||
resMatch := bytes.TrimSpace(match[1])
|
||||
resMatch := strings.TrimSpace(match[1])
|
||||
|
||||
s1 := detectors.Result{
|
||||
DetectorType: detectorspb.DetectorType_Blazemeter,
|
||||
Raw: resMatch,
|
||||
Raw: []byte(resMatch),
|
||||
}
|
||||
|
||||
if verify {
|
||||
@@ -47,13 +53,14 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
continue
|
||||
}
|
||||
req.Header.Add("Content-Type", "application/json")
|
||||
req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", string(resMatch)))
|
||||
req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", resMatch))
|
||||
res, err := client.Do(req)
|
||||
if err == nil {
|
||||
defer res.Body.Close()
|
||||
if res.StatusCode >= 200 && res.StatusCode < 300 {
|
||||
s1.Verified = true
|
||||
} else {
|
||||
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
|
||||
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
|
||||
continue
|
||||
}
|
||||
|
||||
@@ -1,10 +1,10 @@
|
||||
package blitapp
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
@@ -13,30 +13,37 @@ import (
|
||||
|
||||
type Scanner struct{}
|
||||
|
||||
// Ensure the Scanner satisfies the interface at compile time.
|
||||
var _ detectors.Detector = (*Scanner)(nil)
|
||||
|
||||
var (
|
||||
client = common.SaneHttpClient()
|
||||
|
||||
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
|
||||
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"blitapp"}) + `\b([a-zA-Z0-9_-]{39})\b`)
|
||||
)
|
||||
|
||||
func (s Scanner) Keywords() [][]byte {
|
||||
return [][]byte{[]byte("blitapp")}
|
||||
// Keywords are used for efficiently pre-filtering chunks.
|
||||
// Use identifiers in the secret preferably, or the provider name.
|
||||
func (s Scanner) Keywords() []string {
|
||||
return []string{"blitapp"}
|
||||
}
|
||||
|
||||
// FromData will find and optionally verify BlitApp secrets in a given set of bytes.
|
||||
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
|
||||
matches := keyPat.FindAllSubmatch(data, -1)
|
||||
dataStr := string(data)
|
||||
|
||||
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
|
||||
|
||||
for _, match := range matches {
|
||||
if len(match) != 2 {
|
||||
continue
|
||||
}
|
||||
resMatch := bytes.TrimSpace(match[1])
|
||||
resMatch := strings.TrimSpace(match[1])
|
||||
|
||||
s1 := detectors.Result{
|
||||
DetectorType: detectorspb.DetectorType_BlitApp,
|
||||
Raw: resMatch,
|
||||
Raw: []byte(resMatch),
|
||||
}
|
||||
|
||||
if verify {
|
||||
@@ -44,13 +51,14 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
req.Header.Add("API-Key", string(resMatch))
|
||||
req.Header.Add("API-Key", resMatch)
|
||||
res, err := client.Do(req)
|
||||
if err == nil {
|
||||
defer res.Body.Close()
|
||||
if res.StatusCode >= 200 && res.StatusCode < 300 {
|
||||
s1.Verified = true
|
||||
} else {
|
||||
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
|
||||
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
|
||||
continue
|
||||
}
|
||||
|
||||
@@ -43,8 +43,6 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
instanceMatches := instancePat.FindAllStringSubmatch(dataStr, -1)
|
||||
tokenMatches := accessTokenPat.FindAllStringSubmatch(dataStr, -1)
|
||||
|
||||
fmt.Printf("instanceMatches: %v\n", instanceMatches)
|
||||
|
||||
for _, instance := range instanceMatches {
|
||||
if len(instance) != 1 {
|
||||
continue
|
||||
|
||||
@@ -2,6 +2,10 @@ package engine
|
||||
|
||||
import (
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/envoyapikey"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/huggingface"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/salesforce"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/snowflake"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/trufflehogenterprise"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/abbysale"
|
||||
@@ -287,6 +291,7 @@ import (
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/getresponse"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/getsandbox"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/github"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/github_oauth2"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/github_old"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/githubapp"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/gitlab"
|
||||
@@ -556,7 +561,6 @@ import (
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/satismeterwritekey"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/saucelabs"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/scalewaykey"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/github_oauth2"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/scalr"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/scrapeowl"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/scraperapi"
|
||||
@@ -1532,6 +1536,10 @@ func DefaultDetectors() []detectors.Detector {
|
||||
couchbase.Scanner{},
|
||||
envoyapikey.Scanner{},
|
||||
github_oauth2.Scanner{},
|
||||
snowflake.Scanner{},
|
||||
huggingface.Scanner{},
|
||||
trufflehogenterprise.Scanner{},
|
||||
salesforce.Scanner{},
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
+84
-31
@@ -12,6 +12,8 @@ import (
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
v1 "github.com/google/go-containerregistry/pkg/v1"
|
||||
"github.com/google/go-containerregistry/pkg/v1/tarball"
|
||||
"github.com/h2non/filetype"
|
||||
"github.com/mholt/archiver/v4"
|
||||
|
||||
@@ -29,17 +31,40 @@ var (
|
||||
maxDepth = 5
|
||||
maxSize = 250 * 1024 * 1024 // 20MB
|
||||
maxTimeout = time.Duration(30) * time.Second
|
||||
|
||||
defaultBufferSize = 512
|
||||
)
|
||||
|
||||
// Ensure the Archive satisfies the interfaces at compile time.
|
||||
var _ SpecializedHandler = (*Archive)(nil)
|
||||
|
||||
// tempEnv contains the temporary file and directory information for the archive.
|
||||
type tempEnv struct {
|
||||
tempFile *os.File
|
||||
tempFileName string
|
||||
extractPath string
|
||||
}
|
||||
|
||||
// dockerImageDetails contains the details of the Docker image and the temporary environment.
|
||||
type dockerImageDetails struct {
|
||||
image v1.Image
|
||||
temp tempEnv
|
||||
}
|
||||
|
||||
// Archive is a handler for extracting and decompressing archives.
|
||||
type Archive struct {
|
||||
size int
|
||||
currentDepth int
|
||||
}
|
||||
|
||||
// DockerTarReader is a wrapper for io.Reader that also stores the temp filepath and image of the tar file.
|
||||
// Temporary file path information needed for inner-workings of go-containerregistry Image methods.
|
||||
type DockerTarReader struct {
|
||||
io.Reader
|
||||
img v1.Image
|
||||
tmpEnv tempEnv
|
||||
}
|
||||
|
||||
// New sets a default maximum size and current size counter.
|
||||
func (a *Archive) New() {
|
||||
a.size = 0
|
||||
@@ -62,7 +87,7 @@ func SetArchiveMaxTimeout(timeout time.Duration) {
|
||||
|
||||
// FromFile extracts the files from an archive.
|
||||
func (a *Archive) FromFile(originalCtx context.Context, data io.Reader) chan []byte {
|
||||
archiveChan := make(chan []byte, 512)
|
||||
archiveChan := make(chan []byte, defaultBufferSize)
|
||||
go func() {
|
||||
ctx, cancel := context.WithTimeout(originalCtx, maxTimeout)
|
||||
logger := logContext.AddLogger(ctx).Logger()
|
||||
@@ -183,35 +208,35 @@ func (a *Archive) ReadToMax(ctx context.Context, reader io.Reader) (data []byte,
|
||||
logger.Error(err, "Panic occurred when reading archive")
|
||||
}
|
||||
}()
|
||||
fileContent := bytes.Buffer{}
|
||||
logger.V(5).Info("Remaining buffer capacity", "bytes", maxSize-a.size)
|
||||
for i := 0; i <= maxSize/512; i++ {
|
||||
if common.IsDone(ctx) {
|
||||
return nil, ctx.Err()
|
||||
}
|
||||
fileChunk := make([]byte, 512)
|
||||
bRead, err := reader.Read(fileChunk)
|
||||
if err != nil && !errors.Is(err, io.EOF) {
|
||||
return []byte{}, err
|
||||
}
|
||||
a.size += bRead
|
||||
if len(fileChunk) > 0 {
|
||||
fileContent.Write(fileChunk[0:bRead])
|
||||
}
|
||||
if bRead < 512 {
|
||||
return fileContent.Bytes(), nil
|
||||
}
|
||||
if a.size >= maxSize && bRead == 512 {
|
||||
logger.V(2).Info("Max archive size reached.")
|
||||
return fileContent.Bytes(), nil
|
||||
}
|
||||
|
||||
if common.IsDone(ctx) {
|
||||
return nil, ctx.Err()
|
||||
}
|
||||
|
||||
var fileContent bytes.Buffer
|
||||
// Create a limited reader to ensure we don't read more than the max size.
|
||||
lr := io.LimitReader(reader, int64(maxSize))
|
||||
|
||||
// Using io.CopyBuffer for performance advantages. Though buf is mandatory
|
||||
// for the method, due to the internal implementation of io.CopyBuffer, when
|
||||
// *bytes.Buffer implements io.WriterTo or io.ReaderFrom, the provided buf
|
||||
// is simply ignored. Thus, we can pass nil for the buf parameter.
|
||||
_, err = io.CopyBuffer(&fileContent, lr, nil)
|
||||
if err != nil && !errors.Is(err, io.EOF) {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if fileContent.Len() == maxSize {
|
||||
logger.V(2).Info("Max archive size reached.")
|
||||
}
|
||||
|
||||
return fileContent.Bytes(), nil
|
||||
}
|
||||
|
||||
const (
|
||||
arMimeType = "application/x-unix-archive"
|
||||
rpmMimeType = "application/x-rpm"
|
||||
tarMimeType = "application/x-tar"
|
||||
)
|
||||
|
||||
// Define a map of mime types to corresponding command-line tools
|
||||
@@ -265,6 +290,21 @@ func (a *Archive) HandleSpecialized(ctx logContext.Context, reader io.Reader) (i
|
||||
return nil, false, err
|
||||
}
|
||||
reader, err = a.extractRpmContent(ctx, reader)
|
||||
case tarMimeType:
|
||||
//check if tar is a docker image
|
||||
isImg, details, err := a.isDockerImage(ctx, reader)
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
if !isImg {
|
||||
return nil, false, nil
|
||||
}
|
||||
// Build dockerReader containing docker image
|
||||
reader = DockerTarReader{
|
||||
Reader: reader,
|
||||
img: details.image,
|
||||
tmpEnv: details.temp,
|
||||
}
|
||||
default:
|
||||
return reader, false, nil
|
||||
}
|
||||
@@ -289,8 +329,8 @@ func (a *Archive) extractDebContent(ctx logContext.Context, file io.Reader) (io.
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer os.Remove(tmpEnv.tempFileName)
|
||||
defer os.RemoveAll(tmpEnv.extractPath)
|
||||
defer os.Remove(tmpEnv.tempFileName)
|
||||
|
||||
cmd := exec.Command("ar", "x", tmpEnv.tempFile.Name())
|
||||
cmd.Dir = tmpEnv.extractPath
|
||||
@@ -327,8 +367,8 @@ func (a *Archive) extractRpmContent(ctx logContext.Context, file io.Reader) (io.
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer os.Remove(tmpEnv.tempFileName)
|
||||
defer os.RemoveAll(tmpEnv.extractPath)
|
||||
defer os.Remove(tmpEnv.tempFileName)
|
||||
|
||||
// Use rpm2cpio to convert the RPM file to a cpio archive and then extract it using cpio command.
|
||||
cmd := exec.Command("sh", "-c", "rpm2cpio "+tmpEnv.tempFile.Name()+" | cpio -id")
|
||||
@@ -427,12 +467,6 @@ func (a *Archive) handleExtractedFiles(ctx logContext.Context, env tempEnv, hand
|
||||
return dataArchiveName, nil
|
||||
}
|
||||
|
||||
type tempEnv struct {
|
||||
tempFile *os.File
|
||||
tempFileName string
|
||||
extractPath string
|
||||
}
|
||||
|
||||
// createTempEnv creates a temporary file and a temporary directory for extracting archives.
|
||||
// The caller is responsible for removing these temporary resources
|
||||
// (both the file and directory) when they are no longer needed.
|
||||
@@ -476,3 +510,22 @@ func openDataArchive(extractPath string, dataArchiveName string) (io.ReadCloser,
|
||||
}
|
||||
return dataFile, nil
|
||||
}
|
||||
|
||||
// isDockerImage checks if a reader object is a docker image.
|
||||
// Returns true if the tar file is a docker image, dockerImageDetails, and error.
|
||||
// Caller is responsible for removing temporary files and directories IF the file is a docker image.
|
||||
// The docker scanner requires a file path to the image, so we must leave the temporary file on disk.
|
||||
func (a *Archive) isDockerImage(ctx context.Context, file io.Reader) (isImg bool, details dockerImageDetails, err error) {
|
||||
tmpEnv, err := a.createTempEnv(ctx, file)
|
||||
if err != nil {
|
||||
return false, dockerImageDetails{}, err
|
||||
}
|
||||
|
||||
img, err := tarball.ImageFromPath(tmpEnv.tempFileName, nil)
|
||||
if err != nil {
|
||||
os.Remove(tmpEnv.tempFileName)
|
||||
os.RemoveAll(tmpEnv.extractPath)
|
||||
return false, dockerImageDetails{}, err
|
||||
}
|
||||
return true, dockerImageDetails{image: img, temp: tmpEnv}, nil
|
||||
}
|
||||
|
||||
@@ -0,0 +1,102 @@
|
||||
//go:build integration
|
||||
// +build integration
|
||||
|
||||
package handlers
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"testing"
|
||||
|
||||
diskbufferreader "github.com/bill-rich/disk-buffer-reader"
|
||||
"github.com/google/go-containerregistry/pkg/name"
|
||||
"github.com/google/go-containerregistry/pkg/v1/remote"
|
||||
"github.com/google/go-containerregistry/pkg/v1/tarball"
|
||||
"github.com/stretchr/testify/assert"
|
||||
|
||||
logContext "github.com/trufflesecurity/trufflehog/v3/pkg/context"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/source_metadatapb"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/sanitizer"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
|
||||
)
|
||||
|
||||
func TestFoundKeyInDockerTar(t *testing.T) {
|
||||
// URI of the Docker image to clone
|
||||
// imageURI := "ghcr.io/joeleonjr/getting-started-app-with-canary-token:main"
|
||||
imageURI := "trufflesecurity/secrets"
|
||||
|
||||
var imageName name.Reference
|
||||
imageName, err := name.NewTag(imageURI)
|
||||
if err != nil {
|
||||
t.Errorf("%s: error creating image name: %s", imageURI, err)
|
||||
}
|
||||
|
||||
// Pull the image
|
||||
img, err := remote.Image(imageName)
|
||||
if err != nil {
|
||||
t.Errorf("%s: error pulling image: %s", imageURI, err)
|
||||
}
|
||||
|
||||
tempFile, err := os.CreateTemp("", "archive_test_docker_img.tar")
|
||||
if err != nil {
|
||||
t.Errorf("%s: error creating temporary file: %s", imageURI, err)
|
||||
return
|
||||
}
|
||||
|
||||
defer os.Remove(tempFile.Name()) // Clean up the temporary file
|
||||
|
||||
// Save the image as a tar file
|
||||
err = tarball.WriteToFile(tempFile.Name(), imageName, img)
|
||||
if err != nil {
|
||||
t.Errorf("%s: error saving image as tar file: %s", imageURI, err)
|
||||
return
|
||||
}
|
||||
|
||||
ctx := logContext.AddLogger(context.Background())
|
||||
|
||||
inputFile, err := os.Open(tempFile.Name())
|
||||
if err != nil {
|
||||
t.Errorf("%s: error opening tar file: %s", imageURI, err)
|
||||
return
|
||||
}
|
||||
defer inputFile.Close()
|
||||
|
||||
reReader, err := diskbufferreader.New(inputFile)
|
||||
if err != nil {
|
||||
t.Errorf("%s: error creating re-readable reader: %s", imageURI, err)
|
||||
return
|
||||
}
|
||||
defer reReader.Close()
|
||||
|
||||
chunkSkel := &sources.Chunk{
|
||||
SourceType: 1,
|
||||
SourceName: "filesystem",
|
||||
SourceID: 1,
|
||||
SourceMetadata: &source_metadatapb.MetaData{
|
||||
Data: &source_metadatapb.MetaData_Filesystem{
|
||||
Filesystem: &source_metadatapb.Filesystem{
|
||||
File: sanitizer.UTF8(tempFile.Name()),
|
||||
},
|
||||
},
|
||||
},
|
||||
Verify: true,
|
||||
}
|
||||
|
||||
chunksChan := make(chan *sources.Chunk, 1)
|
||||
|
||||
go func() {
|
||||
defer close(chunksChan)
|
||||
HandleFile(ctx, reReader, chunkSkel, chunksChan)
|
||||
}()
|
||||
|
||||
secret := "AKIAXYZDQCEN4B6JSJQI"
|
||||
|
||||
// Read from the channel and validate the secrets.
|
||||
foundSecret := ""
|
||||
for chunkCh := range chunksChan {
|
||||
foundSecret += string(chunkCh.Data)
|
||||
}
|
||||
|
||||
assert.Contains(t, foundSecret, secret)
|
||||
|
||||
}
|
||||
@@ -1,6 +1,7 @@
|
||||
package handlers
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"io"
|
||||
"net/http"
|
||||
@@ -107,7 +108,8 @@ func TestHandleFile(t *testing.T) {
|
||||
ch := make(chan *sources.Chunk, 2)
|
||||
|
||||
// Context cancels the operation.
|
||||
canceledCtx, cancel := context.WithCancel(context.Background())
|
||||
ctx := logContext.AddLogger(context.Background())
|
||||
canceledCtx, cancel := logContext.WithCancel(ctx)
|
||||
cancel()
|
||||
assert.False(t, HandleFile(canceledCtx, strings.NewReader("file"), &sources.Chunk{}, ch))
|
||||
|
||||
@@ -122,10 +124,62 @@ func TestHandleFile(t *testing.T) {
|
||||
assert.NoError(t, err)
|
||||
|
||||
assert.Equal(t, 0, len(ch))
|
||||
assert.True(t, HandleFile(context.Background(), reader, &sources.Chunk{}, ch))
|
||||
assert.True(t, HandleFile(ctx, reader, &sources.Chunk{}, ch))
|
||||
assert.Equal(t, 1, len(ch))
|
||||
}
|
||||
|
||||
func TestReadToMax(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
input []byte
|
||||
expected []byte
|
||||
}{
|
||||
{
|
||||
name: "read full content within maxSize",
|
||||
input: []byte("abcdefg"),
|
||||
expected: []byte("abcdefg"),
|
||||
},
|
||||
{
|
||||
name: "read content larger than maxSize",
|
||||
input: make([]byte, maxSize+10), // this creates a byte slice 10 bytes larger than maxSize
|
||||
expected: make([]byte, maxSize),
|
||||
},
|
||||
{
|
||||
name: "empty input",
|
||||
input: []byte(""),
|
||||
expected: []byte(""),
|
||||
},
|
||||
}
|
||||
|
||||
a := &Archive{}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
reader := bytes.NewReader(tt.input)
|
||||
output, err := a.ReadToMax(context.Background(), reader)
|
||||
assert.Nil(t, err)
|
||||
|
||||
assert.Equal(t, tt.expected, output)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkReadToMax(b *testing.B) {
|
||||
data := bytes.Repeat([]byte("a"), 1024*1000) // 1MB of data.
|
||||
reader := bytes.NewReader(data)
|
||||
a := &Archive{}
|
||||
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
b.StartTimer()
|
||||
_, _ = a.ReadToMax(context.Background(), reader)
|
||||
b.StopTimer()
|
||||
|
||||
_, _ = reader.Seek(0, 0) // Reset the reader position.
|
||||
a.size = 0 // Reset archive size.
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtractDebContent(t *testing.T) {
|
||||
// Open the sample .deb file from the testdata folder.
|
||||
file, err := os.Open("testdata/test.deb")
|
||||
@@ -149,7 +203,7 @@ func TestExtractTarContent(t *testing.T) {
|
||||
assert.Nil(t, err)
|
||||
defer file.Close()
|
||||
|
||||
ctx := context.Background()
|
||||
ctx := logContext.AddLogger(context.Background())
|
||||
|
||||
chunkCh := make(chan *sources.Chunk)
|
||||
go func() {
|
||||
|
||||
@@ -3,11 +3,13 @@ package handlers
|
||||
import (
|
||||
"context"
|
||||
"io"
|
||||
"os"
|
||||
|
||||
diskbufferreader "github.com/bill-rich/disk-buffer-reader"
|
||||
|
||||
logContext "github.com/trufflesecurity/trufflehog/v3/pkg/context"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/sources/docker"
|
||||
)
|
||||
|
||||
func DefaultHandlers() []Handler {
|
||||
@@ -37,7 +39,7 @@ type Handler interface {
|
||||
// packages them in the provided chunk skeleton, and sends them to chunksChan.
|
||||
// The function returns true if processing was successful and false otherwise.
|
||||
// Context is used for cancellation, and the caller is responsible for canceling it if needed.
|
||||
func HandleFile(ctx context.Context, file io.Reader, chunkSkel *sources.Chunk, chunksChan chan *sources.Chunk) bool {
|
||||
func HandleFile(ctx logContext.Context, file io.Reader, chunkSkel *sources.Chunk, chunksChan chan *sources.Chunk) bool {
|
||||
aCtx := logContext.AddLogger(ctx)
|
||||
for _, h := range DefaultHandlers() {
|
||||
h.New()
|
||||
@@ -56,9 +58,15 @@ func HandleFile(ctx context.Context, file io.Reader, chunkSkel *sources.Chunk, c
|
||||
if specialHandler, ok := h.(SpecializedHandler); ok {
|
||||
file, isSpecial, err := specialHandler.HandleSpecialized(aCtx, reReader)
|
||||
if isSpecial {
|
||||
if dockerTarReader, ok := file.(DockerTarReader); ok {
|
||||
// Clean up temporary files when done.
|
||||
defer os.RemoveAll(dockerTarReader.tmpEnv.extractPath)
|
||||
defer os.Remove(dockerTarReader.tmpEnv.tempFileName)
|
||||
err = docker.ScanDockerImg(ctx, dockerTarReader.img, chunksChan, chunkSkel)
|
||||
return err == nil
|
||||
}
|
||||
return handleChunks(aCtx, h.FromFile(ctx, file), chunkSkel, chunksChan)
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
aCtx.Logger().Error(err, "error handling file")
|
||||
}
|
||||
|
||||
@@ -133,7 +133,7 @@ func BranchHeads(repo *gogit.Repository) (map[string]*object.Commit, error) {
|
||||
}
|
||||
headCommit, err := repo.CommitObject(*headHash)
|
||||
if err != nil {
|
||||
logger.Error(err, "unable to get commit", "commit", headCommit.String())
|
||||
logger.Error(err, "unable to get commit", "head_hash", headHash.String())
|
||||
return nil
|
||||
}
|
||||
branches[branchName] = headCommit
|
||||
|
||||
@@ -20,7 +20,7 @@ const (
|
||||
|
||||
// Chunker takes a chunk and splits it into chunks of ChunkSize.
|
||||
func Chunker(originalChunk *Chunk) chan *Chunk {
|
||||
chunkChan := make(chan *Chunk)
|
||||
chunkChan := make(chan *Chunk, 1)
|
||||
go func() {
|
||||
defer close(chunkChan)
|
||||
if len(originalChunk.Data) <= TotalChunkSize {
|
||||
|
||||
@@ -238,6 +238,7 @@ func (s *Source) chunkAction(ctx context.Context, proj project, bld build, act a
|
||||
SourceType: s.Type(),
|
||||
SourceName: s.name,
|
||||
SourceID: s.SourceID(),
|
||||
JobID: s.JobID(),
|
||||
Data: removeCircleSha1Line(data.Bytes()),
|
||||
SourceMetadata: &source_metadatapb.MetaData{
|
||||
Data: &source_metadatapb.MetaData_Circleci{
|
||||
|
||||
+106
-81
@@ -78,7 +78,7 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) err
|
||||
return err
|
||||
}
|
||||
|
||||
logger := ctx.Logger().WithValues("source_type", s.Type(), "source_name", s.name)
|
||||
ctx.Logger().WithValues("source_type", s.Type(), "source_name", s.name)
|
||||
|
||||
for _, image := range s.conn.GetImages() {
|
||||
var img v1.Image
|
||||
@@ -115,95 +115,120 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) err
|
||||
}
|
||||
}
|
||||
|
||||
logger = logger.WithValues("image", base, "tag", tag)
|
||||
logger.V(2).Info("scanning image")
|
||||
|
||||
layers, err := img.Layers()
|
||||
chunkSkel := &sources.Chunk{
|
||||
SourceType: s.Type(),
|
||||
SourceName: s.name,
|
||||
SourceID: s.SourceID(),
|
||||
JobID: s.JobID(),
|
||||
SourceMetadata: &source_metadatapb.MetaData{
|
||||
Data: &source_metadatapb.MetaData_Docker{
|
||||
Docker: &source_metadatapb.Docker{
|
||||
Image: base,
|
||||
Tag: tag,
|
||||
},
|
||||
},
|
||||
},
|
||||
Verify: s.verify,
|
||||
}
|
||||
err = ScanDockerImg(ctx, img, chunksChan, chunkSkel)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
for _, layer := range layers {
|
||||
|
||||
digest, err := layer.Digest()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
logger = logger.WithValues("layer", digest.String())
|
||||
logger.V(2).Info("scanning layer")
|
||||
|
||||
rc, err := layer.Compressed()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
defer rc.Close()
|
||||
|
||||
gzipReader, err := gzip.NewReader(rc)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
defer gzipReader.Close()
|
||||
|
||||
tarReader := tar.NewReader(gzipReader)
|
||||
|
||||
for {
|
||||
header, err := tarReader.Next()
|
||||
if errors.Is(err, io.EOF) {
|
||||
break // End of archive
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Skip files larger than FilesizeLimitBytes
|
||||
if header.Size > FilesizeLimitBytes {
|
||||
continue
|
||||
}
|
||||
|
||||
file := bytes.NewBuffer(make([]byte, 0, header.Size))
|
||||
|
||||
_, err = io.Copy(file, tarReader)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
chunk := &sources.Chunk{
|
||||
SourceType: s.Type(),
|
||||
SourceName: s.name,
|
||||
SourceID: s.SourceID(),
|
||||
Data: file.Bytes(),
|
||||
SourceMetadata: &source_metadatapb.MetaData{
|
||||
Data: &source_metadatapb.MetaData_Docker{
|
||||
Docker: &source_metadatapb.Docker{
|
||||
File: "/" + header.Name, // Prepend /, because the file is relative to the root of the image
|
||||
Image: base,
|
||||
Tag: tag,
|
||||
Layer: digest.String(),
|
||||
},
|
||||
},
|
||||
},
|
||||
Verify: s.verify,
|
||||
}
|
||||
|
||||
select {
|
||||
case chunksChan <- chunk:
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
}
|
||||
}
|
||||
|
||||
dockerLayersScanned.WithLabelValues(s.name).Inc()
|
||||
}
|
||||
|
||||
dockerImagesScanned.WithLabelValues(s.name).Inc()
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func ScanDockerImg(ctx context.Context, img v1.Image, chunksChan chan *sources.Chunk, chunkSkel *sources.Chunk) error {
|
||||
base, tag := chunkSkel.BaseAndTagForDockerImg()
|
||||
|
||||
logger := ctx.Logger().WithValues("image", base, "tag", tag)
|
||||
logger.V(2).Info("scanning image")
|
||||
|
||||
layers, err := img.Layers()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, layer := range layers {
|
||||
digest, err := layer.Digest()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
logger = logger.WithValues("layer", digest.String())
|
||||
logger.V(2).Info("scanning layer")
|
||||
|
||||
rc, err := layer.Compressed()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
defer rc.Close()
|
||||
|
||||
gzipReader, err := gzip.NewReader(rc)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
defer gzipReader.Close()
|
||||
|
||||
tarReader := tar.NewReader(gzipReader)
|
||||
|
||||
for {
|
||||
header, err := tarReader.Next()
|
||||
if errors.Is(err, io.EOF) {
|
||||
break // End of archive
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Skip files larger than FilesizeLimitBytes
|
||||
if header.Size > FilesizeLimitBytes {
|
||||
continue
|
||||
}
|
||||
|
||||
file := bytes.NewBuffer(make([]byte, 0, header.Size))
|
||||
|
||||
_, err = io.Copy(file, tarReader)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
chunk := &sources.Chunk{
|
||||
// This must be TYPE_DOCKER or SetResultLineNumber could panic
|
||||
SourceType: sourcespb.SourceType_SOURCE_TYPE_DOCKER,
|
||||
SourceName: chunkSkel.SourceName,
|
||||
SourceID: chunkSkel.SourceID,
|
||||
Data: file.Bytes(),
|
||||
SourceMetadata: &source_metadatapb.MetaData{
|
||||
Data: &source_metadatapb.MetaData_Docker{
|
||||
Docker: &source_metadatapb.Docker{
|
||||
File: "/" + header.Name, // Prepend /, because the file is relative to the root of the image
|
||||
Image: base,
|
||||
Tag: tag,
|
||||
Layer: digest.String(),
|
||||
},
|
||||
},
|
||||
},
|
||||
Verify: chunkSkel.Verify,
|
||||
}
|
||||
|
||||
select {
|
||||
case chunksChan <- chunk:
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
}
|
||||
}
|
||||
|
||||
dockerLayersScanned.WithLabelValues(chunkSkel.SourceName).Inc()
|
||||
}
|
||||
|
||||
dockerImagesScanned.WithLabelValues(chunkSkel.SourceName).Inc()
|
||||
return nil
|
||||
}
|
||||
|
||||
func baseAndTagFromImage(image string) (base, tag string, hasDigest bool) {
|
||||
regRepoDelimiter := "/"
|
||||
tagDelim := ":"
|
||||
|
||||
@@ -161,6 +161,7 @@ func (s *Source) scanFile(ctx context.Context, path string, chunksChan chan *sou
|
||||
SourceType: s.Type(),
|
||||
SourceName: s.name,
|
||||
SourceID: s.SourceID(),
|
||||
JobID: s.JobID(),
|
||||
SourceMetadata: &source_metadatapb.MetaData{
|
||||
Data: &source_metadatapb.MetaData_Filesystem{
|
||||
Filesystem: &source_metadatapb.Filesystem{
|
||||
@@ -191,6 +192,7 @@ func (s *Source) scanFile(ctx context.Context, path string, chunksChan chan *sou
|
||||
SourceType: s.Type(),
|
||||
SourceName: s.name,
|
||||
SourceID: s.SourceID(),
|
||||
JobID: s.JobID(),
|
||||
Data: data.Bytes(),
|
||||
SourceMetadata: &source_metadatapb.MetaData{
|
||||
Data: &source_metadatapb.MetaData_Filesystem{
|
||||
|
||||
@@ -324,6 +324,7 @@ func (s *Source) processObject(ctx context.Context, o object) error {
|
||||
chunkSkel := &sources.Chunk{
|
||||
SourceName: s.name,
|
||||
SourceType: s.Type(),
|
||||
JobID: s.JobID(),
|
||||
SourceID: s.sourceId,
|
||||
Verify: s.verify,
|
||||
SourceMetadata: &source_metadatapb.MetaData{
|
||||
|
||||
@@ -446,6 +446,7 @@ func (s *Git) ScanCommits(ctx context.Context, repo *git.Repository, path string
|
||||
chunkSkel := &sources.Chunk{
|
||||
SourceName: s.sourceName,
|
||||
SourceID: s.sourceID,
|
||||
JobID: s.jobID,
|
||||
SourceType: s.sourceType,
|
||||
SourceMetadata: metadata,
|
||||
Verify: s.verify,
|
||||
@@ -464,6 +465,7 @@ func (s *Git) ScanCommits(ctx context.Context, repo *git.Repository, path string
|
||||
chunksChan <- &sources.Chunk{
|
||||
SourceName: s.sourceName,
|
||||
SourceID: s.sourceID,
|
||||
JobID: s.jobID,
|
||||
SourceType: s.sourceType,
|
||||
SourceMetadata: metadata,
|
||||
Data: diff.Content.Bytes(),
|
||||
@@ -490,6 +492,7 @@ func (s *Git) gitChunk(ctx context.Context, diff gitparse.Diff, fileName, email,
|
||||
chunksChan <- &sources.Chunk{
|
||||
SourceName: s.sourceName,
|
||||
SourceID: s.sourceID,
|
||||
JobID: s.jobID,
|
||||
SourceType: s.sourceType,
|
||||
SourceMetadata: metadata,
|
||||
Data: append([]byte{}, newChunkBuffer.Bytes()...),
|
||||
@@ -504,6 +507,7 @@ func (s *Git) gitChunk(ctx context.Context, diff gitparse.Diff, fileName, email,
|
||||
chunksChan <- &sources.Chunk{
|
||||
SourceName: s.sourceName,
|
||||
SourceID: s.sourceID,
|
||||
JobID: s.jobID,
|
||||
SourceType: s.sourceType,
|
||||
SourceMetadata: metadata,
|
||||
Data: line,
|
||||
@@ -523,6 +527,7 @@ func (s *Git) gitChunk(ctx context.Context, diff gitparse.Diff, fileName, email,
|
||||
chunksChan <- &sources.Chunk{
|
||||
SourceName: s.sourceName,
|
||||
SourceID: s.sourceID,
|
||||
JobID: s.jobID,
|
||||
SourceType: s.sourceType,
|
||||
SourceMetadata: metadata,
|
||||
Data: append([]byte{}, newChunkBuffer.Bytes()...),
|
||||
@@ -588,6 +593,7 @@ func (s *Git) ScanStaged(ctx context.Context, repo *git.Repository, path string,
|
||||
chunkSkel := &sources.Chunk{
|
||||
SourceName: s.sourceName,
|
||||
SourceID: s.sourceID,
|
||||
JobID: s.jobID,
|
||||
SourceType: s.sourceType,
|
||||
SourceMetadata: metadata,
|
||||
Verify: s.verify,
|
||||
@@ -602,6 +608,7 @@ func (s *Git) ScanStaged(ctx context.Context, repo *git.Repository, path string,
|
||||
chunksChan <- &sources.Chunk{
|
||||
SourceName: s.sourceName,
|
||||
SourceID: s.sourceID,
|
||||
JobID: s.jobID,
|
||||
SourceType: s.sourceType,
|
||||
SourceMetadata: metadata,
|
||||
Data: diff.Content.Bytes(),
|
||||
|
||||
@@ -149,14 +149,16 @@ func (s *Source) newFilteredRepoCache(c cache.Cache, include, exclude []string)
|
||||
for _, ig := range include {
|
||||
g, err := glob.Compile(ig)
|
||||
if err != nil {
|
||||
s.log.V(1).Info("invalid include glob", "glob", g, "err", err)
|
||||
s.log.V(1).Info("invalid include glob", "include_value", ig, "err", err)
|
||||
continue
|
||||
}
|
||||
includeGlobs = append(includeGlobs, g)
|
||||
}
|
||||
for _, eg := range exclude {
|
||||
g, err := glob.Compile(eg)
|
||||
if err != nil {
|
||||
s.log.V(1).Info("invalid exclude glob", "glob", g, "err", err)
|
||||
s.log.V(1).Info("invalid exclude glob", "exclude_value", eg, "err", err)
|
||||
continue
|
||||
}
|
||||
excludeGlobs = append(excludeGlobs, g)
|
||||
}
|
||||
@@ -1223,6 +1225,7 @@ func (s *Source) chunkIssueComments(ctx context.Context, repo, repoPath string,
|
||||
chunk := &sources.Chunk{
|
||||
SourceName: s.name,
|
||||
SourceID: s.SourceID(),
|
||||
JobID: s.JobID(),
|
||||
SourceType: s.Type(),
|
||||
SourceMetadata: &source_metadatapb.MetaData{
|
||||
Data: &source_metadatapb.MetaData_Github{
|
||||
@@ -1256,6 +1259,7 @@ func (s *Source) chunkPullRequestComments(ctx context.Context, repo string, comm
|
||||
SourceName: s.name,
|
||||
SourceID: s.SourceID(),
|
||||
SourceType: s.Type(),
|
||||
JobID: s.JobID(),
|
||||
SourceMetadata: &source_metadatapb.MetaData{
|
||||
Data: &source_metadatapb.MetaData_Github{
|
||||
Github: &source_metadatapb.Github{
|
||||
@@ -1287,6 +1291,7 @@ func (s *Source) chunkGistComments(ctx context.Context, gistUrl string, comments
|
||||
SourceName: s.name,
|
||||
SourceID: s.SourceID(),
|
||||
SourceType: s.Type(),
|
||||
JobID: s.JobID(),
|
||||
SourceMetadata: &source_metadatapb.MetaData{
|
||||
Data: &source_metadatapb.MetaData_Github{
|
||||
Github: &source_metadatapb.Github{
|
||||
|
||||
@@ -71,11 +71,11 @@ func (r *JobProgressRef) Done() <-chan struct{} {
|
||||
// CancelRun requests that the job this is referencing is cancelled and stops
|
||||
// running. This method will have no effect if the job does not allow
|
||||
// cancellation.
|
||||
func (r *JobProgressRef) CancelRun() {
|
||||
func (r *JobProgressRef) CancelRun(cause error) {
|
||||
if r.jobProgress == nil || r.jobProgress.jobCancel == nil {
|
||||
return
|
||||
}
|
||||
r.jobProgress.jobCancel()
|
||||
r.jobProgress.jobCancel(cause)
|
||||
}
|
||||
|
||||
// Fatal is a wrapper around error to differentiate non-fatal errors from fatal
|
||||
@@ -108,7 +108,7 @@ type JobProgress struct {
|
||||
ctx context.Context
|
||||
cancel context.CancelFunc
|
||||
// Requests to cancel the job.
|
||||
jobCancel context.CancelFunc
|
||||
jobCancel context.CancelCauseFunc
|
||||
// Metrics.
|
||||
metrics JobProgressMetrics
|
||||
metricsLock sync.Mutex
|
||||
@@ -150,7 +150,7 @@ func WithHooks(hooks ...JobProgressHook) func(*JobProgress) {
|
||||
}
|
||||
|
||||
// WithCancel allows cancelling the job by the JobProgressRef.
|
||||
func WithCancel(cancel context.CancelFunc) func(*JobProgress) {
|
||||
func WithCancel(cancel context.CancelCauseFunc) func(*JobProgress) {
|
||||
return func(jp *JobProgress) { jp.jobCancel = cancel }
|
||||
}
|
||||
|
||||
|
||||
@@ -376,6 +376,7 @@ func (s *Source) pageChunker(ctx context.Context, client *s3.S3, chunksChan chan
|
||||
SourceType: s.Type(),
|
||||
SourceName: s.name,
|
||||
SourceID: s.SourceID(),
|
||||
JobID: s.JobID(),
|
||||
SourceMetadata: &source_metadatapb.MetaData{
|
||||
Data: &source_metadatapb.MetaData_S3{
|
||||
S3: &source_metadatapb.S3{
|
||||
|
||||
@@ -36,7 +36,10 @@ type SourceManager struct {
|
||||
handles map[handle]sourceInfo
|
||||
handlesLock sync.Mutex
|
||||
// Pool limiting the amount of concurrent sources running.
|
||||
pool errgroup.Group
|
||||
pool errgroup.Group
|
||||
poolLimit int
|
||||
currentRunningCount int32
|
||||
// Max number of units to scan concurrently per source.
|
||||
concurrentUnits int
|
||||
// Run the sources using source unit enumeration / chunking if available.
|
||||
useSourceUnits bool
|
||||
@@ -68,7 +71,10 @@ func WithReportHook(hook JobProgressHook) func(*SourceManager) {
|
||||
|
||||
// WithConcurrentSources limits the concurrent number of sources a manager can run.
|
||||
func WithConcurrentSources(concurrency int) func(*SourceManager) {
|
||||
return func(mgr *SourceManager) { mgr.pool.SetLimit(concurrency) }
|
||||
return func(mgr *SourceManager) {
|
||||
mgr.pool.SetLimit(concurrency)
|
||||
mgr.poolLimit = concurrency
|
||||
}
|
||||
}
|
||||
|
||||
// WithBufferedOutput sets the size of the buffer used for the Chunks() channel.
|
||||
@@ -164,15 +170,17 @@ func (s *SourceManager) asyncRun(ctx context.Context, handle handle) (JobProgres
|
||||
return JobProgressRef{SourceID: int64(handle), SourceName: sourceName}, err
|
||||
}
|
||||
// Create a JobProgress object for tracking progress.
|
||||
ctx, cancel := context.WithCancel(ctx)
|
||||
ctx, cancel := context.WithCancelCause(ctx)
|
||||
progress := NewJobProgress(jobID, int64(handle), sourceName, WithHooks(s.hooks...), WithCancel(cancel))
|
||||
s.pool.Go(func() error {
|
||||
atomic.AddInt32(&s.currentRunningCount, 1)
|
||||
defer atomic.AddInt32(&s.currentRunningCount, -1)
|
||||
ctx := context.WithValues(ctx,
|
||||
"job_id", jobID,
|
||||
"source_manager_worker_id", common.RandomID(5),
|
||||
)
|
||||
defer common.Recover(ctx)
|
||||
defer cancel()
|
||||
defer cancel(nil)
|
||||
return s.run(ctx, handle, jobID, progress)
|
||||
})
|
||||
return progress.Ref(), nil
|
||||
@@ -207,6 +215,16 @@ func (s *SourceManager) ScanChunk(chunk *Chunk) {
|
||||
s.outputChunks <- chunk
|
||||
}
|
||||
|
||||
// AvailableCapacity returns the number of concurrent jobs the manager can
|
||||
// accommodate at this time. If there is no limit, -1 is returned.
|
||||
func (s *SourceManager) AvailableCapacity() int {
|
||||
if s.poolLimit == 0 {
|
||||
return -1
|
||||
}
|
||||
runCount := atomic.LoadInt32(&s.currentRunningCount)
|
||||
return s.poolLimit - int(runCount)
|
||||
}
|
||||
|
||||
// preflightChecks is a helper method to check the Manager or the context isn't
|
||||
// done and that the handle is valid.
|
||||
func (s *SourceManager) preflightChecks(ctx context.Context, handle handle) error {
|
||||
@@ -229,6 +247,12 @@ func (s *SourceManager) run(ctx context.Context, handle handle, jobID int64, rep
|
||||
report.Start(time.Now())
|
||||
defer func() { report.End(time.Now()) }()
|
||||
|
||||
defer func() {
|
||||
if err := context.Cause(ctx); err != nil {
|
||||
report.ReportError(Fatal{err})
|
||||
}
|
||||
}()
|
||||
|
||||
// Initialize the source.
|
||||
sourceInfo, ok := s.getSourceInfo(handle)
|
||||
if !ok {
|
||||
|
||||
@@ -312,8 +312,31 @@ func TestSourceManagerCancelRun(t *testing.T) {
|
||||
ref, err := mgr.ScheduleRun(context.Background(), handle)
|
||||
assert.NoError(t, err)
|
||||
|
||||
ref.CancelRun()
|
||||
cancelErr := fmt.Errorf("abort! abort!")
|
||||
ref.CancelRun(cancelErr)
|
||||
<-ref.Done()
|
||||
assert.Error(t, ref.Snapshot().FatalError())
|
||||
assert.True(t, errors.Is(ref.Snapshot().FatalError(), returnedErr))
|
||||
assert.True(t, errors.Is(ref.Snapshot().FatalErrors(), cancelErr))
|
||||
}
|
||||
|
||||
func TestSourceManagerAvailableCapacity(t *testing.T) {
|
||||
mgr := NewManager(WithConcurrentSources(1337))
|
||||
start, end := make(chan struct{}), make(chan struct{})
|
||||
handle, err := enrollDummy(mgr, callbackChunker{func(context.Context, chan *Chunk) error {
|
||||
start <- struct{}{} // Send start signal.
|
||||
<-end // Wait for end signal.
|
||||
return nil
|
||||
}})
|
||||
assert.NoError(t, err)
|
||||
|
||||
assert.Equal(t, 1337, mgr.AvailableCapacity())
|
||||
ref, err := mgr.ScheduleRun(context.Background(), handle)
|
||||
assert.NoError(t, err)
|
||||
|
||||
<-start // Wait for start signal.
|
||||
assert.Equal(t, 1336, mgr.AvailableCapacity())
|
||||
end <- struct{}{} // Send end signal.
|
||||
<-ref.Done() // Wait for the job to finish.
|
||||
assert.Equal(t, 1337, mgr.AvailableCapacity())
|
||||
}
|
||||
|
||||
@@ -30,6 +30,48 @@ type Chunk struct {
|
||||
Verify bool
|
||||
}
|
||||
|
||||
// BaseAndTagForDockerImg returns a base and tag value for a
|
||||
// docker image regardless of the source type. If the source type
|
||||
// is not a docker image, the base will be a filename/path from the source metadata.
|
||||
// The default is the entire source metadata string. Update for new source types.
|
||||
func (c *Chunk) BaseAndTagForDockerImg() (base string, tag string) {
|
||||
switch c.SourceType {
|
||||
case sourcespb.SourceType_SOURCE_TYPE_DOCKER:
|
||||
base = c.SourceMetadata.GetDocker().GetImage()
|
||||
tag = c.SourceMetadata.GetDocker().GetTag()
|
||||
case sourcespb.SourceType_SOURCE_TYPE_FILESYSTEM:
|
||||
base = c.SourceMetadata.GetFilesystem().GetFile()
|
||||
case sourcespb.SourceType_SOURCE_TYPE_S3:
|
||||
bucket := c.SourceMetadata.GetS3().GetBucket()
|
||||
file := c.SourceMetadata.GetS3().GetFile()
|
||||
base = bucket + "/" + file
|
||||
case sourcespb.SourceType_SOURCE_TYPE_GCS:
|
||||
bucket := c.SourceMetadata.GetGcs().GetBucket()
|
||||
file := c.SourceMetadata.GetGcs().GetFilename()
|
||||
base = bucket + "/" + file
|
||||
case sourcespb.SourceType_SOURCE_TYPE_SYSLOG:
|
||||
host := c.SourceMetadata.GetSyslog().GetHostname()
|
||||
app := c.SourceMetadata.GetSyslog().GetAppname()
|
||||
base = host + "/" + app
|
||||
case sourcespb.SourceType_SOURCE_TYPE_GIT:
|
||||
repo := c.SourceMetadata.GetGit().GetRepository()
|
||||
file := c.SourceMetadata.GetGit().GetFile()
|
||||
base = file + " (repo: " + repo + ")"
|
||||
case sourcespb.SourceType_SOURCE_TYPE_GITLAB:
|
||||
repo := c.SourceMetadata.GetGitlab().GetRepository()
|
||||
file := c.SourceMetadata.GetGitlab().GetFile()
|
||||
base = file + " (repo: " + repo + ")"
|
||||
case sourcespb.SourceType_SOURCE_TYPE_GITHUB:
|
||||
repo := c.SourceMetadata.GetGithub().GetRepository()
|
||||
file := c.SourceMetadata.GetGithub().GetFile()
|
||||
base = file + " (repo: " + repo + ")"
|
||||
default:
|
||||
base = c.SourceMetadata.String()
|
||||
tag = ""
|
||||
}
|
||||
return base, tag
|
||||
}
|
||||
|
||||
// Source defines the interface required to implement a source chunker.
|
||||
type Source interface {
|
||||
// Type returns the source type, used for matching against configuration and jobs.
|
||||
|
||||
@@ -272,6 +272,7 @@ func (s *Source) monitorConnection(ctx context.Context, conn net.Conn, chunksCha
|
||||
SourceName: s.syslog.sourceName,
|
||||
SourceID: s.syslog.sourceID,
|
||||
SourceType: s.syslog.sourceType,
|
||||
JobID: s.JobID(),
|
||||
SourceMetadata: metadata,
|
||||
Data: input,
|
||||
Verify: s.verify,
|
||||
@@ -313,6 +314,7 @@ func (s *Source) acceptUDPConnections(ctx context.Context, netListener net.Packe
|
||||
chunksChan <- &sources.Chunk{
|
||||
SourceName: s.syslog.sourceName,
|
||||
SourceID: s.syslog.sourceID,
|
||||
JobID: s.JobID(),
|
||||
SourceType: s.syslog.sourceType,
|
||||
SourceMetadata: metadata,
|
||||
Data: input,
|
||||
|
||||
Reference in New Issue
Block a user