Compare commits
1 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| ea6768bce9 |
@@ -32,7 +32,7 @@ Have questions? Feedback? Jump in slack or discord and hang out with us
|
||||
|
||||
Join our [Slack Community](https://join.slack.com/t/trufflehog-community/shared_invite/zt-pw2qbi43-Aa86hkiimstfdKH9UCpPzQ)
|
||||
|
||||
Join the [Secret Scanning Discord](https://discord.gg/8Hzbrnkr7E)
|
||||
Join the [Secret Scanning Discord](https://discord.gg/sydS6AHTUP)
|
||||
|
||||
# :tv: Demo
|
||||
|
||||
|
||||
+2
-40
@@ -26,10 +26,9 @@ type Context interface {
|
||||
Logger() logr.Logger
|
||||
}
|
||||
|
||||
// CancelFunc and CancelCauseFunc are type aliases to allow use as if they are
|
||||
// the same types as the standard library variants.
|
||||
// CancelFunc is a type alias to context.CancelFunc to allow use as if they are
|
||||
// the same types.
|
||||
type CancelFunc = context.CancelFunc
|
||||
type CancelCauseFunc = context.CancelCauseFunc
|
||||
|
||||
// logCtx implements Context.
|
||||
type logCtx struct {
|
||||
@@ -69,16 +68,6 @@ func WithCancel(parent Context) (Context, context.CancelFunc) {
|
||||
return lCtx, cancel
|
||||
}
|
||||
|
||||
// WithCancelCause returns context.WithCancelCause with the log object propagated.
|
||||
func WithCancelCause(parent Context) (Context, context.CancelCauseFunc) {
|
||||
ctx, cancel := context.WithCancelCause(parent)
|
||||
lCtx := logCtx{
|
||||
log: parent.Logger(),
|
||||
Context: ctx,
|
||||
}
|
||||
return lCtx, cancel
|
||||
}
|
||||
|
||||
// WithDeadline returns context.WithDeadline with the log object propagated and
|
||||
// the deadline added to the structured log values.
|
||||
func WithDeadline(parent Context, d time.Time) (Context, context.CancelFunc) {
|
||||
@@ -90,17 +79,6 @@ func WithDeadline(parent Context, d time.Time) (Context, context.CancelFunc) {
|
||||
return lCtx, cancel
|
||||
}
|
||||
|
||||
// WithDeadlineCause returns context.WithDeadlineCause with the log object
|
||||
// propagated and the deadline added to the structured log values.
|
||||
func WithDeadlineCause(parent Context, d time.Time, cause error) (Context, context.CancelFunc) {
|
||||
ctx, cancel := context.WithDeadlineCause(parent, d, cause)
|
||||
lCtx := logCtx{
|
||||
log: parent.Logger().WithValues("deadline", d),
|
||||
Context: ctx,
|
||||
}
|
||||
return lCtx, cancel
|
||||
}
|
||||
|
||||
// WithTimeout returns context.WithTimeout with the log object propagated and
|
||||
// the timeout added to the structured log values.
|
||||
func WithTimeout(parent Context, timeout time.Duration) (Context, context.CancelFunc) {
|
||||
@@ -112,22 +90,6 @@ func WithTimeout(parent Context, timeout time.Duration) (Context, context.Cancel
|
||||
return lCtx, cancel
|
||||
}
|
||||
|
||||
// WithTimeoutCause returns context.WithTimeoutCause with the log object
|
||||
// propagated and the timeout added to the structured log values.
|
||||
func WithTimeoutCause(parent Context, timeout time.Duration, cause error) (Context, context.CancelFunc) {
|
||||
ctx, cancel := context.WithTimeoutCause(parent, timeout, cause)
|
||||
lCtx := logCtx{
|
||||
log: parent.Logger().WithValues("timeout", timeout),
|
||||
Context: ctx,
|
||||
}
|
||||
return lCtx, cancel
|
||||
}
|
||||
|
||||
// Cause returns the context.Cause of the context.
|
||||
func Cause(ctx context.Context) error {
|
||||
return context.Cause(ctx)
|
||||
}
|
||||
|
||||
// WithValue returns context.WithValue with the log object propagated and
|
||||
// the value added to the structured log values (if the key is a string).
|
||||
func WithValue(parent Context, key, val any) Context {
|
||||
|
||||
@@ -3,7 +3,6 @@ package context
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
@@ -178,10 +177,3 @@ func TestRace(t *testing.T) {
|
||||
cancel()
|
||||
_ = ctx.Err()
|
||||
}
|
||||
|
||||
func TestCause(t *testing.T) {
|
||||
ctx, cancel := WithCancelCause(Background())
|
||||
err := fmt.Errorf("oh no")
|
||||
cancel(err)
|
||||
assert.Equal(t, err, Cause(ctx))
|
||||
}
|
||||
|
||||
@@ -1,11 +1,11 @@
|
||||
package mattermostpersonaltoken
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
@@ -14,50 +14,44 @@ import (
|
||||
|
||||
type Scanner struct{}
|
||||
|
||||
// Ensure the Scanner satisfies the interface at compile time.
|
||||
var _ detectors.Detector = (*Scanner)(nil)
|
||||
|
||||
var (
|
||||
client = common.SaneHttpClient()
|
||||
|
||||
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
|
||||
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"mattermost"}) + `\b([a-z0-9]{26})\b`)
|
||||
serverPat = regexp.MustCompile(detectors.PrefixRegex([]string{"mattermost"}) + `\b([A-Za-z0-9-_]{1,}.cloud.mattermost.com)\b`)
|
||||
)
|
||||
|
||||
// Keywords are used for efficiently pre-filtering chunks.
|
||||
// Use identifiers in the secret preferably, or the provider name.
|
||||
func (s Scanner) Keywords() []string {
|
||||
return []string{"mattermost"}
|
||||
func (s Scanner) Keywords() [][]byte {
|
||||
return [][]byte{[]byte("mattermost")}
|
||||
}
|
||||
|
||||
// FromData will find and optionally verify MattermostPersonalToken secrets in a given set of bytes.
|
||||
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
|
||||
dataStr := string(data)
|
||||
|
||||
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
|
||||
serverMatches := serverPat.FindAllStringSubmatch(dataStr, -1)
|
||||
matches := keyPat.FindAllSubmatch(data, -1)
|
||||
serverMatches := serverPat.FindAllSubmatch(data, -1)
|
||||
|
||||
for _, match := range matches {
|
||||
if len(match) != 2 {
|
||||
continue
|
||||
}
|
||||
resMatch := strings.TrimSpace(match[1])
|
||||
resMatch := bytes.TrimSpace(match[1])
|
||||
|
||||
for _, serverMatch := range serverMatches {
|
||||
if len(serverMatch) != 2 {
|
||||
continue
|
||||
}
|
||||
serverRes := strings.TrimSpace(serverMatch[1])
|
||||
serverRes := bytes.TrimSpace(serverMatch[1])
|
||||
|
||||
s1 := detectors.Result{
|
||||
DetectorType: detectorspb.DetectorType_MattermostPersonalToken,
|
||||
Raw: []byte(resMatch),
|
||||
RawV2: []byte(resMatch + serverRes),
|
||||
Raw: resMatch,
|
||||
RawV2: append(resMatch, serverRes...),
|
||||
}
|
||||
|
||||
if verify {
|
||||
req, err := http.NewRequestWithContext(ctx, "GET", "https://"+serverRes+"/api/v4/users/stats", nil)
|
||||
req, err := http.NewRequestWithContext(ctx, "GET", fmt.Sprintf("https://%s/api/v4/users/stats", serverRes), nil)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
@@ -68,7 +62,6 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
if res.StatusCode >= 200 && res.StatusCode < 300 {
|
||||
s1.Verified = true
|
||||
} else {
|
||||
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
|
||||
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
|
||||
continue
|
||||
}
|
||||
|
||||
@@ -1,11 +1,11 @@
|
||||
package mavenlink
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
@@ -14,37 +14,30 @@ import (
|
||||
|
||||
type Scanner struct{}
|
||||
|
||||
// Ensure the Scanner satisfies the interface at compile time.
|
||||
var _ detectors.Detector = (*Scanner)(nil)
|
||||
|
||||
var (
|
||||
client = common.SaneHttpClient()
|
||||
|
||||
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
|
||||
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"mavenlink"}) + `\b([0-9a-z]{64})\b`)
|
||||
)
|
||||
|
||||
// Keywords are used for efficiently pre-filtering chunks.
|
||||
// Use identifiers in the secret preferably, or the provider name.
|
||||
func (s Scanner) Keywords() []string {
|
||||
return []string{"mavenlink"}
|
||||
func (s Scanner) Keywords() [][]byte {
|
||||
return [][]byte{[]byte("mavenlink")}
|
||||
}
|
||||
|
||||
// FromData will find and optionally verify Mavenlink secrets in a given set of bytes.
|
||||
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
|
||||
dataStr := string(data)
|
||||
|
||||
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
|
||||
matches := keyPat.FindAllSubmatch(data, -1)
|
||||
|
||||
for _, match := range matches {
|
||||
if len(match) != 2 {
|
||||
continue
|
||||
}
|
||||
resMatch := strings.TrimSpace(match[1])
|
||||
resMatch := bytes.TrimSpace(match[1])
|
||||
|
||||
s1 := detectors.Result{
|
||||
DetectorType: detectorspb.DetectorType_Mavenlink,
|
||||
Raw: []byte(resMatch),
|
||||
Raw: resMatch,
|
||||
}
|
||||
|
||||
if verify {
|
||||
@@ -52,14 +45,13 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", resMatch))
|
||||
req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", string(resMatch)))
|
||||
res, err := client.Do(req)
|
||||
if err == nil {
|
||||
defer res.Body.Close()
|
||||
if res.StatusCode >= 200 && res.StatusCode < 300 {
|
||||
s1.Verified = true
|
||||
} else {
|
||||
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
|
||||
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
|
||||
continue
|
||||
}
|
||||
|
||||
@@ -1,10 +1,10 @@
|
||||
package maxmindlicense
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
@@ -23,32 +23,27 @@ var (
|
||||
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"maxmind", "geoip"}) + `\b([0-9A-Za-z]{16})\b`)
|
||||
)
|
||||
|
||||
// Keywords are used for efficiently pre-filtering chunks.
|
||||
// Use identifiers in the secret preferably, or the provider name.
|
||||
func (s Scanner) Keywords() []string {
|
||||
return []string{"maxmind", "geoip"}
|
||||
func (s Scanner) Keywords() [][]byte {
|
||||
return [][]byte{[]byte("maxmind"), []byte("geoip")}
|
||||
}
|
||||
|
||||
// FromData will find and optionally verify MaxMindLicense secrets in a given set of bytes.
|
||||
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
|
||||
dataStr := string(data)
|
||||
|
||||
keyMatches := keyPat.FindAllStringSubmatch(dataStr, -1)
|
||||
idMatches := idPat.FindAllStringSubmatch(dataStr, -1)
|
||||
keyMatches := keyPat.FindAllSubmatch(data, -1)
|
||||
idMatches := idPat.FindAllSubmatch(data, -1)
|
||||
|
||||
for _, keyMatch := range keyMatches {
|
||||
keyRes := strings.TrimSpace(keyMatch[1])
|
||||
keyRes := bytes.TrimSpace(keyMatch[1])
|
||||
|
||||
for _, idMatch := range idMatches {
|
||||
if len(idMatch) != 2 {
|
||||
continue
|
||||
}
|
||||
idRes := strings.TrimSpace(idMatch[1])
|
||||
idRes := bytes.TrimSpace(idMatch[1])
|
||||
|
||||
s1 := detectors.Result{
|
||||
DetectorType: detectorspb.DetectorType_MaxMindLicense,
|
||||
Redacted: idRes,
|
||||
Raw: []byte(keyRes),
|
||||
Redacted: string(idRes),
|
||||
Raw: keyRes,
|
||||
}
|
||||
|
||||
if verify {
|
||||
@@ -56,7 +51,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
req.SetBasicAuth(idRes, keyRes)
|
||||
req.SetBasicAuth(string(idRes), string(keyRes))
|
||||
res, err := client.Do(req)
|
||||
if err == nil {
|
||||
defer res.Body.Close()
|
||||
|
||||
@@ -7,7 +7,6 @@ import (
|
||||
"mime/multipart"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
@@ -22,31 +21,25 @@ var _ detectors.Detector = (*Scanner)(nil)
|
||||
var (
|
||||
client = common.SaneHttpClient()
|
||||
|
||||
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
|
||||
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"meaningcloud"}) + `\b([a-z0-9]{32})\b`)
|
||||
)
|
||||
|
||||
// Keywords are used for efficiently pre-filtering chunks.
|
||||
// Use identifiers in the secret preferably, or the provider name.
|
||||
func (s Scanner) Keywords() []string {
|
||||
return []string{"meaningcloud"}
|
||||
func (s Scanner) Keywords() [][]byte {
|
||||
return [][]byte{[]byte("meaningcloud")}
|
||||
}
|
||||
|
||||
// FromData will find and optionally verify MeaningCloud secrets in a given set of bytes.
|
||||
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
|
||||
dataStr := string(data)
|
||||
|
||||
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
|
||||
matches := keyPat.FindAllSubmatch(data, -1)
|
||||
|
||||
for _, match := range matches {
|
||||
if len(match) != 2 {
|
||||
continue
|
||||
}
|
||||
resMatch := strings.TrimSpace(match[1])
|
||||
resMatch := bytes.TrimSpace(match[1])
|
||||
|
||||
s1 := detectors.Result{
|
||||
DetectorType: detectorspb.DetectorType_MeaningCloud,
|
||||
Raw: []byte(resMatch),
|
||||
Raw: resMatch,
|
||||
}
|
||||
|
||||
if verify {
|
||||
@@ -56,7 +49,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
_, err = io.Copy(fw, strings.NewReader(resMatch))
|
||||
_, err = io.Copy(fw, bytes.NewReader(resMatch))
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
@@ -64,12 +57,12 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
_, err = io.Copy(fw, strings.NewReader("test"))
|
||||
_, err = io.Copy(fw, bytes.NewReader([]byte("test")))
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
writer.Close()
|
||||
req, err := http.NewRequestWithContext(ctx, "POST", "https://api.meaningcloud.com/lang-4.0/identification", bytes.NewReader(body.Bytes()))
|
||||
req, err := http.NewRequestWithContext(ctx, "POST", "https://api.meaningcloud.com/-4.0/identification", bytes.NewReader(body.Bytes()))
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
@@ -80,7 +73,6 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
if res.StatusCode >= 200 && res.StatusCode < 300 {
|
||||
s1.Verified = true
|
||||
} else {
|
||||
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
|
||||
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
|
||||
continue
|
||||
}
|
||||
|
||||
@@ -1,12 +1,11 @@
|
||||
package mediastack
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
@@ -15,43 +14,35 @@ import (
|
||||
|
||||
type Scanner struct{}
|
||||
|
||||
// Ensure the Scanner satisfies the interface at compile time.
|
||||
var _ detectors.Detector = (*Scanner)(nil)
|
||||
|
||||
var (
|
||||
client = common.SaneHttpClient()
|
||||
|
||||
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
|
||||
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"mediastack"}) + `\b([a-z0-9]{32})\b`)
|
||||
)
|
||||
|
||||
// Keywords are used for efficiently pre-filtering chunks.
|
||||
// Use identifiers in the secret preferably, or the provider name.
|
||||
func (s Scanner) Keywords() []string {
|
||||
return []string{"mediastack"}
|
||||
func (s Scanner) Keywords() [][]byte {
|
||||
return [][]byte{[]byte("mediastack")}
|
||||
}
|
||||
|
||||
// FromData will find and optionally verify MediaStack secrets in a given set of bytes.
|
||||
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
|
||||
dataStr := string(data)
|
||||
|
||||
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
|
||||
matches := keyPat.FindAllSubmatch(data, -1)
|
||||
|
||||
for _, match := range matches {
|
||||
if len(match) != 2 {
|
||||
continue
|
||||
}
|
||||
resMatch := strings.TrimSpace(match[1])
|
||||
resMatch := bytes.TrimSpace(match[1])
|
||||
|
||||
s1 := detectors.Result{
|
||||
DetectorType: detectorspb.DetectorType_MediaStack,
|
||||
Raw: []byte(resMatch),
|
||||
Raw: resMatch,
|
||||
}
|
||||
|
||||
if verify {
|
||||
timeout := 10 * time.Second
|
||||
client.Timeout = timeout
|
||||
req, err := http.NewRequestWithContext(ctx, "GET", fmt.Sprintf("https://api.mediastack.com/v1/news?access_key=%s", resMatch), nil)
|
||||
req, err := http.NewRequestWithContext(ctx, "GET", fmt.Sprintf("https://api.mediastack.com/v1/news?access_key=%s", string(resMatch)), nil)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
@@ -61,7 +52,6 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
if res.StatusCode >= 200 && res.StatusCode < 300 {
|
||||
s1.Verified = true
|
||||
} else {
|
||||
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
|
||||
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
|
||||
continue
|
||||
}
|
||||
|
||||
@@ -1,11 +1,11 @@
|
||||
package meistertask
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
@@ -14,37 +14,29 @@ import (
|
||||
|
||||
type Scanner struct{}
|
||||
|
||||
// Ensure the Scanner satisfies the interface at compile time.
|
||||
var _ detectors.Detector = (*Scanner)(nil)
|
||||
|
||||
var (
|
||||
client = common.SaneHttpClient()
|
||||
|
||||
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
|
||||
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"meistertask"}) + `\b([a-zA-Z0-9]{43})\b`)
|
||||
)
|
||||
|
||||
// Keywords are used for efficiently pre-filtering chunks.
|
||||
// Use identifiers in the secret preferably, or the provider name.
|
||||
func (s Scanner) Keywords() []string {
|
||||
return []string{"meistertask"}
|
||||
func (s Scanner) Keywords() [][]byte {
|
||||
return [][]byte{[]byte("meistertask")}
|
||||
}
|
||||
|
||||
// FromData will find and optionally verify Meistertask secrets in a given set of bytes.
|
||||
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
|
||||
dataStr := string(data)
|
||||
|
||||
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
|
||||
matches := keyPat.FindAllSubmatch(data, -1)
|
||||
|
||||
for _, match := range matches {
|
||||
if len(match) != 2 {
|
||||
continue
|
||||
}
|
||||
resMatch := strings.TrimSpace(match[1])
|
||||
resMatch := bytes.TrimSpace(match[1])
|
||||
|
||||
s1 := detectors.Result{
|
||||
DetectorType: detectorspb.DetectorType_Meistertask,
|
||||
Raw: []byte(resMatch),
|
||||
Raw: resMatch,
|
||||
}
|
||||
|
||||
if verify {
|
||||
@@ -54,19 +46,17 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
}
|
||||
req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", resMatch))
|
||||
res, err := client.Do(req)
|
||||
if err == nil {
|
||||
if err == nil && res != nil {
|
||||
defer res.Body.Close()
|
||||
if res.StatusCode >= 200 && res.StatusCode < 300 {
|
||||
s1.Verified = true
|
||||
} else {
|
||||
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
|
||||
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
|
||||
continue
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
results = append(results, s1)
|
||||
}
|
||||
|
||||
|
||||
@@ -1,11 +1,11 @@
|
||||
package mesibo
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"io"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
@@ -13,44 +13,39 @@ import (
|
||||
)
|
||||
|
||||
type Scanner struct{}
|
||||
// Ensure the Scanner satisfies the interface at compile time.
|
||||
|
||||
var _ detectors.Detector = (*Scanner)(nil)
|
||||
|
||||
var (
|
||||
client = common.SaneHttpClient()
|
||||
|
||||
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
|
||||
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"mesibo"}) + `\b([0-9A-Za-z]{64})\b`)
|
||||
)
|
||||
|
||||
// Keywords are used for efficiently pre-filtering chunks.
|
||||
// Use identifiers in the secret preferably, or the provider name.
|
||||
func (s Scanner) Keywords() []string {
|
||||
return []string{"mesibo"}
|
||||
func (s Scanner) Keywords() [][]byte {
|
||||
return [][]byte{[]byte("mesibo")}
|
||||
}
|
||||
|
||||
// FromData will find and optionally verify Mesibo secrets in a given set of bytes.
|
||||
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
|
||||
dataStr := string(data)
|
||||
|
||||
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
|
||||
matches := keyPat.FindAllSubmatch(data, -1)
|
||||
|
||||
for _, match := range matches {
|
||||
if len(match) != 2 {
|
||||
continue
|
||||
}
|
||||
resMatch := strings.TrimSpace(match[1])
|
||||
|
||||
resMatch := bytes.TrimSpace(match[1])
|
||||
|
||||
s1 := detectors.Result{
|
||||
DetectorType: detectorspb.DetectorType_Mesibo,
|
||||
Raw: []byte(resMatch),
|
||||
Raw: resMatch,
|
||||
}
|
||||
|
||||
if verify {
|
||||
req, err := http.NewRequestWithContext(ctx, "GET", "https://api.mesibo.com/api.php?op=useradd&token="+resMatch, nil)
|
||||
req, err := http.NewRequestWithContext(ctx, "GET", "https://api.mesibo.com/api.php?op=useradd&token="+string(resMatch), nil)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
|
||||
res, err := client.Do(req)
|
||||
if err == nil {
|
||||
defer res.Body.Close()
|
||||
@@ -58,16 +53,14 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
body := string(bodyBytes)
|
||||
|
||||
if !strings.Contains(body, "AUTHFAIL") {
|
||||
if !bytes.Contains(bodyBytes, []byte("AUTHFAIL")) {
|
||||
s1.Verified = true
|
||||
} else {
|
||||
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -1,11 +1,11 @@
|
||||
package messagebird
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
@@ -14,36 +14,29 @@ import (
|
||||
|
||||
type Scanner struct{}
|
||||
|
||||
// Ensure the Scanner satisfies the interface at compile time.
|
||||
var _ detectors.Detector = (*Scanner)(nil)
|
||||
|
||||
var (
|
||||
client = common.SaneHttpClient()
|
||||
|
||||
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"messagebird"}) + `\b([A-Za-z0-9_-]{25})\b`)
|
||||
)
|
||||
|
||||
// Keywords are used for efficiently pre-filtering chunks.
|
||||
// Use identifiers in the secret preferably, or the provider name.
|
||||
func (s Scanner) Keywords() []string {
|
||||
return []string{"messagebird"}
|
||||
func (s Scanner) Keywords() [][]byte {
|
||||
return [][]byte{[]byte("messagebird")}
|
||||
}
|
||||
|
||||
// FromData will find and optionally verify MessageBird secrets in a given set of bytes.
|
||||
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
|
||||
dataStr := string(data)
|
||||
|
||||
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
|
||||
matches := keyPat.FindAllSubmatch(data, -1)
|
||||
|
||||
for _, match := range matches {
|
||||
if len(match) != 2 {
|
||||
continue
|
||||
}
|
||||
resMatch := strings.TrimSpace(match[1])
|
||||
resMatch := bytes.TrimSpace(match[1])
|
||||
|
||||
s1 := detectors.Result{
|
||||
DetectorType: detectorspb.DetectorType_MessageBird,
|
||||
Raw: []byte(resMatch),
|
||||
Raw: resMatch,
|
||||
}
|
||||
|
||||
if verify {
|
||||
@@ -51,7 +44,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
req.Header.Add("Authorization", fmt.Sprintf("AccessKey %s", resMatch))
|
||||
req.Header.Add("Authorization", fmt.Sprintf("AccessKey %s", string(resMatch)))
|
||||
res, err := client.Do(req)
|
||||
if err == nil {
|
||||
defer res.Body.Close()
|
||||
|
||||
@@ -1,12 +1,12 @@
|
||||
package metaapi
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
@@ -28,32 +28,29 @@ var (
|
||||
|
||||
// Keywords are used for efficiently pre-filtering chunks.
|
||||
// Use identifiers in the secret preferably, or the provider name.
|
||||
func (s Scanner) Keywords() []string {
|
||||
return []string{"metaapi", "meta-api"}
|
||||
func (s Scanner) Keywords() [][]byte {
|
||||
return [][]byte{[]byte("metaapi"), []byte("meta-api")}
|
||||
}
|
||||
|
||||
// FromData will find and optionally verify MetaAPI secrets in a given set of bytes.
|
||||
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
|
||||
dataStr := string(data)
|
||||
|
||||
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
|
||||
spellMatches := spellPat.FindAllStringSubmatch(dataStr, -1)
|
||||
matches := keyPat.FindAllSubmatch(data, -1)
|
||||
spellMatches := spellPat.FindAllSubmatch(data, -1)
|
||||
|
||||
for _, spellMatch := range spellMatches {
|
||||
if len(spellMatch) != 2 {
|
||||
continue
|
||||
}
|
||||
resSpellMatch := strings.TrimSpace(spellMatch[1])
|
||||
resSpellMatch := bytes.TrimSpace(spellMatch[1])
|
||||
|
||||
for _, match := range matches {
|
||||
if len(match) != 2 {
|
||||
continue
|
||||
}
|
||||
resMatch := strings.TrimSpace(match[1])
|
||||
resMatch := bytes.TrimSpace(match[1])
|
||||
|
||||
s1 := detectors.Result{
|
||||
DetectorType: detectorspb.DetectorType_MetaAPI,
|
||||
Raw: []byte(resMatch),
|
||||
Raw: resMatch,
|
||||
}
|
||||
|
||||
if verify {
|
||||
@@ -62,16 +59,14 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
continue
|
||||
}
|
||||
req.Header.Add("Content-Type", "application/json")
|
||||
req.Header.Add("apikey", resMatch)
|
||||
req.Header.Add("apikey", string(resMatch))
|
||||
res, err := client.Do(req)
|
||||
if err == nil {
|
||||
defer res.Body.Close()
|
||||
body, errBody := io.ReadAll(res.Body)
|
||||
|
||||
if errBody == nil {
|
||||
bodyStr := string(body)
|
||||
|
||||
if res.StatusCode >= 200 && res.StatusCode < 300 && strings.Contains(bodyStr, `"success":true`) {
|
||||
if res.StatusCode >= 200 && res.StatusCode < 300 && bytes.Contains(body, []byte(`"success":true`)) {
|
||||
s1.Verified = true
|
||||
} else {
|
||||
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
|
||||
|
||||
@@ -1,10 +1,10 @@
|
||||
package metrilo
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
@@ -13,54 +13,50 @@ import (
|
||||
|
||||
type Scanner struct{}
|
||||
|
||||
// Ensure the Scanner satisfies the interface at compile time.
|
||||
var _ detectors.Detector = (*Scanner)(nil)
|
||||
|
||||
var (
|
||||
client = common.SaneHttpClient()
|
||||
|
||||
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
|
||||
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"metrilo"}) + `\b([a-z0-9]{16})\b`)
|
||||
)
|
||||
|
||||
// Keywords are used for efficiently pre-filtering chunks.
|
||||
// Use identifiers in the secret preferably, or the provider name.
|
||||
func (s Scanner) Keywords() []string {
|
||||
return []string{"metrilo"}
|
||||
func (s Scanner) Keywords() [][]byte {
|
||||
return [][]byte{[]byte("metrilo")}
|
||||
}
|
||||
|
||||
// FromData will find and optionally verify Metrilo secrets in a given set of bytes.
|
||||
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
|
||||
dataStr := string(data)
|
||||
|
||||
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
|
||||
matches := keyPat.FindAllSubmatch(data, -1)
|
||||
|
||||
for _, match := range matches {
|
||||
if len(match) != 2 {
|
||||
continue
|
||||
}
|
||||
resMatch := strings.TrimSpace(match[1])
|
||||
resMatch := bytes.TrimSpace(match[1])
|
||||
|
||||
s1 := detectors.Result{
|
||||
DetectorType: detectorspb.DetectorType_Metrilo,
|
||||
Raw: []byte(resMatch),
|
||||
Raw: resMatch,
|
||||
}
|
||||
|
||||
if verify {
|
||||
payload := strings.NewReader(`{"time":1518004715732,"token": "` + resMatch + `" ,"platform":"Wordpress 4.2.7 / Woocommerce 3.5","pluginVersion":"1.1.0","params":{"id":"12","name":"Clothing","url":"https://dummysite.com"}}`)
|
||||
payload := bytes.NewBuffer([]byte(`{"time":1518004715732,"token": "` + string(resMatch) + `" ,"platform":"Wordpress 4.2.7 / Woocommerce 3.5","pluginVersion":"1.1.0","params":{"id":"12","name":"Clothing","url":"https://dummysite.com"}}`))
|
||||
|
||||
req, err := http.NewRequestWithContext(ctx, "POST", "https://trk.mtrl.me/category", payload)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
|
||||
req.Header.Add("Content-Type", "application/json")
|
||||
req.Header.Add("Content-Length", `0`)
|
||||
|
||||
res, err := client.Do(req)
|
||||
if err == nil {
|
||||
defer res.Body.Close()
|
||||
if res.StatusCode >= 200 && res.StatusCode < 300 {
|
||||
s1.Verified = true
|
||||
} else {
|
||||
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
|
||||
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
|
||||
continue
|
||||
}
|
||||
|
||||
@@ -1,11 +1,11 @@
|
||||
package microsoftteamswebhook
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"io"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
@@ -15,41 +15,34 @@ import (
|
||||
|
||||
type Scanner struct{}
|
||||
|
||||
// Ensure the Scanner satisfies the interface at compile time.
|
||||
var _ detectors.Detector = (*Scanner)(nil)
|
||||
|
||||
var (
|
||||
client = common.SaneHttpClientTimeOut(5 * time.Second)
|
||||
|
||||
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
|
||||
keyPat = regexp.MustCompile(`(https:\/\/[a-zA-Z-0-9]+\.webhook\.office\.com\/webhookb2\/[a-zA-Z-0-9]{8}-[a-zA-Z-0-9]{4}-[a-zA-Z-0-9]{4}-[a-zA-Z-0-9]{4}-[a-zA-Z-0-9]{12}\@[a-zA-Z-0-9]{8}-[a-zA-Z-0-9]{4}-[a-zA-Z-0-9]{4}-[a-zA-Z-0-9]{4}-[a-zA-Z-0-9]{12}\/IncomingWebhook\/[a-zA-Z-0-9]{32}\/[a-zA-Z-0-9]{8}-[a-zA-Z-0-9]{4}-[a-zA-Z-0-9]{4}-[a-zA-Z-0-9]{4}-[a-zA-Z-0-9]{12})`)
|
||||
)
|
||||
|
||||
// Keywords are used for efficiently pre-filtering chunks.
|
||||
// Use identifiers in the secret preferably, or the provider name.
|
||||
func (s Scanner) Keywords() []string {
|
||||
return []string{"webhook.office.com"}
|
||||
func (s Scanner) Keywords() [][]byte {
|
||||
return [][]byte{[]byte("webhook.office.com")}
|
||||
}
|
||||
|
||||
// FromData will find and optionally verify MicrosoftTeamsWebhook secrets in a given set of bytes.
|
||||
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
|
||||
dataStr := string(data)
|
||||
|
||||
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
|
||||
matches := keyPat.FindAllSubmatch(data, -1)
|
||||
|
||||
for _, match := range matches {
|
||||
if len(match) != 2 {
|
||||
continue
|
||||
}
|
||||
resMatch := strings.TrimSpace(match[1])
|
||||
|
||||
resMatch := bytes.TrimSpace(match[1])
|
||||
s1 := detectors.Result{
|
||||
DetectorType: detectorspb.DetectorType_MicrosoftTeamsWebhook,
|
||||
Raw: []byte(resMatch),
|
||||
Raw: resMatch,
|
||||
}
|
||||
|
||||
if verify {
|
||||
payload := strings.NewReader(`{'text':''}`)
|
||||
req, err := http.NewRequestWithContext(ctx, "POST", resMatch, payload)
|
||||
payload := bytes.NewReader([]byte("{'text':''}"))
|
||||
req, err := http.NewRequestWithContext(ctx, "POST", string(resMatch), payload)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
@@ -57,9 +50,9 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
res, err := client.Do(req)
|
||||
if err == nil {
|
||||
body, err := io.ReadAll(res.Body)
|
||||
res.Body.Close()
|
||||
defer res.Body.Close()
|
||||
if err == nil {
|
||||
if res.StatusCode >= 200 && strings.Contains(string(body), "Text is required") {
|
||||
if res.StatusCode >= 200 && bytes.Contains(body, []byte("Text is required")) {
|
||||
s1.Verified = true
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,12 +1,12 @@
|
||||
package mindmeister
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
@@ -21,31 +21,25 @@ var _ detectors.Detector = (*Scanner)(nil)
|
||||
var (
|
||||
client = common.SaneHttpClient()
|
||||
|
||||
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
|
||||
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"mindmeister"}) + `\b([a-zA-Z0-9]{43})\b`)
|
||||
)
|
||||
|
||||
// Keywords are used for efficiently pre-filtering chunks.
|
||||
// Use identifiers in the secret preferably, or the provider name.
|
||||
func (s Scanner) Keywords() []string {
|
||||
return []string{"mindmeister"}
|
||||
func (s Scanner) Keywords() [][]byte {
|
||||
return [][]byte{[]byte("mindmeister")}
|
||||
}
|
||||
|
||||
// FromData will find and optionally verify Mindmeister secrets in a given set of bytes.
|
||||
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
|
||||
dataStr := string(data)
|
||||
|
||||
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
|
||||
matches := keyPat.FindAllSubmatch(data, -1)
|
||||
|
||||
for _, match := range matches {
|
||||
if len(match) != 2 {
|
||||
continue
|
||||
}
|
||||
resMatch := strings.TrimSpace(match[1])
|
||||
resMatch := bytes.TrimSpace(match[1])
|
||||
|
||||
s1 := detectors.Result{
|
||||
DetectorType: detectorspb.DetectorType_Mindmeister,
|
||||
Raw: []byte(resMatch),
|
||||
Raw: resMatch,
|
||||
}
|
||||
|
||||
if verify {
|
||||
@@ -53,13 +47,13 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", resMatch))
|
||||
req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", string(resMatch)))
|
||||
res, err := client.Do(req)
|
||||
|
||||
if err == nil {
|
||||
bodyBytes, err := io.ReadAll(res.Body)
|
||||
if err == nil {
|
||||
bodyString := string(bodyBytes)
|
||||
validResponse := strings.Contains(bodyString, `"stat":"ok"`)
|
||||
validResponse := bytes.Contains(bodyBytes, []byte(`"stat":"ok"`))
|
||||
defer res.Body.Close()
|
||||
if res.StatusCode >= 200 && res.StatusCode < 300 {
|
||||
if validResponse {
|
||||
@@ -68,7 +62,6 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
s1.Verified = false
|
||||
}
|
||||
} else {
|
||||
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
|
||||
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
|
||||
continue
|
||||
}
|
||||
|
||||
@@ -1,11 +1,11 @@
|
||||
package miro
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
@@ -14,37 +14,30 @@ import (
|
||||
|
||||
type Scanner struct{}
|
||||
|
||||
// Ensure the Scanner satisfies the interface at compile time.
|
||||
var _ detectors.Detector = (*Scanner)(nil)
|
||||
|
||||
var (
|
||||
client = common.SaneHttpClient()
|
||||
|
||||
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
|
||||
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"miro"}) + `\b([0-9a-zA-Z]{27})\b`)
|
||||
)
|
||||
|
||||
// Keywords are used for efficiently pre-filtering chunks.
|
||||
// Use identifiers in the secret preferably, or the provider name.
|
||||
func (s Scanner) Keywords() []string {
|
||||
return []string{"miro"}
|
||||
func (s Scanner) Keywords() [][]byte {
|
||||
return [][]byte{[]byte("miro")}
|
||||
}
|
||||
|
||||
// FromData will find and optionally verify Miro secrets in a given set of bytes.
|
||||
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
|
||||
dataStr := string(data)
|
||||
|
||||
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
|
||||
matches := keyPat.FindAllSubmatch(data, -1)
|
||||
|
||||
for _, match := range matches {
|
||||
if len(match) != 2 {
|
||||
continue
|
||||
}
|
||||
resMatch := strings.TrimSpace(match[1])
|
||||
resMatch := bytes.TrimSpace(match[1])
|
||||
|
||||
s1 := detectors.Result{
|
||||
DetectorType: detectorspb.DetectorType_Miro,
|
||||
Raw: []byte(resMatch),
|
||||
Raw: resMatch,
|
||||
}
|
||||
|
||||
if verify {
|
||||
@@ -60,7 +53,6 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
if res.StatusCode >= 200 && res.StatusCode < 300 {
|
||||
s1.Verified = true
|
||||
} else {
|
||||
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
|
||||
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
|
||||
continue
|
||||
}
|
||||
|
||||
+10
-18
@@ -1,11 +1,11 @@
|
||||
package mite
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
@@ -14,60 +14,52 @@ import (
|
||||
|
||||
type Scanner struct{}
|
||||
|
||||
// Ensure the Scanner satisfies the interface at compile time.
|
||||
var _ detectors.Detector = (*Scanner)(nil)
|
||||
|
||||
var (
|
||||
client = common.SaneHttpClient()
|
||||
|
||||
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
|
||||
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"mite"}) + `\b([0-9a-z]{16})\b`)
|
||||
urlPat = regexp.MustCompile(`\b([0-9a-z-]{1,}.mite.yo.lk)\b`)
|
||||
)
|
||||
|
||||
// Keywords are used for efficiently pre-filtering chunks.
|
||||
// Use identifiers in the secret preferably, or the provider name.
|
||||
func (s Scanner) Keywords() []string {
|
||||
return []string{"mite"}
|
||||
func (s Scanner) Keywords() [][]byte {
|
||||
return [][]byte{[]byte("mite")}
|
||||
}
|
||||
|
||||
// FromData will find and optionally verify Mite secrets in a given set of bytes.
|
||||
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
|
||||
dataStr := string(data)
|
||||
|
||||
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
|
||||
urlMatches := urlPat.FindAllStringSubmatch(dataStr, -1)
|
||||
matches := keyPat.FindAllSubmatch(data, -1)
|
||||
urlMatches := urlPat.FindAllSubmatch(data, -1)
|
||||
|
||||
for _, match := range matches {
|
||||
if len(match) != 2 {
|
||||
continue
|
||||
}
|
||||
resMatch := strings.TrimSpace(match[1])
|
||||
resMatch := bytes.TrimSpace(match[1])
|
||||
for _, urlMatch := range urlMatches {
|
||||
if len(urlMatch) != 2 {
|
||||
continue
|
||||
}
|
||||
resURL := strings.TrimSpace(urlMatch[1])
|
||||
resURL := bytes.TrimSpace(urlMatch[1])
|
||||
|
||||
s1 := detectors.Result{
|
||||
DetectorType: detectorspb.DetectorType_Mite,
|
||||
Raw: []byte(resMatch),
|
||||
Raw: resMatch,
|
||||
}
|
||||
|
||||
if verify {
|
||||
req, err := http.NewRequestWithContext(ctx, "GET", fmt.Sprintf("https://%s/account.json", resURL), nil)
|
||||
req, err := http.NewRequestWithContext(ctx, "GET", fmt.Sprintf("https://%s/account.json", string(resURL)), nil)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
req.Header.Add("Content-Type", "application/json")
|
||||
req.Header.Add("X-MiteApiKey", resMatch)
|
||||
req.Header.Add("X-MiteApiKey", string(resMatch))
|
||||
res, err := client.Do(req)
|
||||
if err == nil {
|
||||
defer res.Body.Close()
|
||||
if res.StatusCode >= 200 && res.StatusCode < 300 {
|
||||
s1.Verified = true
|
||||
} else {
|
||||
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
|
||||
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
|
||||
continue
|
||||
}
|
||||
|
||||
@@ -1,10 +1,10 @@
|
||||
package mixmax
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
@@ -19,31 +19,25 @@ var _ detectors.Detector = (*Scanner)(nil)
|
||||
var (
|
||||
client = common.SaneHttpClient()
|
||||
|
||||
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
|
||||
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"mixmax"}) + `\b([a-zA-Z0-9_-]{36})\b`)
|
||||
)
|
||||
|
||||
// Keywords are used for efficiently pre-filtering chunks.
|
||||
// Use identifiers in the secret preferably, or the provider name.
|
||||
func (s Scanner) Keywords() []string {
|
||||
return []string{"mixmax"}
|
||||
func (s Scanner) Keywords() [][]byte {
|
||||
return [][]byte{[]byte("mixmax")}
|
||||
}
|
||||
|
||||
// FromData will find and optionally verify Mixmax secrets in a given set of bytes.
|
||||
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
|
||||
dataStr := string(data)
|
||||
|
||||
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
|
||||
matches := keyPat.FindAllSubmatch(data, -1)
|
||||
|
||||
for _, match := range matches {
|
||||
if len(match) != 2 {
|
||||
continue
|
||||
}
|
||||
resMatch := strings.TrimSpace(match[1])
|
||||
resMatch := bytes.TrimSpace(match[1])
|
||||
|
||||
s1 := detectors.Result{
|
||||
DetectorType: detectorspb.DetectorType_Mixmax,
|
||||
Raw: []byte(resMatch),
|
||||
Raw: resMatch,
|
||||
}
|
||||
|
||||
if verify {
|
||||
@@ -52,14 +46,13 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
continue
|
||||
}
|
||||
|
||||
req.Header.Add("X-API-Token", resMatch)
|
||||
req.Header.Add("X-API-Token", string(resMatch))
|
||||
res, err := client.Do(req)
|
||||
if err == nil {
|
||||
defer res.Body.Close()
|
||||
if res.StatusCode >= 200 && res.StatusCode < 300 {
|
||||
s1.Verified = true
|
||||
} else {
|
||||
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
|
||||
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
|
||||
continue
|
||||
}
|
||||
|
||||
@@ -1,10 +1,10 @@
|
||||
package mixpanel
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
@@ -13,46 +13,39 @@ import (
|
||||
|
||||
type Scanner struct{}
|
||||
|
||||
// Ensure the Scanner satisfies the interface at compile time.
|
||||
var _ detectors.Detector = (*Scanner)(nil)
|
||||
|
||||
var (
|
||||
client = common.SaneHttpClient()
|
||||
|
||||
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
|
||||
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"mixpanel"}) + `\b([a-zA-Z0-9-]{32})\b`)
|
||||
idPat = regexp.MustCompile(detectors.PrefixRegex([]string{"mixpanel"}) + `\b([a-zA-Z0-9.-]{30,40})\b`)
|
||||
)
|
||||
|
||||
// Keywords are used for efficiently pre-filtering chunks.
|
||||
// Use identifiers in the secret preferably, or the provider name.
|
||||
func (s Scanner) Keywords() []string {
|
||||
return []string{"mixpanel"}
|
||||
func (s Scanner) Keywords() [][]byte {
|
||||
return [][]byte{[]byte("mixpanel")}
|
||||
}
|
||||
|
||||
// FromData will find and optionally verify Mixpanel secrets in a given set of bytes.
|
||||
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
|
||||
dataStr := string(data)
|
||||
|
||||
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
|
||||
idMatches := idPat.FindAllStringSubmatch(dataStr, -1)
|
||||
matches := keyPat.FindAllSubmatch(data, -1)
|
||||
idMatches := idPat.FindAllSubmatch(data, -1)
|
||||
|
||||
for _, match := range matches {
|
||||
if len(match) != 2 {
|
||||
continue
|
||||
}
|
||||
tokenPatMatch := strings.TrimSpace(match[1])
|
||||
tokenPatMatch := bytes.TrimSpace(match[1])
|
||||
|
||||
for _, idMatch := range idMatches {
|
||||
if len(idMatch) != 2 {
|
||||
continue
|
||||
}
|
||||
|
||||
userPatMatch := strings.TrimSpace(idMatch[1])
|
||||
userPatMatch := bytes.TrimSpace(idMatch[1])
|
||||
|
||||
s1 := detectors.Result{
|
||||
DetectorType: detectorspb.DetectorType_Mixpanel,
|
||||
Raw: []byte(tokenPatMatch),
|
||||
Raw: tokenPatMatch,
|
||||
}
|
||||
|
||||
if verify {
|
||||
@@ -60,14 +53,13 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
req.SetBasicAuth(userPatMatch, tokenPatMatch)
|
||||
req.SetBasicAuth(string(userPatMatch), string(tokenPatMatch))
|
||||
res, err := client.Do(req)
|
||||
if err == nil {
|
||||
defer res.Body.Close()
|
||||
if res.StatusCode >= 200 && res.StatusCode < 300 {
|
||||
s1.Verified = true
|
||||
} else {
|
||||
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
|
||||
if detectors.IsKnownFalsePositive(tokenPatMatch, detectors.DefaultFalsePositives, true) {
|
||||
continue
|
||||
}
|
||||
|
||||
@@ -1,12 +1,11 @@
|
||||
package mockaroo
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
@@ -27,29 +26,27 @@ var (
|
||||
|
||||
// Keywords are used for efficiently pre-filtering chunks.
|
||||
// Use identifiers in the secret preferably, or the provider name.
|
||||
func (s Scanner) Keywords() []string {
|
||||
return []string{"mockaroo"}
|
||||
func (s Scanner) Keywords() [][]byte {
|
||||
return [][]byte{[]byte("mockaroo")}
|
||||
}
|
||||
|
||||
// FromData will find and optionally verify Mockaroo secrets in a given set of bytes.
|
||||
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
|
||||
dataStr := string(data)
|
||||
|
||||
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
|
||||
matches := keyPat.FindAllSubmatch(data, -1)
|
||||
|
||||
for _, match := range matches {
|
||||
if len(match) != 2 {
|
||||
continue
|
||||
}
|
||||
resMatch := strings.TrimSpace(match[1])
|
||||
resMatch := bytes.TrimSpace(match[1])
|
||||
|
||||
s1 := detectors.Result{
|
||||
DetectorType: detectorspb.DetectorType_Mockaroo,
|
||||
Raw: []byte(resMatch),
|
||||
Raw: resMatch,
|
||||
}
|
||||
|
||||
if verify {
|
||||
req, err := http.NewRequestWithContext(ctx, "GET", fmt.Sprintf("https://api.mockaroo.com/api/types?key=%s", resMatch), nil)
|
||||
req, err := http.NewRequestWithContext(ctx, "GET", fmt.Sprintf("https://api.mockaroo.com/api/types?key=%s", string(resMatch)), nil)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
@@ -57,13 +54,8 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
if err == nil {
|
||||
defer res.Body.Close()
|
||||
if res.StatusCode >= 200 && res.StatusCode < 300 {
|
||||
var t typeRes
|
||||
err = json.NewDecoder(res.Body).Decode(&t)
|
||||
if err == nil && len(t.Types) > 0 {
|
||||
s1.Verified = true
|
||||
}
|
||||
s1.Verified = true
|
||||
} else {
|
||||
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
|
||||
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
|
||||
continue
|
||||
}
|
||||
@@ -80,11 +72,3 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
func (s Scanner) Type() detectorspb.DetectorType {
|
||||
return detectorspb.DetectorType_Mockaroo
|
||||
}
|
||||
|
||||
type typeRes struct {
|
||||
Types []struct {
|
||||
Name string `json:"name"`
|
||||
Type interface{} `json:"type"`
|
||||
Parameters []interface{} `json:"parameters"`
|
||||
} `json:"types"`
|
||||
}
|
||||
|
||||
@@ -1,11 +1,11 @@
|
||||
package moderation
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
@@ -27,36 +27,34 @@ var (
|
||||
|
||||
// Keywords are used for efficiently pre-filtering chunks.
|
||||
// Use identifiers in the secret preferably, or the provider name.
|
||||
func (s Scanner) Keywords() []string {
|
||||
return []string{"moderation"}
|
||||
func (s Scanner) Keywords() [][]byte {
|
||||
return [][]byte{[]byte("moderation")}
|
||||
}
|
||||
|
||||
// FromData will find and optionally verify Moderation secrets in a given set of bytes.
|
||||
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
|
||||
dataStr := string(data)
|
||||
|
||||
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
|
||||
matches := keyPat.FindAllSubmatch(data, -1)
|
||||
|
||||
for _, match := range matches {
|
||||
if len(match) != 2 {
|
||||
continue
|
||||
}
|
||||
resMatch := strings.TrimSpace(match[1])
|
||||
resMatch := bytes.TrimSpace(match[1])
|
||||
|
||||
s1 := detectors.Result{
|
||||
DetectorType: detectorspb.DetectorType_Moderation,
|
||||
Raw: []byte(resMatch),
|
||||
Raw: resMatch,
|
||||
}
|
||||
|
||||
if verify {
|
||||
timeout := 5 * time.Second
|
||||
client.Timeout = timeout
|
||||
payload := strings.NewReader(`{"value": "This is an english text"}`)
|
||||
payload := bytes.NewReader([]byte(`{"value": "This is an english text"}`))
|
||||
req, err := http.NewRequestWithContext(ctx, "POST", "https://moderationapi.com/api/v1/analyze/language", payload)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", resMatch))
|
||||
req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", string(resMatch)))
|
||||
req.Header.Add("Content-type", "application/json")
|
||||
res, err := client.Do(req)
|
||||
|
||||
|
||||
@@ -1,10 +1,10 @@
|
||||
package monday
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
@@ -13,54 +13,45 @@ import (
|
||||
|
||||
type Scanner struct{}
|
||||
|
||||
// Ensure the Scanner satisfies the interface at compile time.
|
||||
var _ detectors.Detector = (*Scanner)(nil)
|
||||
|
||||
var (
|
||||
client = common.SaneHttpClient()
|
||||
|
||||
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
|
||||
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"monday"}) + `\b(ey[a-zA-Z0-9_.]{210,225})\b`)
|
||||
)
|
||||
|
||||
// Keywords are used for efficiently pre-filtering chunks.
|
||||
// Use identifiers in the secret preferably, or the provider name.
|
||||
func (s Scanner) Keywords() []string {
|
||||
return []string{"monday"}
|
||||
func (s Scanner) Keywords() [][]byte {
|
||||
return [][]byte{[]byte("monday")}
|
||||
}
|
||||
|
||||
// FromData will find and optionally verify Monday secrets in a given set of bytes.
|
||||
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
|
||||
dataStr := string(data)
|
||||
|
||||
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
|
||||
matches := keyPat.FindAllSubmatch(data, -1)
|
||||
|
||||
for _, match := range matches {
|
||||
if len(match) != 2 {
|
||||
continue
|
||||
}
|
||||
resMatch := strings.TrimSpace(match[1])
|
||||
resMatch := bytes.TrimSpace(match[1])
|
||||
|
||||
s1 := detectors.Result{
|
||||
DetectorType: detectorspb.DetectorType_Monday,
|
||||
Raw: []byte(resMatch),
|
||||
Raw: resMatch,
|
||||
}
|
||||
|
||||
if verify {
|
||||
payload := strings.NewReader(`{"query":"{boards(limit:1){id name}}"}`)
|
||||
payload := bytes.NewReader([]byte(`{"query":"{boards(limit:1){id name}}"}`))
|
||||
req, err := http.NewRequestWithContext(ctx, "POST", "https://api.monday.com/v2", payload)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
req.Header.Add("Content-Type", "application/json")
|
||||
req.Header.Add("Authorization", resMatch)
|
||||
req.Header.Add("Authorization", string(resMatch))
|
||||
res, err := client.Do(req)
|
||||
if err == nil {
|
||||
defer res.Body.Close()
|
||||
if res.StatusCode >= 200 && res.StatusCode < 300 {
|
||||
s1.Verified = true
|
||||
} else {
|
||||
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
|
||||
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
|
||||
continue
|
||||
}
|
||||
|
||||
@@ -1,16 +1,16 @@
|
||||
package mongodb
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"go.mongodb.org/mongo-driver/x/mongo/driver/auth"
|
||||
"go.mongodb.org/mongo-driver/x/mongo/driver/topology"
|
||||
"regexp"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"go.mongodb.org/mongo-driver/mongo"
|
||||
"go.mongodb.org/mongo-driver/mongo/options"
|
||||
"go.mongodb.org/mongo-driver/mongo/readpref"
|
||||
"go.mongodb.org/mongo-driver/x/mongo/driver/auth"
|
||||
"go.mongodb.org/mongo-driver/x/mongo/driver/topology"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
|
||||
@@ -26,28 +26,26 @@ var _ detectors.Detector = (*Scanner)(nil)
|
||||
var (
|
||||
defaultTimeout = 2 * time.Second
|
||||
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
|
||||
keyPat = regexp.MustCompile(`\b(mongodb(\+srv)?://[\S]{3,50}:([\S]{3,88})@[-.%\w\/:]+)\b`)
|
||||
keyPat = regexp.MustCompile(`\b(mongodb(\+srv)?://\S{3,50}:(\S{3,50})@[-.%\w/:]+)\b`)
|
||||
// TODO: Add support for sharded cluster, replica set and Atlas Deployment.
|
||||
)
|
||||
|
||||
// Keywords are used for efficiently pre-filtering chunks.
|
||||
// Use identifiers in the secret preferably, or the provider name.
|
||||
func (s Scanner) Keywords() []string {
|
||||
return []string{"mongodb"}
|
||||
func (s Scanner) Keywords() [][]byte {
|
||||
return [][]byte{[]byte("mongodb")}
|
||||
}
|
||||
|
||||
// FromData will find and optionally verify MongoDB secrets in a given set of bytes.
|
||||
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
|
||||
dataStr := string(data)
|
||||
|
||||
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
|
||||
matches := keyPat.FindAllSubmatch(data, -1)
|
||||
|
||||
for _, match := range matches {
|
||||
resMatch := strings.TrimSpace(match[1])
|
||||
resMatch := bytes.TrimSpace(match[1])
|
||||
|
||||
s1 := detectors.Result{
|
||||
DetectorType: detectorspb.DetectorType_MongoDB,
|
||||
Raw: []byte(resMatch),
|
||||
Raw: resMatch,
|
||||
}
|
||||
|
||||
if verify {
|
||||
@@ -55,7 +53,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
if timeout == 0 {
|
||||
timeout = defaultTimeout
|
||||
}
|
||||
err := verifyUri(resMatch, timeout)
|
||||
err := verifyUri(string(resMatch), timeout)
|
||||
s1.Verified = err == nil
|
||||
if !isErrDeterminate(err) {
|
||||
s1.VerificationError = err
|
||||
|
||||
@@ -43,6 +43,8 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
instanceMatches := instancePat.FindAllStringSubmatch(dataStr, -1)
|
||||
tokenMatches := accessTokenPat.FindAllStringSubmatch(dataStr, -1)
|
||||
|
||||
fmt.Printf("instanceMatches: %v\n", instanceMatches)
|
||||
|
||||
for _, instance := range instanceMatches {
|
||||
if len(instance) != 1 {
|
||||
continue
|
||||
|
||||
@@ -2,7 +2,6 @@ package slackwebhook
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"regexp"
|
||||
@@ -13,15 +12,14 @@ import (
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
|
||||
)
|
||||
|
||||
type Scanner struct {
|
||||
client *http.Client
|
||||
}
|
||||
type Scanner struct{}
|
||||
|
||||
// Ensure the Scanner satisfies the interface at compile time.
|
||||
var _ detectors.Detector = (*Scanner)(nil)
|
||||
|
||||
var (
|
||||
defaultClient = common.SaneHttpClient()
|
||||
client = common.SaneHttpClient()
|
||||
|
||||
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
|
||||
keyPat = regexp.MustCompile(`(https://hooks\.slack\.com/services/T[A-Z0-9]+/B[A-Z0-9]+/[A-Za-z0-9]{23,25})`)
|
||||
)
|
||||
@@ -50,12 +48,6 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
}
|
||||
|
||||
if verify {
|
||||
|
||||
client := s.client
|
||||
if client == nil {
|
||||
client = defaultClient
|
||||
}
|
||||
|
||||
payload := strings.NewReader(`{"text": ""}`)
|
||||
req, err := http.NewRequestWithContext(ctx, "POST", resMatch, payload)
|
||||
if err != nil {
|
||||
@@ -70,18 +62,9 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
continue
|
||||
}
|
||||
body := string(bodyBytes)
|
||||
|
||||
defer res.Body.Close()
|
||||
|
||||
if res.StatusCode >= 200 && res.StatusCode < 300 || (res.StatusCode == 400 && (strings.Contains(body, "no_text") || strings.Contains(body, "missing_text"))) {
|
||||
if (res.StatusCode >= 200 && res.StatusCode < 300) || (res.StatusCode == 400 && (strings.Contains(body, "no_text") || strings.Contains(body, "missing_text"))) {
|
||||
s1.Verified = true
|
||||
} else if res.StatusCode == 401 || res.StatusCode == 403 {
|
||||
// The secret is determinately not verified (nothing to do)
|
||||
} else {
|
||||
s1.VerificationError = fmt.Errorf("unexpected HTTP response status %d", res.StatusCode)
|
||||
}
|
||||
} else {
|
||||
s1.VerificationError = err
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -6,11 +6,10 @@ package slackwebhook
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"github.com/google/go-cmp/cmp"
|
||||
"github.com/google/go-cmp/cmp/cmpopts"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/kylelemons/godebug/pretty"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
@@ -33,12 +32,11 @@ func TestSlackWebhook_FromChunk(t *testing.T) {
|
||||
verify bool
|
||||
}
|
||||
tests := []struct {
|
||||
name string
|
||||
s Scanner
|
||||
args args
|
||||
want []detectors.Result
|
||||
wantErr bool
|
||||
wantVerificationErr bool
|
||||
name string
|
||||
s Scanner
|
||||
args args
|
||||
want []detectors.Result
|
||||
wantErr bool
|
||||
}{
|
||||
{
|
||||
name: "found, verified",
|
||||
@@ -54,8 +52,7 @@ func TestSlackWebhook_FromChunk(t *testing.T) {
|
||||
Verified: true,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
wantVerificationErr: false,
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "found, unverified",
|
||||
@@ -71,8 +68,7 @@ func TestSlackWebhook_FromChunk(t *testing.T) {
|
||||
Verified: false,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
wantVerificationErr: false,
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "not found",
|
||||
@@ -82,48 +78,14 @@ func TestSlackWebhook_FromChunk(t *testing.T) {
|
||||
data: []byte("You cannot find the secret within"),
|
||||
verify: true,
|
||||
},
|
||||
want: nil,
|
||||
wantErr: false,
|
||||
wantVerificationErr: false,
|
||||
},
|
||||
{
|
||||
name: "found, would be verified if not for timeout",
|
||||
s: Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a slackwebhook secret %s within", secret)),
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_SlackWebhook,
|
||||
Verified: false,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
wantVerificationErr: true,
|
||||
},
|
||||
{
|
||||
name: "found, verified but unexpected api surface",
|
||||
s: Scanner{client: common.ConstantResponseHttpClient(404, "")},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a slackwebhook secret %s within", secret)),
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_SlackWebhook,
|
||||
Verified: false,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
wantVerificationErr: true,
|
||||
want: nil,
|
||||
wantErr: false,
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
|
||||
s := Scanner{}
|
||||
got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
|
||||
if (err != nil) != tt.wantErr {
|
||||
t.Errorf("SlackWebhook.FromData() error = %v, wantErr %v", err, tt.wantErr)
|
||||
return
|
||||
@@ -132,12 +94,9 @@ func TestSlackWebhook_FromChunk(t *testing.T) {
|
||||
if len(got[i].Raw) == 0 {
|
||||
t.Fatalf("no raw secret present: \n %+v", got[i])
|
||||
}
|
||||
if (got[i].VerificationError != nil) != tt.wantVerificationErr {
|
||||
t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError)
|
||||
}
|
||||
got[i].Raw = nil
|
||||
}
|
||||
ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "VerificationError")
|
||||
if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {
|
||||
if diff := pretty.Compare(got, tt.want); diff != "" {
|
||||
t.Errorf("SlackWebhook.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
|
||||
}
|
||||
})
|
||||
|
||||
@@ -2,10 +2,6 @@ package engine
|
||||
|
||||
import (
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/envoyapikey"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/huggingface"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/salesforce"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/snowflake"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/trufflehogenterprise"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/abbysale"
|
||||
@@ -291,7 +287,6 @@ import (
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/getresponse"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/getsandbox"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/github"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/github_oauth2"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/github_old"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/githubapp"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/gitlab"
|
||||
@@ -561,6 +556,7 @@ import (
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/satismeterwritekey"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/saucelabs"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/scalewaykey"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/github_oauth2"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/scalr"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/scrapeowl"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/scraperapi"
|
||||
@@ -1536,10 +1532,6 @@ func DefaultDetectors() []detectors.Detector {
|
||||
couchbase.Scanner{},
|
||||
envoyapikey.Scanner{},
|
||||
github_oauth2.Scanner{},
|
||||
snowflake.Scanner{},
|
||||
huggingface.Scanner{},
|
||||
trufflehogenterprise.Scanner{},
|
||||
salesforce.Scanner{},
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
+23
-24
@@ -29,8 +29,6 @@ var (
|
||||
maxDepth = 5
|
||||
maxSize = 250 * 1024 * 1024 // 20MB
|
||||
maxTimeout = time.Duration(30) * time.Second
|
||||
|
||||
defaultBufferSize = 512
|
||||
)
|
||||
|
||||
// Ensure the Archive satisfies the interfaces at compile time.
|
||||
@@ -64,7 +62,7 @@ func SetArchiveMaxTimeout(timeout time.Duration) {
|
||||
|
||||
// FromFile extracts the files from an archive.
|
||||
func (a *Archive) FromFile(originalCtx context.Context, data io.Reader) chan []byte {
|
||||
archiveChan := make(chan []byte, defaultBufferSize)
|
||||
archiveChan := make(chan []byte, 512)
|
||||
go func() {
|
||||
ctx, cancel := context.WithTimeout(originalCtx, maxTimeout)
|
||||
logger := logContext.AddLogger(ctx).Logger()
|
||||
@@ -185,28 +183,29 @@ func (a *Archive) ReadToMax(ctx context.Context, reader io.Reader) (data []byte,
|
||||
logger.Error(err, "Panic occurred when reading archive")
|
||||
}
|
||||
}()
|
||||
|
||||
if common.IsDone(ctx) {
|
||||
return nil, ctx.Err()
|
||||
fileContent := bytes.Buffer{}
|
||||
logger.V(5).Info("Remaining buffer capacity", "bytes", maxSize-a.size)
|
||||
for i := 0; i <= maxSize/512; i++ {
|
||||
if common.IsDone(ctx) {
|
||||
return nil, ctx.Err()
|
||||
}
|
||||
fileChunk := make([]byte, 512)
|
||||
bRead, err := reader.Read(fileChunk)
|
||||
if err != nil && !errors.Is(err, io.EOF) {
|
||||
return []byte{}, err
|
||||
}
|
||||
a.size += bRead
|
||||
if len(fileChunk) > 0 {
|
||||
fileContent.Write(fileChunk[0:bRead])
|
||||
}
|
||||
if bRead < 512 {
|
||||
return fileContent.Bytes(), nil
|
||||
}
|
||||
if a.size >= maxSize && bRead == 512 {
|
||||
logger.V(2).Info("Max archive size reached.")
|
||||
return fileContent.Bytes(), nil
|
||||
}
|
||||
}
|
||||
|
||||
var fileContent bytes.Buffer
|
||||
// Create a limited reader to ensure we don't read more than the max size.
|
||||
lr := io.LimitReader(reader, int64(maxSize))
|
||||
|
||||
// Using io.CopyBuffer for performance advantages. Though buf is mandatory
|
||||
// for the method, due to the internal implementation of io.CopyBuffer, when
|
||||
// *bytes.Buffer implements io.WriterTo or io.ReaderFrom, the provided buf
|
||||
// is simply ignored. Thus, we can pass nil for the buf parameter.
|
||||
_, err = io.CopyBuffer(&fileContent, lr, nil)
|
||||
if err != nil && !errors.Is(err, io.EOF) {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if fileContent.Len() == maxSize {
|
||||
logger.V(2).Info("Max archive size reached.")
|
||||
}
|
||||
|
||||
return fileContent.Bytes(), nil
|
||||
}
|
||||
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
package handlers
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"io"
|
||||
"net/http"
|
||||
@@ -127,58 +126,6 @@ func TestHandleFile(t *testing.T) {
|
||||
assert.Equal(t, 1, len(ch))
|
||||
}
|
||||
|
||||
func TestReadToMax(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
input []byte
|
||||
expected []byte
|
||||
}{
|
||||
{
|
||||
name: "read full content within maxSize",
|
||||
input: []byte("abcdefg"),
|
||||
expected: []byte("abcdefg"),
|
||||
},
|
||||
{
|
||||
name: "read content larger than maxSize",
|
||||
input: make([]byte, maxSize+10), // this creates a byte slice 10 bytes larger than maxSize
|
||||
expected: make([]byte, maxSize),
|
||||
},
|
||||
{
|
||||
name: "empty input",
|
||||
input: []byte(""),
|
||||
expected: []byte(""),
|
||||
},
|
||||
}
|
||||
|
||||
a := &Archive{}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
reader := bytes.NewReader(tt.input)
|
||||
output, err := a.ReadToMax(context.Background(), reader)
|
||||
assert.Nil(t, err)
|
||||
|
||||
assert.Equal(t, tt.expected, output)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkReadToMax(b *testing.B) {
|
||||
data := bytes.Repeat([]byte("a"), 1024*1000) // 1MB of data.
|
||||
reader := bytes.NewReader(data)
|
||||
a := &Archive{}
|
||||
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
b.StartTimer()
|
||||
_, _ = a.ReadToMax(context.Background(), reader)
|
||||
b.StopTimer()
|
||||
|
||||
_, _ = reader.Seek(0, 0) // Reset the reader position.
|
||||
a.size = 0 // Reset archive size.
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtractDebContent(t *testing.T) {
|
||||
// Open the sample .deb file from the testdata folder.
|
||||
file, err := os.Open("testdata/test.deb")
|
||||
|
||||
@@ -133,7 +133,7 @@ func BranchHeads(repo *gogit.Repository) (map[string]*object.Commit, error) {
|
||||
}
|
||||
headCommit, err := repo.CommitObject(*headHash)
|
||||
if err != nil {
|
||||
logger.Error(err, "unable to get commit", "head_hash", headHash.String())
|
||||
logger.Error(err, "unable to get commit", "commit", headCommit.String())
|
||||
return nil
|
||||
}
|
||||
branches[branchName] = headCommit
|
||||
|
||||
@@ -20,7 +20,7 @@ const (
|
||||
|
||||
// Chunker takes a chunk and splits it into chunks of ChunkSize.
|
||||
func Chunker(originalChunk *Chunk) chan *Chunk {
|
||||
chunkChan := make(chan *Chunk, 1)
|
||||
chunkChan := make(chan *Chunk)
|
||||
go func() {
|
||||
defer close(chunkChan)
|
||||
if len(originalChunk.Data) <= TotalChunkSize {
|
||||
|
||||
@@ -238,7 +238,6 @@ func (s *Source) chunkAction(ctx context.Context, proj project, bld build, act a
|
||||
SourceType: s.Type(),
|
||||
SourceName: s.name,
|
||||
SourceID: s.SourceID(),
|
||||
JobID: s.JobID(),
|
||||
Data: removeCircleSha1Line(data.Bytes()),
|
||||
SourceMetadata: &source_metadatapb.MetaData{
|
||||
Data: &source_metadatapb.MetaData_Circleci{
|
||||
|
||||
@@ -174,7 +174,6 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) err
|
||||
SourceType: s.Type(),
|
||||
SourceName: s.name,
|
||||
SourceID: s.SourceID(),
|
||||
JobID: s.JobID(),
|
||||
Data: file.Bytes(),
|
||||
SourceMetadata: &source_metadatapb.MetaData{
|
||||
Data: &source_metadatapb.MetaData_Docker{
|
||||
|
||||
@@ -161,7 +161,6 @@ func (s *Source) scanFile(ctx context.Context, path string, chunksChan chan *sou
|
||||
SourceType: s.Type(),
|
||||
SourceName: s.name,
|
||||
SourceID: s.SourceID(),
|
||||
JobID: s.JobID(),
|
||||
SourceMetadata: &source_metadatapb.MetaData{
|
||||
Data: &source_metadatapb.MetaData_Filesystem{
|
||||
Filesystem: &source_metadatapb.Filesystem{
|
||||
@@ -192,7 +191,6 @@ func (s *Source) scanFile(ctx context.Context, path string, chunksChan chan *sou
|
||||
SourceType: s.Type(),
|
||||
SourceName: s.name,
|
||||
SourceID: s.SourceID(),
|
||||
JobID: s.JobID(),
|
||||
Data: data.Bytes(),
|
||||
SourceMetadata: &source_metadatapb.MetaData{
|
||||
Data: &source_metadatapb.MetaData_Filesystem{
|
||||
|
||||
@@ -324,7 +324,6 @@ func (s *Source) processObject(ctx context.Context, o object) error {
|
||||
chunkSkel := &sources.Chunk{
|
||||
SourceName: s.name,
|
||||
SourceType: s.Type(),
|
||||
JobID: s.JobID(),
|
||||
SourceID: s.sourceId,
|
||||
Verify: s.verify,
|
||||
SourceMetadata: &source_metadatapb.MetaData{
|
||||
|
||||
@@ -446,7 +446,6 @@ func (s *Git) ScanCommits(ctx context.Context, repo *git.Repository, path string
|
||||
chunkSkel := &sources.Chunk{
|
||||
SourceName: s.sourceName,
|
||||
SourceID: s.sourceID,
|
||||
JobID: s.jobID,
|
||||
SourceType: s.sourceType,
|
||||
SourceMetadata: metadata,
|
||||
Verify: s.verify,
|
||||
@@ -465,7 +464,6 @@ func (s *Git) ScanCommits(ctx context.Context, repo *git.Repository, path string
|
||||
chunksChan <- &sources.Chunk{
|
||||
SourceName: s.sourceName,
|
||||
SourceID: s.sourceID,
|
||||
JobID: s.jobID,
|
||||
SourceType: s.sourceType,
|
||||
SourceMetadata: metadata,
|
||||
Data: diff.Content.Bytes(),
|
||||
@@ -492,7 +490,6 @@ func (s *Git) gitChunk(ctx context.Context, diff gitparse.Diff, fileName, email,
|
||||
chunksChan <- &sources.Chunk{
|
||||
SourceName: s.sourceName,
|
||||
SourceID: s.sourceID,
|
||||
JobID: s.jobID,
|
||||
SourceType: s.sourceType,
|
||||
SourceMetadata: metadata,
|
||||
Data: append([]byte{}, newChunkBuffer.Bytes()...),
|
||||
@@ -507,7 +504,6 @@ func (s *Git) gitChunk(ctx context.Context, diff gitparse.Diff, fileName, email,
|
||||
chunksChan <- &sources.Chunk{
|
||||
SourceName: s.sourceName,
|
||||
SourceID: s.sourceID,
|
||||
JobID: s.jobID,
|
||||
SourceType: s.sourceType,
|
||||
SourceMetadata: metadata,
|
||||
Data: line,
|
||||
@@ -527,7 +523,6 @@ func (s *Git) gitChunk(ctx context.Context, diff gitparse.Diff, fileName, email,
|
||||
chunksChan <- &sources.Chunk{
|
||||
SourceName: s.sourceName,
|
||||
SourceID: s.sourceID,
|
||||
JobID: s.jobID,
|
||||
SourceType: s.sourceType,
|
||||
SourceMetadata: metadata,
|
||||
Data: append([]byte{}, newChunkBuffer.Bytes()...),
|
||||
@@ -593,7 +588,6 @@ func (s *Git) ScanStaged(ctx context.Context, repo *git.Repository, path string,
|
||||
chunkSkel := &sources.Chunk{
|
||||
SourceName: s.sourceName,
|
||||
SourceID: s.sourceID,
|
||||
JobID: s.jobID,
|
||||
SourceType: s.sourceType,
|
||||
SourceMetadata: metadata,
|
||||
Verify: s.verify,
|
||||
@@ -608,7 +602,6 @@ func (s *Git) ScanStaged(ctx context.Context, repo *git.Repository, path string,
|
||||
chunksChan <- &sources.Chunk{
|
||||
SourceName: s.sourceName,
|
||||
SourceID: s.sourceID,
|
||||
JobID: s.jobID,
|
||||
SourceType: s.sourceType,
|
||||
SourceMetadata: metadata,
|
||||
Data: diff.Content.Bytes(),
|
||||
|
||||
@@ -149,16 +149,14 @@ func (s *Source) newFilteredRepoCache(c cache.Cache, include, exclude []string)
|
||||
for _, ig := range include {
|
||||
g, err := glob.Compile(ig)
|
||||
if err != nil {
|
||||
s.log.V(1).Info("invalid include glob", "include_value", ig, "err", err)
|
||||
continue
|
||||
s.log.V(1).Info("invalid include glob", "glob", g, "err", err)
|
||||
}
|
||||
includeGlobs = append(includeGlobs, g)
|
||||
}
|
||||
for _, eg := range exclude {
|
||||
g, err := glob.Compile(eg)
|
||||
if err != nil {
|
||||
s.log.V(1).Info("invalid exclude glob", "exclude_value", eg, "err", err)
|
||||
continue
|
||||
s.log.V(1).Info("invalid exclude glob", "glob", g, "err", err)
|
||||
}
|
||||
excludeGlobs = append(excludeGlobs, g)
|
||||
}
|
||||
@@ -1225,7 +1223,6 @@ func (s *Source) chunkIssueComments(ctx context.Context, repo, repoPath string,
|
||||
chunk := &sources.Chunk{
|
||||
SourceName: s.name,
|
||||
SourceID: s.SourceID(),
|
||||
JobID: s.JobID(),
|
||||
SourceType: s.Type(),
|
||||
SourceMetadata: &source_metadatapb.MetaData{
|
||||
Data: &source_metadatapb.MetaData_Github{
|
||||
@@ -1259,7 +1256,6 @@ func (s *Source) chunkPullRequestComments(ctx context.Context, repo string, comm
|
||||
SourceName: s.name,
|
||||
SourceID: s.SourceID(),
|
||||
SourceType: s.Type(),
|
||||
JobID: s.JobID(),
|
||||
SourceMetadata: &source_metadatapb.MetaData{
|
||||
Data: &source_metadatapb.MetaData_Github{
|
||||
Github: &source_metadatapb.Github{
|
||||
@@ -1291,7 +1287,6 @@ func (s *Source) chunkGistComments(ctx context.Context, gistUrl string, comments
|
||||
SourceName: s.name,
|
||||
SourceID: s.SourceID(),
|
||||
SourceType: s.Type(),
|
||||
JobID: s.JobID(),
|
||||
SourceMetadata: &source_metadatapb.MetaData{
|
||||
Data: &source_metadatapb.MetaData_Github{
|
||||
Github: &source_metadatapb.Github{
|
||||
|
||||
@@ -71,11 +71,11 @@ func (r *JobProgressRef) Done() <-chan struct{} {
|
||||
// CancelRun requests that the job this is referencing is cancelled and stops
|
||||
// running. This method will have no effect if the job does not allow
|
||||
// cancellation.
|
||||
func (r *JobProgressRef) CancelRun(cause error) {
|
||||
func (r *JobProgressRef) CancelRun() {
|
||||
if r.jobProgress == nil || r.jobProgress.jobCancel == nil {
|
||||
return
|
||||
}
|
||||
r.jobProgress.jobCancel(cause)
|
||||
r.jobProgress.jobCancel()
|
||||
}
|
||||
|
||||
// Fatal is a wrapper around error to differentiate non-fatal errors from fatal
|
||||
@@ -108,7 +108,7 @@ type JobProgress struct {
|
||||
ctx context.Context
|
||||
cancel context.CancelFunc
|
||||
// Requests to cancel the job.
|
||||
jobCancel context.CancelCauseFunc
|
||||
jobCancel context.CancelFunc
|
||||
// Metrics.
|
||||
metrics JobProgressMetrics
|
||||
metricsLock sync.Mutex
|
||||
@@ -150,7 +150,7 @@ func WithHooks(hooks ...JobProgressHook) func(*JobProgress) {
|
||||
}
|
||||
|
||||
// WithCancel allows cancelling the job by the JobProgressRef.
|
||||
func WithCancel(cancel context.CancelCauseFunc) func(*JobProgress) {
|
||||
func WithCancel(cancel context.CancelFunc) func(*JobProgress) {
|
||||
return func(jp *JobProgress) { jp.jobCancel = cancel }
|
||||
}
|
||||
|
||||
|
||||
+69
-34
@@ -104,34 +104,22 @@ func (s *Source) setMaxObjectSize(maxObjectSize int64) {
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Source) newClient(region, roleArn string) (*s3.S3, error) {
|
||||
func (s *Source) newClient(region string) (*s3.S3, error) {
|
||||
cfg := aws.NewConfig()
|
||||
cfg.CredentialsChainVerboseErrors = aws.Bool(true)
|
||||
cfg.Region = aws.String(region)
|
||||
|
||||
if roleArn == "" {
|
||||
switch cred := s.conn.GetCredential().(type) {
|
||||
case *sourcespb.S3_SessionToken:
|
||||
cfg.Credentials = credentials.NewStaticCredentials(cred.SessionToken.Key, cred.SessionToken.Secret, cred.SessionToken.SessionToken)
|
||||
case *sourcespb.S3_AccessKey:
|
||||
cfg.Credentials = credentials.NewStaticCredentials(cred.AccessKey.Key, cred.AccessKey.Secret, "")
|
||||
case *sourcespb.S3_Unauthenticated:
|
||||
cfg.Credentials = credentials.AnonymousCredentials
|
||||
case *sourcespb.S3_CloudEnvironment:
|
||||
// Nothing needs to be done!
|
||||
default:
|
||||
return nil, errors.Errorf("invalid configuration given for %s source", s.name)
|
||||
}
|
||||
} else {
|
||||
sess, err := session.NewSession(cfg)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
stsClient := sts.New(sess)
|
||||
cfg.Credentials = stscreds.NewCredentialsWithClient(stsClient, roleArn, func(p *stscreds.AssumeRoleProvider) {
|
||||
p.RoleSessionName = "trufflehog"
|
||||
})
|
||||
switch cred := s.conn.GetCredential().(type) {
|
||||
case *sourcespb.S3_SessionToken:
|
||||
cfg.Credentials = credentials.NewStaticCredentials(cred.SessionToken.Key, cred.SessionToken.Secret, cred.SessionToken.SessionToken)
|
||||
case *sourcespb.S3_AccessKey:
|
||||
cfg.Credentials = credentials.NewStaticCredentials(cred.AccessKey.Key, cred.AccessKey.Secret, "")
|
||||
case *sourcespb.S3_Unauthenticated:
|
||||
cfg.Credentials = credentials.AnonymousCredentials
|
||||
case *sourcespb.S3_CloudEnvironment:
|
||||
// Nothing needs to be done!
|
||||
default:
|
||||
return nil, errors.Errorf("invalid configuration given for %s source", s.name)
|
||||
}
|
||||
|
||||
sess, err := session.NewSessionWithOptions(session.Options{
|
||||
@@ -145,6 +133,40 @@ func (s *Source) newClient(region, roleArn string) (*s3.S3, error) {
|
||||
return s3.New(sess), nil
|
||||
}
|
||||
|
||||
// Separate role assumption functionality into a different newClient function
|
||||
func (s *Source) newRoleClient(region string, roleArn string) (*s3.S3, error) {
|
||||
|
||||
cfg := aws.NewConfig()
|
||||
cfg.CredentialsChainVerboseErrors = aws.Bool(true)
|
||||
cfg.Region = aws.String(region)
|
||||
|
||||
var baseCredentials *credentials.Credentials
|
||||
|
||||
sess, err := session.NewSession(cfg)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
stsClient := sts.New(sess)
|
||||
baseCredentials = stscreds.NewCredentialsWithClient(stsClient, roleArn, func(p *stscreds.AssumeRoleProvider) {
|
||||
p.RoleSessionName = "trufflehog"
|
||||
})
|
||||
|
||||
cfg.Credentials = baseCredentials
|
||||
|
||||
sess, err = session.NewSessionWithOptions(session.Options{
|
||||
SharedConfigState: session.SharedConfigEnable,
|
||||
Config: *cfg,
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Create and return the S3 client with the final configuration
|
||||
return s3.New(sess), nil
|
||||
}
|
||||
|
||||
// IAM identity needs s3:ListBuckets permission
|
||||
func (s *Source) getBucketsToScan(client *s3.S3) ([]string, error) {
|
||||
if len(s.conn.Buckets) > 0 {
|
||||
@@ -182,10 +204,13 @@ func (s *Source) scanBuckets(ctx context.Context, client *s3.S3, role string, bu
|
||||
|
||||
var regionalClient *s3.S3
|
||||
if region != defaultAWSRegion {
|
||||
regionalClient, err = s.newClient(region, role)
|
||||
if role != "" {
|
||||
regionalClient, err = s.newRoleClient(region, role)
|
||||
} else {
|
||||
regionalClient, err = s.newClient(region)
|
||||
}
|
||||
if err != nil {
|
||||
s.log.Error(err, "could not make regional s3 client")
|
||||
continue
|
||||
}
|
||||
} else {
|
||||
regionalClient = client
|
||||
@@ -213,13 +238,24 @@ func (s *Source) scanBuckets(ctx context.Context, client *s3.S3, role string, bu
|
||||
func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) error {
|
||||
const defaultAWSRegion = "us-east-1"
|
||||
|
||||
roles := s.conn.Roles
|
||||
if len(roles) == 0 {
|
||||
roles = []string{""}
|
||||
}
|
||||
if len(s.conn.Roles) > 0 {
|
||||
for _, role := range s.conn.Roles {
|
||||
client, err := s.newRoleClient(defaultAWSRegion, role)
|
||||
if err != nil {
|
||||
return errors.WrapPrefix(err, "could not create s3 client", 0)
|
||||
}
|
||||
|
||||
for _, role := range roles {
|
||||
client, err := s.newClient(defaultAWSRegion, role)
|
||||
bucketsToScan, err := s.getBucketsToScan(client)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := s.scanBuckets(ctx, client, role, bucketsToScan, chunksChan); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
} else {
|
||||
client, err := s.newClient(defaultAWSRegion)
|
||||
if err != nil {
|
||||
return errors.WrapPrefix(err, "could not create s3 client", 0)
|
||||
}
|
||||
@@ -229,7 +265,7 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) err
|
||||
return err
|
||||
}
|
||||
|
||||
if err := s.scanBuckets(ctx, client, role, bucketsToScan, chunksChan); err != nil {
|
||||
if err := s.scanBuckets(ctx, client, "", bucketsToScan, chunksChan); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
@@ -340,7 +376,6 @@ func (s *Source) pageChunker(ctx context.Context, client *s3.S3, chunksChan chan
|
||||
SourceType: s.Type(),
|
||||
SourceName: s.name,
|
||||
SourceID: s.SourceID(),
|
||||
JobID: s.JobID(),
|
||||
SourceMetadata: &source_metadatapb.MetaData{
|
||||
Data: &source_metadatapb.MetaData_S3{
|
||||
S3: &source_metadatapb.S3{
|
||||
|
||||
@@ -26,7 +26,6 @@ type SourceInitFunc func(ctx context.Context, jobID, sourceID int64) (Source, er
|
||||
// initialization.
|
||||
type sourceInfo struct {
|
||||
initFunc SourceInitFunc
|
||||
sourceID int64
|
||||
name string
|
||||
}
|
||||
|
||||
@@ -37,10 +36,7 @@ type SourceManager struct {
|
||||
handles map[handle]sourceInfo
|
||||
handlesLock sync.Mutex
|
||||
// Pool limiting the amount of concurrent sources running.
|
||||
pool errgroup.Group
|
||||
poolLimit int
|
||||
currentRunningCount int32
|
||||
// Max number of units to scan concurrently per source.
|
||||
pool errgroup.Group
|
||||
concurrentUnits int
|
||||
// Run the sources using source unit enumeration / chunking if available.
|
||||
useSourceUnits bool
|
||||
@@ -72,10 +68,7 @@ func WithReportHook(hook JobProgressHook) func(*SourceManager) {
|
||||
|
||||
// WithConcurrentSources limits the concurrent number of sources a manager can run.
|
||||
func WithConcurrentSources(concurrency int) func(*SourceManager) {
|
||||
return func(mgr *SourceManager) {
|
||||
mgr.pool.SetLimit(concurrency)
|
||||
mgr.poolLimit = concurrency
|
||||
}
|
||||
return func(mgr *SourceManager) { mgr.pool.SetLimit(concurrency) }
|
||||
}
|
||||
|
||||
// WithBufferedOutput sets the size of the buffer used for the Chunks() channel.
|
||||
@@ -127,7 +120,6 @@ func (s *SourceManager) Enroll(ctx context.Context, name string, kind sourcespb.
|
||||
}
|
||||
s.handles[handleID] = sourceInfo{
|
||||
initFunc: f,
|
||||
sourceID: int64(handleID),
|
||||
name: name,
|
||||
}
|
||||
return handleID, nil
|
||||
@@ -136,18 +128,7 @@ func (s *SourceManager) Enroll(ctx context.Context, name string, kind sourcespb.
|
||||
// Run blocks until a resource is available to run the source, then
|
||||
// synchronously runs it. The first fatal error, if any, will be returned.
|
||||
func (s *SourceManager) Run(ctx context.Context, handle handle) (JobProgressRef, error) {
|
||||
progress, err := s.ScheduleRun(ctx, handle)
|
||||
if err != nil {
|
||||
return progress, err
|
||||
}
|
||||
<-progress.Done()
|
||||
return progress, progress.Snapshot().FatalError()
|
||||
}
|
||||
|
||||
// RunOnce blocks until a resource is available to run the source, then
|
||||
// synchronously runs it. The first fatal error, if any, will be returned.
|
||||
func (s *SourceManager) RunOnce(ctx context.Context, name string, source Source) (JobProgressRef, error) {
|
||||
progress, err := s.ScheduleRunOnce(ctx, name, source)
|
||||
progress, err := s.asyncRun(ctx, handle)
|
||||
if err != nil {
|
||||
return progress, err
|
||||
}
|
||||
@@ -162,22 +143,15 @@ func (s *SourceManager) ScheduleRun(ctx context.Context, handle handle) (JobProg
|
||||
return s.asyncRun(ctx, handle)
|
||||
}
|
||||
|
||||
// ScheduleRunOnce blocks until a resource is available to run the source, then
|
||||
// asynchronously runs it. Error information is stored and accessible via the
|
||||
// JobProgressRef as it becomes available.
|
||||
func (s *SourceManager) ScheduleRunOnce(ctx context.Context, name string, source Source) (JobProgressRef, error) {
|
||||
return s.asyncRunOnce(ctx, name, source)
|
||||
}
|
||||
|
||||
// asyncRun is a helper method to asynchronously run the Source. It calls out
|
||||
// to the API to get a job ID for this run, creates a JobProgress object, then
|
||||
// waits for an available goroutine to asynchronously run it.
|
||||
func (s *SourceManager) asyncRun(ctx context.Context, handle handle) (JobProgressRef, error) {
|
||||
// Do preflight checks before waiting on the pool.
|
||||
if err := s.preflightChecks(ctx); err != nil {
|
||||
if err := s.preflightChecks(ctx, handle); err != nil {
|
||||
return JobProgressRef{}, err
|
||||
}
|
||||
// Get the name.
|
||||
// Get the name. Should never fail due to preflight checks.
|
||||
sourceInfo, ok := s.getSourceInfo(handle)
|
||||
if !ok {
|
||||
return JobProgressRef{SourceID: int64(handle)}, fmt.Errorf("unrecognized handle")
|
||||
@@ -190,47 +164,16 @@ func (s *SourceManager) asyncRun(ctx context.Context, handle handle) (JobProgres
|
||||
return JobProgressRef{SourceID: int64(handle), SourceName: sourceName}, err
|
||||
}
|
||||
// Create a JobProgress object for tracking progress.
|
||||
ctx, cancel := context.WithCancelCause(ctx)
|
||||
ctx, cancel := context.WithCancel(ctx)
|
||||
progress := NewJobProgress(jobID, int64(handle), sourceName, WithHooks(s.hooks...), WithCancel(cancel))
|
||||
s.pool.Go(func() error {
|
||||
atomic.AddInt32(&s.currentRunningCount, 1)
|
||||
defer atomic.AddInt32(&s.currentRunningCount, -1)
|
||||
ctx := context.WithValues(ctx,
|
||||
"job_id", jobID,
|
||||
"source_manager_worker_id", common.RandomID(5),
|
||||
)
|
||||
defer common.Recover(ctx)
|
||||
defer cancel(nil)
|
||||
return s.run(ctx, sourceInfo, jobID, progress)
|
||||
})
|
||||
return progress.Ref(), nil
|
||||
}
|
||||
|
||||
// asyncRunOnce is a helper method to asynchronously run an already initialized
|
||||
// Source once. Unlike asyncRun, this method doesn't call out to the API for a
|
||||
// job ID since it's already available in the Source.
|
||||
func (s *SourceManager) asyncRunOnce(ctx context.Context, sourceName string, source Source) (JobProgressRef, error) {
|
||||
if err := s.preflightChecks(ctx); err != nil {
|
||||
return JobProgressRef{}, err
|
||||
}
|
||||
ctx = context.WithValue(ctx, "source_id", source.SourceID())
|
||||
jobID := source.JobID()
|
||||
// Create a JobProgress object for tracking progress.
|
||||
ctx, cancel := context.WithCancelCause(ctx)
|
||||
progress := NewJobProgress(jobID, source.SourceID(), sourceName, WithHooks(s.hooks...), WithCancel(cancel))
|
||||
s.pool.Go(func() error {
|
||||
ctx := context.WithValues(ctx,
|
||||
"job_id", jobID,
|
||||
"source_manager_worker_id", common.RandomID(5),
|
||||
)
|
||||
defer common.Recover(ctx)
|
||||
defer cancel(nil)
|
||||
sourceInfo := sourceInfo{
|
||||
initFunc: func(context.Context, int64, int64) (Source, error) { return source, nil },
|
||||
sourceID: source.SourceID(),
|
||||
name: sourceName,
|
||||
}
|
||||
return s.run(ctx, sourceInfo, jobID, progress)
|
||||
defer cancel()
|
||||
return s.run(ctx, handle, jobID, progress)
|
||||
})
|
||||
return progress.Ref(), nil
|
||||
}
|
||||
@@ -264,41 +207,37 @@ func (s *SourceManager) ScanChunk(chunk *Chunk) {
|
||||
s.outputChunks <- chunk
|
||||
}
|
||||
|
||||
// AvailableCapacity returns the number of concurrent jobs the manager can
|
||||
// accommodate at this time. If there is no limit, -1 is returned.
|
||||
func (s *SourceManager) AvailableCapacity() int {
|
||||
if s.poolLimit == 0 {
|
||||
return -1
|
||||
}
|
||||
runCount := atomic.LoadInt32(&s.currentRunningCount)
|
||||
return s.poolLimit - int(runCount)
|
||||
}
|
||||
|
||||
// preflightChecks is a helper method to check the Manager or the context isn't done.
|
||||
func (s *SourceManager) preflightChecks(ctx context.Context) error {
|
||||
// preflightChecks is a helper method to check the Manager or the context isn't
|
||||
// done and that the handle is valid.
|
||||
func (s *SourceManager) preflightChecks(ctx context.Context, handle handle) error {
|
||||
// Check if the manager has been Waited.
|
||||
if s.done {
|
||||
return fmt.Errorf("manager is done")
|
||||
}
|
||||
// Check the handle is valid.
|
||||
if _, ok := s.getSourceInfo(handle); !ok {
|
||||
return fmt.Errorf("unrecognized handle")
|
||||
}
|
||||
return ctx.Err()
|
||||
}
|
||||
|
||||
// run is a helper method to sychronously run the source. It does not check for
|
||||
// acquired resources. An error is returned if there was a fatal error during
|
||||
// the run. This information is also recorded in the JobProgress.
|
||||
func (s *SourceManager) run(ctx context.Context, sourceInfo sourceInfo, jobID int64, report *JobProgress) error {
|
||||
func (s *SourceManager) run(ctx context.Context, handle handle, jobID int64, report *JobProgress) error {
|
||||
defer report.Finish()
|
||||
report.Start(time.Now())
|
||||
defer func() { report.End(time.Now()) }()
|
||||
|
||||
defer func() {
|
||||
if err := context.Cause(ctx); err != nil {
|
||||
report.ReportError(Fatal{err})
|
||||
}
|
||||
}()
|
||||
|
||||
// Initialize the source.
|
||||
source, err := sourceInfo.initFunc(ctx, jobID, sourceInfo.sourceID)
|
||||
sourceInfo, ok := s.getSourceInfo(handle)
|
||||
if !ok {
|
||||
// Shouldn't happen due to preflight checks.
|
||||
err := fmt.Errorf("unrecognized handle")
|
||||
report.ReportError(Fatal{err})
|
||||
return Fatal{err}
|
||||
}
|
||||
source, err := sourceInfo.initFunc(ctx, jobID, int64(handle))
|
||||
if err != nil {
|
||||
report.ReportError(Fatal{err})
|
||||
return Fatal{err}
|
||||
@@ -310,14 +249,14 @@ func (s *SourceManager) run(ctx context.Context, sourceInfo sourceInfo, jobID in
|
||||
)
|
||||
// Check for the preferred method of tracking source units.
|
||||
if enumChunker, ok := source.(SourceUnitEnumChunker); ok && s.useSourceUnits {
|
||||
return s.runWithUnits(ctx, enumChunker, report)
|
||||
return s.runWithUnits(ctx, handle, enumChunker, report)
|
||||
}
|
||||
return s.runWithoutUnits(ctx, source, report)
|
||||
return s.runWithoutUnits(ctx, handle, source, report)
|
||||
}
|
||||
|
||||
// runWithoutUnits is a helper method to run a Source. It has coarse-grained
|
||||
// job reporting.
|
||||
func (s *SourceManager) runWithoutUnits(ctx context.Context, source Source, report *JobProgress) error {
|
||||
func (s *SourceManager) runWithoutUnits(ctx context.Context, handle handle, source Source, report *JobProgress) error {
|
||||
// Introspect on the chunks we get from the Chunks method.
|
||||
ch := make(chan *Chunk, 1)
|
||||
var wg sync.WaitGroup
|
||||
@@ -347,7 +286,7 @@ func (s *SourceManager) runWithoutUnits(ctx context.Context, source Source, repo
|
||||
// runWithUnits is a helper method to run a Source that is also a
|
||||
// SourceUnitEnumChunker. This allows better introspection of what is getting
|
||||
// scanned and any errors encountered.
|
||||
func (s *SourceManager) runWithUnits(ctx context.Context, source SourceUnitEnumChunker, report *JobProgress) error {
|
||||
func (s *SourceManager) runWithUnits(ctx context.Context, handle handle, source SourceUnitEnumChunker, report *JobProgress) error {
|
||||
unitReporter := &mgrUnitReporter{
|
||||
unitCh: make(chan SourceUnit, 1),
|
||||
report: report,
|
||||
|
||||
@@ -14,9 +14,8 @@ import (
|
||||
|
||||
// DummySource implements Source and is used for testing a SourceManager.
|
||||
type DummySource struct {
|
||||
sourceID int64
|
||||
jobID int64
|
||||
initCalled bool
|
||||
sourceID int64
|
||||
jobID int64
|
||||
chunker
|
||||
}
|
||||
|
||||
@@ -24,7 +23,6 @@ func (d *DummySource) Type() sourcespb.SourceType { return 1337 }
|
||||
func (d *DummySource) SourceID() int64 { return d.sourceID }
|
||||
func (d *DummySource) JobID() int64 { return d.jobID }
|
||||
func (d *DummySource) Init(_ context.Context, _ string, jobID, sourceID int64, _ bool, _ *anypb.Any, _ int) error {
|
||||
d.initCalled = true
|
||||
d.sourceID = sourceID
|
||||
d.jobID = jobID
|
||||
return nil
|
||||
@@ -314,49 +312,8 @@ func TestSourceManagerCancelRun(t *testing.T) {
|
||||
ref, err := mgr.ScheduleRun(context.Background(), handle)
|
||||
assert.NoError(t, err)
|
||||
|
||||
cancelErr := fmt.Errorf("abort! abort!")
|
||||
ref.CancelRun(cancelErr)
|
||||
ref.CancelRun()
|
||||
<-ref.Done()
|
||||
assert.Error(t, ref.Snapshot().FatalError())
|
||||
assert.True(t, errors.Is(ref.Snapshot().FatalError(), returnedErr))
|
||||
assert.True(t, errors.Is(ref.Snapshot().FatalErrors(), cancelErr))
|
||||
}
|
||||
|
||||
func TestSourceManagerAvailableCapacity(t *testing.T) {
|
||||
mgr := NewManager(WithConcurrentSources(1337))
|
||||
start, end := make(chan struct{}), make(chan struct{})
|
||||
handle, err := enrollDummy(mgr, callbackChunker{func(context.Context, chan *Chunk) error {
|
||||
start <- struct{}{} // Send start signal.
|
||||
<-end // Wait for end signal.
|
||||
return nil
|
||||
}})
|
||||
assert.NoError(t, err)
|
||||
|
||||
assert.Equal(t, 1337, mgr.AvailableCapacity())
|
||||
ref, err := mgr.ScheduleRun(context.Background(), handle)
|
||||
assert.NoError(t, err)
|
||||
|
||||
<-start // Wait for start signal.
|
||||
assert.Equal(t, 1336, mgr.AvailableCapacity())
|
||||
end <- struct{}{} // Send end signal.
|
||||
<-ref.Done() // Wait for the job to finish.
|
||||
assert.Equal(t, 1337, mgr.AvailableCapacity())
|
||||
}
|
||||
|
||||
func TestSourceManagerRunOnce(t *testing.T) {
|
||||
mgr := NewManager()
|
||||
var chunksCalled bool
|
||||
source := &DummySource{sourceID: 123, jobID: 456, chunker: callbackChunker{
|
||||
func(context.Context, chan *Chunk) error {
|
||||
chunksCalled = true
|
||||
return nil
|
||||
},
|
||||
}}
|
||||
ref, err := mgr.RunOnce(context.Background(), "test", source)
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, source.jobID, ref.JobID)
|
||||
assert.Equal(t, source.sourceID, ref.SourceID)
|
||||
assert.Equal(t, "test", ref.SourceName)
|
||||
assert.True(t, chunksCalled)
|
||||
assert.False(t, source.initCalled)
|
||||
}
|
||||
|
||||
@@ -272,7 +272,6 @@ func (s *Source) monitorConnection(ctx context.Context, conn net.Conn, chunksCha
|
||||
SourceName: s.syslog.sourceName,
|
||||
SourceID: s.syslog.sourceID,
|
||||
SourceType: s.syslog.sourceType,
|
||||
JobID: s.JobID(),
|
||||
SourceMetadata: metadata,
|
||||
Data: input,
|
||||
Verify: s.verify,
|
||||
@@ -314,7 +313,6 @@ func (s *Source) acceptUDPConnections(ctx context.Context, netListener net.Packe
|
||||
chunksChan <- &sources.Chunk{
|
||||
SourceName: s.syslog.sourceName,
|
||||
SourceID: s.syslog.sourceID,
|
||||
JobID: s.JobID(),
|
||||
SourceType: s.syslog.sourceType,
|
||||
SourceMetadata: metadata,
|
||||
Data: input,
|
||||
|
||||
Reference in New Issue
Block a user