Compare commits
1 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| ea6768bce9 |
@@ -32,7 +32,7 @@ Have questions? Feedback? Jump in slack or discord and hang out with us
|
||||
|
||||
Join our [Slack Community](https://join.slack.com/t/trufflehog-community/shared_invite/zt-pw2qbi43-Aa86hkiimstfdKH9UCpPzQ)
|
||||
|
||||
Join the [Secret Scanning Discord](https://discord.gg/8Hzbrnkr7E)
|
||||
Join the [Secret Scanning Discord](https://discord.gg/sydS6AHTUP)
|
||||
|
||||
# :tv: Demo
|
||||
|
||||
|
||||
@@ -1,11 +1,11 @@
|
||||
package mattermostpersonaltoken
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
@@ -14,50 +14,44 @@ import (
|
||||
|
||||
type Scanner struct{}
|
||||
|
||||
// Ensure the Scanner satisfies the interface at compile time.
|
||||
var _ detectors.Detector = (*Scanner)(nil)
|
||||
|
||||
var (
|
||||
client = common.SaneHttpClient()
|
||||
|
||||
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
|
||||
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"mattermost"}) + `\b([a-z0-9]{26})\b`)
|
||||
serverPat = regexp.MustCompile(detectors.PrefixRegex([]string{"mattermost"}) + `\b([A-Za-z0-9-_]{1,}.cloud.mattermost.com)\b`)
|
||||
)
|
||||
|
||||
// Keywords are used for efficiently pre-filtering chunks.
|
||||
// Use identifiers in the secret preferably, or the provider name.
|
||||
func (s Scanner) Keywords() []string {
|
||||
return []string{"mattermost"}
|
||||
func (s Scanner) Keywords() [][]byte {
|
||||
return [][]byte{[]byte("mattermost")}
|
||||
}
|
||||
|
||||
// FromData will find and optionally verify MattermostPersonalToken secrets in a given set of bytes.
|
||||
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
|
||||
dataStr := string(data)
|
||||
|
||||
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
|
||||
serverMatches := serverPat.FindAllStringSubmatch(dataStr, -1)
|
||||
matches := keyPat.FindAllSubmatch(data, -1)
|
||||
serverMatches := serverPat.FindAllSubmatch(data, -1)
|
||||
|
||||
for _, match := range matches {
|
||||
if len(match) != 2 {
|
||||
continue
|
||||
}
|
||||
resMatch := strings.TrimSpace(match[1])
|
||||
resMatch := bytes.TrimSpace(match[1])
|
||||
|
||||
for _, serverMatch := range serverMatches {
|
||||
if len(serverMatch) != 2 {
|
||||
continue
|
||||
}
|
||||
serverRes := strings.TrimSpace(serverMatch[1])
|
||||
serverRes := bytes.TrimSpace(serverMatch[1])
|
||||
|
||||
s1 := detectors.Result{
|
||||
DetectorType: detectorspb.DetectorType_MattermostPersonalToken,
|
||||
Raw: []byte(resMatch),
|
||||
RawV2: []byte(resMatch + serverRes),
|
||||
Raw: resMatch,
|
||||
RawV2: append(resMatch, serverRes...),
|
||||
}
|
||||
|
||||
if verify {
|
||||
req, err := http.NewRequestWithContext(ctx, "GET", "https://"+serverRes+"/api/v4/users/stats", nil)
|
||||
req, err := http.NewRequestWithContext(ctx, "GET", fmt.Sprintf("https://%s/api/v4/users/stats", serverRes), nil)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
@@ -68,7 +62,6 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
if res.StatusCode >= 200 && res.StatusCode < 300 {
|
||||
s1.Verified = true
|
||||
} else {
|
||||
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
|
||||
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
|
||||
continue
|
||||
}
|
||||
|
||||
@@ -1,11 +1,11 @@
|
||||
package mavenlink
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
@@ -14,37 +14,30 @@ import (
|
||||
|
||||
type Scanner struct{}
|
||||
|
||||
// Ensure the Scanner satisfies the interface at compile time.
|
||||
var _ detectors.Detector = (*Scanner)(nil)
|
||||
|
||||
var (
|
||||
client = common.SaneHttpClient()
|
||||
|
||||
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
|
||||
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"mavenlink"}) + `\b([0-9a-z]{64})\b`)
|
||||
)
|
||||
|
||||
// Keywords are used for efficiently pre-filtering chunks.
|
||||
// Use identifiers in the secret preferably, or the provider name.
|
||||
func (s Scanner) Keywords() []string {
|
||||
return []string{"mavenlink"}
|
||||
func (s Scanner) Keywords() [][]byte {
|
||||
return [][]byte{[]byte("mavenlink")}
|
||||
}
|
||||
|
||||
// FromData will find and optionally verify Mavenlink secrets in a given set of bytes.
|
||||
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
|
||||
dataStr := string(data)
|
||||
|
||||
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
|
||||
matches := keyPat.FindAllSubmatch(data, -1)
|
||||
|
||||
for _, match := range matches {
|
||||
if len(match) != 2 {
|
||||
continue
|
||||
}
|
||||
resMatch := strings.TrimSpace(match[1])
|
||||
resMatch := bytes.TrimSpace(match[1])
|
||||
|
||||
s1 := detectors.Result{
|
||||
DetectorType: detectorspb.DetectorType_Mavenlink,
|
||||
Raw: []byte(resMatch),
|
||||
Raw: resMatch,
|
||||
}
|
||||
|
||||
if verify {
|
||||
@@ -52,14 +45,13 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", resMatch))
|
||||
req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", string(resMatch)))
|
||||
res, err := client.Do(req)
|
||||
if err == nil {
|
||||
defer res.Body.Close()
|
||||
if res.StatusCode >= 200 && res.StatusCode < 300 {
|
||||
s1.Verified = true
|
||||
} else {
|
||||
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
|
||||
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
|
||||
continue
|
||||
}
|
||||
|
||||
@@ -1,10 +1,10 @@
|
||||
package maxmindlicense
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
@@ -23,32 +23,27 @@ var (
|
||||
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"maxmind", "geoip"}) + `\b([0-9A-Za-z]{16})\b`)
|
||||
)
|
||||
|
||||
// Keywords are used for efficiently pre-filtering chunks.
|
||||
// Use identifiers in the secret preferably, or the provider name.
|
||||
func (s Scanner) Keywords() []string {
|
||||
return []string{"maxmind", "geoip"}
|
||||
func (s Scanner) Keywords() [][]byte {
|
||||
return [][]byte{[]byte("maxmind"), []byte("geoip")}
|
||||
}
|
||||
|
||||
// FromData will find and optionally verify MaxMindLicense secrets in a given set of bytes.
|
||||
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
|
||||
dataStr := string(data)
|
||||
|
||||
keyMatches := keyPat.FindAllStringSubmatch(dataStr, -1)
|
||||
idMatches := idPat.FindAllStringSubmatch(dataStr, -1)
|
||||
keyMatches := keyPat.FindAllSubmatch(data, -1)
|
||||
idMatches := idPat.FindAllSubmatch(data, -1)
|
||||
|
||||
for _, keyMatch := range keyMatches {
|
||||
keyRes := strings.TrimSpace(keyMatch[1])
|
||||
keyRes := bytes.TrimSpace(keyMatch[1])
|
||||
|
||||
for _, idMatch := range idMatches {
|
||||
if len(idMatch) != 2 {
|
||||
continue
|
||||
}
|
||||
idRes := strings.TrimSpace(idMatch[1])
|
||||
idRes := bytes.TrimSpace(idMatch[1])
|
||||
|
||||
s1 := detectors.Result{
|
||||
DetectorType: detectorspb.DetectorType_MaxMindLicense,
|
||||
Redacted: idRes,
|
||||
Raw: []byte(keyRes),
|
||||
Redacted: string(idRes),
|
||||
Raw: keyRes,
|
||||
}
|
||||
|
||||
if verify {
|
||||
@@ -56,7 +51,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
req.SetBasicAuth(idRes, keyRes)
|
||||
req.SetBasicAuth(string(idRes), string(keyRes))
|
||||
res, err := client.Do(req)
|
||||
if err == nil {
|
||||
defer res.Body.Close()
|
||||
|
||||
@@ -7,7 +7,6 @@ import (
|
||||
"mime/multipart"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
@@ -22,31 +21,25 @@ var _ detectors.Detector = (*Scanner)(nil)
|
||||
var (
|
||||
client = common.SaneHttpClient()
|
||||
|
||||
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
|
||||
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"meaningcloud"}) + `\b([a-z0-9]{32})\b`)
|
||||
)
|
||||
|
||||
// Keywords are used for efficiently pre-filtering chunks.
|
||||
// Use identifiers in the secret preferably, or the provider name.
|
||||
func (s Scanner) Keywords() []string {
|
||||
return []string{"meaningcloud"}
|
||||
func (s Scanner) Keywords() [][]byte {
|
||||
return [][]byte{[]byte("meaningcloud")}
|
||||
}
|
||||
|
||||
// FromData will find and optionally verify MeaningCloud secrets in a given set of bytes.
|
||||
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
|
||||
dataStr := string(data)
|
||||
|
||||
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
|
||||
matches := keyPat.FindAllSubmatch(data, -1)
|
||||
|
||||
for _, match := range matches {
|
||||
if len(match) != 2 {
|
||||
continue
|
||||
}
|
||||
resMatch := strings.TrimSpace(match[1])
|
||||
resMatch := bytes.TrimSpace(match[1])
|
||||
|
||||
s1 := detectors.Result{
|
||||
DetectorType: detectorspb.DetectorType_MeaningCloud,
|
||||
Raw: []byte(resMatch),
|
||||
Raw: resMatch,
|
||||
}
|
||||
|
||||
if verify {
|
||||
@@ -56,7 +49,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
_, err = io.Copy(fw, strings.NewReader(resMatch))
|
||||
_, err = io.Copy(fw, bytes.NewReader(resMatch))
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
@@ -64,12 +57,12 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
_, err = io.Copy(fw, strings.NewReader("test"))
|
||||
_, err = io.Copy(fw, bytes.NewReader([]byte("test")))
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
writer.Close()
|
||||
req, err := http.NewRequestWithContext(ctx, "POST", "https://api.meaningcloud.com/lang-4.0/identification", bytes.NewReader(body.Bytes()))
|
||||
req, err := http.NewRequestWithContext(ctx, "POST", "https://api.meaningcloud.com/-4.0/identification", bytes.NewReader(body.Bytes()))
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
@@ -80,7 +73,6 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
if res.StatusCode >= 200 && res.StatusCode < 300 {
|
||||
s1.Verified = true
|
||||
} else {
|
||||
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
|
||||
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
|
||||
continue
|
||||
}
|
||||
|
||||
@@ -1,12 +1,11 @@
|
||||
package mediastack
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
@@ -15,43 +14,35 @@ import (
|
||||
|
||||
type Scanner struct{}
|
||||
|
||||
// Ensure the Scanner satisfies the interface at compile time.
|
||||
var _ detectors.Detector = (*Scanner)(nil)
|
||||
|
||||
var (
|
||||
client = common.SaneHttpClient()
|
||||
|
||||
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
|
||||
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"mediastack"}) + `\b([a-z0-9]{32})\b`)
|
||||
)
|
||||
|
||||
// Keywords are used for efficiently pre-filtering chunks.
|
||||
// Use identifiers in the secret preferably, or the provider name.
|
||||
func (s Scanner) Keywords() []string {
|
||||
return []string{"mediastack"}
|
||||
func (s Scanner) Keywords() [][]byte {
|
||||
return [][]byte{[]byte("mediastack")}
|
||||
}
|
||||
|
||||
// FromData will find and optionally verify MediaStack secrets in a given set of bytes.
|
||||
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
|
||||
dataStr := string(data)
|
||||
|
||||
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
|
||||
matches := keyPat.FindAllSubmatch(data, -1)
|
||||
|
||||
for _, match := range matches {
|
||||
if len(match) != 2 {
|
||||
continue
|
||||
}
|
||||
resMatch := strings.TrimSpace(match[1])
|
||||
resMatch := bytes.TrimSpace(match[1])
|
||||
|
||||
s1 := detectors.Result{
|
||||
DetectorType: detectorspb.DetectorType_MediaStack,
|
||||
Raw: []byte(resMatch),
|
||||
Raw: resMatch,
|
||||
}
|
||||
|
||||
if verify {
|
||||
timeout := 10 * time.Second
|
||||
client.Timeout = timeout
|
||||
req, err := http.NewRequestWithContext(ctx, "GET", fmt.Sprintf("https://api.mediastack.com/v1/news?access_key=%s", resMatch), nil)
|
||||
req, err := http.NewRequestWithContext(ctx, "GET", fmt.Sprintf("https://api.mediastack.com/v1/news?access_key=%s", string(resMatch)), nil)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
@@ -61,7 +52,6 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
if res.StatusCode >= 200 && res.StatusCode < 300 {
|
||||
s1.Verified = true
|
||||
} else {
|
||||
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
|
||||
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
|
||||
continue
|
||||
}
|
||||
|
||||
@@ -1,11 +1,11 @@
|
||||
package meistertask
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
@@ -14,37 +14,29 @@ import (
|
||||
|
||||
type Scanner struct{}
|
||||
|
||||
// Ensure the Scanner satisfies the interface at compile time.
|
||||
var _ detectors.Detector = (*Scanner)(nil)
|
||||
|
||||
var (
|
||||
client = common.SaneHttpClient()
|
||||
|
||||
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
|
||||
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"meistertask"}) + `\b([a-zA-Z0-9]{43})\b`)
|
||||
)
|
||||
|
||||
// Keywords are used for efficiently pre-filtering chunks.
|
||||
// Use identifiers in the secret preferably, or the provider name.
|
||||
func (s Scanner) Keywords() []string {
|
||||
return []string{"meistertask"}
|
||||
func (s Scanner) Keywords() [][]byte {
|
||||
return [][]byte{[]byte("meistertask")}
|
||||
}
|
||||
|
||||
// FromData will find and optionally verify Meistertask secrets in a given set of bytes.
|
||||
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
|
||||
dataStr := string(data)
|
||||
|
||||
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
|
||||
matches := keyPat.FindAllSubmatch(data, -1)
|
||||
|
||||
for _, match := range matches {
|
||||
if len(match) != 2 {
|
||||
continue
|
||||
}
|
||||
resMatch := strings.TrimSpace(match[1])
|
||||
resMatch := bytes.TrimSpace(match[1])
|
||||
|
||||
s1 := detectors.Result{
|
||||
DetectorType: detectorspb.DetectorType_Meistertask,
|
||||
Raw: []byte(resMatch),
|
||||
Raw: resMatch,
|
||||
}
|
||||
|
||||
if verify {
|
||||
@@ -54,19 +46,17 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
}
|
||||
req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", resMatch))
|
||||
res, err := client.Do(req)
|
||||
if err == nil {
|
||||
if err == nil && res != nil {
|
||||
defer res.Body.Close()
|
||||
if res.StatusCode >= 200 && res.StatusCode < 300 {
|
||||
s1.Verified = true
|
||||
} else {
|
||||
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
|
||||
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
|
||||
continue
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
results = append(results, s1)
|
||||
}
|
||||
|
||||
|
||||
@@ -1,11 +1,11 @@
|
||||
package mesibo
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"io"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
@@ -13,44 +13,39 @@ import (
|
||||
)
|
||||
|
||||
type Scanner struct{}
|
||||
// Ensure the Scanner satisfies the interface at compile time.
|
||||
|
||||
var _ detectors.Detector = (*Scanner)(nil)
|
||||
|
||||
var (
|
||||
client = common.SaneHttpClient()
|
||||
|
||||
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
|
||||
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"mesibo"}) + `\b([0-9A-Za-z]{64})\b`)
|
||||
)
|
||||
|
||||
// Keywords are used for efficiently pre-filtering chunks.
|
||||
// Use identifiers in the secret preferably, or the provider name.
|
||||
func (s Scanner) Keywords() []string {
|
||||
return []string{"mesibo"}
|
||||
func (s Scanner) Keywords() [][]byte {
|
||||
return [][]byte{[]byte("mesibo")}
|
||||
}
|
||||
|
||||
// FromData will find and optionally verify Mesibo secrets in a given set of bytes.
|
||||
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
|
||||
dataStr := string(data)
|
||||
|
||||
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
|
||||
matches := keyPat.FindAllSubmatch(data, -1)
|
||||
|
||||
for _, match := range matches {
|
||||
if len(match) != 2 {
|
||||
continue
|
||||
}
|
||||
resMatch := strings.TrimSpace(match[1])
|
||||
|
||||
resMatch := bytes.TrimSpace(match[1])
|
||||
|
||||
s1 := detectors.Result{
|
||||
DetectorType: detectorspb.DetectorType_Mesibo,
|
||||
Raw: []byte(resMatch),
|
||||
Raw: resMatch,
|
||||
}
|
||||
|
||||
if verify {
|
||||
req, err := http.NewRequestWithContext(ctx, "GET", "https://api.mesibo.com/api.php?op=useradd&token="+resMatch, nil)
|
||||
req, err := http.NewRequestWithContext(ctx, "GET", "https://api.mesibo.com/api.php?op=useradd&token="+string(resMatch), nil)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
|
||||
res, err := client.Do(req)
|
||||
if err == nil {
|
||||
defer res.Body.Close()
|
||||
@@ -58,16 +53,14 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
body := string(bodyBytes)
|
||||
|
||||
if !strings.Contains(body, "AUTHFAIL") {
|
||||
if !bytes.Contains(bodyBytes, []byte("AUTHFAIL")) {
|
||||
s1.Verified = true
|
||||
} else {
|
||||
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -1,11 +1,11 @@
|
||||
package messagebird
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
@@ -14,36 +14,29 @@ import (
|
||||
|
||||
type Scanner struct{}
|
||||
|
||||
// Ensure the Scanner satisfies the interface at compile time.
|
||||
var _ detectors.Detector = (*Scanner)(nil)
|
||||
|
||||
var (
|
||||
client = common.SaneHttpClient()
|
||||
|
||||
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"messagebird"}) + `\b([A-Za-z0-9_-]{25})\b`)
|
||||
)
|
||||
|
||||
// Keywords are used for efficiently pre-filtering chunks.
|
||||
// Use identifiers in the secret preferably, or the provider name.
|
||||
func (s Scanner) Keywords() []string {
|
||||
return []string{"messagebird"}
|
||||
func (s Scanner) Keywords() [][]byte {
|
||||
return [][]byte{[]byte("messagebird")}
|
||||
}
|
||||
|
||||
// FromData will find and optionally verify MessageBird secrets in a given set of bytes.
|
||||
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
|
||||
dataStr := string(data)
|
||||
|
||||
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
|
||||
matches := keyPat.FindAllSubmatch(data, -1)
|
||||
|
||||
for _, match := range matches {
|
||||
if len(match) != 2 {
|
||||
continue
|
||||
}
|
||||
resMatch := strings.TrimSpace(match[1])
|
||||
resMatch := bytes.TrimSpace(match[1])
|
||||
|
||||
s1 := detectors.Result{
|
||||
DetectorType: detectorspb.DetectorType_MessageBird,
|
||||
Raw: []byte(resMatch),
|
||||
Raw: resMatch,
|
||||
}
|
||||
|
||||
if verify {
|
||||
@@ -51,7 +44,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
req.Header.Add("Authorization", fmt.Sprintf("AccessKey %s", resMatch))
|
||||
req.Header.Add("Authorization", fmt.Sprintf("AccessKey %s", string(resMatch)))
|
||||
res, err := client.Do(req)
|
||||
if err == nil {
|
||||
defer res.Body.Close()
|
||||
|
||||
@@ -1,12 +1,12 @@
|
||||
package metaapi
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
@@ -28,32 +28,29 @@ var (
|
||||
|
||||
// Keywords are used for efficiently pre-filtering chunks.
|
||||
// Use identifiers in the secret preferably, or the provider name.
|
||||
func (s Scanner) Keywords() []string {
|
||||
return []string{"metaapi", "meta-api"}
|
||||
func (s Scanner) Keywords() [][]byte {
|
||||
return [][]byte{[]byte("metaapi"), []byte("meta-api")}
|
||||
}
|
||||
|
||||
// FromData will find and optionally verify MetaAPI secrets in a given set of bytes.
|
||||
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
|
||||
dataStr := string(data)
|
||||
|
||||
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
|
||||
spellMatches := spellPat.FindAllStringSubmatch(dataStr, -1)
|
||||
matches := keyPat.FindAllSubmatch(data, -1)
|
||||
spellMatches := spellPat.FindAllSubmatch(data, -1)
|
||||
|
||||
for _, spellMatch := range spellMatches {
|
||||
if len(spellMatch) != 2 {
|
||||
continue
|
||||
}
|
||||
resSpellMatch := strings.TrimSpace(spellMatch[1])
|
||||
resSpellMatch := bytes.TrimSpace(spellMatch[1])
|
||||
|
||||
for _, match := range matches {
|
||||
if len(match) != 2 {
|
||||
continue
|
||||
}
|
||||
resMatch := strings.TrimSpace(match[1])
|
||||
resMatch := bytes.TrimSpace(match[1])
|
||||
|
||||
s1 := detectors.Result{
|
||||
DetectorType: detectorspb.DetectorType_MetaAPI,
|
||||
Raw: []byte(resMatch),
|
||||
Raw: resMatch,
|
||||
}
|
||||
|
||||
if verify {
|
||||
@@ -62,16 +59,14 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
continue
|
||||
}
|
||||
req.Header.Add("Content-Type", "application/json")
|
||||
req.Header.Add("apikey", resMatch)
|
||||
req.Header.Add("apikey", string(resMatch))
|
||||
res, err := client.Do(req)
|
||||
if err == nil {
|
||||
defer res.Body.Close()
|
||||
body, errBody := io.ReadAll(res.Body)
|
||||
|
||||
if errBody == nil {
|
||||
bodyStr := string(body)
|
||||
|
||||
if res.StatusCode >= 200 && res.StatusCode < 300 && strings.Contains(bodyStr, `"success":true`) {
|
||||
if res.StatusCode >= 200 && res.StatusCode < 300 && bytes.Contains(body, []byte(`"success":true`)) {
|
||||
s1.Verified = true
|
||||
} else {
|
||||
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
|
||||
|
||||
@@ -1,10 +1,10 @@
|
||||
package metrilo
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
@@ -13,54 +13,50 @@ import (
|
||||
|
||||
type Scanner struct{}
|
||||
|
||||
// Ensure the Scanner satisfies the interface at compile time.
|
||||
var _ detectors.Detector = (*Scanner)(nil)
|
||||
|
||||
var (
|
||||
client = common.SaneHttpClient()
|
||||
|
||||
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
|
||||
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"metrilo"}) + `\b([a-z0-9]{16})\b`)
|
||||
)
|
||||
|
||||
// Keywords are used for efficiently pre-filtering chunks.
|
||||
// Use identifiers in the secret preferably, or the provider name.
|
||||
func (s Scanner) Keywords() []string {
|
||||
return []string{"metrilo"}
|
||||
func (s Scanner) Keywords() [][]byte {
|
||||
return [][]byte{[]byte("metrilo")}
|
||||
}
|
||||
|
||||
// FromData will find and optionally verify Metrilo secrets in a given set of bytes.
|
||||
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
|
||||
dataStr := string(data)
|
||||
|
||||
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
|
||||
matches := keyPat.FindAllSubmatch(data, -1)
|
||||
|
||||
for _, match := range matches {
|
||||
if len(match) != 2 {
|
||||
continue
|
||||
}
|
||||
resMatch := strings.TrimSpace(match[1])
|
||||
resMatch := bytes.TrimSpace(match[1])
|
||||
|
||||
s1 := detectors.Result{
|
||||
DetectorType: detectorspb.DetectorType_Metrilo,
|
||||
Raw: []byte(resMatch),
|
||||
Raw: resMatch,
|
||||
}
|
||||
|
||||
if verify {
|
||||
payload := strings.NewReader(`{"time":1518004715732,"token": "` + resMatch + `" ,"platform":"Wordpress 4.2.7 / Woocommerce 3.5","pluginVersion":"1.1.0","params":{"id":"12","name":"Clothing","url":"https://dummysite.com"}}`)
|
||||
payload := bytes.NewBuffer([]byte(`{"time":1518004715732,"token": "` + string(resMatch) + `" ,"platform":"Wordpress 4.2.7 / Woocommerce 3.5","pluginVersion":"1.1.0","params":{"id":"12","name":"Clothing","url":"https://dummysite.com"}}`))
|
||||
|
||||
req, err := http.NewRequestWithContext(ctx, "POST", "https://trk.mtrl.me/category", payload)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
|
||||
req.Header.Add("Content-Type", "application/json")
|
||||
req.Header.Add("Content-Length", `0`)
|
||||
|
||||
res, err := client.Do(req)
|
||||
if err == nil {
|
||||
defer res.Body.Close()
|
||||
if res.StatusCode >= 200 && res.StatusCode < 300 {
|
||||
s1.Verified = true
|
||||
} else {
|
||||
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
|
||||
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
|
||||
continue
|
||||
}
|
||||
|
||||
@@ -1,11 +1,11 @@
|
||||
package microsoftteamswebhook
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"io"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
@@ -15,41 +15,34 @@ import (
|
||||
|
||||
type Scanner struct{}
|
||||
|
||||
// Ensure the Scanner satisfies the interface at compile time.
|
||||
var _ detectors.Detector = (*Scanner)(nil)
|
||||
|
||||
var (
|
||||
client = common.SaneHttpClientTimeOut(5 * time.Second)
|
||||
|
||||
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
|
||||
keyPat = regexp.MustCompile(`(https:\/\/[a-zA-Z-0-9]+\.webhook\.office\.com\/webhookb2\/[a-zA-Z-0-9]{8}-[a-zA-Z-0-9]{4}-[a-zA-Z-0-9]{4}-[a-zA-Z-0-9]{4}-[a-zA-Z-0-9]{12}\@[a-zA-Z-0-9]{8}-[a-zA-Z-0-9]{4}-[a-zA-Z-0-9]{4}-[a-zA-Z-0-9]{4}-[a-zA-Z-0-9]{12}\/IncomingWebhook\/[a-zA-Z-0-9]{32}\/[a-zA-Z-0-9]{8}-[a-zA-Z-0-9]{4}-[a-zA-Z-0-9]{4}-[a-zA-Z-0-9]{4}-[a-zA-Z-0-9]{12})`)
|
||||
)
|
||||
|
||||
// Keywords are used for efficiently pre-filtering chunks.
|
||||
// Use identifiers in the secret preferably, or the provider name.
|
||||
func (s Scanner) Keywords() []string {
|
||||
return []string{"webhook.office.com"}
|
||||
func (s Scanner) Keywords() [][]byte {
|
||||
return [][]byte{[]byte("webhook.office.com")}
|
||||
}
|
||||
|
||||
// FromData will find and optionally verify MicrosoftTeamsWebhook secrets in a given set of bytes.
|
||||
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
|
||||
dataStr := string(data)
|
||||
|
||||
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
|
||||
matches := keyPat.FindAllSubmatch(data, -1)
|
||||
|
||||
for _, match := range matches {
|
||||
if len(match) != 2 {
|
||||
continue
|
||||
}
|
||||
resMatch := strings.TrimSpace(match[1])
|
||||
|
||||
resMatch := bytes.TrimSpace(match[1])
|
||||
s1 := detectors.Result{
|
||||
DetectorType: detectorspb.DetectorType_MicrosoftTeamsWebhook,
|
||||
Raw: []byte(resMatch),
|
||||
Raw: resMatch,
|
||||
}
|
||||
|
||||
if verify {
|
||||
payload := strings.NewReader(`{'text':''}`)
|
||||
req, err := http.NewRequestWithContext(ctx, "POST", resMatch, payload)
|
||||
payload := bytes.NewReader([]byte("{'text':''}"))
|
||||
req, err := http.NewRequestWithContext(ctx, "POST", string(resMatch), payload)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
@@ -57,9 +50,9 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
res, err := client.Do(req)
|
||||
if err == nil {
|
||||
body, err := io.ReadAll(res.Body)
|
||||
res.Body.Close()
|
||||
defer res.Body.Close()
|
||||
if err == nil {
|
||||
if res.StatusCode >= 200 && strings.Contains(string(body), "Text is required") {
|
||||
if res.StatusCode >= 200 && bytes.Contains(body, []byte("Text is required")) {
|
||||
s1.Verified = true
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,12 +1,12 @@
|
||||
package mindmeister
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
@@ -21,31 +21,25 @@ var _ detectors.Detector = (*Scanner)(nil)
|
||||
var (
|
||||
client = common.SaneHttpClient()
|
||||
|
||||
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
|
||||
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"mindmeister"}) + `\b([a-zA-Z0-9]{43})\b`)
|
||||
)
|
||||
|
||||
// Keywords are used for efficiently pre-filtering chunks.
|
||||
// Use identifiers in the secret preferably, or the provider name.
|
||||
func (s Scanner) Keywords() []string {
|
||||
return []string{"mindmeister"}
|
||||
func (s Scanner) Keywords() [][]byte {
|
||||
return [][]byte{[]byte("mindmeister")}
|
||||
}
|
||||
|
||||
// FromData will find and optionally verify Mindmeister secrets in a given set of bytes.
|
||||
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
|
||||
dataStr := string(data)
|
||||
|
||||
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
|
||||
matches := keyPat.FindAllSubmatch(data, -1)
|
||||
|
||||
for _, match := range matches {
|
||||
if len(match) != 2 {
|
||||
continue
|
||||
}
|
||||
resMatch := strings.TrimSpace(match[1])
|
||||
resMatch := bytes.TrimSpace(match[1])
|
||||
|
||||
s1 := detectors.Result{
|
||||
DetectorType: detectorspb.DetectorType_Mindmeister,
|
||||
Raw: []byte(resMatch),
|
||||
Raw: resMatch,
|
||||
}
|
||||
|
||||
if verify {
|
||||
@@ -53,13 +47,13 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", resMatch))
|
||||
req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", string(resMatch)))
|
||||
res, err := client.Do(req)
|
||||
|
||||
if err == nil {
|
||||
bodyBytes, err := io.ReadAll(res.Body)
|
||||
if err == nil {
|
||||
bodyString := string(bodyBytes)
|
||||
validResponse := strings.Contains(bodyString, `"stat":"ok"`)
|
||||
validResponse := bytes.Contains(bodyBytes, []byte(`"stat":"ok"`))
|
||||
defer res.Body.Close()
|
||||
if res.StatusCode >= 200 && res.StatusCode < 300 {
|
||||
if validResponse {
|
||||
@@ -68,7 +62,6 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
s1.Verified = false
|
||||
}
|
||||
} else {
|
||||
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
|
||||
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
|
||||
continue
|
||||
}
|
||||
|
||||
@@ -1,11 +1,11 @@
|
||||
package miro
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
@@ -14,37 +14,30 @@ import (
|
||||
|
||||
type Scanner struct{}
|
||||
|
||||
// Ensure the Scanner satisfies the interface at compile time.
|
||||
var _ detectors.Detector = (*Scanner)(nil)
|
||||
|
||||
var (
|
||||
client = common.SaneHttpClient()
|
||||
|
||||
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
|
||||
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"miro"}) + `\b([0-9a-zA-Z]{27})\b`)
|
||||
)
|
||||
|
||||
// Keywords are used for efficiently pre-filtering chunks.
|
||||
// Use identifiers in the secret preferably, or the provider name.
|
||||
func (s Scanner) Keywords() []string {
|
||||
return []string{"miro"}
|
||||
func (s Scanner) Keywords() [][]byte {
|
||||
return [][]byte{[]byte("miro")}
|
||||
}
|
||||
|
||||
// FromData will find and optionally verify Miro secrets in a given set of bytes.
|
||||
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
|
||||
dataStr := string(data)
|
||||
|
||||
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
|
||||
matches := keyPat.FindAllSubmatch(data, -1)
|
||||
|
||||
for _, match := range matches {
|
||||
if len(match) != 2 {
|
||||
continue
|
||||
}
|
||||
resMatch := strings.TrimSpace(match[1])
|
||||
resMatch := bytes.TrimSpace(match[1])
|
||||
|
||||
s1 := detectors.Result{
|
||||
DetectorType: detectorspb.DetectorType_Miro,
|
||||
Raw: []byte(resMatch),
|
||||
Raw: resMatch,
|
||||
}
|
||||
|
||||
if verify {
|
||||
@@ -60,7 +53,6 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
if res.StatusCode >= 200 && res.StatusCode < 300 {
|
||||
s1.Verified = true
|
||||
} else {
|
||||
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
|
||||
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
|
||||
continue
|
||||
}
|
||||
|
||||
+10
-18
@@ -1,11 +1,11 @@
|
||||
package mite
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
@@ -14,60 +14,52 @@ import (
|
||||
|
||||
type Scanner struct{}
|
||||
|
||||
// Ensure the Scanner satisfies the interface at compile time.
|
||||
var _ detectors.Detector = (*Scanner)(nil)
|
||||
|
||||
var (
|
||||
client = common.SaneHttpClient()
|
||||
|
||||
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
|
||||
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"mite"}) + `\b([0-9a-z]{16})\b`)
|
||||
urlPat = regexp.MustCompile(`\b([0-9a-z-]{1,}.mite.yo.lk)\b`)
|
||||
)
|
||||
|
||||
// Keywords are used for efficiently pre-filtering chunks.
|
||||
// Use identifiers in the secret preferably, or the provider name.
|
||||
func (s Scanner) Keywords() []string {
|
||||
return []string{"mite"}
|
||||
func (s Scanner) Keywords() [][]byte {
|
||||
return [][]byte{[]byte("mite")}
|
||||
}
|
||||
|
||||
// FromData will find and optionally verify Mite secrets in a given set of bytes.
|
||||
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
|
||||
dataStr := string(data)
|
||||
|
||||
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
|
||||
urlMatches := urlPat.FindAllStringSubmatch(dataStr, -1)
|
||||
matches := keyPat.FindAllSubmatch(data, -1)
|
||||
urlMatches := urlPat.FindAllSubmatch(data, -1)
|
||||
|
||||
for _, match := range matches {
|
||||
if len(match) != 2 {
|
||||
continue
|
||||
}
|
||||
resMatch := strings.TrimSpace(match[1])
|
||||
resMatch := bytes.TrimSpace(match[1])
|
||||
for _, urlMatch := range urlMatches {
|
||||
if len(urlMatch) != 2 {
|
||||
continue
|
||||
}
|
||||
resURL := strings.TrimSpace(urlMatch[1])
|
||||
resURL := bytes.TrimSpace(urlMatch[1])
|
||||
|
||||
s1 := detectors.Result{
|
||||
DetectorType: detectorspb.DetectorType_Mite,
|
||||
Raw: []byte(resMatch),
|
||||
Raw: resMatch,
|
||||
}
|
||||
|
||||
if verify {
|
||||
req, err := http.NewRequestWithContext(ctx, "GET", fmt.Sprintf("https://%s/account.json", resURL), nil)
|
||||
req, err := http.NewRequestWithContext(ctx, "GET", fmt.Sprintf("https://%s/account.json", string(resURL)), nil)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
req.Header.Add("Content-Type", "application/json")
|
||||
req.Header.Add("X-MiteApiKey", resMatch)
|
||||
req.Header.Add("X-MiteApiKey", string(resMatch))
|
||||
res, err := client.Do(req)
|
||||
if err == nil {
|
||||
defer res.Body.Close()
|
||||
if res.StatusCode >= 200 && res.StatusCode < 300 {
|
||||
s1.Verified = true
|
||||
} else {
|
||||
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
|
||||
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
|
||||
continue
|
||||
}
|
||||
|
||||
@@ -1,10 +1,10 @@
|
||||
package mixmax
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
@@ -19,31 +19,25 @@ var _ detectors.Detector = (*Scanner)(nil)
|
||||
var (
|
||||
client = common.SaneHttpClient()
|
||||
|
||||
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
|
||||
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"mixmax"}) + `\b([a-zA-Z0-9_-]{36})\b`)
|
||||
)
|
||||
|
||||
// Keywords are used for efficiently pre-filtering chunks.
|
||||
// Use identifiers in the secret preferably, or the provider name.
|
||||
func (s Scanner) Keywords() []string {
|
||||
return []string{"mixmax"}
|
||||
func (s Scanner) Keywords() [][]byte {
|
||||
return [][]byte{[]byte("mixmax")}
|
||||
}
|
||||
|
||||
// FromData will find and optionally verify Mixmax secrets in a given set of bytes.
|
||||
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
|
||||
dataStr := string(data)
|
||||
|
||||
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
|
||||
matches := keyPat.FindAllSubmatch(data, -1)
|
||||
|
||||
for _, match := range matches {
|
||||
if len(match) != 2 {
|
||||
continue
|
||||
}
|
||||
resMatch := strings.TrimSpace(match[1])
|
||||
resMatch := bytes.TrimSpace(match[1])
|
||||
|
||||
s1 := detectors.Result{
|
||||
DetectorType: detectorspb.DetectorType_Mixmax,
|
||||
Raw: []byte(resMatch),
|
||||
Raw: resMatch,
|
||||
}
|
||||
|
||||
if verify {
|
||||
@@ -52,14 +46,13 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
continue
|
||||
}
|
||||
|
||||
req.Header.Add("X-API-Token", resMatch)
|
||||
req.Header.Add("X-API-Token", string(resMatch))
|
||||
res, err := client.Do(req)
|
||||
if err == nil {
|
||||
defer res.Body.Close()
|
||||
if res.StatusCode >= 200 && res.StatusCode < 300 {
|
||||
s1.Verified = true
|
||||
} else {
|
||||
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
|
||||
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
|
||||
continue
|
||||
}
|
||||
|
||||
@@ -1,10 +1,10 @@
|
||||
package mixpanel
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
@@ -13,46 +13,39 @@ import (
|
||||
|
||||
type Scanner struct{}
|
||||
|
||||
// Ensure the Scanner satisfies the interface at compile time.
|
||||
var _ detectors.Detector = (*Scanner)(nil)
|
||||
|
||||
var (
|
||||
client = common.SaneHttpClient()
|
||||
|
||||
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
|
||||
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"mixpanel"}) + `\b([a-zA-Z0-9-]{32})\b`)
|
||||
idPat = regexp.MustCompile(detectors.PrefixRegex([]string{"mixpanel"}) + `\b([a-zA-Z0-9.-]{30,40})\b`)
|
||||
)
|
||||
|
||||
// Keywords are used for efficiently pre-filtering chunks.
|
||||
// Use identifiers in the secret preferably, or the provider name.
|
||||
func (s Scanner) Keywords() []string {
|
||||
return []string{"mixpanel"}
|
||||
func (s Scanner) Keywords() [][]byte {
|
||||
return [][]byte{[]byte("mixpanel")}
|
||||
}
|
||||
|
||||
// FromData will find and optionally verify Mixpanel secrets in a given set of bytes.
|
||||
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
|
||||
dataStr := string(data)
|
||||
|
||||
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
|
||||
idMatches := idPat.FindAllStringSubmatch(dataStr, -1)
|
||||
matches := keyPat.FindAllSubmatch(data, -1)
|
||||
idMatches := idPat.FindAllSubmatch(data, -1)
|
||||
|
||||
for _, match := range matches {
|
||||
if len(match) != 2 {
|
||||
continue
|
||||
}
|
||||
tokenPatMatch := strings.TrimSpace(match[1])
|
||||
tokenPatMatch := bytes.TrimSpace(match[1])
|
||||
|
||||
for _, idMatch := range idMatches {
|
||||
if len(idMatch) != 2 {
|
||||
continue
|
||||
}
|
||||
|
||||
userPatMatch := strings.TrimSpace(idMatch[1])
|
||||
userPatMatch := bytes.TrimSpace(idMatch[1])
|
||||
|
||||
s1 := detectors.Result{
|
||||
DetectorType: detectorspb.DetectorType_Mixpanel,
|
||||
Raw: []byte(tokenPatMatch),
|
||||
Raw: tokenPatMatch,
|
||||
}
|
||||
|
||||
if verify {
|
||||
@@ -60,14 +53,13 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
req.SetBasicAuth(userPatMatch, tokenPatMatch)
|
||||
req.SetBasicAuth(string(userPatMatch), string(tokenPatMatch))
|
||||
res, err := client.Do(req)
|
||||
if err == nil {
|
||||
defer res.Body.Close()
|
||||
if res.StatusCode >= 200 && res.StatusCode < 300 {
|
||||
s1.Verified = true
|
||||
} else {
|
||||
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
|
||||
if detectors.IsKnownFalsePositive(tokenPatMatch, detectors.DefaultFalsePositives, true) {
|
||||
continue
|
||||
}
|
||||
|
||||
@@ -1,12 +1,11 @@
|
||||
package mockaroo
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
@@ -27,29 +26,27 @@ var (
|
||||
|
||||
// Keywords are used for efficiently pre-filtering chunks.
|
||||
// Use identifiers in the secret preferably, or the provider name.
|
||||
func (s Scanner) Keywords() []string {
|
||||
return []string{"mockaroo"}
|
||||
func (s Scanner) Keywords() [][]byte {
|
||||
return [][]byte{[]byte("mockaroo")}
|
||||
}
|
||||
|
||||
// FromData will find and optionally verify Mockaroo secrets in a given set of bytes.
|
||||
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
|
||||
dataStr := string(data)
|
||||
|
||||
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
|
||||
matches := keyPat.FindAllSubmatch(data, -1)
|
||||
|
||||
for _, match := range matches {
|
||||
if len(match) != 2 {
|
||||
continue
|
||||
}
|
||||
resMatch := strings.TrimSpace(match[1])
|
||||
resMatch := bytes.TrimSpace(match[1])
|
||||
|
||||
s1 := detectors.Result{
|
||||
DetectorType: detectorspb.DetectorType_Mockaroo,
|
||||
Raw: []byte(resMatch),
|
||||
Raw: resMatch,
|
||||
}
|
||||
|
||||
if verify {
|
||||
req, err := http.NewRequestWithContext(ctx, "GET", fmt.Sprintf("https://api.mockaroo.com/api/types?key=%s", resMatch), nil)
|
||||
req, err := http.NewRequestWithContext(ctx, "GET", fmt.Sprintf("https://api.mockaroo.com/api/types?key=%s", string(resMatch)), nil)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
@@ -57,13 +54,8 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
if err == nil {
|
||||
defer res.Body.Close()
|
||||
if res.StatusCode >= 200 && res.StatusCode < 300 {
|
||||
var t typeRes
|
||||
err = json.NewDecoder(res.Body).Decode(&t)
|
||||
if err == nil && len(t.Types) > 0 {
|
||||
s1.Verified = true
|
||||
}
|
||||
s1.Verified = true
|
||||
} else {
|
||||
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
|
||||
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
|
||||
continue
|
||||
}
|
||||
@@ -80,11 +72,3 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
func (s Scanner) Type() detectorspb.DetectorType {
|
||||
return detectorspb.DetectorType_Mockaroo
|
||||
}
|
||||
|
||||
type typeRes struct {
|
||||
Types []struct {
|
||||
Name string `json:"name"`
|
||||
Type interface{} `json:"type"`
|
||||
Parameters []interface{} `json:"parameters"`
|
||||
} `json:"types"`
|
||||
}
|
||||
|
||||
@@ -1,11 +1,11 @@
|
||||
package moderation
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
@@ -27,36 +27,34 @@ var (
|
||||
|
||||
// Keywords are used for efficiently pre-filtering chunks.
|
||||
// Use identifiers in the secret preferably, or the provider name.
|
||||
func (s Scanner) Keywords() []string {
|
||||
return []string{"moderation"}
|
||||
func (s Scanner) Keywords() [][]byte {
|
||||
return [][]byte{[]byte("moderation")}
|
||||
}
|
||||
|
||||
// FromData will find and optionally verify Moderation secrets in a given set of bytes.
|
||||
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
|
||||
dataStr := string(data)
|
||||
|
||||
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
|
||||
matches := keyPat.FindAllSubmatch(data, -1)
|
||||
|
||||
for _, match := range matches {
|
||||
if len(match) != 2 {
|
||||
continue
|
||||
}
|
||||
resMatch := strings.TrimSpace(match[1])
|
||||
resMatch := bytes.TrimSpace(match[1])
|
||||
|
||||
s1 := detectors.Result{
|
||||
DetectorType: detectorspb.DetectorType_Moderation,
|
||||
Raw: []byte(resMatch),
|
||||
Raw: resMatch,
|
||||
}
|
||||
|
||||
if verify {
|
||||
timeout := 5 * time.Second
|
||||
client.Timeout = timeout
|
||||
payload := strings.NewReader(`{"value": "This is an english text"}`)
|
||||
payload := bytes.NewReader([]byte(`{"value": "This is an english text"}`))
|
||||
req, err := http.NewRequestWithContext(ctx, "POST", "https://moderationapi.com/api/v1/analyze/language", payload)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", resMatch))
|
||||
req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", string(resMatch)))
|
||||
req.Header.Add("Content-type", "application/json")
|
||||
res, err := client.Do(req)
|
||||
|
||||
|
||||
@@ -1,10 +1,10 @@
|
||||
package monday
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
@@ -13,54 +13,45 @@ import (
|
||||
|
||||
type Scanner struct{}
|
||||
|
||||
// Ensure the Scanner satisfies the interface at compile time.
|
||||
var _ detectors.Detector = (*Scanner)(nil)
|
||||
|
||||
var (
|
||||
client = common.SaneHttpClient()
|
||||
|
||||
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
|
||||
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"monday"}) + `\b(ey[a-zA-Z0-9_.]{210,225})\b`)
|
||||
)
|
||||
|
||||
// Keywords are used for efficiently pre-filtering chunks.
|
||||
// Use identifiers in the secret preferably, or the provider name.
|
||||
func (s Scanner) Keywords() []string {
|
||||
return []string{"monday"}
|
||||
func (s Scanner) Keywords() [][]byte {
|
||||
return [][]byte{[]byte("monday")}
|
||||
}
|
||||
|
||||
// FromData will find and optionally verify Monday secrets in a given set of bytes.
|
||||
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
|
||||
dataStr := string(data)
|
||||
|
||||
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
|
||||
matches := keyPat.FindAllSubmatch(data, -1)
|
||||
|
||||
for _, match := range matches {
|
||||
if len(match) != 2 {
|
||||
continue
|
||||
}
|
||||
resMatch := strings.TrimSpace(match[1])
|
||||
resMatch := bytes.TrimSpace(match[1])
|
||||
|
||||
s1 := detectors.Result{
|
||||
DetectorType: detectorspb.DetectorType_Monday,
|
||||
Raw: []byte(resMatch),
|
||||
Raw: resMatch,
|
||||
}
|
||||
|
||||
if verify {
|
||||
payload := strings.NewReader(`{"query":"{boards(limit:1){id name}}"}`)
|
||||
payload := bytes.NewReader([]byte(`{"query":"{boards(limit:1){id name}}"}`))
|
||||
req, err := http.NewRequestWithContext(ctx, "POST", "https://api.monday.com/v2", payload)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
req.Header.Add("Content-Type", "application/json")
|
||||
req.Header.Add("Authorization", resMatch)
|
||||
req.Header.Add("Authorization", string(resMatch))
|
||||
res, err := client.Do(req)
|
||||
if err == nil {
|
||||
defer res.Body.Close()
|
||||
if res.StatusCode >= 200 && res.StatusCode < 300 {
|
||||
s1.Verified = true
|
||||
} else {
|
||||
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
|
||||
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
|
||||
continue
|
||||
}
|
||||
|
||||
@@ -1,16 +1,16 @@
|
||||
package mongodb
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"go.mongodb.org/mongo-driver/x/mongo/driver/auth"
|
||||
"go.mongodb.org/mongo-driver/x/mongo/driver/topology"
|
||||
"regexp"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"go.mongodb.org/mongo-driver/mongo"
|
||||
"go.mongodb.org/mongo-driver/mongo/options"
|
||||
"go.mongodb.org/mongo-driver/mongo/readpref"
|
||||
"go.mongodb.org/mongo-driver/x/mongo/driver/auth"
|
||||
"go.mongodb.org/mongo-driver/x/mongo/driver/topology"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
|
||||
@@ -26,28 +26,26 @@ var _ detectors.Detector = (*Scanner)(nil)
|
||||
var (
|
||||
defaultTimeout = 2 * time.Second
|
||||
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
|
||||
keyPat = regexp.MustCompile(`\b(mongodb(\+srv)?://[\S]{3,50}:([\S]{3,88})@[-.%\w\/:]+)\b`)
|
||||
keyPat = regexp.MustCompile(`\b(mongodb(\+srv)?://\S{3,50}:(\S{3,50})@[-.%\w/:]+)\b`)
|
||||
// TODO: Add support for sharded cluster, replica set and Atlas Deployment.
|
||||
)
|
||||
|
||||
// Keywords are used for efficiently pre-filtering chunks.
|
||||
// Use identifiers in the secret preferably, or the provider name.
|
||||
func (s Scanner) Keywords() []string {
|
||||
return []string{"mongodb"}
|
||||
func (s Scanner) Keywords() [][]byte {
|
||||
return [][]byte{[]byte("mongodb")}
|
||||
}
|
||||
|
||||
// FromData will find and optionally verify MongoDB secrets in a given set of bytes.
|
||||
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
|
||||
dataStr := string(data)
|
||||
|
||||
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
|
||||
matches := keyPat.FindAllSubmatch(data, -1)
|
||||
|
||||
for _, match := range matches {
|
||||
resMatch := strings.TrimSpace(match[1])
|
||||
resMatch := bytes.TrimSpace(match[1])
|
||||
|
||||
s1 := detectors.Result{
|
||||
DetectorType: detectorspb.DetectorType_MongoDB,
|
||||
Raw: []byte(resMatch),
|
||||
Raw: resMatch,
|
||||
}
|
||||
|
||||
if verify {
|
||||
@@ -55,7 +53,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
if timeout == 0 {
|
||||
timeout = defaultTimeout
|
||||
}
|
||||
err := verifyUri(resMatch, timeout)
|
||||
err := verifyUri(string(resMatch), timeout)
|
||||
s1.Verified = err == nil
|
||||
if !isErrDeterminate(err) {
|
||||
s1.VerificationError = err
|
||||
|
||||
@@ -43,6 +43,8 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
instanceMatches := instancePat.FindAllStringSubmatch(dataStr, -1)
|
||||
tokenMatches := accessTokenPat.FindAllStringSubmatch(dataStr, -1)
|
||||
|
||||
fmt.Printf("instanceMatches: %v\n", instanceMatches)
|
||||
|
||||
for _, instance := range instanceMatches {
|
||||
if len(instance) != 1 {
|
||||
continue
|
||||
|
||||
@@ -2,10 +2,6 @@ package engine
|
||||
|
||||
import (
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/envoyapikey"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/huggingface"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/salesforce"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/snowflake"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/trufflehogenterprise"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/abbysale"
|
||||
@@ -291,7 +287,6 @@ import (
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/getresponse"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/getsandbox"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/github"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/github_oauth2"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/github_old"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/githubapp"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/gitlab"
|
||||
@@ -561,6 +556,7 @@ import (
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/satismeterwritekey"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/saucelabs"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/scalewaykey"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/github_oauth2"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/scalr"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/scrapeowl"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/scraperapi"
|
||||
@@ -1536,10 +1532,6 @@ func DefaultDetectors() []detectors.Detector {
|
||||
couchbase.Scanner{},
|
||||
envoyapikey.Scanner{},
|
||||
github_oauth2.Scanner{},
|
||||
snowflake.Scanner{},
|
||||
huggingface.Scanner{},
|
||||
trufflehogenterprise.Scanner{},
|
||||
salesforce.Scanner{},
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
+23
-24
@@ -29,8 +29,6 @@ var (
|
||||
maxDepth = 5
|
||||
maxSize = 250 * 1024 * 1024 // 20MB
|
||||
maxTimeout = time.Duration(30) * time.Second
|
||||
|
||||
defaultBufferSize = 512
|
||||
)
|
||||
|
||||
// Ensure the Archive satisfies the interfaces at compile time.
|
||||
@@ -64,7 +62,7 @@ func SetArchiveMaxTimeout(timeout time.Duration) {
|
||||
|
||||
// FromFile extracts the files from an archive.
|
||||
func (a *Archive) FromFile(originalCtx context.Context, data io.Reader) chan []byte {
|
||||
archiveChan := make(chan []byte, defaultBufferSize)
|
||||
archiveChan := make(chan []byte, 512)
|
||||
go func() {
|
||||
ctx, cancel := context.WithTimeout(originalCtx, maxTimeout)
|
||||
logger := logContext.AddLogger(ctx).Logger()
|
||||
@@ -185,28 +183,29 @@ func (a *Archive) ReadToMax(ctx context.Context, reader io.Reader) (data []byte,
|
||||
logger.Error(err, "Panic occurred when reading archive")
|
||||
}
|
||||
}()
|
||||
|
||||
if common.IsDone(ctx) {
|
||||
return nil, ctx.Err()
|
||||
fileContent := bytes.Buffer{}
|
||||
logger.V(5).Info("Remaining buffer capacity", "bytes", maxSize-a.size)
|
||||
for i := 0; i <= maxSize/512; i++ {
|
||||
if common.IsDone(ctx) {
|
||||
return nil, ctx.Err()
|
||||
}
|
||||
fileChunk := make([]byte, 512)
|
||||
bRead, err := reader.Read(fileChunk)
|
||||
if err != nil && !errors.Is(err, io.EOF) {
|
||||
return []byte{}, err
|
||||
}
|
||||
a.size += bRead
|
||||
if len(fileChunk) > 0 {
|
||||
fileContent.Write(fileChunk[0:bRead])
|
||||
}
|
||||
if bRead < 512 {
|
||||
return fileContent.Bytes(), nil
|
||||
}
|
||||
if a.size >= maxSize && bRead == 512 {
|
||||
logger.V(2).Info("Max archive size reached.")
|
||||
return fileContent.Bytes(), nil
|
||||
}
|
||||
}
|
||||
|
||||
var fileContent bytes.Buffer
|
||||
// Create a limited reader to ensure we don't read more than the max size.
|
||||
lr := io.LimitReader(reader, int64(maxSize))
|
||||
|
||||
// Using io.CopyBuffer for performance advantages. Though buf is mandatory
|
||||
// for the method, due to the internal implementation of io.CopyBuffer, when
|
||||
// *bytes.Buffer implements io.WriterTo or io.ReaderFrom, the provided buf
|
||||
// is simply ignored. Thus, we can pass nil for the buf parameter.
|
||||
_, err = io.CopyBuffer(&fileContent, lr, nil)
|
||||
if err != nil && !errors.Is(err, io.EOF) {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if fileContent.Len() == maxSize {
|
||||
logger.V(2).Info("Max archive size reached.")
|
||||
}
|
||||
|
||||
return fileContent.Bytes(), nil
|
||||
}
|
||||
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
package handlers
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"io"
|
||||
"net/http"
|
||||
@@ -127,58 +126,6 @@ func TestHandleFile(t *testing.T) {
|
||||
assert.Equal(t, 1, len(ch))
|
||||
}
|
||||
|
||||
func TestReadToMax(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
input []byte
|
||||
expected []byte
|
||||
}{
|
||||
{
|
||||
name: "read full content within maxSize",
|
||||
input: []byte("abcdefg"),
|
||||
expected: []byte("abcdefg"),
|
||||
},
|
||||
{
|
||||
name: "read content larger than maxSize",
|
||||
input: make([]byte, maxSize+10), // this creates a byte slice 10 bytes larger than maxSize
|
||||
expected: make([]byte, maxSize),
|
||||
},
|
||||
{
|
||||
name: "empty input",
|
||||
input: []byte(""),
|
||||
expected: []byte(""),
|
||||
},
|
||||
}
|
||||
|
||||
a := &Archive{}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
reader := bytes.NewReader(tt.input)
|
||||
output, err := a.ReadToMax(context.Background(), reader)
|
||||
assert.Nil(t, err)
|
||||
|
||||
assert.Equal(t, tt.expected, output)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkReadToMax(b *testing.B) {
|
||||
data := bytes.Repeat([]byte("a"), 1024*1000) // 1MB of data.
|
||||
reader := bytes.NewReader(data)
|
||||
a := &Archive{}
|
||||
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
b.StartTimer()
|
||||
_, _ = a.ReadToMax(context.Background(), reader)
|
||||
b.StopTimer()
|
||||
|
||||
_, _ = reader.Seek(0, 0) // Reset the reader position.
|
||||
a.size = 0 // Reset archive size.
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtractDebContent(t *testing.T) {
|
||||
// Open the sample .deb file from the testdata folder.
|
||||
file, err := os.Open("testdata/test.deb")
|
||||
|
||||
@@ -133,7 +133,7 @@ func BranchHeads(repo *gogit.Repository) (map[string]*object.Commit, error) {
|
||||
}
|
||||
headCommit, err := repo.CommitObject(*headHash)
|
||||
if err != nil {
|
||||
logger.Error(err, "unable to get commit", "head_hash", headHash.String())
|
||||
logger.Error(err, "unable to get commit", "commit", headCommit.String())
|
||||
return nil
|
||||
}
|
||||
branches[branchName] = headCommit
|
||||
|
||||
@@ -20,7 +20,7 @@ const (
|
||||
|
||||
// Chunker takes a chunk and splits it into chunks of ChunkSize.
|
||||
func Chunker(originalChunk *Chunk) chan *Chunk {
|
||||
chunkChan := make(chan *Chunk, 1)
|
||||
chunkChan := make(chan *Chunk)
|
||||
go func() {
|
||||
defer close(chunkChan)
|
||||
if len(originalChunk.Data) <= TotalChunkSize {
|
||||
|
||||
@@ -238,7 +238,6 @@ func (s *Source) chunkAction(ctx context.Context, proj project, bld build, act a
|
||||
SourceType: s.Type(),
|
||||
SourceName: s.name,
|
||||
SourceID: s.SourceID(),
|
||||
JobID: s.JobID(),
|
||||
Data: removeCircleSha1Line(data.Bytes()),
|
||||
SourceMetadata: &source_metadatapb.MetaData{
|
||||
Data: &source_metadatapb.MetaData_Circleci{
|
||||
|
||||
@@ -174,7 +174,6 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) err
|
||||
SourceType: s.Type(),
|
||||
SourceName: s.name,
|
||||
SourceID: s.SourceID(),
|
||||
JobID: s.JobID(),
|
||||
Data: file.Bytes(),
|
||||
SourceMetadata: &source_metadatapb.MetaData{
|
||||
Data: &source_metadatapb.MetaData_Docker{
|
||||
|
||||
@@ -161,7 +161,6 @@ func (s *Source) scanFile(ctx context.Context, path string, chunksChan chan *sou
|
||||
SourceType: s.Type(),
|
||||
SourceName: s.name,
|
||||
SourceID: s.SourceID(),
|
||||
JobID: s.JobID(),
|
||||
SourceMetadata: &source_metadatapb.MetaData{
|
||||
Data: &source_metadatapb.MetaData_Filesystem{
|
||||
Filesystem: &source_metadatapb.Filesystem{
|
||||
@@ -192,7 +191,6 @@ func (s *Source) scanFile(ctx context.Context, path string, chunksChan chan *sou
|
||||
SourceType: s.Type(),
|
||||
SourceName: s.name,
|
||||
SourceID: s.SourceID(),
|
||||
JobID: s.JobID(),
|
||||
Data: data.Bytes(),
|
||||
SourceMetadata: &source_metadatapb.MetaData{
|
||||
Data: &source_metadatapb.MetaData_Filesystem{
|
||||
|
||||
@@ -324,7 +324,6 @@ func (s *Source) processObject(ctx context.Context, o object) error {
|
||||
chunkSkel := &sources.Chunk{
|
||||
SourceName: s.name,
|
||||
SourceType: s.Type(),
|
||||
JobID: s.JobID(),
|
||||
SourceID: s.sourceId,
|
||||
Verify: s.verify,
|
||||
SourceMetadata: &source_metadatapb.MetaData{
|
||||
|
||||
@@ -446,7 +446,6 @@ func (s *Git) ScanCommits(ctx context.Context, repo *git.Repository, path string
|
||||
chunkSkel := &sources.Chunk{
|
||||
SourceName: s.sourceName,
|
||||
SourceID: s.sourceID,
|
||||
JobID: s.jobID,
|
||||
SourceType: s.sourceType,
|
||||
SourceMetadata: metadata,
|
||||
Verify: s.verify,
|
||||
@@ -465,7 +464,6 @@ func (s *Git) ScanCommits(ctx context.Context, repo *git.Repository, path string
|
||||
chunksChan <- &sources.Chunk{
|
||||
SourceName: s.sourceName,
|
||||
SourceID: s.sourceID,
|
||||
JobID: s.jobID,
|
||||
SourceType: s.sourceType,
|
||||
SourceMetadata: metadata,
|
||||
Data: diff.Content.Bytes(),
|
||||
@@ -492,7 +490,6 @@ func (s *Git) gitChunk(ctx context.Context, diff gitparse.Diff, fileName, email,
|
||||
chunksChan <- &sources.Chunk{
|
||||
SourceName: s.sourceName,
|
||||
SourceID: s.sourceID,
|
||||
JobID: s.jobID,
|
||||
SourceType: s.sourceType,
|
||||
SourceMetadata: metadata,
|
||||
Data: append([]byte{}, newChunkBuffer.Bytes()...),
|
||||
@@ -507,7 +504,6 @@ func (s *Git) gitChunk(ctx context.Context, diff gitparse.Diff, fileName, email,
|
||||
chunksChan <- &sources.Chunk{
|
||||
SourceName: s.sourceName,
|
||||
SourceID: s.sourceID,
|
||||
JobID: s.jobID,
|
||||
SourceType: s.sourceType,
|
||||
SourceMetadata: metadata,
|
||||
Data: line,
|
||||
@@ -527,7 +523,6 @@ func (s *Git) gitChunk(ctx context.Context, diff gitparse.Diff, fileName, email,
|
||||
chunksChan <- &sources.Chunk{
|
||||
SourceName: s.sourceName,
|
||||
SourceID: s.sourceID,
|
||||
JobID: s.jobID,
|
||||
SourceType: s.sourceType,
|
||||
SourceMetadata: metadata,
|
||||
Data: append([]byte{}, newChunkBuffer.Bytes()...),
|
||||
@@ -593,7 +588,6 @@ func (s *Git) ScanStaged(ctx context.Context, repo *git.Repository, path string,
|
||||
chunkSkel := &sources.Chunk{
|
||||
SourceName: s.sourceName,
|
||||
SourceID: s.sourceID,
|
||||
JobID: s.jobID,
|
||||
SourceType: s.sourceType,
|
||||
SourceMetadata: metadata,
|
||||
Verify: s.verify,
|
||||
@@ -608,7 +602,6 @@ func (s *Git) ScanStaged(ctx context.Context, repo *git.Repository, path string,
|
||||
chunksChan <- &sources.Chunk{
|
||||
SourceName: s.sourceName,
|
||||
SourceID: s.sourceID,
|
||||
JobID: s.jobID,
|
||||
SourceType: s.sourceType,
|
||||
SourceMetadata: metadata,
|
||||
Data: diff.Content.Bytes(),
|
||||
|
||||
@@ -149,16 +149,14 @@ func (s *Source) newFilteredRepoCache(c cache.Cache, include, exclude []string)
|
||||
for _, ig := range include {
|
||||
g, err := glob.Compile(ig)
|
||||
if err != nil {
|
||||
s.log.V(1).Info("invalid include glob", "include_value", ig, "err", err)
|
||||
continue
|
||||
s.log.V(1).Info("invalid include glob", "glob", g, "err", err)
|
||||
}
|
||||
includeGlobs = append(includeGlobs, g)
|
||||
}
|
||||
for _, eg := range exclude {
|
||||
g, err := glob.Compile(eg)
|
||||
if err != nil {
|
||||
s.log.V(1).Info("invalid exclude glob", "exclude_value", eg, "err", err)
|
||||
continue
|
||||
s.log.V(1).Info("invalid exclude glob", "glob", g, "err", err)
|
||||
}
|
||||
excludeGlobs = append(excludeGlobs, g)
|
||||
}
|
||||
@@ -1225,7 +1223,6 @@ func (s *Source) chunkIssueComments(ctx context.Context, repo, repoPath string,
|
||||
chunk := &sources.Chunk{
|
||||
SourceName: s.name,
|
||||
SourceID: s.SourceID(),
|
||||
JobID: s.JobID(),
|
||||
SourceType: s.Type(),
|
||||
SourceMetadata: &source_metadatapb.MetaData{
|
||||
Data: &source_metadatapb.MetaData_Github{
|
||||
@@ -1259,7 +1256,6 @@ func (s *Source) chunkPullRequestComments(ctx context.Context, repo string, comm
|
||||
SourceName: s.name,
|
||||
SourceID: s.SourceID(),
|
||||
SourceType: s.Type(),
|
||||
JobID: s.JobID(),
|
||||
SourceMetadata: &source_metadatapb.MetaData{
|
||||
Data: &source_metadatapb.MetaData_Github{
|
||||
Github: &source_metadatapb.Github{
|
||||
@@ -1291,7 +1287,6 @@ func (s *Source) chunkGistComments(ctx context.Context, gistUrl string, comments
|
||||
SourceName: s.name,
|
||||
SourceID: s.SourceID(),
|
||||
SourceType: s.Type(),
|
||||
JobID: s.JobID(),
|
||||
SourceMetadata: &source_metadatapb.MetaData{
|
||||
Data: &source_metadatapb.MetaData_Github{
|
||||
Github: &source_metadatapb.Github{
|
||||
|
||||
@@ -376,7 +376,6 @@ func (s *Source) pageChunker(ctx context.Context, client *s3.S3, chunksChan chan
|
||||
SourceType: s.Type(),
|
||||
SourceName: s.name,
|
||||
SourceID: s.SourceID(),
|
||||
JobID: s.JobID(),
|
||||
SourceMetadata: &source_metadatapb.MetaData{
|
||||
Data: &source_metadatapb.MetaData_S3{
|
||||
S3: &source_metadatapb.S3{
|
||||
|
||||
@@ -36,10 +36,7 @@ type SourceManager struct {
|
||||
handles map[handle]sourceInfo
|
||||
handlesLock sync.Mutex
|
||||
// Pool limiting the amount of concurrent sources running.
|
||||
pool errgroup.Group
|
||||
poolLimit int
|
||||
currentRunningCount int32
|
||||
// Max number of units to scan concurrently per source.
|
||||
pool errgroup.Group
|
||||
concurrentUnits int
|
||||
// Run the sources using source unit enumeration / chunking if available.
|
||||
useSourceUnits bool
|
||||
@@ -71,10 +68,7 @@ func WithReportHook(hook JobProgressHook) func(*SourceManager) {
|
||||
|
||||
// WithConcurrentSources limits the concurrent number of sources a manager can run.
|
||||
func WithConcurrentSources(concurrency int) func(*SourceManager) {
|
||||
return func(mgr *SourceManager) {
|
||||
mgr.pool.SetLimit(concurrency)
|
||||
mgr.poolLimit = concurrency
|
||||
}
|
||||
return func(mgr *SourceManager) { mgr.pool.SetLimit(concurrency) }
|
||||
}
|
||||
|
||||
// WithBufferedOutput sets the size of the buffer used for the Chunks() channel.
|
||||
@@ -173,8 +167,6 @@ func (s *SourceManager) asyncRun(ctx context.Context, handle handle) (JobProgres
|
||||
ctx, cancel := context.WithCancel(ctx)
|
||||
progress := NewJobProgress(jobID, int64(handle), sourceName, WithHooks(s.hooks...), WithCancel(cancel))
|
||||
s.pool.Go(func() error {
|
||||
atomic.AddInt32(&s.currentRunningCount, 1)
|
||||
defer atomic.AddInt32(&s.currentRunningCount, -1)
|
||||
ctx := context.WithValues(ctx,
|
||||
"job_id", jobID,
|
||||
"source_manager_worker_id", common.RandomID(5),
|
||||
@@ -215,16 +207,6 @@ func (s *SourceManager) ScanChunk(chunk *Chunk) {
|
||||
s.outputChunks <- chunk
|
||||
}
|
||||
|
||||
// AvailableCapacity returns the number of concurrent jobs the manager can
|
||||
// accommodate at this time. If there is no limit, -1 is returned.
|
||||
func (s *SourceManager) AvailableCapacity() int {
|
||||
if s.poolLimit == 0 {
|
||||
return -1
|
||||
}
|
||||
runCount := atomic.LoadInt32(&s.currentRunningCount)
|
||||
return s.poolLimit - int(runCount)
|
||||
}
|
||||
|
||||
// preflightChecks is a helper method to check the Manager or the context isn't
|
||||
// done and that the handle is valid.
|
||||
func (s *SourceManager) preflightChecks(ctx context.Context, handle handle) error {
|
||||
|
||||
@@ -317,24 +317,3 @@ func TestSourceManagerCancelRun(t *testing.T) {
|
||||
assert.Error(t, ref.Snapshot().FatalError())
|
||||
assert.True(t, errors.Is(ref.Snapshot().FatalError(), returnedErr))
|
||||
}
|
||||
|
||||
func TestSourceManagerAvailableCapacity(t *testing.T) {
|
||||
mgr := NewManager(WithConcurrentSources(1337))
|
||||
start, end := make(chan struct{}), make(chan struct{})
|
||||
handle, err := enrollDummy(mgr, callbackChunker{func(context.Context, chan *Chunk) error {
|
||||
start <- struct{}{} // Send start signal.
|
||||
<-end // Wait for end signal.
|
||||
return nil
|
||||
}})
|
||||
assert.NoError(t, err)
|
||||
|
||||
assert.Equal(t, 1337, mgr.AvailableCapacity())
|
||||
ref, err := mgr.ScheduleRun(context.Background(), handle)
|
||||
assert.NoError(t, err)
|
||||
|
||||
<-start // Wait for start signal.
|
||||
assert.Equal(t, 1336, mgr.AvailableCapacity())
|
||||
end <- struct{}{} // Send end signal.
|
||||
<-ref.Done() // Wait for the job to finish.
|
||||
assert.Equal(t, 1337, mgr.AvailableCapacity())
|
||||
}
|
||||
|
||||
@@ -272,7 +272,6 @@ func (s *Source) monitorConnection(ctx context.Context, conn net.Conn, chunksCha
|
||||
SourceName: s.syslog.sourceName,
|
||||
SourceID: s.syslog.sourceID,
|
||||
SourceType: s.syslog.sourceType,
|
||||
JobID: s.JobID(),
|
||||
SourceMetadata: metadata,
|
||||
Data: input,
|
||||
Verify: s.verify,
|
||||
@@ -314,7 +313,6 @@ func (s *Source) acceptUDPConnections(ctx context.Context, netListener net.Packe
|
||||
chunksChan <- &sources.Chunk{
|
||||
SourceName: s.syslog.sourceName,
|
||||
SourceID: s.syslog.sourceID,
|
||||
JobID: s.JobID(),
|
||||
SourceType: s.syslog.sourceType,
|
||||
SourceMetadata: metadata,
|
||||
Data: input,
|
||||
|
||||
Reference in New Issue
Block a user