Compare commits

..

1 Commits

Author SHA1 Message Date
Ahrav Dutta 2e9420cfc5 PR 19 of 40 2023-08-25 21:40:45 -07:00
37 changed files with 217 additions and 482 deletions
+1 -1
View File
@@ -32,7 +32,7 @@ Have questions? Feedback? Jump in slack or discord and hang out with us
Join our [Slack Community](https://join.slack.com/t/trufflehog-community/shared_invite/zt-pw2qbi43-Aa86hkiimstfdKH9UCpPzQ)
Join the [Secret Scanning Discord](https://discord.gg/8Hzbrnkr7E)
Join the [Secret Scanning Discord](https://discord.gg/sydS6AHTUP)
# :tv: Demo
+7 -10
View File
@@ -1,10 +1,10 @@
package lastfm
import (
"bytes"
"context"
"net/http"
"regexp"
"strings"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
@@ -25,29 +25,27 @@ var (
// Keywords are used for efficiently pre-filtering chunks.
// Use identifiers in the secret preferably, or the provider name.
func (s Scanner) Keywords() []string {
return []string{"lastfm"}
func (s Scanner) Keywords() [][]byte {
return [][]byte{[]byte("lastfm")}
}
// FromData will find and optionally verify Lastfm secrets in a given set of bytes.
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
dataStr := string(data)
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
matches := keyPat.FindAllSubmatch(data, -1)
for _, match := range matches {
if len(match) != 2 {
continue
}
resMatch := strings.TrimSpace(match[1])
resMatch := bytes.TrimSpace(match[1])
s1 := detectors.Result{
DetectorType: detectorspb.DetectorType_Lastfm,
Raw: []byte(resMatch),
Raw: resMatch,
}
if verify {
req, err := http.NewRequestWithContext(ctx, "GET", "https://ws.audioscrobbler.com/2.0/?method=artist.getSimilar&api_key="+resMatch, nil)
req, err := http.NewRequestWithContext(ctx, "GET", "https://ws.audioscrobbler.com/2.0/?method=artist.getSimilar&api_key="+string(resMatch), nil)
if err != nil {
continue
}
@@ -57,7 +55,6 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
if res.StatusCode >= 200 && res.StatusCode < 300 {
s1.Verified = true
} else {
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
continue
}
+22 -66
View File
@@ -1,108 +1,64 @@
package launchdarkly
import (
"bytes"
"context"
"fmt"
"net/http"
"regexp"
"strings"
"time"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
ldclient "github.com/launchdarkly/go-server-sdk/v6"
"github.com/launchdarkly/go-server-sdk/v6/ldcomponents"
)
type Scanner struct {
client *http.Client
}
type Scanner struct{}
// Ensure the Scanner satisfies the interface at compile time.
var _ detectors.Detector = (*Scanner)(nil)
var (
defaultClient = common.SaneHttpClient()
defaultSDKConfig = ldclient.Config{
Logging: ldcomponents.NoLogging(),
}
defaultSDKTimeout = 10 * time.Second
invalidSDKKeyError = "SDK key contains invalid characters"
client = common.SaneHttpClient()
// Launchdarkly keys are UUIDv4s with either api- or sdk- prefixes.
// mob- keys are possible, but are not sensitive credentials.
keyPat = regexp.MustCompile(`\b((?:api|sdk)-[a-z0-9]{8}-[a-z0-9]{4}-4[a-z0-9]{3}-[a-z0-9]{4}-[a-z0-9]{12})\b`)
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"launchdarkly", "launch_darkly"}) + `\b([a-z0-9-]{40})\b`)
)
// We are not including "mob-" because client keys are not sensitive.
// They are expected to be public.
func (s Scanner) Keywords() []string {
return []string{"api-", "sdk-"}
func (s Scanner) Keywords() [][]byte {
return [][]byte{[]byte("launchdarkly"), []byte("launch_darkly")}
}
// FromData will find and optionally verify LaunchDarkly secrets in a given set of bytes.
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
dataStr := string(data)
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
matches := keyPat.FindAllSubmatch(data, -1)
for _, match := range matches {
if len(match) != 2 {
continue
}
resMatch := strings.TrimSpace(match[1])
resMatch := bytes.TrimSpace(match[1])
s1 := detectors.Result{
DetectorType: detectorspb.DetectorType_LaunchDarkly,
Raw: []byte(resMatch),
Raw: resMatch,
}
if verify {
if strings.HasPrefix(resMatch, "api-") {
req, err := http.NewRequestWithContext(ctx, "GET", "https://app.launchdarkly.com/api/v2/tokens", nil)
if err != nil {
continue
}
client := s.client
if client == nil {
client = defaultClient
}
req.Header.Add("Authorization", resMatch)
res, err := client.Do(req)
if err == nil {
defer res.Body.Close()
if res.StatusCode >= 200 && res.StatusCode < 300 {
s1.Verified = true
} else if res.StatusCode == 401 {
// 401 is expected for an invalid token, so there is nothing to do here.
} else {
s1.VerificationError = fmt.Errorf("unexpected HTTP response status %d", res.StatusCode)
}
} else {
s1.VerificationError = err
}
} else {
// This is a server SDK key. Try to initialize using the SDK.
_, err := ldclient.MakeCustomClient(resMatch, defaultSDKConfig, defaultSDKTimeout)
if err == nil {
req, err := http.NewRequestWithContext(ctx, "GET", "https://app.launchdarkly.com/api/v2/tokens", nil)
if err != nil {
continue
}
newHeader := "Bearer " + string(resMatch)
req.Header.Add("Authorization", newHeader)
res, err := client.Do(req)
if err == nil {
defer res.Body.Close()
if res.StatusCode >= 200 && res.StatusCode < 300 {
s1.Verified = true
} else if err == ldclient.ErrInitializationFailed || err.Error() == invalidSDKKeyError {
// If initialization fails, the key is not valid, so do nothing.
} else {
// If the error isn't nil or known, then this is likely a timeout error: ldclient.ErrInitializationTimeout
// But any other error here means we don't know if this key is valid.
s1.VerificationError = err
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
continue
}
}
}
}
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
if !s1.Verified && detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
continue
}
results = append(results, s1)
}
+22 -31
View File
@@ -11,61 +11,52 @@ import (
"time"
"github.com/go-ldap/ldap/v3"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
)
type Scanner struct{}
// Ensure the Scanner satisfies the interface at compile time.
var _ detectors.Detector = (*Scanner)(nil)
var (
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
uriPat = regexp.MustCompile(`\b(?i)ldaps?://[\S]+\b`)
// ldap://127.0.0.1:389
// ldap://127.0.0.1
// ldap://mydomain.test
// ldaps://[fe80:4049:92ff:fe44:4bd1]:5060
// ldap://[fe80::4bd1]:5060
// ldap://ds.example.com:389/dc=example,dc=com?givenName,sn,cn?sub?(uid=john.doe)
uriPat = regexp.MustCompile(`\b(?i)ldaps?://[\S]+\b`)
usernamePat = regexp.MustCompile(detectors.PrefixRegex([]string{"user", "bind"}) + `["']([a-zA-Z=,]{4,150})["']`)
passwordPat = regexp.MustCompile(detectors.PrefixRegex([]string{"pass"}) + `["']([\S]{4,48})["']`)
// https://learn.microsoft.com/en-us/windows/win32/api/iads/nf-iads-iadsopendsobject-opendsobject?redirectedfrom=MSDN
// I.E. Set ou = dso.OpenDSObject("LDAP://DC.business.com/OU=IT,DC=Business,DC=com", "Business\administrator", "Pa$$word01", 1)
iadPat = regexp.MustCompile(`OpenDSObject\(\"(?i)(ldaps?://[\S]+)\", ?\"([\S]+)\", ?\"([\S]+)\",[ \d]+\)`)
iadPat = regexp.MustCompile(`OpenDSObject\(\"(?i)(ldaps?://[\S]+)\", ?\"([\S]+)\", ?\"([\S]+)\",[ \d]+\)`)
)
// Keywords are used for efficiently pre-filtering chunks.
// Use identifiers in the secret preferably, or the provider name.
func (s Scanner) Keywords() []string {
return []string{"ldaps://", "ldap://"}
func (s Scanner) Keywords() [][]byte {
return [][]byte{
[]byte("ldaps://"),
[]byte("ldap://"),
}
}
// FromData will find and optionally verify Ldap secrets in a given set of bytes.
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
dataStr := string(data)
// Check for matches in the URI + username + password format
uriMatches := uriPat.FindAllString(dataStr, -1)
uriMatches := uriPat.FindAllSubmatch(data, -1)
for _, uri := range uriMatches {
ldapURL, err := url.Parse(uri)
ldapURL, err := url.Parse(string(uri[0]))
if err != nil {
continue
}
usernameMatches := usernamePat.FindAllStringSubmatch(dataStr, -1)
usernameMatches := usernamePat.FindAllSubmatch(data, -1)
for _, username := range usernameMatches {
passwordMatches := passwordPat.FindAllStringSubmatch(dataStr, -1)
if len(username) != 2 {
continue
}
passwordMatches := passwordPat.FindAllSubmatch(data, -1)
for _, password := range passwordMatches {
s1 := detectors.Result{
DetectorType: detectorspb.DetectorType_LDAP,
Raw: []byte(strings.Join([]string{ldapURL.String(), username[1], password[1]}, "\t")),
Raw: []byte(strings.Join([]string{ldapURL.String(), string(username[1]), string(password[1])}, "\t")),
}
if verify {
verificationErr := verifyLDAP(username[1], password[1], ldapURL)
verificationErr := verifyLDAP(string(username[1]), string(password[1]), ldapURL)
s1.Verified = verificationErr == nil
if !isErrDeterminate(verificationErr) {
s1.VerificationError = verificationErr
@@ -78,24 +69,24 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
}
// Check for matches for the IAD library format
iadMatches := iadPat.FindAllStringSubmatch(dataStr, -1)
iadMatches := iadPat.FindAllSubmatch(data, -1)
for _, iad := range iadMatches {
uri := iad[1]
username := iad[2]
password := iad[3]
ldapURL, err := url.Parse(uri)
ldapURL, err := url.Parse(string(uri))
if err != nil {
continue
}
s1 := detectors.Result{
DetectorType: detectorspb.DetectorType_LDAP,
Raw: []byte(strings.Join([]string{ldapURL.String(), username, password}, "\t")),
Raw: []byte(strings.Join([]string{ldapURL.String(), string(username), string(password)}, "\t")),
}
if verify {
verificationError := verifyLDAP(username, password, ldapURL)
verificationError := verifyLDAP(string(username), string(password), ldapURL)
s1.Verified = verificationError == nil
if !isErrDeterminate(verificationError) {
+15 -20
View File
@@ -1,11 +1,11 @@
package leadfeeder
import (
"bytes"
"context"
"fmt"
"net/http"
"regexp"
"strings"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
@@ -14,55 +14,50 @@ import (
type Scanner struct{}
// Ensure the Scanner satisfies the interface at compile time.
var _ detectors.Detector = (*Scanner)(nil)
var (
client = common.SaneHttpClient()
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"leadfeeder"}) + `\b([a-zA-Z0-9-]{43})\b`)
)
// Keywords are used for efficiently pre-filtering chunks.
// Use identifiers in the secret preferably, or the provider name.
func (s Scanner) Keywords() []string {
return []string{"leadfeeder"}
func (s Scanner) Keywords() [][]byte {
return [][]byte{[]byte("leadfeeder")}
}
// FromData will find and optionally verify Leadfeeder secrets in a given set of bytes.
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
dataStr := string(data)
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
matches := keyPat.FindAllSubmatch(data, -1)
for _, match := range matches {
if len(match) != 2 {
continue
}
resMatch := strings.TrimSpace(match[1])
resMatch := bytes.TrimSpace(match[1])
s1 := detectors.Result{
DetectorType: detectorspb.DetectorType_Leadfeeder,
Raw: []byte(resMatch),
Raw: resMatch,
}
if verify {
req, err := http.NewRequestWithContext(ctx, "GET", "https://api.leadfeeder.com/accounts", nil)
if err != nil {
continue
}
req.Header.Add("Authorization", fmt.Sprintf("Token token=%s", resMatch))
req.Header.Add("Authorization", fmt.Sprintf("Token token=%s", string(resMatch)))
res, err := client.Do(req)
if err == nil {
defer res.Body.Close()
if res.StatusCode >= 200 && res.StatusCode < 300 {
s1.Verified = true
} else {
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
continue
}
} else if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
continue
}
}
}
+8 -17
View File
@@ -1,12 +1,12 @@
package lemlist
import (
"bytes"
"context"
b64 "encoding/base64"
"fmt"
"net/http"
"regexp"
"strings"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
@@ -15,42 +15,34 @@ import (
type Scanner struct{}
// Ensure the Scanner satisfies the interface at compile time
var _ detectors.Detector = (*Scanner)(nil)
var (
client = common.SaneHttpClient()
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"lemlist"}) + `\b([a-f0-9]{32})\b`)
)
// Keywords are used for efficiently pre-filtering chunks.
// Use identifiers in the secret preferably, or the provider name.
func (s Scanner) Keywords() []string {
return []string{"lemlist"}
func (s Scanner) Keywords() [][]byte {
return [][]byte{[]byte("lemlist")}
}
// FromData will find and optionally verify Lemlist secrets in a given set of bytes.
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
dataStr := string(data)
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
matches := keyPat.FindAllSubmatch(data, -1)
for _, match := range matches {
if len(match) != 2 {
continue
}
resMatch := strings.TrimSpace(match[1])
resMatch := bytes.TrimSpace(match[1])
s1 := detectors.Result{
DetectorType: detectorspb.DetectorType_Lemlist,
Raw: []byte(resMatch),
Raw: resMatch,
}
if verify {
data := fmt.Sprintf(":%s", resMatch)
sEnc := b64.StdEncoding.EncodeToString([]byte(data))
data := []byte(fmt.Sprintf(":%s", resMatch))
sEnc := b64.StdEncoding.EncodeToString(data)
req, err := http.NewRequestWithContext(ctx, "GET", "https://api.lemlist.com/api/team", nil)
if err != nil {
continue
@@ -62,7 +54,6 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
if res.StatusCode >= 200 && res.StatusCode < 300 {
s1.Verified = true
} else {
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
continue
}
+7 -9
View File
@@ -1,11 +1,11 @@
package lendflow
import (
"bytes"
"context"
"fmt"
"net/http"
"regexp"
"strings"
"time"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
@@ -21,31 +21,29 @@ var _ detectors.Detector = (*Scanner)(nil)
var (
client = common.SaneHttpClient()
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
// Keeping the regexp patterns unchanged
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"lendflow"}) + `\b([a-zA-Z0-9]{36}\.[a-zA-Z0-9]{235}\.[a-zA-Z0-9]{32}\-[a-zA-Z0-9]{47}\-[a-zA-Z0-9_]{162}\-[a-zA-Z0-9]{42}\-[a-zA-Z0-9_]{40}\-[a-zA-Z0-9_]{66}\-[a-zA-Z0-9_]{59}\-[a-zA-Z0-9]{7}\-[a-zA-Z0-9_]{220})\b`)
)
// Keywords are used for efficiently pre-filtering chunks.
// Use identifiers in the secret preferably, or the provider name.
func (s Scanner) Keywords() []string {
return []string{"lendflow"}
func (s Scanner) Keywords() [][]byte {
return [][]byte{[]byte("lendflow")}
}
// FromData will find and optionally verify Lendflow secrets in a given set of bytes.
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
dataStr := string(data)
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
matches := keyPat.FindAllSubmatch(data, -1)
for _, match := range matches {
if len(match) != 2 {
continue
}
resMatch := strings.TrimSpace(match[1])
resMatch := bytes.TrimSpace(match[1])
s1 := detectors.Result{
DetectorType: detectorspb.DetectorType_Lendflow,
Raw: []byte(resMatch),
Raw: resMatch,
}
if verify {
@@ -1,11 +1,11 @@
package lessannoyingcrm
import (
"bytes"
"context"
"fmt"
"net/http"
"regexp"
"strings"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
@@ -14,42 +14,35 @@ import (
type Scanner struct{}
// Ensure the Scanner satisfies the interface at compile time.
var _ detectors.Detector = (*Scanner)(nil)
var (
client = common.SaneHttpClient()
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"less"}) + `\b([a-zA-Z0-9-]{57})\b`)
)
// Keywords are used for efficiently pre-filtering chunks.
// Use identifiers in the secret preferably, or the provider name.
func (s Scanner) Keywords() []string {
return []string{"lessannoyingcrm"}
func (s Scanner) Keywords() [][]byte {
return [][]byte{[]byte("lessannoyingcrm")}
}
// FromData will find and optionally verify LessAnnoyingCRM secrets in a given set of bytes.
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
dataStr := string(data)
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
matches := keyPat.FindAllSubmatch(data, -1)
for _, match := range matches {
if len(match) != 2 {
continue
}
resMatch := strings.TrimSpace(match[1])
resMatch := bytes.TrimSpace(match[1])
s1 := detectors.Result{
DetectorType: detectorspb.DetectorType_LessAnnoyingCRM,
Raw: []byte(resMatch),
Raw: resMatch,
}
if verify {
userCode := strings.Split(resMatch, "-")
req, err := http.NewRequestWithContext(ctx, "GET", fmt.Sprintf("https://api.lessannoyingcrm.com?UserCode=%s&APIToken=%s&Function=GetUserInfo", userCode, resMatch), nil)
userCode := bytes.Split(resMatch, []byte("-"))
req, err := http.NewRequestWithContext(ctx, "GET", fmt.Sprintf("https://api.lessannoyingcrm.com?UserCode=%s&APIToken=%s&Function=GetUserInfo", userCode[0], resMatch), nil)
if err != nil {
continue
}
@@ -60,7 +53,6 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
if res.StatusCode >= 200 && res.StatusCode < 300 {
s1.Verified = true
} else {
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
continue
}
+7 -15
View File
@@ -1,11 +1,11 @@
package lexigram
import (
"bytes"
"context"
"fmt"
"net/http"
"regexp"
"strings"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
@@ -14,37 +14,30 @@ import (
type Scanner struct{}
// Ensure the Scanner satisfies the interface at compile time.
var _ detectors.Detector = (*Scanner)(nil)
var (
client = common.SaneHttpClient()
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"lexigram"}) + `\b([a-zA-Z0-9\S]{301})\b`)
)
// Keywords are used for efficiently pre-filtering chunks.
// Use identifiers in the secret preferably, or the provider name.
func (s Scanner) Keywords() []string {
return []string{"lexigram"}
func (s Scanner) Keywords() [][]byte {
return [][]byte{[]byte("lexigram")}
}
// FromData will find and optionally verify Lexigram secrets in a given set of bytes.
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
dataStr := string(data)
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
matches := keyPat.FindAllSubmatch(data, -1)
for _, match := range matches {
if len(match) != 2 {
continue
}
resMatch := strings.TrimSpace(match[1])
resMatch := bytes.TrimSpace(match[1])
s1 := detectors.Result{
DetectorType: detectorspb.DetectorType_Lexigram,
Raw: []byte(resMatch),
Raw: resMatch,
}
if verify {
@@ -52,14 +45,13 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
if err != nil {
continue
}
req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", resMatch))
req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", string(resMatch)))
res, err := client.Do(req)
if err == nil {
defer res.Body.Close()
if res.StatusCode >= 200 && res.StatusCode < 300 {
s1.Verified = true
} else {
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
continue
}
+10 -14
View File
@@ -1,11 +1,11 @@
package linearapi
import (
"bytes"
"context"
"io"
"net/http"
"regexp"
"strings"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
@@ -20,41 +20,39 @@ var _ detectors.Detector = (*Scanner)(nil)
var (
client = common.SaneHttpClient()
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
// Keeping the regexp patterns unchanged
keyPat = regexp.MustCompile(`\b(lin_api_[0-9A-Za-z]{40})\b`)
)
// Keywords are used for efficiently pre-filtering chunks.
// Use identifiers in the secret preferably, or the provider name.
func (s Scanner) Keywords() []string {
return []string{"lin_api_"}
func (s Scanner) Keywords() [][]byte {
return [][]byte{[]byte("lin_api_")}
}
// FromData will find and optionally verify LinearAPI secrets in a given set of bytes.
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
dataStr := string(data)
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
matches := keyPat.FindAllSubmatch(data, -1)
for _, match := range matches {
if len(match) != 2 {
continue
}
resMatch := strings.TrimSpace(match[1])
resMatch := bytes.TrimSpace(match[1])
s1 := detectors.Result{
DetectorType: detectorspb.DetectorType_LinearAPI,
Raw: []byte(resMatch),
Raw: resMatch,
}
if verify {
reqDataQuery := strings.NewReader(`{ "query": "{ viewer { id name } }" }`)
reqDataQuery := bytes.NewReader([]byte(`{ "query": "{ viewer { id name } }" }`))
req, err := http.NewRequestWithContext(ctx, "POST", "https://api.linear.app/graphql", reqDataQuery)
if err != nil {
continue
}
req.Header.Add("Content-Type", "application/json")
req.Header.Add("Authorization", resMatch)
req.Header.Add("Authorization", string(resMatch))
res, err := client.Do(req)
if err == nil {
@@ -63,11 +61,9 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
if err != nil {
continue
}
body := string(bodyBytes)
if res.StatusCode >= 200 && res.StatusCode < 300 && strings.Contains(body, `"data":`) {
if res.StatusCode >= 200 && res.StatusCode < 300 && bytes.Contains(bodyBytes, []byte(`"data":`)) {
s1.Verified = true
} else {
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
continue
}
+7 -10
View File
@@ -1,11 +1,11 @@
package linemessaging
import (
"bytes"
"context"
"fmt"
"net/http"
"regexp"
"strings"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
@@ -20,31 +20,29 @@ var _ detectors.Detector = (*Scanner)(nil)
var (
client = common.SaneHttpClient()
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
// Keeping the regexp patterns unchanged
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"line"}) + `\b([A-Za-z0-9+/]{171,172})\b`)
)
// Keywords are used for efficiently pre-filtering chunks.
// Use identifiers in the secret preferably, or the provider name.
func (s Scanner) Keywords() []string {
return []string{"line"}
func (s Scanner) Keywords() [][]byte {
return [][]byte{[]byte("line")}
}
// FromData will find and optionally verify LineMessaging secrets in a given set of bytes.
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
dataStr := string(data)
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
matches := keyPat.FindAllSubmatch(data, -1)
for _, match := range matches {
if len(match) != 2 {
continue
}
resMatch := strings.TrimSpace(match[1])
resMatch := bytes.TrimSpace(match[1])
s1 := detectors.Result{
DetectorType: detectorspb.DetectorType_LineMessaging,
Raw: []byte(resMatch),
Raw: resMatch,
}
if verify {
@@ -59,7 +57,6 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
if res.StatusCode >= 200 && res.StatusCode < 300 {
s1.Verified = true
} else {
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
continue
}
+6 -14
View File
@@ -1,11 +1,11 @@
package linenotify
import (
"bytes"
"context"
"fmt"
"net/http"
"regexp"
"strings"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
@@ -14,37 +14,30 @@ import (
type Scanner struct{}
// Ensure the Scanner satisfies the interface at compile time.
var _ detectors.Detector = (*Scanner)(nil)
var (
client = common.SaneHttpClient()
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"linenotify"}) + `\b([0-9A-Za-z]{43})\b`)
)
// Keywords are used for efficiently pre-filtering chunks.
// Use identifiers in the secret preferably, or the provider name.
func (s Scanner) Keywords() []string {
return []string{"linenotify"}
func (s Scanner) Keywords() [][]byte {
return [][]byte{[]byte("linenotify")}
}
// FromData will find and optionally verify LineNotify secrets in a given set of bytes.
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
dataStr := string(data)
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
matches := keyPat.FindAllSubmatch(data, -1)
for _, match := range matches {
if len(match) != 2 {
continue
}
resMatch := strings.TrimSpace(match[1])
resMatch := bytes.TrimSpace(match[1])
s1 := detectors.Result{
DetectorType: detectorspb.DetectorType_LineNotify,
Raw: []byte(resMatch),
Raw: resMatch,
}
if verify {
@@ -59,7 +52,6 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
if res.StatusCode >= 200 && res.StatusCode < 300 {
s1.Verified = true
} else {
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
continue
}
+7 -13
View File
@@ -1,11 +1,11 @@
package linkpreview
import (
"bytes"
"context"
"fmt"
"net/http"
"regexp"
"strings"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
@@ -20,33 +20,28 @@ var _ detectors.Detector = (*Scanner)(nil)
var (
client = common.SaneHttpClient()
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"linkpreview"}) + `\b([a-zA-Z0-9]{32})\b`)
)
// Keywords are used for efficiently pre-filtering chunks.
// Use identifiers in the secret preferably, or the provider name.
func (s Scanner) Keywords() []string {
return []string{"linkpreview"}
func (s Scanner) Keywords() [][]byte {
return [][]byte{[]byte("linkpreview")}
}
// FromData will find and optionally verify LinkPreview secrets in a given set of bytes.
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
dataStr := string(data)
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
matches := keyPat.FindAllSubmatch(data, -1)
for _, match := range matches {
if len(match) != 2 {
continue
}
resMatch := strings.TrimSpace(match[1])
resMatch := bytes.TrimSpace(match[1])
s1 := detectors.Result{
DetectorType: detectorspb.DetectorType_LinkPreview,
Raw: []byte(resMatch),
Raw: resMatch,
}
if verify {
req, err := http.NewRequestWithContext(ctx, "GET", fmt.Sprintf("https://api.linkpreview.net/?key=%s&q=https://google.com", resMatch), nil)
req, err := http.NewRequestWithContext(ctx, "GET", fmt.Sprintf("https://api.linkpreview.net/?key=%s&q=https://google.com", string(resMatch)), nil)
if err != nil {
continue
}
@@ -56,7 +51,6 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
if res.StatusCode >= 200 && res.StatusCode < 300 {
s1.Verified = true
} else {
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
continue
}
+7 -8
View File
@@ -1,10 +1,10 @@
package liveagent
import (
"bytes"
"context"
"net/http"
"regexp"
"strings"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
@@ -25,25 +25,24 @@ var (
// Keywords are used for efficiently pre-filtering chunks.
// Use identifiers in the secret preferably, or the provider name.
func (s Scanner) Keywords() []string {
return []string{"liveagent"}
func (s Scanner) Keywords() [][]byte {
return [][]byte{[]byte("liveagent")}
}
// FromData will find and optionally verify LiveAgent secrets in a given set of bytes.
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
dataStr := string(data)
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
matches := keyPat.FindAllSubmatch(data, -1)
for _, match := range matches {
if len(match) != 2 {
continue
}
resMatch := strings.TrimSpace(match[1])
resMatch := bytes.TrimSpace(match[1])
s1 := detectors.Result{
DetectorType: detectorspb.DetectorType_LiveAgent,
Raw: []byte(resMatch),
Raw: resMatch,
}
if verify {
@@ -51,7 +50,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
if err != nil {
continue
}
req.Header.Add("apikey", resMatch)
req.Header.Add("apikey", string(resMatch))
res, err := client.Do(req)
if err == nil {
defer res.Body.Close()
+8 -17
View File
@@ -1,10 +1,10 @@
package livestorm
import (
"bytes"
"context"
"net/http"
"regexp"
"strings"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
@@ -13,52 +13,43 @@ import (
type Scanner struct{}
// Ensure the Scanner satisfies the interface at compile time.
var _ detectors.Detector = (*Scanner)(nil)
var (
client = common.SaneHttpClient()
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"livestorm"}) + `\b(eyJhbGciOiJIUzI1NiJ9\.eyJhdWQiOiJhcGkubGl2ZXN0b3JtLmNvIiwianRpIjoi[0-9A-Z-a-z]{134}\.[0-9A-Za-z\-\_]{43}[ \r\n]{1})`)
)
// Keywords are used for efficiently pre-filtering chunks.
// Use identifiers in the secret preferably, or the provider name.
func (s Scanner) Keywords() []string {
return []string{"livestorm"}
func (s Scanner) Keywords() [][]byte {
return [][]byte{[]byte("livestorm")}
}
// FromData will find and optionally verify Livestorm secrets in a given set of bytes.
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
dataStr := string(data)
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
matches := keyPat.FindAllSubmatch(data, -1)
for _, match := range matches {
if len(match) != 2 {
continue
}
resMatch := strings.TrimSpace(match[1])
resMatch := bytes.TrimSpace(match[1])
s1 := detectors.Result{
DetectorType: detectorspb.DetectorType_Livestorm,
Raw: []byte(resMatch),
Raw: resMatch,
}
if verify {
req, err := http.NewRequest("GET", "https://api.livestorm.co/v1/ping", nil)
req, err := http.NewRequestWithContext(ctx, "GET", "https://api.livestorm.co/v1/ping", nil)
if err != nil {
continue
}
req.Header.Add("Authorization", resMatch)
req.Header.Add("Authorization", string(resMatch))
res, err := client.Do(req)
if err == nil {
defer res.Body.Close()
if res.StatusCode >= 200 && res.StatusCode < 300 {
s1.Verified = true
} else {
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
continue
}
+7 -16
View File
@@ -1,11 +1,11 @@
package loadmill
import (
"bytes"
"context"
"fmt"
"net/http"
"regexp"
"strings"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
@@ -14,37 +14,29 @@ import (
type Scanner struct{}
// Ensure the Scanner satisfies the interface at compile time.
var _ detectors.Detector = (*Scanner)(nil)
var (
client = common.SaneHttpClient()
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"loadmill"}) + `\b([0-9a-zA-Z]{40})\b`)
)
// Keywords are used for efficiently pre-filtering chunks.
// Use identifiers in the secret preferably, or the provider name.
func (s Scanner) Keywords() []string {
return []string{"loadmill"}
func (s Scanner) Keywords() [][]byte {
return [][]byte{[]byte("loadmill")}
}
// FromData will find and optionally verify Loadmill secrets in a given set of bytes.
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
dataStr := string(data)
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
matches := keyPat.FindAllSubmatch(data, -1)
for _, match := range matches {
if len(match) != 2 {
continue
}
resMatch := strings.TrimSpace(match[1])
resMatch := bytes.TrimSpace(match[1])
s1 := detectors.Result{
DetectorType: detectorspb.DetectorType_Loadmill,
Raw: []byte(resMatch),
Raw: resMatch,
}
if verify {
@@ -53,14 +45,13 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
continue
}
req.Header.Add("Accept", "application/vnd.loadmill+json; version=3")
req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", resMatch))
req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", string(resMatch)))
res, err := client.Do(req)
if err == nil {
defer res.Body.Close()
if res.StatusCode >= 200 && res.StatusCode < 300 {
s1.Verified = true
} else {
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
continue
}
+7 -8
View File
@@ -1,10 +1,10 @@
package lob
import (
"bytes"
"context"
"net/http"
"regexp"
"strings"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
@@ -25,25 +25,24 @@ var (
// Keywords are used for efficiently pre-filtering chunks.
// Use identifiers in the secret preferably, or the provider name.
func (s Scanner) Keywords() []string {
return []string{"lob"}
func (s Scanner) Keywords() [][]byte {
return [][]byte{[]byte("lob")}
}
// FromData will find and optionally verify Lob secrets in a given set of bytes.
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
dataStr := string(data)
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
matches := keyPat.FindAllSubmatch(data, -1)
for _, match := range matches {
if len(match) != 2 {
continue
}
resMatch := strings.TrimSpace(match[1])
resMatch := bytes.TrimSpace(match[1])
s1 := detectors.Result{
DetectorType: detectorspb.DetectorType_Lob,
Raw: []byte(resMatch),
Raw: resMatch,
}
if verify {
@@ -51,7 +50,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
if err != nil {
continue
}
req.SetBasicAuth(resMatch, "")
req.SetBasicAuth(string(resMatch), "")
res, err := client.Do(req)
if err == nil {
defer res.Body.Close()
+7 -9
View File
@@ -1,10 +1,10 @@
package locationiq
import (
"bytes"
"context"
"net/http"
"regexp"
"strings"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
@@ -25,29 +25,27 @@ var (
// Keywords are used for efficiently pre-filtering chunks.
// Use identifiers in the secret preferably, or the provider name.
func (s Scanner) Keywords() []string {
return []string{"locationiq"}
func (s Scanner) Keywords() [][]byte {
return [][]byte{[]byte("locationiq")}
}
// FromData will find and optionally verify LocationIQ secrets in a given set of bytes.
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
dataStr := string(data)
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
matches := keyPat.FindAllSubmatch(data, -1)
for _, match := range matches {
if len(match) != 2 {
continue
}
resMatch := strings.TrimSpace(match[1])
resMatch := bytes.TrimSpace(match[1])
s1 := detectors.Result{
DetectorType: detectorspb.DetectorType_LocationIQ,
Raw: []byte(resMatch),
Raw: resMatch,
}
if verify {
req, err := http.NewRequestWithContext(ctx, "GET", "https://us1.locationiq.com/v1/reverse.php?key="+resMatch+"&lat=-37.870662&lon=144.9803321&format=json", nil)
req, err := http.NewRequestWithContext(ctx, "GET", "https://us1.locationiq.com/v1/reverse.php?key="+string(resMatch)+"&lat=-37.870662&lon=144.9803321&format=json", nil)
if err != nil {
continue
}
+8 -11
View File
@@ -1,10 +1,10 @@
package loginradius
import (
"bytes"
"context"
"net/http"
"regexp"
"strings"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
@@ -19,35 +19,33 @@ var _ detectors.Detector = (*Scanner)(nil)
var (
client = common.SaneHttpClient()
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
// Keeping the regexp patterns unchanged
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"loginradius"}) + `\b([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})\b`)
)
// Keywords are used for efficiently pre-filtering chunks.
// Use identifiers in the secret preferably, or the provider name.
func (s Scanner) Keywords() []string {
return []string{"loginradius"}
func (s Scanner) Keywords() [][]byte {
return [][]byte{[]byte("loginradius")}
}
// FromData will find and optionally verify Loginradius secrets in a given set of bytes.
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
dataStr := string(data)
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
matches := keyPat.FindAllSubmatch(data, -1)
for _, match := range matches {
if len(match) != 2 {
continue
}
resMatch := strings.TrimSpace(match[1])
resMatch := bytes.TrimSpace(match[1])
s1 := detectors.Result{
DetectorType: detectorspb.DetectorType_Loginradius,
Raw: []byte(resMatch),
Raw: resMatch,
}
if verify {
req, err := http.NewRequestWithContext(ctx, "GET", "https://api.loginradius.com/identity/v2/serverinfo?apikey="+resMatch, nil)
req, err := http.NewRequestWithContext(ctx, "GET", "https://api.loginradius.com/identity/v2/serverinfo?apikey="+string(resMatch), nil)
if err != nil {
continue
}
@@ -57,7 +55,6 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
if res.StatusCode >= 200 && res.StatusCode < 300 {
s1.Verified = true
} else {
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
continue
}
+7 -16
View File
@@ -1,10 +1,10 @@
package lokalisetoken
import (
"bytes"
"context"
"net/http"
"regexp"
"strings"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
@@ -13,38 +13,30 @@ import (
type Scanner struct{}
// Ensure the Scanner satisfies the interface at compile time.
var _ detectors.Detector = (*Scanner)(nil)
var (
client = common.SaneHttpClient()
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"lokalise"}) + `\b([a-z0-9]{40})\b`)
)
// Keywords are used for efficiently pre-filtering chunks.
// Use identifiers in the secret preferably, or the provider name.
func (s Scanner) Keywords() []string {
return []string{"lokalise"}
func (s Scanner) Keywords() [][]byte {
return [][]byte{[]byte("lokalise")}
}
// FromData will find and optionally verify LokaliseToken secrets in a given set of bytes.
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
dataStr := string(data)
// log.Println(dataStr)
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
matches := keyPat.FindAllSubmatch(data, -1)
for _, match := range matches {
if len(match) != 2 {
continue
}
resMatch := strings.TrimSpace(match[1])
resMatch := bytes.TrimSpace(match[1])
s1 := detectors.Result{
DetectorType: detectorspb.DetectorType_LokaliseToken,
Raw: []byte(resMatch),
Raw: resMatch,
}
if verify {
@@ -52,14 +44,13 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
if err != nil {
continue
}
req.Header.Add("x-api-token", resMatch)
req.Header.Add("x-api-token", string(resMatch))
res, err := client.Do(req)
if err == nil {
defer res.Body.Close()
if res.StatusCode >= 200 && res.StatusCode < 300 {
s1.Verified = true
} else {
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
continue
}
+7 -10
View File
@@ -1,11 +1,11 @@
package loyverse
import (
"bytes"
"context"
"fmt"
"net/http"
"regexp"
"strings"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
@@ -26,25 +26,23 @@ var (
// Keywords are used for efficiently pre-filtering chunks.
// Use identifiers in the secret preferably, or the provider name.
func (s Scanner) Keywords() []string {
return []string{"loyverse"}
func (s Scanner) Keywords() [][]byte {
return [][]byte{[]byte("loyverse")}
}
// FromData will find and optionally verify Loyverse secrets in a given set of bytes.
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
dataStr := string(data)
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
matches := keyPat.FindAllSubmatch(data, -1)
for _, match := range matches {
if len(match) != 2 {
continue
}
resMatch := strings.TrimSpace(match[1])
resMatch := bytes.TrimSpace(match[1])
s1 := detectors.Result{
DetectorType: detectorspb.DetectorType_Loyverse,
Raw: []byte(resMatch),
Raw: resMatch,
}
if verify {
@@ -52,14 +50,13 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
if err != nil {
continue
}
req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", resMatch))
req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", string(resMatch)))
res, err := client.Do(req)
if err == nil {
defer res.Body.Close()
if res.StatusCode >= 200 && res.StatusCode < 300 {
s1.Verified = true
} else {
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
continue
}
+2
View File
@@ -43,6 +43,8 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
instanceMatches := instancePat.FindAllStringSubmatch(dataStr, -1)
tokenMatches := accessTokenPat.FindAllStringSubmatch(dataStr, -1)
fmt.Printf("instanceMatches: %v\n", instanceMatches)
for _, instance := range instanceMatches {
if len(instance) != 1 {
continue
+1 -9
View File
@@ -2,10 +2,6 @@ package engine
import (
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/envoyapikey"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/huggingface"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/salesforce"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/snowflake"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/trufflehogenterprise"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/abbysale"
@@ -291,7 +287,6 @@ import (
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/getresponse"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/getsandbox"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/github"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/github_oauth2"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/github_old"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/githubapp"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/gitlab"
@@ -561,6 +556,7 @@ import (
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/satismeterwritekey"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/saucelabs"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/scalewaykey"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/github_oauth2"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/scalr"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/scrapeowl"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/scraperapi"
@@ -1536,10 +1532,6 @@ func DefaultDetectors() []detectors.Detector {
couchbase.Scanner{},
envoyapikey.Scanner{},
github_oauth2.Scanner{},
snowflake.Scanner{},
huggingface.Scanner{},
trufflehogenterprise.Scanner{},
salesforce.Scanner{},
}
}
+23 -24
View File
@@ -29,8 +29,6 @@ var (
maxDepth = 5
maxSize = 250 * 1024 * 1024 // 20MB
maxTimeout = time.Duration(30) * time.Second
defaultBufferSize = 512
)
// Ensure the Archive satisfies the interfaces at compile time.
@@ -64,7 +62,7 @@ func SetArchiveMaxTimeout(timeout time.Duration) {
// FromFile extracts the files from an archive.
func (a *Archive) FromFile(originalCtx context.Context, data io.Reader) chan []byte {
archiveChan := make(chan []byte, defaultBufferSize)
archiveChan := make(chan []byte, 512)
go func() {
ctx, cancel := context.WithTimeout(originalCtx, maxTimeout)
logger := logContext.AddLogger(ctx).Logger()
@@ -185,28 +183,29 @@ func (a *Archive) ReadToMax(ctx context.Context, reader io.Reader) (data []byte,
logger.Error(err, "Panic occurred when reading archive")
}
}()
if common.IsDone(ctx) {
return nil, ctx.Err()
fileContent := bytes.Buffer{}
logger.V(5).Info("Remaining buffer capacity", "bytes", maxSize-a.size)
for i := 0; i <= maxSize/512; i++ {
if common.IsDone(ctx) {
return nil, ctx.Err()
}
fileChunk := make([]byte, 512)
bRead, err := reader.Read(fileChunk)
if err != nil && !errors.Is(err, io.EOF) {
return []byte{}, err
}
a.size += bRead
if len(fileChunk) > 0 {
fileContent.Write(fileChunk[0:bRead])
}
if bRead < 512 {
return fileContent.Bytes(), nil
}
if a.size >= maxSize && bRead == 512 {
logger.V(2).Info("Max archive size reached.")
return fileContent.Bytes(), nil
}
}
var fileContent bytes.Buffer
// Create a limited reader to ensure we don't read more than the max size.
lr := io.LimitReader(reader, int64(maxSize))
// Using io.CopyBuffer for performance advantages. Though buf is mandatory
// for the method, due to the internal implementation of io.CopyBuffer, when
// *bytes.Buffer implements io.WriterTo or io.ReaderFrom, the provided buf
// is simply ignored. Thus, we can pass nil for the buf parameter.
_, err = io.CopyBuffer(&fileContent, lr, nil)
if err != nil && !errors.Is(err, io.EOF) {
return nil, err
}
if fileContent.Len() == maxSize {
logger.V(2).Info("Max archive size reached.")
}
return fileContent.Bytes(), nil
}
-53
View File
@@ -1,7 +1,6 @@
package handlers
import (
"bytes"
"context"
"io"
"net/http"
@@ -127,58 +126,6 @@ func TestHandleFile(t *testing.T) {
assert.Equal(t, 1, len(ch))
}
func TestReadToMax(t *testing.T) {
tests := []struct {
name string
input []byte
expected []byte
}{
{
name: "read full content within maxSize",
input: []byte("abcdefg"),
expected: []byte("abcdefg"),
},
{
name: "read content larger than maxSize",
input: make([]byte, maxSize+10), // this creates a byte slice 10 bytes larger than maxSize
expected: make([]byte, maxSize),
},
{
name: "empty input",
input: []byte(""),
expected: []byte(""),
},
}
a := &Archive{}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
reader := bytes.NewReader(tt.input)
output, err := a.ReadToMax(context.Background(), reader)
assert.Nil(t, err)
assert.Equal(t, tt.expected, output)
})
}
}
func BenchmarkReadToMax(b *testing.B) {
data := bytes.Repeat([]byte("a"), 1024*1000) // 1MB of data.
reader := bytes.NewReader(data)
a := &Archive{}
b.ResetTimer()
for i := 0; i < b.N; i++ {
b.StartTimer()
_, _ = a.ReadToMax(context.Background(), reader)
b.StopTimer()
_, _ = reader.Seek(0, 0) // Reset the reader position.
a.size = 0 // Reset archive size.
}
}
func TestExtractDebContent(t *testing.T) {
// Open the sample .deb file from the testdata folder.
file, err := os.Open("testdata/test.deb")
+1 -1
View File
@@ -133,7 +133,7 @@ func BranchHeads(repo *gogit.Repository) (map[string]*object.Commit, error) {
}
headCommit, err := repo.CommitObject(*headHash)
if err != nil {
logger.Error(err, "unable to get commit", "head_hash", headHash.String())
logger.Error(err, "unable to get commit", "commit", headCommit.String())
return nil
}
branches[branchName] = headCommit
+1 -1
View File
@@ -20,7 +20,7 @@ const (
// Chunker takes a chunk and splits it into chunks of ChunkSize.
func Chunker(originalChunk *Chunk) chan *Chunk {
chunkChan := make(chan *Chunk, 1)
chunkChan := make(chan *Chunk)
go func() {
defer close(chunkChan)
if len(originalChunk.Data) <= TotalChunkSize {
-1
View File
@@ -238,7 +238,6 @@ func (s *Source) chunkAction(ctx context.Context, proj project, bld build, act a
SourceType: s.Type(),
SourceName: s.name,
SourceID: s.SourceID(),
JobID: s.JobID(),
Data: removeCircleSha1Line(data.Bytes()),
SourceMetadata: &source_metadatapb.MetaData{
Data: &source_metadatapb.MetaData_Circleci{
-1
View File
@@ -174,7 +174,6 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) err
SourceType: s.Type(),
SourceName: s.name,
SourceID: s.SourceID(),
JobID: s.JobID(),
Data: file.Bytes(),
SourceMetadata: &source_metadatapb.MetaData{
Data: &source_metadatapb.MetaData_Docker{
-2
View File
@@ -161,7 +161,6 @@ func (s *Source) scanFile(ctx context.Context, path string, chunksChan chan *sou
SourceType: s.Type(),
SourceName: s.name,
SourceID: s.SourceID(),
JobID: s.JobID(),
SourceMetadata: &source_metadatapb.MetaData{
Data: &source_metadatapb.MetaData_Filesystem{
Filesystem: &source_metadatapb.Filesystem{
@@ -192,7 +191,6 @@ func (s *Source) scanFile(ctx context.Context, path string, chunksChan chan *sou
SourceType: s.Type(),
SourceName: s.name,
SourceID: s.SourceID(),
JobID: s.JobID(),
Data: data.Bytes(),
SourceMetadata: &source_metadatapb.MetaData{
Data: &source_metadatapb.MetaData_Filesystem{
-1
View File
@@ -324,7 +324,6 @@ func (s *Source) processObject(ctx context.Context, o object) error {
chunkSkel := &sources.Chunk{
SourceName: s.name,
SourceType: s.Type(),
JobID: s.JobID(),
SourceID: s.sourceId,
Verify: s.verify,
SourceMetadata: &source_metadatapb.MetaData{
-7
View File
@@ -446,7 +446,6 @@ func (s *Git) ScanCommits(ctx context.Context, repo *git.Repository, path string
chunkSkel := &sources.Chunk{
SourceName: s.sourceName,
SourceID: s.sourceID,
JobID: s.jobID,
SourceType: s.sourceType,
SourceMetadata: metadata,
Verify: s.verify,
@@ -465,7 +464,6 @@ func (s *Git) ScanCommits(ctx context.Context, repo *git.Repository, path string
chunksChan <- &sources.Chunk{
SourceName: s.sourceName,
SourceID: s.sourceID,
JobID: s.jobID,
SourceType: s.sourceType,
SourceMetadata: metadata,
Data: diff.Content.Bytes(),
@@ -492,7 +490,6 @@ func (s *Git) gitChunk(ctx context.Context, diff gitparse.Diff, fileName, email,
chunksChan <- &sources.Chunk{
SourceName: s.sourceName,
SourceID: s.sourceID,
JobID: s.jobID,
SourceType: s.sourceType,
SourceMetadata: metadata,
Data: append([]byte{}, newChunkBuffer.Bytes()...),
@@ -507,7 +504,6 @@ func (s *Git) gitChunk(ctx context.Context, diff gitparse.Diff, fileName, email,
chunksChan <- &sources.Chunk{
SourceName: s.sourceName,
SourceID: s.sourceID,
JobID: s.jobID,
SourceType: s.sourceType,
SourceMetadata: metadata,
Data: line,
@@ -527,7 +523,6 @@ func (s *Git) gitChunk(ctx context.Context, diff gitparse.Diff, fileName, email,
chunksChan <- &sources.Chunk{
SourceName: s.sourceName,
SourceID: s.sourceID,
JobID: s.jobID,
SourceType: s.sourceType,
SourceMetadata: metadata,
Data: append([]byte{}, newChunkBuffer.Bytes()...),
@@ -593,7 +588,6 @@ func (s *Git) ScanStaged(ctx context.Context, repo *git.Repository, path string,
chunkSkel := &sources.Chunk{
SourceName: s.sourceName,
SourceID: s.sourceID,
JobID: s.jobID,
SourceType: s.sourceType,
SourceMetadata: metadata,
Verify: s.verify,
@@ -608,7 +602,6 @@ func (s *Git) ScanStaged(ctx context.Context, repo *git.Repository, path string,
chunksChan <- &sources.Chunk{
SourceName: s.sourceName,
SourceID: s.sourceID,
JobID: s.jobID,
SourceType: s.sourceType,
SourceMetadata: metadata,
Data: diff.Content.Bytes(),
+2 -7
View File
@@ -149,16 +149,14 @@ func (s *Source) newFilteredRepoCache(c cache.Cache, include, exclude []string)
for _, ig := range include {
g, err := glob.Compile(ig)
if err != nil {
s.log.V(1).Info("invalid include glob", "include_value", ig, "err", err)
continue
s.log.V(1).Info("invalid include glob", "glob", g, "err", err)
}
includeGlobs = append(includeGlobs, g)
}
for _, eg := range exclude {
g, err := glob.Compile(eg)
if err != nil {
s.log.V(1).Info("invalid exclude glob", "exclude_value", eg, "err", err)
continue
s.log.V(1).Info("invalid exclude glob", "glob", g, "err", err)
}
excludeGlobs = append(excludeGlobs, g)
}
@@ -1225,7 +1223,6 @@ func (s *Source) chunkIssueComments(ctx context.Context, repo, repoPath string,
chunk := &sources.Chunk{
SourceName: s.name,
SourceID: s.SourceID(),
JobID: s.JobID(),
SourceType: s.Type(),
SourceMetadata: &source_metadatapb.MetaData{
Data: &source_metadatapb.MetaData_Github{
@@ -1259,7 +1256,6 @@ func (s *Source) chunkPullRequestComments(ctx context.Context, repo string, comm
SourceName: s.name,
SourceID: s.SourceID(),
SourceType: s.Type(),
JobID: s.JobID(),
SourceMetadata: &source_metadatapb.MetaData{
Data: &source_metadatapb.MetaData_Github{
Github: &source_metadatapb.Github{
@@ -1291,7 +1287,6 @@ func (s *Source) chunkGistComments(ctx context.Context, gistUrl string, comments
SourceName: s.name,
SourceID: s.SourceID(),
SourceType: s.Type(),
JobID: s.JobID(),
SourceMetadata: &source_metadatapb.MetaData{
Data: &source_metadatapb.MetaData_Github{
Github: &source_metadatapb.Github{
-1
View File
@@ -376,7 +376,6 @@ func (s *Source) pageChunker(ctx context.Context, client *s3.S3, chunksChan chan
SourceType: s.Type(),
SourceName: s.name,
SourceID: s.SourceID(),
JobID: s.JobID(),
SourceMetadata: &source_metadatapb.MetaData{
Data: &source_metadatapb.MetaData_S3{
S3: &source_metadatapb.S3{
+2 -20
View File
@@ -36,10 +36,7 @@ type SourceManager struct {
handles map[handle]sourceInfo
handlesLock sync.Mutex
// Pool limiting the amount of concurrent sources running.
pool errgroup.Group
poolLimit int
currentRunningCount int32
// Max number of units to scan concurrently per source.
pool errgroup.Group
concurrentUnits int
// Run the sources using source unit enumeration / chunking if available.
useSourceUnits bool
@@ -71,10 +68,7 @@ func WithReportHook(hook JobProgressHook) func(*SourceManager) {
// WithConcurrentSources limits the concurrent number of sources a manager can run.
func WithConcurrentSources(concurrency int) func(*SourceManager) {
return func(mgr *SourceManager) {
mgr.pool.SetLimit(concurrency)
mgr.poolLimit = concurrency
}
return func(mgr *SourceManager) { mgr.pool.SetLimit(concurrency) }
}
// WithBufferedOutput sets the size of the buffer used for the Chunks() channel.
@@ -173,8 +167,6 @@ func (s *SourceManager) asyncRun(ctx context.Context, handle handle) (JobProgres
ctx, cancel := context.WithCancel(ctx)
progress := NewJobProgress(jobID, int64(handle), sourceName, WithHooks(s.hooks...), WithCancel(cancel))
s.pool.Go(func() error {
atomic.AddInt32(&s.currentRunningCount, 1)
defer atomic.AddInt32(&s.currentRunningCount, -1)
ctx := context.WithValues(ctx,
"job_id", jobID,
"source_manager_worker_id", common.RandomID(5),
@@ -215,16 +207,6 @@ func (s *SourceManager) ScanChunk(chunk *Chunk) {
s.outputChunks <- chunk
}
// AvailableCapacity returns the number of concurrent jobs the manager can
// accommodate at this time. If there is no limit, -1 is returned.
func (s *SourceManager) AvailableCapacity() int {
if s.poolLimit == 0 {
return -1
}
runCount := atomic.LoadInt32(&s.currentRunningCount)
return s.poolLimit - int(runCount)
}
// preflightChecks is a helper method to check the Manager or the context isn't
// done and that the handle is valid.
func (s *SourceManager) preflightChecks(ctx context.Context, handle handle) error {
-21
View File
@@ -317,24 +317,3 @@ func TestSourceManagerCancelRun(t *testing.T) {
assert.Error(t, ref.Snapshot().FatalError())
assert.True(t, errors.Is(ref.Snapshot().FatalError(), returnedErr))
}
func TestSourceManagerAvailableCapacity(t *testing.T) {
mgr := NewManager(WithConcurrentSources(1337))
start, end := make(chan struct{}), make(chan struct{})
handle, err := enrollDummy(mgr, callbackChunker{func(context.Context, chan *Chunk) error {
start <- struct{}{} // Send start signal.
<-end // Wait for end signal.
return nil
}})
assert.NoError(t, err)
assert.Equal(t, 1337, mgr.AvailableCapacity())
ref, err := mgr.ScheduleRun(context.Background(), handle)
assert.NoError(t, err)
<-start // Wait for start signal.
assert.Equal(t, 1336, mgr.AvailableCapacity())
end <- struct{}{} // Send end signal.
<-ref.Done() // Wait for the job to finish.
assert.Equal(t, 1337, mgr.AvailableCapacity())
}
-2
View File
@@ -272,7 +272,6 @@ func (s *Source) monitorConnection(ctx context.Context, conn net.Conn, chunksCha
SourceName: s.syslog.sourceName,
SourceID: s.syslog.sourceID,
SourceType: s.syslog.sourceType,
JobID: s.JobID(),
SourceMetadata: metadata,
Data: input,
Verify: s.verify,
@@ -314,7 +313,6 @@ func (s *Source) acceptUDPConnections(ctx context.Context, netListener net.Packe
chunksChan <- &sources.Chunk{
SourceName: s.syslog.sourceName,
SourceID: s.syslog.sourceID,
JobID: s.JobID(),
SourceType: s.syslog.sourceType,
SourceMetadata: metadata,
Data: input,