Compare commits

..

1 Commits

Author SHA1 Message Date
Ahrav Dutta abf3e4283c PR 24 of 40 2023-08-25 21:41:00 -07:00
37 changed files with 197 additions and 463 deletions
+1 -1
View File
@@ -32,7 +32,7 @@ Have questions? Feedback? Jump in slack or discord and hang out with us
Join our [Slack Community](https://join.slack.com/t/trufflehog-community/shared_invite/zt-pw2qbi43-Aa86hkiimstfdKH9UCpPzQ)
Join the [Secret Scanning Discord](https://discord.gg/8Hzbrnkr7E)
Join the [Secret Scanning Discord](https://discord.gg/sydS6AHTUP)
# :tv: Demo
+11 -16
View File
@@ -1,10 +1,10 @@
package oopspam
import (
"bytes"
"context"
"net/http"
"regexp"
"strings"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
@@ -13,54 +13,49 @@ import (
type Scanner struct{}
// Ensure the Scanner satisfies the interface at compile time.
var _ detectors.Detector = (*Scanner)(nil)
var (
client = common.SaneHttpClient()
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"oopspam"}) + `\b([a-zA-Z0-9]{40})\b`)
)
// Keywords are used for efficiently pre-filtering chunks.
// Use identifiers in the secret preferably, or the provider name.
func (s Scanner) Keywords() []string {
return []string{"oopspam"}
func (s Scanner) Keywords() [][]byte {
return [][]byte{[]byte("oopspam")}
}
// FromData will find and optionally verify OOPSpam secrets in a given set of bytes.
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
dataStr := string(data)
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
matches := keyPat.FindAllSubmatch(data, -1)
for _, match := range matches {
if len(match) != 2 {
continue
}
resMatch := strings.TrimSpace(match[1])
resMatch := bytes.TrimSpace(match[1])
s1 := detectors.Result{
DetectorType: detectorspb.DetectorType_OOPSpam,
Raw: []byte(resMatch),
Raw: resMatch,
}
if verify {
payload := strings.NewReader(`{"checkForLength":true,"content":"Dear Agent, We are a manufacturing company which specializes in supplying Aluminum Rod with Zinc Alloy Rod to customers worldwide, based in Japan, Asia. We have been unable to follow up payments effectively for transactions with debtor customers in your country due to our distant locations, thus our reason for requesting for your services representation.","senderIP":"185.234.219.246","allowedCountries":["it","us"],"allowedLanguages":["en"]}`)
payload := bytes.NewBuffer([]byte(`{"checkForLength":true,"content":"Dear Agent, We are a manufacturing company which specializes in supplying Aluminum Rod with Zinc Alloy Rod to customers worldwide, based in Japan, Asia. We have been unable to follow up payments effectively for transactions with debtor customers in your country due to our distant locations, thus our reason for requesting for your services representation.","senderIP":"185.234.219.246","allowedCountries":["it","us"],"allowedLanguages":["en"]}`))
req, err := http.NewRequestWithContext(ctx, "POST", "https://api.oopspam.com/v1/spamdetection", payload)
if err != nil {
continue
}
req.Header.Add("Content-Type", "application/json")
req.Header.Add("X-Api-Key", resMatch)
req.Header.Add("X-Api-Key", string(resMatch))
res, err := client.Do(req)
if err == nil {
defer res.Body.Close()
if res.StatusCode >= 200 && res.StatusCode < 300 {
s1.Verified = true
} else {
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
continue
}
+15 -20
View File
@@ -1,6 +1,7 @@
package openai
import (
"bytes"
"context"
"encoding/json"
"fmt"
@@ -19,10 +20,8 @@ type Scanner struct{}
// Ensure the Scanner satisfies the interface at compile time.
var _ detectors.Detector = (*Scanner)(nil)
// The magic string T3BlbkFJ is the base64-encoded string: OpenAI
var keyPat = regexp.MustCompile(`\b(sk-[[:alnum:]]{20}T3BlbkFJ[[:alnum:]]{20})\b`)
var keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"T3BlbkFJ"}) + `(\bsk-[[:alnum:]]{20}T3BlbkFJ[[:alnum:]]{20}\b)`)
// TODO: Add secret context?? Information about access, ownership etc
type orgResponse struct {
Data []organization `json:"data"`
}
@@ -37,48 +36,44 @@ type organization struct {
Role string `json:"role"`
}
// Keywords are used for efficiently pre-filtering chunks.
// Use identifiers in the secret preferably, or the provider name.
func (s Scanner) Keywords() []string {
return []string{"T3BlbkFJ"}
func (s Scanner) Keywords() [][]byte {
return [][]byte{[]byte("T3BlbkFJ")}
}
// FromData will find and optionally verify OpenAI secrets in a given set of bytes.
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
dataStr := string(data)
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
matches := keyPat.FindAllSubmatch(data, -1)
for _, match := range matches {
// First match is entire regex, second is the first group.
if len(match) != 2 {
continue
}
token := match[1]
token := bytes.TrimSpace(match[1])
redacted := append(append(token[:3], []byte("...")...), token[47:]...)
s1 := detectors.Result{
DetectorType: detectorspb.DetectorType_OpenAI,
Redacted: token[:3] + "..." + token[47:],
Raw: []byte(token),
Redacted: string(redacted),
Raw: token,
}
if verify {
client := common.SaneHttpClient()
// Undocumented API
// https://api.openai.com/v1/organizations
req, err := http.NewRequestWithContext(ctx, "GET", "https://api.openai.com/v1/organizations", nil)
if err != nil {
continue
}
req.Header.Add("Content-Type", "application/json; charset=utf-8")
req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", token))
req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", string(token)))
res, err := client.Do(req)
if err == nil {
defer res.Body.Close()
if res.StatusCode >= 200 && res.StatusCode < 300 {
var orgs orgResponse
err = json.NewDecoder(res.Body).Decode(&orgs)
res.Body.Close()
if err == nil {
s1.Verified = true
org := orgs.Data[0]
@@ -97,7 +92,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
}
}
if !s1.Verified && detectors.IsKnownFalsePositive(string(s1.Raw), detectors.DefaultFalsePositives, true) {
if !s1.Verified && detectors.IsKnownFalsePositive(token, detectors.DefaultFalsePositives, true) {
continue
}
+9 -11
View File
@@ -1,11 +1,11 @@
package opencagedata
import (
"bytes"
"context"
"fmt"
"net/http"
"regexp"
"strings"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
@@ -21,34 +21,34 @@ var (
client = common.SaneHttpClient()
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"opencagedata"}) + `\b([a-z0-9]{32})\b`)
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"opencagedata"}) + `\b([0-9a-z]{32})\b`)
)
// Keywords are used for efficiently pre-filtering chunks.
// Use identifiers in the secret preferably, or the provider name.
func (s Scanner) Keywords() []string {
return []string{"opencagedata"}
func (s Scanner) Keywords() [][]byte {
return [][]byte{[]byte("opencagedata")}
}
// FromData will find and optionally verify OpenCageData secrets in a given set of bytes.
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
dataStr := string(data)
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
matches := keyPat.FindAllSubmatch(data, -1)
for _, match := range matches {
if len(match) != 2 {
continue
}
resMatch := strings.TrimSpace(match[1])
resMatch := bytes.TrimSpace(match[1])
s1 := detectors.Result{
DetectorType: detectorspb.DetectorType_OpenCageData,
Raw: []byte(resMatch),
Raw: resMatch,
}
if verify {
req, err := http.NewRequestWithContext(ctx, "GET", fmt.Sprintf("https://api.opencagedata.com/geocode/v1/json?q=12.8797,121.7740&key=%s", resMatch), nil)
req, err := http.NewRequestWithContext(ctx, "GET", fmt.Sprintf("https://api.opencagedata.com/geocode/v1/json?q=12.8797,121.7740&key=%s", string(resMatch)), nil)
if err != nil {
continue
}
@@ -58,14 +58,12 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
if res.StatusCode >= 200 && res.StatusCode < 300 {
s1.Verified = true
} else {
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
continue
}
}
}
}
results = append(results, s1)
}
+7 -15
View File
@@ -1,10 +1,10 @@
package opengraphr
import (
"bytes"
"context"
"net/http"
"regexp"
"strings"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
@@ -13,41 +13,34 @@ import (
type Scanner struct{}
// Ensure the Scanner satisfies the interface at compile time.
var _ detectors.Detector = (*Scanner)(nil)
var (
client = common.SaneHttpClient()
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"opengraphr"}) + `\b([0-9Aa-zA-Z]{80})\b`)
)
// Keywords are used for efficiently pre-filtering chunks.
// Use identifiers in the secret preferably, or the provider name.
func (s Scanner) Keywords() []string {
return []string{"opengraphr"}
func (s Scanner) Keywords() [][]byte {
return [][]byte{[]byte("opengraphr")}
}
// FromData will find and optionally verify Opengraphr secrets in a given set of bytes.
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
dataStr := string(data)
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
matches := keyPat.FindAllSubmatch(data, -1)
for _, match := range matches {
if len(match) != 2 {
continue
}
resMatch := strings.TrimSpace(match[1])
resMatch := bytes.TrimSpace(match[1])
s1 := detectors.Result{
DetectorType: detectorspb.DetectorType_Opengraphr,
Raw: []byte(resMatch),
Raw: resMatch,
}
if verify {
req, err := http.NewRequestWithContext(ctx, "GET", "https://api.opengraphr.com/v1/og?api_token="+resMatch+"&url=https://www.marlonpamisa.com/", nil)
req, err := http.NewRequestWithContext(ctx, "GET", "https://api.opengraphr.com/v1/og?api_token="+string(resMatch)+"&url=https://www.marlonpamisa.com/", nil)
if err != nil {
continue
}
@@ -57,7 +50,6 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
if res.StatusCode >= 200 && res.StatusCode < 300 {
s1.Verified = true
} else {
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
continue
}
+7 -16
View File
@@ -1,10 +1,10 @@
package openuv
import (
"bytes"
"context"
"net/http"
"regexp"
"strings"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
@@ -13,37 +13,29 @@ import (
type Scanner struct{}
// Ensure the Scanner satisfies the interface at compile time.
var _ detectors.Detector = (*Scanner)(nil)
var (
client = common.SaneHttpClient()
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"openuv"}) + `\b([0-9a-z]{32})\b`)
)
// Keywords are used for efficiently pre-filtering chunks.
// Use identifiers in the secret preferably, or the provider name.
func (s Scanner) Keywords() []string {
return []string{"openuv"}
func (s Scanner) Keywords() [][]byte {
return [][]byte{[]byte("openuv")}
}
// FromData will find and optionally verify Openuv secrets in a given set of bytes.
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
dataStr := string(data)
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
matches := keyPat.FindAllSubmatch(data, -1)
for _, match := range matches {
if len(match) != 2 {
continue
}
resMatch := strings.TrimSpace(match[1])
resMatch := bytes.TrimSpace(match[1])
s1 := detectors.Result{
DetectorType: detectorspb.DetectorType_Openuv,
Raw: []byte(resMatch),
Raw: resMatch,
}
if verify {
@@ -51,14 +43,13 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
if err != nil {
continue
}
req.Header.Add("x-access-token", resMatch)
req.Header.Add("x-access-token", string(resMatch))
res, err := client.Do(req)
if err == nil {
defer res.Body.Close()
if res.StatusCode >= 200 && res.StatusCode < 300 {
s1.Verified = true
} else {
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
continue
}
+7 -16
View File
@@ -1,10 +1,10 @@
package openweather
import (
"bytes"
"context"
"net/http"
"regexp"
"strings"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
@@ -13,41 +13,33 @@ import (
type Scanner struct{}
// Ensure the Scanner satisfies the interface at compile time.
var _ detectors.Detector = (*Scanner)(nil)
var (
client = common.SaneHttpClient()
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"openweather"}) + `\b([a-z0-9]{32})\b`)
)
// Keywords are used for efficiently pre-filtering chunks.
// Use identifiers in the secret preferably, or the provider name.
func (s Scanner) Keywords() []string {
return []string{"openweather"}
func (s Scanner) Keywords() [][]byte {
return [][]byte{[]byte("openweather")}
}
// FromData will find and optionally verify OpenWeather secrets in a given set of bytes.
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
dataStr := string(data)
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
matches := keyPat.FindAllSubmatch(data, -1)
for _, match := range matches {
if len(match) != 2 {
continue
}
resMatch := strings.TrimSpace(match[1])
resMatch := bytes.TrimSpace(match[1])
s1 := detectors.Result{
DetectorType: detectorspb.DetectorType_OpenWeather,
Raw: []byte(resMatch),
Raw: resMatch,
}
if verify {
req, err := http.NewRequestWithContext(ctx, "GET", "https://api.openweathermap.org/data/2.5/weather?id=524901&lang=fr&appid="+resMatch, nil)
req, err := http.NewRequestWithContext(ctx, "GET", "https://api.openweathermap.org/data/2.5/weather?id=524901&=fr&appid="+string(resMatch), nil)
if err != nil {
continue
}
@@ -58,7 +50,6 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
if res.StatusCode >= 200 && res.StatusCode < 300 {
s1.Verified = true
} else {
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
continue
}
+8 -17
View File
@@ -1,11 +1,11 @@
package opsgenie
import (
"bytes"
"context"
"fmt"
"net/http"
"regexp"
"strings"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
@@ -14,13 +14,10 @@ import (
type Scanner struct{}
// Ensure the Scanner satisfies the interface at compile time.
var _ detectors.Detector = (*Scanner)(nil)
var (
client = common.SaneHttpClient()
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"opsgenie"}) + `\b([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})\b`)
)
@@ -28,42 +25,36 @@ func (s Scanner) Type() detectorspb.DetectorType {
return detectorspb.DetectorType_Opsgenie
}
// Keywords are used for efficiently pre-filtering chunks.
// Use identifiers in the secret preferably, or the provider name.
func (s Scanner) Keywords() []string {
return []string{"opsgenie"}
func (s Scanner) Keywords() [][]byte {
return [][]byte{[]byte("opsgenie")}
}
// FromData will find and optionally verify Opsgenie secrets in a given set of bytes.
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
dataStr := string(data)
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
matches := keyPat.FindAllSubmatch(data, -1)
for _, match := range matches {
if len(match) != 2 {
continue
}
resMatch := strings.TrimSpace(match[1])
resMatch := bytes.TrimSpace(match[1])
s1 := detectors.Result{
DetectorType: detectorspb.DetectorType_Opsgenie,
Raw: []byte(resMatch),
Raw: resMatch,
}
if verify {
req, err := http.NewRequestWithContext(ctx, "GET", "https://api.opsgenie.com/v2/alerts", nil)
req, err := http.NewRequestWithContext(ctx, "GET", "https://api.opsgenie.com/v2/users", nil)
if err != nil {
continue
}
req.Header.Add("Authorization", fmt.Sprintf("GenieKey %s", resMatch))
req.Header.Add("Authorization", fmt.Sprintf("GenieKey %s", string(resMatch)))
res, err := client.Do(req)
if err == nil {
defer res.Body.Close()
if res.StatusCode >= 200 && res.StatusCode < 300 {
s1.Verified = true
} else {
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
continue
}
+8 -12
View File
@@ -1,11 +1,11 @@
package optimizely
import (
"bytes"
"context"
"fmt"
"net/http"
"regexp"
"strings"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
@@ -24,27 +24,23 @@ var (
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"optimizely"}) + `\b([0-9A-Za-z-:]{54})\b`)
)
// Keywords are used for efficiently pre-filtering chunks.
// Use identifiers in the secret preferably, or the provider name.
func (s Scanner) Keywords() []string {
return []string{"optimizely"}
func (s Scanner) Keywords() [][]byte {
return [][]byte{[]byte("optimizely")}
}
// FromData will find and optionally verify Optimizely secrets in a given set of bytes.
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
dataStr := string(data)
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
matches := keyPat.FindAllSubmatch(data, -1)
for _, match := range matches {
if len(match) != 2 {
continue
}
resMatch := strings.TrimSpace(match[1])
resMatch := bytes.TrimSpace(match[1])
s1 := detectors.Result{
DetectorType: detectorspb.DetectorType_Optimizely,
Raw: []byte(resMatch),
Raw: resMatch,
}
if verify {
@@ -52,19 +48,19 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
if err != nil {
continue
}
req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", resMatch))
req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", string(resMatch)))
res, err := client.Do(req)
if err == nil {
defer res.Body.Close()
if res.StatusCode >= 200 && res.StatusCode < 300 {
s1.Verified = true
} else {
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
continue
}
}
}
}
results = append(results, s1)
+7 -16
View File
@@ -1,11 +1,11 @@
package owlbot
import (
"bytes"
"context"
"fmt"
"net/http"
"regexp"
"strings"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
@@ -14,37 +14,29 @@ import (
type Scanner struct{}
// Ensure the Scanner satisfies the interface at compile time.
var _ detectors.Detector = (*Scanner)(nil)
var (
client = common.SaneHttpClient()
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"owlbot"}) + `\b([a-z0-9]{40})\b`)
)
// Keywords are used for efficiently pre-filtering chunks.
// Use identifiers in the secret preferably, or the provider name.
func (s Scanner) Keywords() []string {
return []string{"owlbot"}
func (s Scanner) Keywords() [][]byte {
return [][]byte{[]byte("owlbot")}
}
// FromData will find and optionally verify Owlbot secrets in a given set of bytes.
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
dataStr := string(data)
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
matches := keyPat.FindAllSubmatch(data, -1)
for _, match := range matches {
if len(match) != 2 {
continue
}
resMatch := strings.TrimSpace(match[1])
resMatch := bytes.TrimSpace(match[1])
s1 := detectors.Result{
DetectorType: detectorspb.DetectorType_Owlbot,
Raw: []byte(resMatch),
Raw: resMatch,
}
if verify {
@@ -52,14 +44,13 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
if err != nil {
continue
}
req.Header.Add("Authorization", fmt.Sprintf("Token %s", resMatch))
req.Header.Add("Authorization", fmt.Sprintf("Token %s", string(resMatch)))
res, err := client.Do(req)
if err == nil {
defer res.Body.Close()
if res.StatusCode >= 200 && res.StatusCode < 300 {
s1.Verified = true
} else {
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
continue
}
+8 -17
View File
@@ -1,10 +1,10 @@
package packagecloud
import (
"bytes"
"context"
"net/http"
"regexp"
"strings"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
@@ -13,37 +13,29 @@ import (
type Scanner struct{}
// Ensure the Scanner satisfies the interface at compile time.
var _ detectors.Detector = (*Scanner)(nil)
var (
client = common.SaneHttpClient()
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"packagecloud"}) + `\b([0-9a-f]{48})\b`)
)
// Keywords are used for efficiently pre-filtering chunks.
// Use identifiers in the secret preferably, or the provider name.
func (s Scanner) Keywords() []string {
return []string{"packagecloud"}
func (s Scanner) Keywords() [][]byte {
return [][]byte{[]byte("packagecloud")}
}
// FromData will find and optionally verify PackageCloud secrets in a given set of bytes.
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
dataStr := string(data)
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
matches := keyPat.FindAllSubmatch(data, -1)
for _, match := range matches {
if len(match) != 2 {
continue
}
resMatch := strings.TrimSpace(match[1])
resMatch := bytes.TrimSpace(match[1])
s1 := detectors.Result{
DetectorType: detectorspb.DetectorType_PackageCloud,
Raw: []byte(resMatch),
Raw: resMatch,
}
if verify {
@@ -51,15 +43,14 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
if err != nil {
continue
}
req.SetBasicAuth(resMatch, "")
req.SetBasicAuth(string(resMatch), "")
res, err := client.Do(req)
if err == nil {
defer res.Body.Close()
if res.StatusCode >= 200 && res.StatusCode < 300 {
s1.Verified = true
} else {
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
if detectors.IsKnownFalsePositive(match[1], detectors.DefaultFalsePositives, true) {
continue
}
}
@@ -1,11 +1,11 @@
package pagerdutyapikey
import (
"bytes"
"context"
"fmt"
"net/http"
"regexp"
"strings"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
@@ -14,37 +14,30 @@ import (
type Scanner struct{}
// Ensure the Scanner satisfies the interface at compile time.
var _ detectors.Detector = (*Scanner)(nil)
var (
client = common.SaneHttpClient()
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"pagerduty"}) + `\b([a-z]{1}\+[a-zA-Z]{9}\-[a-z]{2}\-[a-z0-9]{5})\b`)
)
// Keywords are used for efficiently pre-filtering chunks.
// Use identifiers in the secret preferably, or the provider name.
func (s Scanner) Keywords() []string {
return []string{"pagerduty"}
func (s Scanner) Keywords() [][]byte {
return [][]byte{[]byte("pagerduty")}
}
// FromData will find and optionally verify PagerDutyApiKey secrets in a given set of bytes.
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
dataStr := string(data)
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
matches := keyPat.FindAllSubmatch(data, -1)
for _, match := range matches {
if len(match) != 2 {
continue
}
resMatch := strings.TrimSpace(match[1])
resMatch := bytes.TrimSpace(match[1])
s1 := detectors.Result{
DetectorType: detectorspb.DetectorType_PagerDutyApiKey,
Raw: []byte(resMatch),
Raw: resMatch,
}
if verify {
@@ -65,7 +58,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
}
if !s1.Verified {
if detectors.IsKnownFalsePositive(string(s1.Raw), detectors.DefaultFalsePositives, true) {
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
continue
}
}
+6 -9
View File
@@ -5,7 +5,6 @@ import (
"fmt"
"net/http"
"regexp"
"strings"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
@@ -26,25 +25,23 @@ var (
// Keywords are used for efficiently pre-filtering chunks.
// Use identifiers in the secret preferably, or the provider name.
func (s Scanner) Keywords() []string {
return []string{"pandadoc"}
func (s Scanner) Keywords() [][]byte {
return [][]byte{[]byte("pandadoc")}
}
// FromData will find and optionally verify Pandadoc secrets in a given set of bytes.
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
dataStr := string(data)
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
matches := keyPat.FindAllSubmatch(data, -1)
for _, match := range matches {
if len(match) != 2 {
continue
}
resMatch := strings.TrimSpace(match[1])
resMatch := match[1]
s1 := detectors.Result{
DetectorType: detectorspb.DetectorType_Pandadoc,
Raw: []byte(resMatch),
Raw: resMatch,
}
if verify {
@@ -52,7 +49,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
if err != nil {
continue
}
req.Header.Add("Authorization", fmt.Sprintf("API-Key %s", resMatch))
req.Header.Add("Authorization", fmt.Sprintf("API-Key %s", string(resMatch)))
res, err := client.Do(req)
if err == nil {
defer res.Body.Close()
+11 -15
View File
@@ -1,11 +1,11 @@
package pandascore
import (
"bytes"
"context"
"fmt"
"net/http"
"regexp"
"strings"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
@@ -14,46 +14,39 @@ import (
type Scanner struct{}
// Ensure the Scanner satisfies the interface at compile time.
var _ detectors.Detector = (*Scanner)(nil)
var (
client = common.SaneHttpClient()
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"pandascore"}) + `([ \r\n]{0,1}[0-9A-Za-z\-\_]{51}[ \r\n]{1})`)
)
// Keywords are used for efficiently pre-filtering chunks.
// Use identifiers in the secret preferably, or the provider name.
func (s Scanner) Keywords() []string {
return []string{"pandascore"}
func (s Scanner) Keywords() [][]byte {
return [][]byte{[]byte("pandascore")}
}
// FromData will find and optionally verify PandaScore secrets in a given set of bytes.
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
dataStr := string(data)
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
matches := keyPat.FindAllSubmatch(data, -1)
for _, match := range matches {
if len(match) != 2 {
continue
}
resMatch := strings.TrimSpace(match[1])
resMatch := bytes.TrimSpace(match[1])
s1 := detectors.Result{
DetectorType: detectorspb.DetectorType_PandaScore,
Raw: []byte(resMatch),
Raw: resMatch,
}
if verify {
req, err := http.NewRequest("GET", "https://api.pandascore.co/videogames", nil)
req, err := http.NewRequestWithContext(ctx, "GET", "https://api.pandascore.co/videogames", nil)
if err != nil {
continue
}
req.Header.Add("Accept", "application/json")
req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", resMatch))
req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", string(resMatch)))
res, err := client.Do(req)
if err == nil {
defer res.Body.Close()
@@ -61,6 +54,9 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
s1.Verified = true
} else {
s1.Verified = false
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
continue
}
}
}
}
+7 -13
View File
@@ -1,11 +1,11 @@
package paperform
import (
"bytes"
"context"
"fmt"
"net/http"
"regexp"
"strings"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
@@ -20,31 +20,26 @@ var _ detectors.Detector = (*Scanner)(nil)
var (
client = common.SaneHttpClient()
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"paperform"}) + `\b([a-z0-9A-Z-._]{850,1000})\b`)
)
// Keywords are used for efficiently pre-filtering chunks.
// Use identifiers in the secret preferably, or the provider name.
func (s Scanner) Keywords() []string {
return []string{"paperform"}
func (s Scanner) Keywords() [][]byte {
return [][]byte{[]byte("paperform")}
}
// FromData will find and optionally verify Paperform secrets in a given set of bytes.
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
dataStr := string(data)
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
matches := keyPat.FindAllSubmatch(data, -1)
for _, match := range matches {
if len(match) != 2 {
continue
}
resMatch := strings.TrimSpace(match[1])
resMatch := bytes.TrimSpace(match[1])
s1 := detectors.Result{
DetectorType: detectorspb.DetectorType_Paperform,
Raw: []byte(resMatch),
Raw: resMatch,
}
if verify {
@@ -52,14 +47,13 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
if err != nil {
continue
}
req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", resMatch))
req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", string(resMatch)))
res, err := client.Do(req)
if err == nil {
defer res.Body.Close()
if res.StatusCode >= 200 && res.StatusCode < 300 {
s1.Verified = true
} else {
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
continue
}
+8 -20
View File
@@ -7,7 +7,6 @@ import (
"mime/multipart"
"net/http"
"regexp"
"strings"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
@@ -16,37 +15,29 @@ import (
type Scanner struct{}
// Ensure the Scanner satisfies the interface at compile time.
var _ detectors.Detector = (*Scanner)(nil)
var (
client = common.SaneHttpClient()
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"paralleldots"}) + `\b([0-9A-Za-z]{43})\b`)
)
// Keywords are used for efficiently pre-filtering chunks.
// Use identifiers in the secret preferably, or the provider name.
func (s Scanner) Keywords() []string {
return []string{"paralleldots"}
func (s Scanner) Keywords() [][]byte {
return [][]byte{[]byte("paralleldots")}
}
// FromData will find and optionally verify Paralleldots secrets in a given set of bytes.
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
dataStr := string(data)
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
matches := keyPat.FindAllSubmatch(data, -1)
for _, match := range matches {
if len(match) != 2 {
continue
}
resMatch := strings.TrimSpace(match[1])
resMatch := bytes.TrimSpace(match[1])
s1 := detectors.Result{
DetectorType: detectorspb.DetectorType_ParallelDots,
Raw: []byte(resMatch),
Raw: resMatch,
}
if verify {
@@ -56,7 +47,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
if err != nil {
continue
}
_, err = io.Copy(fw, strings.NewReader(resMatch))
_, err = io.Copy(fw, bytes.NewReader(resMatch))
if err != nil {
continue
}
@@ -64,7 +55,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
if err != nil {
continue
}
_, err = io.Copy(fw, strings.NewReader("sample text"))
_, err = io.Copy(fw, bytes.NewReader([]byte("sample text")))
if err != nil {
continue
}
@@ -81,11 +72,9 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
if err != nil {
continue
}
body := string(bodyBytes)
if (res.StatusCode >= 200 && res.StatusCode < 300) && strings.Contains(body, "intent") {
if (res.StatusCode >= 200 && res.StatusCode < 300) && bytes.Contains(bodyBytes, []byte("intent")) {
s1.Verified = true
} else {
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
continue
}
@@ -95,7 +84,6 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
results = append(results, s1)
}
return results, nil
}
+7 -15
View File
@@ -1,11 +1,11 @@
package parsehub
import (
"bytes"
"context"
"fmt"
"net/http"
"regexp"
"strings"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
@@ -14,41 +14,34 @@ import (
type Scanner struct{}
// Ensure the Scanner satisfies the interface at compile time.
var _ detectors.Detector = (*Scanner)(nil)
var (
client = common.SaneHttpClient()
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"parsehub"}) + `\b([0-9a-zA-Z]{12})\b`)
)
// Keywords are used for efficiently pre-filtering chunks.
// Use identifiers in the secret preferably, or the provider name.
func (s Scanner) Keywords() []string {
return []string{"parsehub"}
func (s Scanner) Keywords() [][]byte {
return [][]byte{[]byte("parsehub")}
}
// FromData will find and optionally verify Parsehub secrets in a given set of bytes.
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
dataStr := string(data)
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
matches := keyPat.FindAllSubmatch(data, -1)
for _, match := range matches {
if len(match) != 2 {
continue
}
resMatch := strings.TrimSpace(match[1])
resMatch := bytes.TrimSpace(match[1])
s1 := detectors.Result{
DetectorType: detectorspb.DetectorType_Parsehub,
Raw: []byte(resMatch),
Raw: resMatch,
}
if verify {
req, err := http.NewRequestWithContext(ctx, "GET", fmt.Sprintf("https://www.parsehub.com/api/v2/projects?api_key=%s&limit=1", resMatch), nil)
req, err := http.NewRequestWithContext(ctx, "GET", fmt.Sprintf("https://www.parsehub.com/api/v2/projects?api_key=%s&limit=1", string(resMatch)), nil)
if err != nil {
continue
}
@@ -58,7 +51,6 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
if res.StatusCode >= 200 && res.StatusCode < 300 {
s1.Verified = true
} else {
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
continue
}
+8 -18
View File
@@ -1,10 +1,10 @@
package parsers
import (
"bytes"
"context"
"net/http"
"regexp"
"strings"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
@@ -13,64 +13,54 @@ import (
type Scanner struct{}
// Ensure the Scanner satisfies the interface at compile time.
var _ detectors.Detector = (*Scanner)(nil)
var (
client = common.SaneHttpClient()
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"parsers"}) + `\b([0-9a-z]{64})\b`)
)
// Keywords are used for efficiently pre-filtering chunks.
// Use identifiers in the secret preferably, or the provider name.
func (s Scanner) Keywords() []string {
return []string{"parsers"}
func (s Scanner) Keywords() [][]byte {
return [][]byte{[]byte("parsers")}
}
// FromData will find and optionally verify Parsers secrets in a given set of bytes.
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
dataStr := string(data)
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
matches := keyPat.FindAllSubmatch(data, -1)
for _, match := range matches {
if len(match) != 2 {
continue
}
resMatch := strings.TrimSpace(match[1])
resMatch := bytes.TrimSpace(match[1])
s1 := detectors.Result{
DetectorType: detectorspb.DetectorType_Parsers,
Raw: []byte(resMatch),
Raw: resMatch,
}
if verify {
payload := strings.NewReader(`{"sql": "CREATE TABLE t();SELECT 1; SELECT 1 FROM ; SELECT 2"}`)
payload := bytes.NewBuffer([]byte(`{"sql": "CREATE TABLE t();SELECT 1; SELECT 1 FROM ; SELECT 2"}`))
req, err := http.NewRequestWithContext(ctx, "POST", "https://api.parsers.dev/api/v1/parse/postgresql/", payload)
if err != nil {
continue
}
req.Header.Add("Content-Type", "application/json")
req.Header.Add("x-api-key", resMatch)
req.Header.Add("x-api-key", string(resMatch))
res, err := client.Do(req)
if err == nil {
defer res.Body.Close()
if res.StatusCode >= 200 && res.StatusCode < 300 {
s1.Verified = true
} else {
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
continue
}
}
}
}
results = append(results, s1)
}
return results, nil
}
+7 -16
View File
@@ -1,11 +1,11 @@
package parseur
import (
"bytes"
"context"
"fmt"
"net/http"
"regexp"
"strings"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
@@ -14,37 +14,29 @@ import (
type Scanner struct{}
// Ensure the Scanner satisfies the interface at compile time
var _ detectors.Detector = (*Scanner)(nil)
var (
client = common.SaneHttpClient()
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"parseur"}) + `\b([a-f0-9]{40})\b`)
)
// Keywords are used for efficiently pre-filtering chunks.
// Use identifiers in the secret preferably, or the provider name.
func (s Scanner) Keywords() []string {
return []string{"parseur"}
func (s Scanner) Keywords() [][]byte {
return [][]byte{[]byte("parseur")}
}
// FromData will find and optionally verify Parseur secrets in a given set of bytes.
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
dataStr := string(data)
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
matches := keyPat.FindAllSubmatch(data, -1)
for _, match := range matches {
if len(match) != 2 {
continue
}
resMatch := strings.TrimSpace(match[1])
resMatch := bytes.TrimSpace(match[1])
s1 := detectors.Result{
DetectorType: detectorspb.DetectorType_Parseur,
Raw: []byte(resMatch),
Raw: resMatch,
}
if verify {
@@ -52,14 +44,13 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
if err != nil {
continue
}
req.Header.Add("Authorization", fmt.Sprintf("Token %s", resMatch))
req.Header.Add("Authorization", fmt.Sprintf("Token %s", string(resMatch)))
res, err := client.Do(req)
if err == nil {
defer res.Body.Close()
if res.StatusCode >= 200 && res.StatusCode < 300 {
s1.Verified = true
} else {
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
continue
}
+8 -18
View File
@@ -1,11 +1,11 @@
package partnerstack
import (
"bytes"
"context"
"fmt"
"net/http"
"regexp"
"strings"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
@@ -14,37 +14,30 @@ import (
type Scanner struct{}
// Ensure the Scanner satisfies the interface at compile time.
var _ detectors.Detector = (*Scanner)(nil)
var (
client = common.SaneHttpClient()
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"partnerstack"}) + `\b([0-9A-Za-z]{64})\b`)
)
// Keywords are used for efficiently pre-filtering chunks.
// Use identifiers in the secret preferably, or the provider name.
func (s Scanner) Keywords() []string {
return []string{"partnerstack"}
func (s Scanner) Keywords() [][]byte {
return [][]byte{[]byte("partnerstack")}
}
// FromData will find and optionally verify Partnerstack secrets in a given set of bytes.
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
dataStr := string(data)
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
matches := keyPat.FindAllSubmatch(data, -1)
for _, match := range matches {
if len(match) != 2 {
continue
}
resMatch := strings.TrimSpace(match[1])
resMatch := bytes.TrimSpace(match[1])
s1 := detectors.Result{
DetectorType: detectorspb.DetectorType_Partnerstack,
Raw: []byte(resMatch),
Raw: resMatch,
}
if verify {
@@ -53,24 +46,21 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
continue
}
req.Header.Add("Accept", "application/json")
req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", resMatch))
req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", string(resMatch)))
res, err := client.Do(req)
if err == nil {
defer res.Body.Close()
if res.StatusCode >= 200 && res.StatusCode < 300 {
s1.Verified = true
} else {
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
continue
}
}
}
}
results = append(results, s1)
}
return results, nil
}
+7 -16
View File
@@ -1,10 +1,10 @@
package passbase
import (
"bytes"
"context"
"net/http"
"regexp"
"strings"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
@@ -13,37 +13,29 @@ import (
type Scanner struct{}
// Ensure the Scanner satisfies the interface at compile time.
var _ detectors.Detector = (*Scanner)(nil)
var (
client = common.SaneHttpClient()
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"passbase"}) + `\b([a-zA-Z0-9]{128})\b`)
)
// Keywords are used for efficiently pre-filtering chunks.
// Use identifiers in the secret preferably, or the provider name.
func (s Scanner) Keywords() []string {
return []string{"passbase"}
func (s Scanner) Keywords() [][]byte {
return [][]byte{[]byte("passbase")}
}
// FromData will find and optionally verify Passbase secrets in a given set of bytes.
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
dataStr := string(data)
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
matches := keyPat.FindAllSubmatch(data, -1)
for _, match := range matches {
if len(match) != 2 {
continue
}
resMatch := strings.TrimSpace(match[1])
resMatch := bytes.TrimSpace(match[1])
s1 := detectors.Result{
DetectorType: detectorspb.DetectorType_Passbase,
Raw: []byte(resMatch),
Raw: resMatch,
}
if verify {
@@ -51,14 +43,13 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
if err != nil {
continue
}
req.Header.Add("X-API-KEY", resMatch)
req.Header.Add("X-API-KEY", string(resMatch))
res, err := client.Do(req)
if err == nil {
defer res.Body.Close()
if res.StatusCode >= 200 && res.StatusCode < 300 {
s1.Verified = true
} else {
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
continue
}
+2
View File
@@ -43,6 +43,8 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
instanceMatches := instancePat.FindAllStringSubmatch(dataStr, -1)
tokenMatches := accessTokenPat.FindAllStringSubmatch(dataStr, -1)
fmt.Printf("instanceMatches: %v\n", instanceMatches)
for _, instance := range instanceMatches {
if len(instance) != 1 {
continue
+1 -9
View File
@@ -2,10 +2,6 @@ package engine
import (
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/envoyapikey"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/huggingface"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/salesforce"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/snowflake"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/trufflehogenterprise"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/abbysale"
@@ -291,7 +287,6 @@ import (
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/getresponse"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/getsandbox"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/github"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/github_oauth2"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/github_old"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/githubapp"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/gitlab"
@@ -561,6 +556,7 @@ import (
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/satismeterwritekey"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/saucelabs"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/scalewaykey"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/github_oauth2"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/scalr"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/scrapeowl"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/scraperapi"
@@ -1536,10 +1532,6 @@ func DefaultDetectors() []detectors.Detector {
couchbase.Scanner{},
envoyapikey.Scanner{},
github_oauth2.Scanner{},
snowflake.Scanner{},
huggingface.Scanner{},
trufflehogenterprise.Scanner{},
salesforce.Scanner{},
}
}
+23 -24
View File
@@ -29,8 +29,6 @@ var (
maxDepth = 5
maxSize = 250 * 1024 * 1024 // 20MB
maxTimeout = time.Duration(30) * time.Second
defaultBufferSize = 512
)
// Ensure the Archive satisfies the interfaces at compile time.
@@ -64,7 +62,7 @@ func SetArchiveMaxTimeout(timeout time.Duration) {
// FromFile extracts the files from an archive.
func (a *Archive) FromFile(originalCtx context.Context, data io.Reader) chan []byte {
archiveChan := make(chan []byte, defaultBufferSize)
archiveChan := make(chan []byte, 512)
go func() {
ctx, cancel := context.WithTimeout(originalCtx, maxTimeout)
logger := logContext.AddLogger(ctx).Logger()
@@ -185,28 +183,29 @@ func (a *Archive) ReadToMax(ctx context.Context, reader io.Reader) (data []byte,
logger.Error(err, "Panic occurred when reading archive")
}
}()
if common.IsDone(ctx) {
return nil, ctx.Err()
fileContent := bytes.Buffer{}
logger.V(5).Info("Remaining buffer capacity", "bytes", maxSize-a.size)
for i := 0; i <= maxSize/512; i++ {
if common.IsDone(ctx) {
return nil, ctx.Err()
}
fileChunk := make([]byte, 512)
bRead, err := reader.Read(fileChunk)
if err != nil && !errors.Is(err, io.EOF) {
return []byte{}, err
}
a.size += bRead
if len(fileChunk) > 0 {
fileContent.Write(fileChunk[0:bRead])
}
if bRead < 512 {
return fileContent.Bytes(), nil
}
if a.size >= maxSize && bRead == 512 {
logger.V(2).Info("Max archive size reached.")
return fileContent.Bytes(), nil
}
}
var fileContent bytes.Buffer
// Create a limited reader to ensure we don't read more than the max size.
lr := io.LimitReader(reader, int64(maxSize))
// Using io.CopyBuffer for performance advantages. Though buf is mandatory
// for the method, due to the internal implementation of io.CopyBuffer, when
// *bytes.Buffer implements io.WriterTo or io.ReaderFrom, the provided buf
// is simply ignored. Thus, we can pass nil for the buf parameter.
_, err = io.CopyBuffer(&fileContent, lr, nil)
if err != nil && !errors.Is(err, io.EOF) {
return nil, err
}
if fileContent.Len() == maxSize {
logger.V(2).Info("Max archive size reached.")
}
return fileContent.Bytes(), nil
}
-53
View File
@@ -1,7 +1,6 @@
package handlers
import (
"bytes"
"context"
"io"
"net/http"
@@ -127,58 +126,6 @@ func TestHandleFile(t *testing.T) {
assert.Equal(t, 1, len(ch))
}
func TestReadToMax(t *testing.T) {
tests := []struct {
name string
input []byte
expected []byte
}{
{
name: "read full content within maxSize",
input: []byte("abcdefg"),
expected: []byte("abcdefg"),
},
{
name: "read content larger than maxSize",
input: make([]byte, maxSize+10), // this creates a byte slice 10 bytes larger than maxSize
expected: make([]byte, maxSize),
},
{
name: "empty input",
input: []byte(""),
expected: []byte(""),
},
}
a := &Archive{}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
reader := bytes.NewReader(tt.input)
output, err := a.ReadToMax(context.Background(), reader)
assert.Nil(t, err)
assert.Equal(t, tt.expected, output)
})
}
}
func BenchmarkReadToMax(b *testing.B) {
data := bytes.Repeat([]byte("a"), 1024*1000) // 1MB of data.
reader := bytes.NewReader(data)
a := &Archive{}
b.ResetTimer()
for i := 0; i < b.N; i++ {
b.StartTimer()
_, _ = a.ReadToMax(context.Background(), reader)
b.StopTimer()
_, _ = reader.Seek(0, 0) // Reset the reader position.
a.size = 0 // Reset archive size.
}
}
func TestExtractDebContent(t *testing.T) {
// Open the sample .deb file from the testdata folder.
file, err := os.Open("testdata/test.deb")
+1 -1
View File
@@ -133,7 +133,7 @@ func BranchHeads(repo *gogit.Repository) (map[string]*object.Commit, error) {
}
headCommit, err := repo.CommitObject(*headHash)
if err != nil {
logger.Error(err, "unable to get commit", "head_hash", headHash.String())
logger.Error(err, "unable to get commit", "commit", headCommit.String())
return nil
}
branches[branchName] = headCommit
+1 -1
View File
@@ -20,7 +20,7 @@ const (
// Chunker takes a chunk and splits it into chunks of ChunkSize.
func Chunker(originalChunk *Chunk) chan *Chunk {
chunkChan := make(chan *Chunk, 1)
chunkChan := make(chan *Chunk)
go func() {
defer close(chunkChan)
if len(originalChunk.Data) <= TotalChunkSize {
-1
View File
@@ -238,7 +238,6 @@ func (s *Source) chunkAction(ctx context.Context, proj project, bld build, act a
SourceType: s.Type(),
SourceName: s.name,
SourceID: s.SourceID(),
JobID: s.JobID(),
Data: removeCircleSha1Line(data.Bytes()),
SourceMetadata: &source_metadatapb.MetaData{
Data: &source_metadatapb.MetaData_Circleci{
-1
View File
@@ -174,7 +174,6 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) err
SourceType: s.Type(),
SourceName: s.name,
SourceID: s.SourceID(),
JobID: s.JobID(),
Data: file.Bytes(),
SourceMetadata: &source_metadatapb.MetaData{
Data: &source_metadatapb.MetaData_Docker{
-2
View File
@@ -161,7 +161,6 @@ func (s *Source) scanFile(ctx context.Context, path string, chunksChan chan *sou
SourceType: s.Type(),
SourceName: s.name,
SourceID: s.SourceID(),
JobID: s.JobID(),
SourceMetadata: &source_metadatapb.MetaData{
Data: &source_metadatapb.MetaData_Filesystem{
Filesystem: &source_metadatapb.Filesystem{
@@ -192,7 +191,6 @@ func (s *Source) scanFile(ctx context.Context, path string, chunksChan chan *sou
SourceType: s.Type(),
SourceName: s.name,
SourceID: s.SourceID(),
JobID: s.JobID(),
Data: data.Bytes(),
SourceMetadata: &source_metadatapb.MetaData{
Data: &source_metadatapb.MetaData_Filesystem{
-1
View File
@@ -324,7 +324,6 @@ func (s *Source) processObject(ctx context.Context, o object) error {
chunkSkel := &sources.Chunk{
SourceName: s.name,
SourceType: s.Type(),
JobID: s.JobID(),
SourceID: s.sourceId,
Verify: s.verify,
SourceMetadata: &source_metadatapb.MetaData{
-7
View File
@@ -446,7 +446,6 @@ func (s *Git) ScanCommits(ctx context.Context, repo *git.Repository, path string
chunkSkel := &sources.Chunk{
SourceName: s.sourceName,
SourceID: s.sourceID,
JobID: s.jobID,
SourceType: s.sourceType,
SourceMetadata: metadata,
Verify: s.verify,
@@ -465,7 +464,6 @@ func (s *Git) ScanCommits(ctx context.Context, repo *git.Repository, path string
chunksChan <- &sources.Chunk{
SourceName: s.sourceName,
SourceID: s.sourceID,
JobID: s.jobID,
SourceType: s.sourceType,
SourceMetadata: metadata,
Data: diff.Content.Bytes(),
@@ -492,7 +490,6 @@ func (s *Git) gitChunk(ctx context.Context, diff gitparse.Diff, fileName, email,
chunksChan <- &sources.Chunk{
SourceName: s.sourceName,
SourceID: s.sourceID,
JobID: s.jobID,
SourceType: s.sourceType,
SourceMetadata: metadata,
Data: append([]byte{}, newChunkBuffer.Bytes()...),
@@ -507,7 +504,6 @@ func (s *Git) gitChunk(ctx context.Context, diff gitparse.Diff, fileName, email,
chunksChan <- &sources.Chunk{
SourceName: s.sourceName,
SourceID: s.sourceID,
JobID: s.jobID,
SourceType: s.sourceType,
SourceMetadata: metadata,
Data: line,
@@ -527,7 +523,6 @@ func (s *Git) gitChunk(ctx context.Context, diff gitparse.Diff, fileName, email,
chunksChan <- &sources.Chunk{
SourceName: s.sourceName,
SourceID: s.sourceID,
JobID: s.jobID,
SourceType: s.sourceType,
SourceMetadata: metadata,
Data: append([]byte{}, newChunkBuffer.Bytes()...),
@@ -593,7 +588,6 @@ func (s *Git) ScanStaged(ctx context.Context, repo *git.Repository, path string,
chunkSkel := &sources.Chunk{
SourceName: s.sourceName,
SourceID: s.sourceID,
JobID: s.jobID,
SourceType: s.sourceType,
SourceMetadata: metadata,
Verify: s.verify,
@@ -608,7 +602,6 @@ func (s *Git) ScanStaged(ctx context.Context, repo *git.Repository, path string,
chunksChan <- &sources.Chunk{
SourceName: s.sourceName,
SourceID: s.sourceID,
JobID: s.jobID,
SourceType: s.sourceType,
SourceMetadata: metadata,
Data: diff.Content.Bytes(),
+2 -7
View File
@@ -149,16 +149,14 @@ func (s *Source) newFilteredRepoCache(c cache.Cache, include, exclude []string)
for _, ig := range include {
g, err := glob.Compile(ig)
if err != nil {
s.log.V(1).Info("invalid include glob", "include_value", ig, "err", err)
continue
s.log.V(1).Info("invalid include glob", "glob", g, "err", err)
}
includeGlobs = append(includeGlobs, g)
}
for _, eg := range exclude {
g, err := glob.Compile(eg)
if err != nil {
s.log.V(1).Info("invalid exclude glob", "exclude_value", eg, "err", err)
continue
s.log.V(1).Info("invalid exclude glob", "glob", g, "err", err)
}
excludeGlobs = append(excludeGlobs, g)
}
@@ -1225,7 +1223,6 @@ func (s *Source) chunkIssueComments(ctx context.Context, repo, repoPath string,
chunk := &sources.Chunk{
SourceName: s.name,
SourceID: s.SourceID(),
JobID: s.JobID(),
SourceType: s.Type(),
SourceMetadata: &source_metadatapb.MetaData{
Data: &source_metadatapb.MetaData_Github{
@@ -1259,7 +1256,6 @@ func (s *Source) chunkPullRequestComments(ctx context.Context, repo string, comm
SourceName: s.name,
SourceID: s.SourceID(),
SourceType: s.Type(),
JobID: s.JobID(),
SourceMetadata: &source_metadatapb.MetaData{
Data: &source_metadatapb.MetaData_Github{
Github: &source_metadatapb.Github{
@@ -1291,7 +1287,6 @@ func (s *Source) chunkGistComments(ctx context.Context, gistUrl string, comments
SourceName: s.name,
SourceID: s.SourceID(),
SourceType: s.Type(),
JobID: s.JobID(),
SourceMetadata: &source_metadatapb.MetaData{
Data: &source_metadatapb.MetaData_Github{
Github: &source_metadatapb.Github{
-1
View File
@@ -376,7 +376,6 @@ func (s *Source) pageChunker(ctx context.Context, client *s3.S3, chunksChan chan
SourceType: s.Type(),
SourceName: s.name,
SourceID: s.SourceID(),
JobID: s.JobID(),
SourceMetadata: &source_metadatapb.MetaData{
Data: &source_metadatapb.MetaData_S3{
S3: &source_metadatapb.S3{
+2 -20
View File
@@ -36,10 +36,7 @@ type SourceManager struct {
handles map[handle]sourceInfo
handlesLock sync.Mutex
// Pool limiting the amount of concurrent sources running.
pool errgroup.Group
poolLimit int
currentRunningCount int32
// Max number of units to scan concurrently per source.
pool errgroup.Group
concurrentUnits int
// Run the sources using source unit enumeration / chunking if available.
useSourceUnits bool
@@ -71,10 +68,7 @@ func WithReportHook(hook JobProgressHook) func(*SourceManager) {
// WithConcurrentSources limits the concurrent number of sources a manager can run.
func WithConcurrentSources(concurrency int) func(*SourceManager) {
return func(mgr *SourceManager) {
mgr.pool.SetLimit(concurrency)
mgr.poolLimit = concurrency
}
return func(mgr *SourceManager) { mgr.pool.SetLimit(concurrency) }
}
// WithBufferedOutput sets the size of the buffer used for the Chunks() channel.
@@ -173,8 +167,6 @@ func (s *SourceManager) asyncRun(ctx context.Context, handle handle) (JobProgres
ctx, cancel := context.WithCancel(ctx)
progress := NewJobProgress(jobID, int64(handle), sourceName, WithHooks(s.hooks...), WithCancel(cancel))
s.pool.Go(func() error {
atomic.AddInt32(&s.currentRunningCount, 1)
defer atomic.AddInt32(&s.currentRunningCount, -1)
ctx := context.WithValues(ctx,
"job_id", jobID,
"source_manager_worker_id", common.RandomID(5),
@@ -215,16 +207,6 @@ func (s *SourceManager) ScanChunk(chunk *Chunk) {
s.outputChunks <- chunk
}
// AvailableCapacity returns the number of concurrent jobs the manager can
// accommodate at this time. If there is no limit, -1 is returned.
func (s *SourceManager) AvailableCapacity() int {
if s.poolLimit == 0 {
return -1
}
runCount := atomic.LoadInt32(&s.currentRunningCount)
return s.poolLimit - int(runCount)
}
// preflightChecks is a helper method to check the Manager or the context isn't
// done and that the handle is valid.
func (s *SourceManager) preflightChecks(ctx context.Context, handle handle) error {
-21
View File
@@ -317,24 +317,3 @@ func TestSourceManagerCancelRun(t *testing.T) {
assert.Error(t, ref.Snapshot().FatalError())
assert.True(t, errors.Is(ref.Snapshot().FatalError(), returnedErr))
}
func TestSourceManagerAvailableCapacity(t *testing.T) {
mgr := NewManager(WithConcurrentSources(1337))
start, end := make(chan struct{}), make(chan struct{})
handle, err := enrollDummy(mgr, callbackChunker{func(context.Context, chan *Chunk) error {
start <- struct{}{} // Send start signal.
<-end // Wait for end signal.
return nil
}})
assert.NoError(t, err)
assert.Equal(t, 1337, mgr.AvailableCapacity())
ref, err := mgr.ScheduleRun(context.Background(), handle)
assert.NoError(t, err)
<-start // Wait for start signal.
assert.Equal(t, 1336, mgr.AvailableCapacity())
end <- struct{}{} // Send end signal.
<-ref.Done() // Wait for the job to finish.
assert.Equal(t, 1337, mgr.AvailableCapacity())
}
-2
View File
@@ -272,7 +272,6 @@ func (s *Source) monitorConnection(ctx context.Context, conn net.Conn, chunksCha
SourceName: s.syslog.sourceName,
SourceID: s.syslog.sourceID,
SourceType: s.syslog.sourceType,
JobID: s.JobID(),
SourceMetadata: metadata,
Data: input,
Verify: s.verify,
@@ -314,7 +313,6 @@ func (s *Source) acceptUDPConnections(ctx context.Context, netListener net.Packe
chunksChan <- &sources.Chunk{
SourceName: s.syslog.sourceName,
SourceID: s.syslog.sourceID,
JobID: s.JobID(),
SourceType: s.syslog.sourceType,
SourceMetadata: metadata,
Data: input,