Compare commits
1 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 67f9677b16 |
@@ -32,7 +32,7 @@ Have questions? Feedback? Jump in slack or discord and hang out with us
|
||||
|
||||
Join our [Slack Community](https://join.slack.com/t/trufflehog-community/shared_invite/zt-pw2qbi43-Aa86hkiimstfdKH9UCpPzQ)
|
||||
|
||||
Join the [Secret Scanning Discord](https://discord.gg/8Hzbrnkr7E)
|
||||
Join the [Secret Scanning Discord](https://discord.gg/sydS6AHTUP)
|
||||
|
||||
# :tv: Demo
|
||||
|
||||
|
||||
@@ -1,11 +1,11 @@
|
||||
package nimble
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
@@ -21,45 +21,42 @@ var (
|
||||
client = common.SaneHttpClient()
|
||||
|
||||
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
|
||||
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"nimble"}) + `\b([a-zA-Z0-9]{30})\b`)
|
||||
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"nimble"}) + `\b([0-9a-zA-Z]{30})\b`)
|
||||
)
|
||||
|
||||
// Keywords are used for efficiently pre-filtering chunks.
|
||||
// Use identifiers in the secret preferably, or the provider name.
|
||||
func (s Scanner) Keywords() []string {
|
||||
return []string{"nimble"}
|
||||
func (s Scanner) Keywords() [][]byte {
|
||||
return [][]byte{[]byte("nimble")}
|
||||
}
|
||||
|
||||
// FromData will find and optionally verify Nimble secrets in a given set of bytes.
|
||||
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
|
||||
dataStr := string(data)
|
||||
|
||||
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
|
||||
matches := keyPat.FindAllSubmatch(data, -1)
|
||||
|
||||
for _, match := range matches {
|
||||
if len(match) != 2 {
|
||||
continue
|
||||
}
|
||||
resMatch := strings.TrimSpace(match[1])
|
||||
resMatch := bytes.TrimSpace(match[1])
|
||||
|
||||
s1 := detectors.Result{
|
||||
DetectorType: detectorspb.DetectorType_Nimble,
|
||||
Raw: []byte(resMatch),
|
||||
Raw: resMatch,
|
||||
}
|
||||
|
||||
if verify {
|
||||
req, err := http.NewRequestWithContext(ctx, "GET", "https://app.nimble.com/api/v1/myself", nil)
|
||||
req, err := http.NewRequestWithContext(ctx, "GET", fmt.Sprintf("https://app.nimble.com/api/v1/myself"), nil)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", resMatch))
|
||||
req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", string(resMatch)))
|
||||
res, err := client.Do(req)
|
||||
if err == nil {
|
||||
defer res.Body.Close()
|
||||
if res.StatusCode >= 200 && res.StatusCode < 300 {
|
||||
s1.Verified = true
|
||||
} else {
|
||||
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
|
||||
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
|
||||
continue
|
||||
}
|
||||
|
||||
@@ -1,10 +1,10 @@
|
||||
package nitro
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
@@ -25,25 +25,22 @@ var (
|
||||
|
||||
// Keywords are used for efficiently pre-filtering chunks.
|
||||
// Use identifiers in the secret preferably, or the provider name.
|
||||
func (s Scanner) Keywords() []string {
|
||||
return []string{"nitro"}
|
||||
func (s Scanner) Keywords() [][]byte {
|
||||
return [][]byte{[]byte("nitro")}
|
||||
}
|
||||
|
||||
// FromData will find and optionally verify Nitro secrets in a given set of bytes.
|
||||
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
|
||||
dataStr := string(data)
|
||||
|
||||
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
|
||||
matches := keyPat.FindAllSubmatch(data, -1)
|
||||
|
||||
for _, match := range matches {
|
||||
if len(match) != 2 {
|
||||
continue
|
||||
}
|
||||
resMatch := strings.TrimSpace(match[1])
|
||||
resMatch := bytes.TrimSpace(match[1])
|
||||
|
||||
s1 := detectors.Result{
|
||||
DetectorType: detectorspb.DetectorType_Nitro,
|
||||
Raw: []byte(resMatch),
|
||||
Raw: resMatch,
|
||||
}
|
||||
|
||||
if verify {
|
||||
@@ -52,7 +49,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
continue
|
||||
}
|
||||
req.Header.Add("Content-Type", "application/json")
|
||||
req.SetBasicAuth(resMatch, "")
|
||||
req.SetBasicAuth(string(resMatch), "")
|
||||
res, err := client.Do(req)
|
||||
if err == nil {
|
||||
defer res.Body.Close()
|
||||
|
||||
@@ -1,11 +1,11 @@
|
||||
package noticeable
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
@@ -14,54 +14,47 @@ import (
|
||||
|
||||
type Scanner struct{}
|
||||
|
||||
// Ensure the Scanner satisfies the interface at compile time.
|
||||
var _ detectors.Detector = (*Scanner)(nil)
|
||||
|
||||
var (
|
||||
client = common.SaneHttpClient()
|
||||
|
||||
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
|
||||
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"noticeable"}) + `\b([0-9a-zA-Z]{20})\b`)
|
||||
)
|
||||
|
||||
// Keywords are used for efficiently pre-filtering chunks.
|
||||
// Use identifiers in the secret preferably, or the provider name.
|
||||
func (s Scanner) Keywords() []string {
|
||||
return []string{"noticeable"}
|
||||
func (s Scanner) Keywords() [][]byte {
|
||||
return [][]byte{[]byte("noticeable")}
|
||||
}
|
||||
|
||||
// FromData will find and optionally verify Noticeable secrets in a given set of bytes.
|
||||
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
|
||||
dataStr := string(data)
|
||||
|
||||
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
|
||||
matches := keyPat.FindAllSubmatch(data, -1)
|
||||
|
||||
for _, match := range matches {
|
||||
if len(match) != 2 {
|
||||
continue
|
||||
}
|
||||
resMatch := strings.TrimSpace(match[1])
|
||||
|
||||
resMatch := bytes.TrimSpace(match[1])
|
||||
|
||||
s1 := detectors.Result{
|
||||
DetectorType: detectorspb.DetectorType_Noticeable,
|
||||
Raw: []byte(resMatch),
|
||||
Raw: resMatch,
|
||||
}
|
||||
|
||||
if verify {
|
||||
payload := strings.NewReader(`{"query": "YOUR_GRAPHQL_PAYLOAD"}`)
|
||||
payload := bytes.NewReader([]byte(`{"query": "YOUR_GRAPHQL_PAYLOAD"}`))
|
||||
req, err := http.NewRequestWithContext(ctx, "POST", "https://api.noticeable.io/graphql", payload)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
req.Header.Add("Content-Type", "application/json")
|
||||
req.Header.Add("Authorization", fmt.Sprintf("Apikey %s", resMatch))
|
||||
req.Header.Add("Authorization", fmt.Sprintf("Apikey %s", string(resMatch)))
|
||||
res, err := client.Do(req)
|
||||
if err == nil {
|
||||
defer res.Body.Close()
|
||||
if res.StatusCode >= 200 && res.StatusCode < 300 {
|
||||
s1.Verified = true
|
||||
} else {
|
||||
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
|
||||
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
|
||||
continue
|
||||
}
|
||||
|
||||
@@ -1,11 +1,11 @@
|
||||
package notion
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
@@ -14,36 +14,29 @@ import (
|
||||
|
||||
type Scanner struct{}
|
||||
|
||||
// Ensure the Scanner satisfies the interface at compile time.
|
||||
var _ detectors.Detector = (*Scanner)(nil)
|
||||
|
||||
var (
|
||||
client = common.SaneHttpClient()
|
||||
|
||||
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
|
||||
keyPat = regexp.MustCompile(`\b(secret_[A-Za-z0-9]{43})\b`)
|
||||
)
|
||||
|
||||
// Keywords are used for efficiently pre-filtering chunks.
|
||||
// Use identifiers in the secret preferably, or the provider name.
|
||||
func (s Scanner) Keywords() []string {
|
||||
return []string{"notion"}
|
||||
func (s Scanner) Keywords() [][]byte {
|
||||
return [][]byte{[]byte("notion")}
|
||||
}
|
||||
|
||||
// FromData will find and optionally verify Notion secrets in a given set of bytes.
|
||||
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
|
||||
dataStr := string(data)
|
||||
|
||||
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
|
||||
matches := keyPat.FindAllSubmatch(data, -1)
|
||||
|
||||
for _, match := range matches {
|
||||
if len(match) != 2 {
|
||||
continue
|
||||
}
|
||||
resMatch := strings.TrimSpace(match[1])
|
||||
resMatch := bytes.TrimSpace(match[1])
|
||||
|
||||
s1 := detectors.Result{
|
||||
DetectorType: detectorspb.DetectorType_Notion,
|
||||
Raw: []byte(resMatch),
|
||||
Raw: resMatch,
|
||||
}
|
||||
|
||||
if verify {
|
||||
@@ -52,14 +45,13 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
continue
|
||||
}
|
||||
req.Header.Add("Notion-Version", "2021-05-11")
|
||||
req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", resMatch))
|
||||
req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", string(resMatch)))
|
||||
res, err := client.Do(req)
|
||||
if err == nil {
|
||||
defer res.Body.Close()
|
||||
if res.StatusCode >= 200 && res.StatusCode < 300 {
|
||||
s1.Verified = true
|
||||
} else {
|
||||
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
|
||||
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
|
||||
continue
|
||||
}
|
||||
|
||||
@@ -1,10 +1,10 @@
|
||||
package nozbeteams
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
@@ -13,63 +13,56 @@ import (
|
||||
|
||||
type Scanner struct{}
|
||||
|
||||
// Ensure the Scanner satisfies the interface at compile time.
|
||||
var _ detectors.Detector = (*Scanner)(nil)
|
||||
|
||||
var (
|
||||
client = common.SaneHttpClient()
|
||||
|
||||
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
|
||||
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"nozbe", "nozbeteams"}) + `\b([0-9A-Za-z]{16}_[0-9A-Za-z\-_]{64}[ \r\n]{1})`)
|
||||
)
|
||||
|
||||
// Keywords are used for efficiently pre-filtering chunks.
|
||||
// Use identifiers in the secret preferably, or the provider name.
|
||||
func (s Scanner) Keywords() []string {
|
||||
return []string{"nozbe", "nozbeteams"}
|
||||
func (s Scanner) Keywords() [][]byte {
|
||||
return [][]byte{[]byte("nozbe"), []byte("nozbeteams")}
|
||||
}
|
||||
|
||||
// FromData will find and optionally verify NozbeTeams secrets in a given set of bytes.
|
||||
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
|
||||
dataStr := string(data)
|
||||
|
||||
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
|
||||
matches := keyPat.FindAllSubmatch(data, -1)
|
||||
|
||||
for _, match := range matches {
|
||||
if len(match) != 2 {
|
||||
continue
|
||||
}
|
||||
resMatch := strings.TrimSpace(match[1])
|
||||
|
||||
resMatch := bytes.TrimSpace(match[1])
|
||||
s1 := detectors.Result{
|
||||
DetectorType: detectorspb.DetectorType_NozbeTeams,
|
||||
Raw: []byte(resMatch),
|
||||
Raw: resMatch,
|
||||
}
|
||||
|
||||
if verify {
|
||||
req, err := http.NewRequestWithContext(ctx, "GET", "https://api4.nozbe.com/v1/api/users", nil)
|
||||
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
|
||||
req.Header.Add("Accept", "application/json")
|
||||
req.Header.Add("Authorization", resMatch)
|
||||
req.Header.Add("Authorization", string(resMatch))
|
||||
|
||||
res, err := client.Do(req)
|
||||
|
||||
if err == nil {
|
||||
defer res.Body.Close()
|
||||
if res.StatusCode >= 200 && res.StatusCode < 300 {
|
||||
s1.Verified = true
|
||||
} else {
|
||||
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
|
||||
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
|
||||
continue
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
results = append(results, s1)
|
||||
}
|
||||
|
||||
return results, nil
|
||||
}
|
||||
|
||||
|
||||
@@ -1,11 +1,11 @@
|
||||
package npmtoken
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
@@ -29,23 +29,23 @@ var (
|
||||
|
||||
// Keywords are used for efficiently pre-filtering chunks.
|
||||
// Use identifiers in the secret preferably, or the provider name.
|
||||
func (s Scanner) Keywords() []string {
|
||||
return []string{"npm"}
|
||||
func (s Scanner) Keywords() [][]byte {
|
||||
return [][]byte{[]byte("npm")}
|
||||
}
|
||||
|
||||
// FromData will find and optionally verify NpmToken secrets in a given set of bytes.
|
||||
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
|
||||
dataStr := string(data)
|
||||
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
|
||||
|
||||
matches := keyPat.FindAllSubmatch(data, -1)
|
||||
for _, match := range matches {
|
||||
if len(match) != 2 {
|
||||
continue
|
||||
}
|
||||
resMatch := strings.TrimSpace(match[1])
|
||||
resMatch := bytes.TrimSpace(match[1])
|
||||
|
||||
s1 := detectors.Result{
|
||||
DetectorType: detectorspb.DetectorType_NpmToken,
|
||||
Raw: []byte(resMatch),
|
||||
Raw: resMatch,
|
||||
}
|
||||
|
||||
if verify {
|
||||
@@ -53,7 +53,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", resMatch))
|
||||
req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", string(resMatch)))
|
||||
res, err := client.Do(req)
|
||||
if err == nil {
|
||||
defer res.Body.Close()
|
||||
|
||||
@@ -2,7 +2,6 @@ package npmtokenv2
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"regexp"
|
||||
|
||||
@@ -13,7 +12,6 @@ import (
|
||||
|
||||
type Scanner struct{}
|
||||
|
||||
// Ensure the Scanner satisfies the interfaces at compile time.
|
||||
var _ detectors.Detector = (*Scanner)(nil)
|
||||
var _ detectors.Versioner = (*Scanner)(nil)
|
||||
|
||||
@@ -21,32 +19,25 @@ func (s Scanner) Version() int { return 2 }
|
||||
|
||||
var (
|
||||
client = common.SaneHttpClient()
|
||||
|
||||
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
|
||||
keyPat = regexp.MustCompile(`(npm_[0-9a-zA-Z]{36})`)
|
||||
)
|
||||
|
||||
// Keywords are used for efficiently pre-filtering chunks.
|
||||
// Use identifiers in the secret preferably, or the provider name.
|
||||
func (s Scanner) Keywords() []string {
|
||||
return []string{"npm_"}
|
||||
func (s Scanner) Keywords() [][]byte {
|
||||
return [][]byte{[]byte("npm_")}
|
||||
}
|
||||
|
||||
// FromData will find and optionally verify NpmTokenV2 secrets in a given set of bytes.
|
||||
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
|
||||
dataStr := string(data)
|
||||
|
||||
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
|
||||
matches := keyPat.FindAllSubmatch(data, -1)
|
||||
|
||||
for _, match := range matches {
|
||||
if len(match) != 2 {
|
||||
continue
|
||||
}
|
||||
resMatch := match[1]
|
||||
|
||||
resMatch := match[1]
|
||||
s1 := detectors.Result{
|
||||
DetectorType: detectorspb.DetectorType_NpmToken,
|
||||
Raw: []byte(resMatch),
|
||||
Raw: resMatch,
|
||||
}
|
||||
|
||||
if verify {
|
||||
@@ -54,14 +45,13 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", resMatch))
|
||||
req.Header.Add("Authorization", "Bearer "+string(resMatch))
|
||||
res, err := client.Do(req)
|
||||
if err == nil {
|
||||
defer res.Body.Close()
|
||||
if res.StatusCode >= 200 && res.StatusCode < 300 {
|
||||
s1.Verified = true
|
||||
} else {
|
||||
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
|
||||
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
|
||||
continue
|
||||
}
|
||||
|
||||
@@ -1,10 +1,10 @@
|
||||
package nugetapikey
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
@@ -13,37 +13,30 @@ import (
|
||||
|
||||
type Scanner struct{}
|
||||
|
||||
// Ensure the Scanner satisfies the interface at compile time.
|
||||
var _ detectors.Detector = (*Scanner)(nil)
|
||||
|
||||
var (
|
||||
client = common.SaneHttpClient()
|
||||
|
||||
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
|
||||
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"nuget"}) + `\b([a-z0-9]{46})\b`)
|
||||
)
|
||||
|
||||
// Keywords are used for efficiently pre-filtering chunks.
|
||||
// Use identifiers in the secret preferably, or the provider name.
|
||||
func (s Scanner) Keywords() []string {
|
||||
return []string{"nuget"}
|
||||
func (s Scanner) Keywords() [][]byte {
|
||||
return [][]byte{[]byte("nuget")}
|
||||
}
|
||||
|
||||
// FromData will find and optionally verify Nugetapikey secrets in a given set of bytes.
|
||||
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
|
||||
dataStr := string(data)
|
||||
|
||||
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
|
||||
matches := keyPat.FindAllSubmatch(data, -1)
|
||||
|
||||
for _, match := range matches {
|
||||
if len(match) != 2 {
|
||||
continue
|
||||
}
|
||||
resMatch := strings.TrimSpace(match[1])
|
||||
resMatch := bytes.TrimSpace(match[1])
|
||||
|
||||
s1 := detectors.Result{
|
||||
DetectorType: detectorspb.DetectorType_NuGetApiKey,
|
||||
Raw: []byte(resMatch),
|
||||
Raw: resMatch,
|
||||
}
|
||||
|
||||
if verify {
|
||||
@@ -52,15 +45,14 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
continue
|
||||
}
|
||||
req.Header.Add("Content-Length", "0")
|
||||
req.Header.Add("X-NuGet-ApiKey", resMatch)
|
||||
req.Header.Add("X-NuGet-ApiKey", string(resMatch))
|
||||
res, err := client.Do(req)
|
||||
if err == nil {
|
||||
defer res.Body.Close()
|
||||
// we can either match on response code "400" or "Bad Request" response
|
||||
|
||||
if res.StatusCode == 400 {
|
||||
s1.Verified = true
|
||||
} else {
|
||||
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
|
||||
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
|
||||
continue
|
||||
}
|
||||
|
||||
@@ -1,12 +1,12 @@
|
||||
package numverify
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
@@ -15,46 +15,44 @@ import (
|
||||
|
||||
type Scanner struct{}
|
||||
|
||||
// Ensure the Scanner satisfies the interface at compile time.
|
||||
var _ detectors.Detector = (*Scanner)(nil)
|
||||
|
||||
var (
|
||||
client = common.SaneHttpClient()
|
||||
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
|
||||
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"numverify"}) + `\b([a-z0-9]{32})\b`)
|
||||
)
|
||||
|
||||
// Keywords are used for efficiently pre-filtering chunks.
|
||||
// Use identifiers in the secret preferably, or the provider name.
|
||||
func (s Scanner) Keywords() []string {
|
||||
return []string{"numverify"}
|
||||
func (s Scanner) Keywords() [][]byte {
|
||||
return [][]byte{[]byte("numverify")}
|
||||
}
|
||||
|
||||
// FromData will find and optionally verify Numverify secrets in a given set of bytes.
|
||||
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
|
||||
dataStr := string(data)
|
||||
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
|
||||
matches := keyPat.FindAllSubmatch(data, -1)
|
||||
|
||||
for _, match := range matches {
|
||||
if len(match) != 2 {
|
||||
continue
|
||||
}
|
||||
resMatch := strings.TrimSpace(match[1])
|
||||
resMatch := bytes.TrimSpace(match[1])
|
||||
|
||||
s1 := detectors.Result{
|
||||
DetectorType: detectorspb.DetectorType_Numverify,
|
||||
Raw: []byte(resMatch),
|
||||
Raw: resMatch,
|
||||
}
|
||||
|
||||
if verify {
|
||||
req, err := http.NewRequestWithContext(ctx, "GET", fmt.Sprintf("https://apilayer.net/api/validate?access_key=%s&number=14158586273", resMatch), nil)
|
||||
req, err := http.NewRequestWithContext(ctx, "GET", fmt.Sprintf("https://apilayer.net/api/validate?access_key=%s&number=14158586273", string(resMatch)), nil)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
|
||||
res, err := client.Do(req)
|
||||
if err == nil {
|
||||
bodyBytes, err := io.ReadAll(res.Body)
|
||||
if err == nil {
|
||||
bodyString := string(bodyBytes)
|
||||
validResponse := strings.Contains(bodyString, `country_code`) || strings.Contains(bodyString, `"info":"Access Restricted - Your current Subscription Plan does not support HTTPS Encryption."`)
|
||||
validResponse := bytes.Contains(bodyBytes, []byte(`country_code`)) || bytes.Contains(bodyBytes, []byte(`"info":"Access Restricted - Your current Subscription Plan does not support HTTPS Encryption."`))
|
||||
defer res.Body.Close()
|
||||
|
||||
if res.StatusCode >= 200 && res.StatusCode < 300 {
|
||||
if validResponse {
|
||||
s1.Verified = true
|
||||
@@ -62,7 +60,6 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
s1.Verified = false
|
||||
}
|
||||
} else {
|
||||
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
|
||||
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
|
||||
continue
|
||||
}
|
||||
@@ -73,6 +70,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
|
||||
results = append(results, s1)
|
||||
}
|
||||
|
||||
return results, nil
|
||||
}
|
||||
|
||||
|
||||
@@ -1,10 +1,10 @@
|
||||
package nutritionix
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
@@ -13,64 +13,56 @@ import (
|
||||
|
||||
type Scanner struct{}
|
||||
|
||||
// Ensure the Scanner satisfies the interface at compile time.
|
||||
var _ detectors.Detector = (*Scanner)(nil)
|
||||
|
||||
var (
|
||||
client = common.SaneHttpClient()
|
||||
|
||||
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
|
||||
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"nutritionix"}) + `\b([a-z0-9]{32})\b`)
|
||||
idPat = regexp.MustCompile(detectors.PrefixRegex([]string{"nutritionix"}) + `\b([a-z0-9]{8})\b`)
|
||||
)
|
||||
|
||||
// Keywords are used for efficiently pre-filtering chunks.
|
||||
// Use identifiers in the secret preferably, or the provider name.
|
||||
func (s Scanner) Keywords() []string {
|
||||
return []string{"nutritionix"}
|
||||
func (s Scanner) Keywords() [][]byte {
|
||||
return [][]byte{[]byte("nutritionix")}
|
||||
}
|
||||
|
||||
// FromData will find and optionally verify Nutritionix secrets in a given set of bytes.
|
||||
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
|
||||
dataStr := string(data)
|
||||
|
||||
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
|
||||
idMatches := idPat.FindAllStringSubmatch(dataStr, -1)
|
||||
matches := keyPat.FindAllSubmatch(data, -1)
|
||||
idMatches := idPat.FindAllSubmatch(data, -1)
|
||||
|
||||
for _, match := range matches {
|
||||
if len(match) != 2 {
|
||||
continue
|
||||
}
|
||||
resMatch := strings.TrimSpace(match[1])
|
||||
resMatch := bytes.TrimSpace(match[1])
|
||||
|
||||
for _, idMatch := range idMatches {
|
||||
if len(idMatch) != 2 {
|
||||
continue
|
||||
}
|
||||
|
||||
resIdMatch := strings.TrimSpace(idMatch[1])
|
||||
resIdMatch := bytes.TrimSpace(idMatch[1])
|
||||
|
||||
s1 := detectors.Result{
|
||||
DetectorType: detectorspb.DetectorType_Nutritionix,
|
||||
Raw: []byte(resMatch),
|
||||
Raw: resMatch,
|
||||
}
|
||||
|
||||
if verify {
|
||||
payload := strings.NewReader(`{"query":"for breakfast i ate 2 eggs, bacon, and french toast","timezone":"US/Eastern"}`)
|
||||
payload := bytes.NewBuffer([]byte(`{"query":"for breakfast i ate 2 eggs, bacon, and french toast","timezone":"US/Eastern"}`))
|
||||
req, err := http.NewRequestWithContext(ctx, "POST", "https://trackapi.nutritionix.com/v2/natural/nutrients", payload)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
req.Header.Add("Content-Type", "application/json")
|
||||
req.Header.Add("x-app-id", resIdMatch)
|
||||
req.Header.Add("x-app-key", resMatch)
|
||||
req.Header.Add("x-app-id", string(resIdMatch))
|
||||
req.Header.Add("x-app-key", string(resMatch))
|
||||
res, err := client.Do(req)
|
||||
if err == nil {
|
||||
defer res.Body.Close()
|
||||
if res.StatusCode >= 200 && res.StatusCode < 300 {
|
||||
s1.Verified = true
|
||||
} else {
|
||||
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
|
||||
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
|
||||
continue
|
||||
}
|
||||
|
||||
@@ -1,11 +1,11 @@
|
||||
package nylas
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
@@ -26,25 +26,23 @@ var (
|
||||
|
||||
// Keywords are used for efficiently pre-filtering chunks.
|
||||
// Use identifiers in the secret preferably, or the provider name.
|
||||
func (s Scanner) Keywords() []string {
|
||||
return []string{"nylas"}
|
||||
func (s Scanner) Keywords() [][]byte {
|
||||
return [][]byte{[]byte("nylas")}
|
||||
}
|
||||
|
||||
// FromData will find and optionally verify Nylas secrets in a given set of bytes.
|
||||
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
|
||||
dataStr := string(data)
|
||||
|
||||
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
|
||||
matches := keyPat.FindAllSubmatch(data, -1)
|
||||
|
||||
for _, match := range matches {
|
||||
if len(match) != 2 {
|
||||
continue
|
||||
}
|
||||
resMatch := strings.TrimSpace(match[1])
|
||||
resMatch := bytes.TrimSpace(match[1])
|
||||
|
||||
s1 := detectors.Result{
|
||||
DetectorType: detectorspb.DetectorType_Nylas,
|
||||
Raw: []byte(resMatch),
|
||||
Raw: resMatch,
|
||||
}
|
||||
|
||||
if verify {
|
||||
@@ -53,14 +51,13 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
continue
|
||||
}
|
||||
req.Header.Add("Accept", "application/vnd.nylas+json; version=3")
|
||||
req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", resMatch))
|
||||
req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", string(resMatch)))
|
||||
res, err := client.Do(req)
|
||||
if err == nil {
|
||||
defer res.Body.Close()
|
||||
if res.StatusCode >= 200 && res.StatusCode < 300 {
|
||||
s1.Verified = true
|
||||
} else {
|
||||
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
|
||||
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
|
||||
continue
|
||||
}
|
||||
|
||||
@@ -1,10 +1,10 @@
|
||||
package nytimes
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
@@ -25,29 +25,26 @@ var (
|
||||
|
||||
// Keywords are used for efficiently pre-filtering chunks.
|
||||
// Use identifiers in the secret preferably, or the provider name.
|
||||
func (s Scanner) Keywords() []string {
|
||||
return []string{"nytimes"}
|
||||
func (s Scanner) Keywords() [][]byte {
|
||||
return [][]byte{[]byte("nytimes")}
|
||||
}
|
||||
|
||||
// FromData will find and optionally verify Nytimes secrets in a given set of bytes.
|
||||
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
|
||||
dataStr := string(data)
|
||||
|
||||
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
|
||||
matches := keyPat.FindAllSubmatch(data, -1)
|
||||
|
||||
for _, match := range matches {
|
||||
if len(match) != 2 {
|
||||
continue
|
||||
}
|
||||
resMatch := strings.TrimSpace(match[1])
|
||||
resMatch := bytes.TrimSpace(match[1])
|
||||
|
||||
s1 := detectors.Result{
|
||||
DetectorType: detectorspb.DetectorType_Nytimes,
|
||||
Raw: []byte(resMatch),
|
||||
Raw: resMatch,
|
||||
}
|
||||
|
||||
if verify {
|
||||
req, err := http.NewRequestWithContext(ctx, "GET", "https://api.nytimes.com/svc/archive/v1/2019/1.json?api-key="+resMatch, nil)
|
||||
req, err := http.NewRequestWithContext(ctx, "GET", "https://api.nytimes.com/svc/archive/v1/2019/1.json?api-key="+string(resMatch), nil)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
|
||||
@@ -1,11 +1,11 @@
|
||||
package oanda
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
@@ -14,37 +14,29 @@ import (
|
||||
|
||||
type Scanner struct{}
|
||||
|
||||
// Ensure the Scanner satisfies the interface at compile time.
|
||||
var _ detectors.Detector = (*Scanner)(nil)
|
||||
|
||||
var (
|
||||
client = common.SaneHttpClient()
|
||||
|
||||
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
|
||||
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"oanda"}) + `\b([a-zA-Z0-9]{24})\b`)
|
||||
)
|
||||
|
||||
// Keywords are used for efficiently pre-filtering chunks.
|
||||
// Use identifiers in the secret preferably, or the provider name.
|
||||
func (s Scanner) Keywords() []string {
|
||||
return []string{"oanda"}
|
||||
func (s Scanner) Keywords() [][]byte {
|
||||
return [][]byte{[]byte("oanda")}
|
||||
}
|
||||
|
||||
// FromData will find and optionally verify Oanda secrets in a given set of bytes.
|
||||
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
|
||||
dataStr := string(data)
|
||||
|
||||
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
|
||||
matches := keyPat.FindAllSubmatch(data, -1)
|
||||
|
||||
for _, match := range matches {
|
||||
if len(match) != 2 {
|
||||
continue
|
||||
}
|
||||
resMatch := strings.TrimSpace(match[1])
|
||||
resMatch := bytes.TrimSpace(match[1])
|
||||
|
||||
s1 := detectors.Result{
|
||||
DetectorType: detectorspb.DetectorType_Oanda,
|
||||
Raw: []byte(resMatch),
|
||||
Raw: resMatch,
|
||||
}
|
||||
|
||||
if verify {
|
||||
@@ -52,14 +44,13 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", resMatch))
|
||||
req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", string(resMatch)))
|
||||
res, err := client.Do(req)
|
||||
if err == nil {
|
||||
defer res.Body.Close()
|
||||
if res.StatusCode >= 200 && res.StatusCode < 300 {
|
||||
s1.Verified = true
|
||||
} else {
|
||||
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
|
||||
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
|
||||
continue
|
||||
}
|
||||
|
||||
@@ -1,12 +1,12 @@
|
||||
package okta
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
@@ -15,51 +15,35 @@ import (
|
||||
|
||||
type Scanner struct{}
|
||||
|
||||
// Ensure the Scanner satisfies the interface at compile time.
|
||||
var _ detectors.Detector = (*Scanner)(nil)
|
||||
|
||||
var (
|
||||
domainPat = regexp.MustCompile(`\b[a-z0-9-]{1,40}\.okta(?:preview|-emea){0,1}\.com\b`)
|
||||
tokenPat = regexp.MustCompile(`\b00[a-zA-Z0-9_-]{40}\b`)
|
||||
// TODO: Oauth client secrets
|
||||
)
|
||||
|
||||
// Keywords are used for efficiently pre-filtering chunks.
|
||||
// Use identifiers in the secret preferably, or the provider name.
|
||||
func (s Scanner) Keywords() []string {
|
||||
return []string{"okta"}
|
||||
func (s Scanner) Keywords() [][]byte {
|
||||
return [][]byte{[]byte("okta")}
|
||||
}
|
||||
|
||||
// FromData will find and optionally verify Okta secrets in a given set of bytes.
|
||||
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
|
||||
for _, tokenMatch := range tokenPat.FindAll(data, -1) {
|
||||
token := string(tokenMatch)
|
||||
|
||||
for _, domainMatch := range domainPat.FindAll(data, -1) {
|
||||
domain := string(domainMatch)
|
||||
|
||||
s := detectors.Result{
|
||||
DetectorType: detectorspb.DetectorType_Okta,
|
||||
Raw: []byte(token),
|
||||
RawV2: []byte(fmt.Sprintf("%s:%s", domain, token)),
|
||||
Raw: tokenMatch,
|
||||
RawV2: []byte(fmt.Sprintf("%s:%s", string(domainMatch), string(tokenMatch))),
|
||||
}
|
||||
|
||||
if verify {
|
||||
// curl -v -X GET \
|
||||
// -H "Accept: application/json" \
|
||||
// -H "Content-Type: application/json" \
|
||||
// -H "Authorization: Bearer token" \
|
||||
// "https://subdomain.okta.com/api/v1/users/me"
|
||||
//
|
||||
|
||||
url := fmt.Sprintf("https://%s/api/v1/users/me", domain)
|
||||
url := fmt.Sprintf("https://%s/api/v1/users/me", string(domainMatch))
|
||||
req, err := http.NewRequestWithContext(ctx, "GET", url, nil)
|
||||
if err != nil {
|
||||
return results, err
|
||||
}
|
||||
req.Header.Set("Accept", "application/json")
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
req.Header.Set("Authorization", fmt.Sprintf("SSWS %s", token))
|
||||
req.Header.Set("Authorization", fmt.Sprintf("SSWS %s", string(tokenMatch)))
|
||||
|
||||
resp, err := common.SaneHttpClient().Do(req)
|
||||
if err != nil {
|
||||
@@ -68,14 +52,14 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode >= 200 && resp.StatusCode < 300 {
|
||||
body, _ := io.ReadAll(resp.Body)
|
||||
if strings.Contains(string(body), "activated") {
|
||||
if bytes.Contains(body, []byte("activated")) {
|
||||
s.Verified = true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if !s.Verified {
|
||||
if detectors.IsKnownFalsePositive(string(s.Raw), detectors.DefaultFalsePositives, true) {
|
||||
if detectors.IsKnownFalsePositive(s.Raw, detectors.DefaultFalsePositives, true) {
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,10 +1,10 @@
|
||||
package omnisend
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
@@ -13,37 +13,29 @@ import (
|
||||
|
||||
type Scanner struct{}
|
||||
|
||||
// Ensure the Scanner satisfies the interface at compile time.
|
||||
var _ detectors.Detector = (*Scanner)(nil)
|
||||
|
||||
var (
|
||||
client = common.SaneHttpClient()
|
||||
|
||||
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
|
||||
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"omnisend"}) + `\b([a-z0-9A-Z-]{75})\b`)
|
||||
)
|
||||
|
||||
// Keywords are used for efficiently pre-filtering chunks.
|
||||
// Use identifiers in the secret preferably, or the provider name.
|
||||
func (s Scanner) Keywords() []string {
|
||||
return []string{"omnisend"}
|
||||
func (s Scanner) Keywords() [][]byte {
|
||||
return [][]byte{[]byte("omnisend")}
|
||||
}
|
||||
|
||||
// FromData will find and optionally verify Omnisend secrets in a given set of bytes.
|
||||
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
|
||||
dataStr := string(data)
|
||||
|
||||
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
|
||||
matches := keyPat.FindAllSubmatch(data, -1)
|
||||
|
||||
for _, match := range matches {
|
||||
if len(match) != 2 {
|
||||
continue
|
||||
}
|
||||
resMatch := strings.TrimSpace(match[1])
|
||||
resMatch := bytes.TrimSpace(match[1])
|
||||
|
||||
s1 := detectors.Result{
|
||||
DetectorType: detectorspb.DetectorType_Omnisend,
|
||||
Raw: []byte(resMatch),
|
||||
Raw: resMatch,
|
||||
}
|
||||
|
||||
if verify {
|
||||
@@ -51,14 +43,13 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
req.Header.Add("X-API-KEY", resMatch)
|
||||
req.Header.Add("X-API-KEY", string(resMatch))
|
||||
res, err := client.Do(req)
|
||||
if err == nil {
|
||||
defer res.Body.Close()
|
||||
if res.StatusCode >= 200 && res.StatusCode < 300 {
|
||||
s1.Verified = true
|
||||
} else {
|
||||
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
|
||||
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
|
||||
continue
|
||||
}
|
||||
|
||||
@@ -1,12 +1,12 @@
|
||||
package onedesk
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
@@ -23,40 +23,38 @@ var (
|
||||
|
||||
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
|
||||
emailPat = regexp.MustCompile(`\b([a-zA-Z0-9._-]+@[a-zA-Z0-9._-]+\.[a-z]+)\b`)
|
||||
pwordPat = regexp.MustCompile(detectors.PrefixRegex([]string{"onedesk"}) + `\b([a-zA-Z0-9!=@#$%^]{8,64})`)
|
||||
pwordPat = regexp.MustCompile(detectors.PrefixRegex([]string{"onedesk"}) + `\b([a-zA-Z0-9!=@#$%^]{8,64})\b`)
|
||||
)
|
||||
|
||||
// Keywords are used for efficiently pre-filtering chunks.
|
||||
// Use identifiers in the secret preferably, or the provider name.
|
||||
func (s Scanner) Keywords() []string {
|
||||
return []string{"onedesk"}
|
||||
func (s Scanner) Keywords() [][]byte {
|
||||
return [][]byte{[]byte("onedesk")}
|
||||
}
|
||||
|
||||
// FromData will find and optionally verify Onedesk secrets in a given set of bytes.
|
||||
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
|
||||
dataStr := string(data)
|
||||
|
||||
matches := emailPat.FindAllStringSubmatch(dataStr, -1)
|
||||
pwordMatches := pwordPat.FindAllStringSubmatch(dataStr, -1)
|
||||
matches := emailPat.FindAllSubmatch(data, -1)
|
||||
pwordMatches := pwordPat.FindAllSubmatch(data, -1)
|
||||
|
||||
for _, match := range matches {
|
||||
if len(match) != 2 {
|
||||
continue
|
||||
}
|
||||
resMatch := strings.TrimSpace(match[1])
|
||||
resMatch := bytes.TrimSpace(match[1])
|
||||
for _, pwordMatch := range pwordMatches {
|
||||
if len(pwordMatch) != 2 {
|
||||
continue
|
||||
}
|
||||
resPword := strings.TrimSpace(pwordMatch[1])
|
||||
resPword := bytes.TrimSpace(pwordMatch[1])
|
||||
|
||||
s1 := detectors.Result{
|
||||
DetectorType: detectorspb.DetectorType_Onedesk,
|
||||
Raw: []byte(resMatch),
|
||||
Raw: resMatch,
|
||||
}
|
||||
|
||||
if verify {
|
||||
payload := strings.NewReader(fmt.Sprintf(`{"email": "%s", "password": "%s"}`, resMatch, resPword))
|
||||
payload := bytes.NewBufferString(fmt.Sprintf(`{"email": "%s", "password": "%s"}`, string(resMatch), string(resPword)))
|
||||
req, err := http.NewRequestWithContext(ctx, "POST", "https://app.onedesk.com/rest/2.0/login/loginUser", payload)
|
||||
if err != nil {
|
||||
continue
|
||||
@@ -69,8 +67,8 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
body := string(bodyBytes)
|
||||
if res.StatusCode >= 200 && res.StatusCode < 300 && strings.Contains(body, `"code":"SUCCESS"`) {
|
||||
body := bodyBytes
|
||||
if res.StatusCode >= 200 && res.StatusCode < 300 && bytes.Contains(body, []byte(`"code":"SUCCESS"`)) {
|
||||
s1.Verified = true
|
||||
} else {
|
||||
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
|
||||
|
||||
@@ -1,11 +1,11 @@
|
||||
package onelogin
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
@@ -14,58 +14,54 @@ import (
|
||||
|
||||
type Scanner struct{}
|
||||
|
||||
// Ensure the Scanner satisfies the interface at compile time.
|
||||
// Ensure the Scanner satisfies the detector interface at compile time.
|
||||
var _ detectors.Detector = (*Scanner)(nil)
|
||||
|
||||
var (
|
||||
oauthClientIDPat = regexp.MustCompile(`(?i)id[a-zA-Z0-9_' "=]{0,20}([a-z0-9]{64})`)
|
||||
oauthClientSecretPat = regexp.MustCompile(`(?i)secret[a-zA-Z0-9_' "=]{0,20}([a-z0-9]{64})`)
|
||||
|
||||
// TODO: Legacy API tokens
|
||||
|
||||
apiDomains = []string{"api.us.onelogin.com", "api.eu.onelogin.com"}
|
||||
apiDomains = [][]byte{[]byte("api.us.onelogin.com"), []byte("api.eu.onelogin.com")}
|
||||
|
||||
client = http.Client{Timeout: time.Second * 5}
|
||||
)
|
||||
|
||||
// Keywords are used for efficiently pre-filtering chunks.
|
||||
// Use identifiers in the secret preferably, or the provider name.
|
||||
func (s Scanner) Keywords() []string {
|
||||
return []string{"onelogin"}
|
||||
func (s Scanner) Keywords() [][]byte {
|
||||
return [][]byte{[]byte("onelogin")}
|
||||
}
|
||||
|
||||
// FromData will find and optionally verify Onelogin secrets in a given set of bytes.
|
||||
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
|
||||
dataStr := string(data)
|
||||
idSubMatches := oauthClientIDPat.FindAllSubmatch(data, -1)
|
||||
secretSubMatches := oauthClientSecretPat.FindAllSubmatch(data, -1)
|
||||
|
||||
for _, clientID := range oauthClientIDPat.FindAllStringSubmatch(dataStr, -1) {
|
||||
if len(clientID) != 2 {
|
||||
for _, idMatch := range idSubMatches {
|
||||
if len(idMatch) != 2 {
|
||||
continue
|
||||
}
|
||||
for _, clientSecret := range oauthClientSecretPat.FindAllStringSubmatch(dataStr, -1) {
|
||||
if len(clientSecret) != 2 {
|
||||
for _, secretMatch := range secretSubMatches {
|
||||
if len(secretMatch) != 2 {
|
||||
continue
|
||||
}
|
||||
|
||||
s := detectors.Result{
|
||||
DetectorType: detectorspb.DetectorType_OneLogin,
|
||||
Raw: []byte(clientID[1]),
|
||||
RawV2: []byte(fmt.Sprintf("%s:%s", clientID[1], clientSecret[1])),
|
||||
Redacted: clientID[1],
|
||||
Raw: bytes.TrimSpace(idMatch[1]),
|
||||
RawV2: bytes.TrimSpace(secretMatch[1]),
|
||||
Redacted: string(idMatch[1]),
|
||||
}
|
||||
|
||||
if verify {
|
||||
for _, domain := range apiDomains {
|
||||
tokenURL := fmt.Sprintf("https://%s/auth/oauth2/v2/token", domain)
|
||||
req, err := http.NewRequestWithContext(ctx, "POST", tokenURL, strings.NewReader(`{"grant_type":"client_credentials"}`))
|
||||
req, err := http.NewRequestWithContext(ctx, "POST", tokenURL, bytes.NewBuffer([]byte(`{"grant_type":"client_credentials"}`)))
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
req.Header.Add("Authorization", fmt.Sprintf("client_id:%s, client_secret:%s", clientID[1], clientSecret[1]))
|
||||
req.Header.Add("Authorization", fmt.Sprintf("client_id:%s, client_secret:%s", idMatch[1], secretMatch[1]))
|
||||
req.Header.Add("Content-Type", "application/json; charset=utf-8")
|
||||
res, err := client.Do(req)
|
||||
if err == nil {
|
||||
res.Body.Close() // The request body is unused.
|
||||
res.Body.Close()
|
||||
|
||||
if res.StatusCode >= 200 && res.StatusCode < 300 {
|
||||
s.Verified = true
|
||||
@@ -74,7 +70,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
}
|
||||
}
|
||||
|
||||
if !s.Verified && detectors.IsKnownFalsePositive(string(s.Raw), detectors.DefaultFalsePositives, true) {
|
||||
if !s.Verified && detectors.IsKnownFalsePositive(s.Raw, detectors.DefaultFalsePositives, true) {
|
||||
continue
|
||||
}
|
||||
|
||||
|
||||
@@ -1,10 +1,10 @@
|
||||
package onepagecrm
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
@@ -13,60 +13,53 @@ import (
|
||||
|
||||
type Scanner struct{}
|
||||
|
||||
// Ensure the Scanner satisfies the interface at compile time.
|
||||
var _ detectors.Detector = (*Scanner)(nil)
|
||||
|
||||
var (
|
||||
client = common.SaneHttpClient()
|
||||
|
||||
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
|
||||
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"onepagecrm"}) + `\b([a-zA-Z0-9=]{44})`)
|
||||
idPat = regexp.MustCompile(detectors.PrefixRegex([]string{"onepagecrm"}) + `\b([a-z0-9]{24})\b`)
|
||||
)
|
||||
|
||||
// Keywords are used for efficiently pre-filtering chunks.
|
||||
// Use identifiers in the secret preferably, or the provider name.
|
||||
func (s Scanner) Keywords() []string {
|
||||
return []string{"onepagecrm"}
|
||||
func (s Scanner) Keywords() [][]byte {
|
||||
return [][]byte{[]byte("onepagecrm")}
|
||||
}
|
||||
|
||||
// FromData will find and optionally verify OnepageCRM secrets in a given set of bytes.
|
||||
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
|
||||
dataStr := string(data)
|
||||
|
||||
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
|
||||
idMatches := idPat.FindAllStringSubmatch(dataStr, -1)
|
||||
matches := keyPat.FindAllSubmatch(data, -1)
|
||||
idMatches := idPat.FindAllSubmatch(data, -1)
|
||||
|
||||
for _, match := range matches {
|
||||
if len(match) != 2 {
|
||||
continue
|
||||
}
|
||||
tokenPatMatch := strings.TrimSpace(match[1])
|
||||
tokenPatMatch := bytes.TrimSpace(match[1])
|
||||
|
||||
for _, idMatch := range idMatches {
|
||||
if len(idMatch) != 2 {
|
||||
continue
|
||||
}
|
||||
|
||||
userPatMatch := strings.TrimSpace(idMatch[1])
|
||||
userPatMatch := bytes.TrimSpace(idMatch[1])
|
||||
|
||||
s1 := detectors.Result{
|
||||
DetectorType: detectorspb.DetectorType_OnepageCRM,
|
||||
Raw: []byte(tokenPatMatch),
|
||||
Raw: tokenPatMatch,
|
||||
}
|
||||
|
||||
if verify {
|
||||
req, err := http.NewRequestWithContext(ctx, "GET", "https://app.onepagecrm.com/api/v3/contacts.json?per_page=20&page=1", nil)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
req.SetBasicAuth(userPatMatch, tokenPatMatch)
|
||||
req.SetBasicAuth(string(userPatMatch), string(tokenPatMatch))
|
||||
res, err := client.Do(req)
|
||||
if err == nil {
|
||||
defer res.Body.Close()
|
||||
if res.StatusCode >= 200 && res.StatusCode < 300 {
|
||||
s1.Verified = true
|
||||
} else {
|
||||
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
|
||||
if detectors.IsKnownFalsePositive(tokenPatMatch, detectors.DefaultFalsePositives, true) {
|
||||
continue
|
||||
}
|
||||
|
||||
@@ -1,12 +1,12 @@
|
||||
package onesignal
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
b64 "encoding/base64"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
@@ -25,32 +25,27 @@ var (
|
||||
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"onesignal"}) + common.UUIDPattern)
|
||||
)
|
||||
|
||||
// Keywords are used for efficiently pre-filtering chunks.
|
||||
// Use identifiers in the secret preferably, or the provider name.
|
||||
func (s Scanner) Keywords() []string {
|
||||
return []string{"onesignal"}
|
||||
func (s Scanner) Keywords() [][]byte {
|
||||
return [][]byte{[]byte("onesignal")}
|
||||
}
|
||||
|
||||
// FromData will find and optionally verify Onesignal secrets in a given set of bytes.
|
||||
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
|
||||
dataStr := string(data)
|
||||
|
||||
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
|
||||
matches := keyPat.FindAllSubmatch(data, -1)
|
||||
|
||||
for _, match := range matches {
|
||||
if len(match) != 2 {
|
||||
continue
|
||||
}
|
||||
resMatch := strings.TrimSpace(match[1])
|
||||
resMatch := bytes.TrimSpace(match[1])
|
||||
|
||||
s1 := detectors.Result{
|
||||
DetectorType: detectorspb.DetectorType_Onesignal,
|
||||
Raw: []byte(resMatch),
|
||||
Raw: resMatch,
|
||||
}
|
||||
|
||||
if verify {
|
||||
data := fmt.Sprintf("%s:", resMatch)
|
||||
sEnc := b64.StdEncoding.EncodeToString([]byte(data))
|
||||
dataF := append(resMatch, ':')
|
||||
sEnc := b64.StdEncoding.EncodeToString(dataF)
|
||||
req, err := http.NewRequestWithContext(ctx, "GET", "https://onesignal.com/api/v1/apps", nil)
|
||||
if err != nil {
|
||||
continue
|
||||
@@ -63,7 +58,6 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
if res.StatusCode >= 200 && res.StatusCode < 300 {
|
||||
s1.Verified = true
|
||||
} else {
|
||||
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key
|
||||
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
|
||||
continue
|
||||
}
|
||||
|
||||
@@ -1,11 +1,11 @@
|
||||
package onwaterio
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
@@ -14,41 +14,35 @@ import (
|
||||
|
||||
type Scanner struct{}
|
||||
|
||||
// Ensure the Scanner satisfies the interface at compile time.
|
||||
var _ detectors.Detector = (*Scanner)(nil)
|
||||
|
||||
var (
|
||||
client = common.SaneHttpClient()
|
||||
|
||||
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
|
||||
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"onwater"}) + `\b([a-zA-Z0-9_-]{20})\b`)
|
||||
)
|
||||
|
||||
// Keywords are used for efficiently pre-filtering chunks.
|
||||
// Use identifiers in the secret preferably, or the provider name.
|
||||
func (s Scanner) Keywords() []string {
|
||||
return []string{"onwater"}
|
||||
func (s Scanner) Keywords() [][]byte {
|
||||
return [][]byte{[]byte("onwater")}
|
||||
}
|
||||
|
||||
// FromData will find and optionally verify OnWaterIO secrets in a given set of bytes.
|
||||
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
|
||||
dataStr := string(data)
|
||||
|
||||
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
|
||||
matches := keyPat.FindAllSubmatch(data, -1)
|
||||
|
||||
for _, match := range matches {
|
||||
if len(match) != 2 {
|
||||
continue
|
||||
}
|
||||
resMatch := strings.TrimSpace(match[1])
|
||||
resMatch := bytes.TrimSpace(match[1])
|
||||
|
||||
s1 := detectors.Result{
|
||||
DetectorType: detectorspb.DetectorType_OnWaterIO,
|
||||
Raw: []byte(resMatch),
|
||||
Raw: resMatch,
|
||||
}
|
||||
|
||||
if verify {
|
||||
req, err := http.NewRequestWithContext(ctx, "GET", fmt.Sprintf("https://api.onwater.io/api/v1/results/23.92323,-66.3?access_token=%s", resMatch), nil)
|
||||
req, err := http.NewRequestWithContext(ctx, "GET", fmt.Sprintf("https://api.onwater.io/api/v1/results/23.92323,-66.3?access_token=%s", string(resMatch)), nil)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
@@ -58,7 +52,6 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
if res.StatusCode >= 200 && res.StatusCode < 300 {
|
||||
s1.Verified = true
|
||||
} else {
|
||||
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
|
||||
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
|
||||
continue
|
||||
}
|
||||
|
||||
@@ -43,6 +43,8 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
instanceMatches := instancePat.FindAllStringSubmatch(dataStr, -1)
|
||||
tokenMatches := accessTokenPat.FindAllStringSubmatch(dataStr, -1)
|
||||
|
||||
fmt.Printf("instanceMatches: %v\n", instanceMatches)
|
||||
|
||||
for _, instance := range instanceMatches {
|
||||
if len(instance) != 1 {
|
||||
continue
|
||||
|
||||
@@ -2,10 +2,6 @@ package engine
|
||||
|
||||
import (
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/envoyapikey"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/huggingface"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/salesforce"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/snowflake"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/trufflehogenterprise"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/abbysale"
|
||||
@@ -291,7 +287,6 @@ import (
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/getresponse"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/getsandbox"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/github"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/github_oauth2"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/github_old"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/githubapp"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/gitlab"
|
||||
@@ -561,6 +556,7 @@ import (
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/satismeterwritekey"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/saucelabs"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/scalewaykey"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/github_oauth2"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/scalr"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/scrapeowl"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/scraperapi"
|
||||
@@ -1536,10 +1532,6 @@ func DefaultDetectors() []detectors.Detector {
|
||||
couchbase.Scanner{},
|
||||
envoyapikey.Scanner{},
|
||||
github_oauth2.Scanner{},
|
||||
snowflake.Scanner{},
|
||||
huggingface.Scanner{},
|
||||
trufflehogenterprise.Scanner{},
|
||||
salesforce.Scanner{},
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
+23
-24
@@ -29,8 +29,6 @@ var (
|
||||
maxDepth = 5
|
||||
maxSize = 250 * 1024 * 1024 // 20MB
|
||||
maxTimeout = time.Duration(30) * time.Second
|
||||
|
||||
defaultBufferSize = 512
|
||||
)
|
||||
|
||||
// Ensure the Archive satisfies the interfaces at compile time.
|
||||
@@ -64,7 +62,7 @@ func SetArchiveMaxTimeout(timeout time.Duration) {
|
||||
|
||||
// FromFile extracts the files from an archive.
|
||||
func (a *Archive) FromFile(originalCtx context.Context, data io.Reader) chan []byte {
|
||||
archiveChan := make(chan []byte, defaultBufferSize)
|
||||
archiveChan := make(chan []byte, 512)
|
||||
go func() {
|
||||
ctx, cancel := context.WithTimeout(originalCtx, maxTimeout)
|
||||
logger := logContext.AddLogger(ctx).Logger()
|
||||
@@ -185,28 +183,29 @@ func (a *Archive) ReadToMax(ctx context.Context, reader io.Reader) (data []byte,
|
||||
logger.Error(err, "Panic occurred when reading archive")
|
||||
}
|
||||
}()
|
||||
|
||||
if common.IsDone(ctx) {
|
||||
return nil, ctx.Err()
|
||||
fileContent := bytes.Buffer{}
|
||||
logger.V(5).Info("Remaining buffer capacity", "bytes", maxSize-a.size)
|
||||
for i := 0; i <= maxSize/512; i++ {
|
||||
if common.IsDone(ctx) {
|
||||
return nil, ctx.Err()
|
||||
}
|
||||
fileChunk := make([]byte, 512)
|
||||
bRead, err := reader.Read(fileChunk)
|
||||
if err != nil && !errors.Is(err, io.EOF) {
|
||||
return []byte{}, err
|
||||
}
|
||||
a.size += bRead
|
||||
if len(fileChunk) > 0 {
|
||||
fileContent.Write(fileChunk[0:bRead])
|
||||
}
|
||||
if bRead < 512 {
|
||||
return fileContent.Bytes(), nil
|
||||
}
|
||||
if a.size >= maxSize && bRead == 512 {
|
||||
logger.V(2).Info("Max archive size reached.")
|
||||
return fileContent.Bytes(), nil
|
||||
}
|
||||
}
|
||||
|
||||
var fileContent bytes.Buffer
|
||||
// Create a limited reader to ensure we don't read more than the max size.
|
||||
lr := io.LimitReader(reader, int64(maxSize))
|
||||
|
||||
// Using io.CopyBuffer for performance advantages. Though buf is mandatory
|
||||
// for the method, due to the internal implementation of io.CopyBuffer, when
|
||||
// *bytes.Buffer implements io.WriterTo or io.ReaderFrom, the provided buf
|
||||
// is simply ignored. Thus, we can pass nil for the buf parameter.
|
||||
_, err = io.CopyBuffer(&fileContent, lr, nil)
|
||||
if err != nil && !errors.Is(err, io.EOF) {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if fileContent.Len() == maxSize {
|
||||
logger.V(2).Info("Max archive size reached.")
|
||||
}
|
||||
|
||||
return fileContent.Bytes(), nil
|
||||
}
|
||||
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
package handlers
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"io"
|
||||
"net/http"
|
||||
@@ -127,58 +126,6 @@ func TestHandleFile(t *testing.T) {
|
||||
assert.Equal(t, 1, len(ch))
|
||||
}
|
||||
|
||||
func TestReadToMax(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
input []byte
|
||||
expected []byte
|
||||
}{
|
||||
{
|
||||
name: "read full content within maxSize",
|
||||
input: []byte("abcdefg"),
|
||||
expected: []byte("abcdefg"),
|
||||
},
|
||||
{
|
||||
name: "read content larger than maxSize",
|
||||
input: make([]byte, maxSize+10), // this creates a byte slice 10 bytes larger than maxSize
|
||||
expected: make([]byte, maxSize),
|
||||
},
|
||||
{
|
||||
name: "empty input",
|
||||
input: []byte(""),
|
||||
expected: []byte(""),
|
||||
},
|
||||
}
|
||||
|
||||
a := &Archive{}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
reader := bytes.NewReader(tt.input)
|
||||
output, err := a.ReadToMax(context.Background(), reader)
|
||||
assert.Nil(t, err)
|
||||
|
||||
assert.Equal(t, tt.expected, output)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkReadToMax(b *testing.B) {
|
||||
data := bytes.Repeat([]byte("a"), 1024*1000) // 1MB of data.
|
||||
reader := bytes.NewReader(data)
|
||||
a := &Archive{}
|
||||
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
b.StartTimer()
|
||||
_, _ = a.ReadToMax(context.Background(), reader)
|
||||
b.StopTimer()
|
||||
|
||||
_, _ = reader.Seek(0, 0) // Reset the reader position.
|
||||
a.size = 0 // Reset archive size.
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtractDebContent(t *testing.T) {
|
||||
// Open the sample .deb file from the testdata folder.
|
||||
file, err := os.Open("testdata/test.deb")
|
||||
|
||||
@@ -133,7 +133,7 @@ func BranchHeads(repo *gogit.Repository) (map[string]*object.Commit, error) {
|
||||
}
|
||||
headCommit, err := repo.CommitObject(*headHash)
|
||||
if err != nil {
|
||||
logger.Error(err, "unable to get commit", "head_hash", headHash.String())
|
||||
logger.Error(err, "unable to get commit", "commit", headCommit.String())
|
||||
return nil
|
||||
}
|
||||
branches[branchName] = headCommit
|
||||
|
||||
@@ -20,7 +20,7 @@ const (
|
||||
|
||||
// Chunker takes a chunk and splits it into chunks of ChunkSize.
|
||||
func Chunker(originalChunk *Chunk) chan *Chunk {
|
||||
chunkChan := make(chan *Chunk, 1)
|
||||
chunkChan := make(chan *Chunk)
|
||||
go func() {
|
||||
defer close(chunkChan)
|
||||
if len(originalChunk.Data) <= TotalChunkSize {
|
||||
|
||||
@@ -238,7 +238,6 @@ func (s *Source) chunkAction(ctx context.Context, proj project, bld build, act a
|
||||
SourceType: s.Type(),
|
||||
SourceName: s.name,
|
||||
SourceID: s.SourceID(),
|
||||
JobID: s.JobID(),
|
||||
Data: removeCircleSha1Line(data.Bytes()),
|
||||
SourceMetadata: &source_metadatapb.MetaData{
|
||||
Data: &source_metadatapb.MetaData_Circleci{
|
||||
|
||||
@@ -174,7 +174,6 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) err
|
||||
SourceType: s.Type(),
|
||||
SourceName: s.name,
|
||||
SourceID: s.SourceID(),
|
||||
JobID: s.JobID(),
|
||||
Data: file.Bytes(),
|
||||
SourceMetadata: &source_metadatapb.MetaData{
|
||||
Data: &source_metadatapb.MetaData_Docker{
|
||||
|
||||
@@ -161,7 +161,6 @@ func (s *Source) scanFile(ctx context.Context, path string, chunksChan chan *sou
|
||||
SourceType: s.Type(),
|
||||
SourceName: s.name,
|
||||
SourceID: s.SourceID(),
|
||||
JobID: s.JobID(),
|
||||
SourceMetadata: &source_metadatapb.MetaData{
|
||||
Data: &source_metadatapb.MetaData_Filesystem{
|
||||
Filesystem: &source_metadatapb.Filesystem{
|
||||
@@ -192,7 +191,6 @@ func (s *Source) scanFile(ctx context.Context, path string, chunksChan chan *sou
|
||||
SourceType: s.Type(),
|
||||
SourceName: s.name,
|
||||
SourceID: s.SourceID(),
|
||||
JobID: s.JobID(),
|
||||
Data: data.Bytes(),
|
||||
SourceMetadata: &source_metadatapb.MetaData{
|
||||
Data: &source_metadatapb.MetaData_Filesystem{
|
||||
|
||||
@@ -324,7 +324,6 @@ func (s *Source) processObject(ctx context.Context, o object) error {
|
||||
chunkSkel := &sources.Chunk{
|
||||
SourceName: s.name,
|
||||
SourceType: s.Type(),
|
||||
JobID: s.JobID(),
|
||||
SourceID: s.sourceId,
|
||||
Verify: s.verify,
|
||||
SourceMetadata: &source_metadatapb.MetaData{
|
||||
|
||||
@@ -446,7 +446,6 @@ func (s *Git) ScanCommits(ctx context.Context, repo *git.Repository, path string
|
||||
chunkSkel := &sources.Chunk{
|
||||
SourceName: s.sourceName,
|
||||
SourceID: s.sourceID,
|
||||
JobID: s.jobID,
|
||||
SourceType: s.sourceType,
|
||||
SourceMetadata: metadata,
|
||||
Verify: s.verify,
|
||||
@@ -465,7 +464,6 @@ func (s *Git) ScanCommits(ctx context.Context, repo *git.Repository, path string
|
||||
chunksChan <- &sources.Chunk{
|
||||
SourceName: s.sourceName,
|
||||
SourceID: s.sourceID,
|
||||
JobID: s.jobID,
|
||||
SourceType: s.sourceType,
|
||||
SourceMetadata: metadata,
|
||||
Data: diff.Content.Bytes(),
|
||||
@@ -492,7 +490,6 @@ func (s *Git) gitChunk(ctx context.Context, diff gitparse.Diff, fileName, email,
|
||||
chunksChan <- &sources.Chunk{
|
||||
SourceName: s.sourceName,
|
||||
SourceID: s.sourceID,
|
||||
JobID: s.jobID,
|
||||
SourceType: s.sourceType,
|
||||
SourceMetadata: metadata,
|
||||
Data: append([]byte{}, newChunkBuffer.Bytes()...),
|
||||
@@ -507,7 +504,6 @@ func (s *Git) gitChunk(ctx context.Context, diff gitparse.Diff, fileName, email,
|
||||
chunksChan <- &sources.Chunk{
|
||||
SourceName: s.sourceName,
|
||||
SourceID: s.sourceID,
|
||||
JobID: s.jobID,
|
||||
SourceType: s.sourceType,
|
||||
SourceMetadata: metadata,
|
||||
Data: line,
|
||||
@@ -527,7 +523,6 @@ func (s *Git) gitChunk(ctx context.Context, diff gitparse.Diff, fileName, email,
|
||||
chunksChan <- &sources.Chunk{
|
||||
SourceName: s.sourceName,
|
||||
SourceID: s.sourceID,
|
||||
JobID: s.jobID,
|
||||
SourceType: s.sourceType,
|
||||
SourceMetadata: metadata,
|
||||
Data: append([]byte{}, newChunkBuffer.Bytes()...),
|
||||
@@ -593,7 +588,6 @@ func (s *Git) ScanStaged(ctx context.Context, repo *git.Repository, path string,
|
||||
chunkSkel := &sources.Chunk{
|
||||
SourceName: s.sourceName,
|
||||
SourceID: s.sourceID,
|
||||
JobID: s.jobID,
|
||||
SourceType: s.sourceType,
|
||||
SourceMetadata: metadata,
|
||||
Verify: s.verify,
|
||||
@@ -608,7 +602,6 @@ func (s *Git) ScanStaged(ctx context.Context, repo *git.Repository, path string,
|
||||
chunksChan <- &sources.Chunk{
|
||||
SourceName: s.sourceName,
|
||||
SourceID: s.sourceID,
|
||||
JobID: s.jobID,
|
||||
SourceType: s.sourceType,
|
||||
SourceMetadata: metadata,
|
||||
Data: diff.Content.Bytes(),
|
||||
|
||||
@@ -149,16 +149,14 @@ func (s *Source) newFilteredRepoCache(c cache.Cache, include, exclude []string)
|
||||
for _, ig := range include {
|
||||
g, err := glob.Compile(ig)
|
||||
if err != nil {
|
||||
s.log.V(1).Info("invalid include glob", "include_value", ig, "err", err)
|
||||
continue
|
||||
s.log.V(1).Info("invalid include glob", "glob", g, "err", err)
|
||||
}
|
||||
includeGlobs = append(includeGlobs, g)
|
||||
}
|
||||
for _, eg := range exclude {
|
||||
g, err := glob.Compile(eg)
|
||||
if err != nil {
|
||||
s.log.V(1).Info("invalid exclude glob", "exclude_value", eg, "err", err)
|
||||
continue
|
||||
s.log.V(1).Info("invalid exclude glob", "glob", g, "err", err)
|
||||
}
|
||||
excludeGlobs = append(excludeGlobs, g)
|
||||
}
|
||||
@@ -1225,7 +1223,6 @@ func (s *Source) chunkIssueComments(ctx context.Context, repo, repoPath string,
|
||||
chunk := &sources.Chunk{
|
||||
SourceName: s.name,
|
||||
SourceID: s.SourceID(),
|
||||
JobID: s.JobID(),
|
||||
SourceType: s.Type(),
|
||||
SourceMetadata: &source_metadatapb.MetaData{
|
||||
Data: &source_metadatapb.MetaData_Github{
|
||||
@@ -1259,7 +1256,6 @@ func (s *Source) chunkPullRequestComments(ctx context.Context, repo string, comm
|
||||
SourceName: s.name,
|
||||
SourceID: s.SourceID(),
|
||||
SourceType: s.Type(),
|
||||
JobID: s.JobID(),
|
||||
SourceMetadata: &source_metadatapb.MetaData{
|
||||
Data: &source_metadatapb.MetaData_Github{
|
||||
Github: &source_metadatapb.Github{
|
||||
@@ -1291,7 +1287,6 @@ func (s *Source) chunkGistComments(ctx context.Context, gistUrl string, comments
|
||||
SourceName: s.name,
|
||||
SourceID: s.SourceID(),
|
||||
SourceType: s.Type(),
|
||||
JobID: s.JobID(),
|
||||
SourceMetadata: &source_metadatapb.MetaData{
|
||||
Data: &source_metadatapb.MetaData_Github{
|
||||
Github: &source_metadatapb.Github{
|
||||
|
||||
@@ -376,7 +376,6 @@ func (s *Source) pageChunker(ctx context.Context, client *s3.S3, chunksChan chan
|
||||
SourceType: s.Type(),
|
||||
SourceName: s.name,
|
||||
SourceID: s.SourceID(),
|
||||
JobID: s.JobID(),
|
||||
SourceMetadata: &source_metadatapb.MetaData{
|
||||
Data: &source_metadatapb.MetaData_S3{
|
||||
S3: &source_metadatapb.S3{
|
||||
|
||||
@@ -36,10 +36,7 @@ type SourceManager struct {
|
||||
handles map[handle]sourceInfo
|
||||
handlesLock sync.Mutex
|
||||
// Pool limiting the amount of concurrent sources running.
|
||||
pool errgroup.Group
|
||||
poolLimit int
|
||||
currentRunningCount int32
|
||||
// Max number of units to scan concurrently per source.
|
||||
pool errgroup.Group
|
||||
concurrentUnits int
|
||||
// Run the sources using source unit enumeration / chunking if available.
|
||||
useSourceUnits bool
|
||||
@@ -71,10 +68,7 @@ func WithReportHook(hook JobProgressHook) func(*SourceManager) {
|
||||
|
||||
// WithConcurrentSources limits the concurrent number of sources a manager can run.
|
||||
func WithConcurrentSources(concurrency int) func(*SourceManager) {
|
||||
return func(mgr *SourceManager) {
|
||||
mgr.pool.SetLimit(concurrency)
|
||||
mgr.poolLimit = concurrency
|
||||
}
|
||||
return func(mgr *SourceManager) { mgr.pool.SetLimit(concurrency) }
|
||||
}
|
||||
|
||||
// WithBufferedOutput sets the size of the buffer used for the Chunks() channel.
|
||||
@@ -173,8 +167,6 @@ func (s *SourceManager) asyncRun(ctx context.Context, handle handle) (JobProgres
|
||||
ctx, cancel := context.WithCancel(ctx)
|
||||
progress := NewJobProgress(jobID, int64(handle), sourceName, WithHooks(s.hooks...), WithCancel(cancel))
|
||||
s.pool.Go(func() error {
|
||||
atomic.AddInt32(&s.currentRunningCount, 1)
|
||||
defer atomic.AddInt32(&s.currentRunningCount, -1)
|
||||
ctx := context.WithValues(ctx,
|
||||
"job_id", jobID,
|
||||
"source_manager_worker_id", common.RandomID(5),
|
||||
@@ -215,16 +207,6 @@ func (s *SourceManager) ScanChunk(chunk *Chunk) {
|
||||
s.outputChunks <- chunk
|
||||
}
|
||||
|
||||
// AvailableCapacity returns the number of concurrent jobs the manager can
|
||||
// accommodate at this time. If there is no limit, -1 is returned.
|
||||
func (s *SourceManager) AvailableCapacity() int {
|
||||
if s.poolLimit == 0 {
|
||||
return -1
|
||||
}
|
||||
runCount := atomic.LoadInt32(&s.currentRunningCount)
|
||||
return s.poolLimit - int(runCount)
|
||||
}
|
||||
|
||||
// preflightChecks is a helper method to check the Manager or the context isn't
|
||||
// done and that the handle is valid.
|
||||
func (s *SourceManager) preflightChecks(ctx context.Context, handle handle) error {
|
||||
|
||||
@@ -317,24 +317,3 @@ func TestSourceManagerCancelRun(t *testing.T) {
|
||||
assert.Error(t, ref.Snapshot().FatalError())
|
||||
assert.True(t, errors.Is(ref.Snapshot().FatalError(), returnedErr))
|
||||
}
|
||||
|
||||
func TestSourceManagerAvailableCapacity(t *testing.T) {
|
||||
mgr := NewManager(WithConcurrentSources(1337))
|
||||
start, end := make(chan struct{}), make(chan struct{})
|
||||
handle, err := enrollDummy(mgr, callbackChunker{func(context.Context, chan *Chunk) error {
|
||||
start <- struct{}{} // Send start signal.
|
||||
<-end // Wait for end signal.
|
||||
return nil
|
||||
}})
|
||||
assert.NoError(t, err)
|
||||
|
||||
assert.Equal(t, 1337, mgr.AvailableCapacity())
|
||||
ref, err := mgr.ScheduleRun(context.Background(), handle)
|
||||
assert.NoError(t, err)
|
||||
|
||||
<-start // Wait for start signal.
|
||||
assert.Equal(t, 1336, mgr.AvailableCapacity())
|
||||
end <- struct{}{} // Send end signal.
|
||||
<-ref.Done() // Wait for the job to finish.
|
||||
assert.Equal(t, 1337, mgr.AvailableCapacity())
|
||||
}
|
||||
|
||||
@@ -272,7 +272,6 @@ func (s *Source) monitorConnection(ctx context.Context, conn net.Conn, chunksCha
|
||||
SourceName: s.syslog.sourceName,
|
||||
SourceID: s.syslog.sourceID,
|
||||
SourceType: s.syslog.sourceType,
|
||||
JobID: s.JobID(),
|
||||
SourceMetadata: metadata,
|
||||
Data: input,
|
||||
Verify: s.verify,
|
||||
@@ -314,7 +313,6 @@ func (s *Source) acceptUDPConnections(ctx context.Context, netListener net.Packe
|
||||
chunksChan <- &sources.Chunk{
|
||||
SourceName: s.syslog.sourceName,
|
||||
SourceID: s.syslog.sourceID,
|
||||
JobID: s.JobID(),
|
||||
SourceType: s.syslog.sourceType,
|
||||
SourceMetadata: metadata,
|
||||
Data: input,
|
||||
|
||||
Reference in New Issue
Block a user