Compare commits

..

1 Commits

Author SHA1 Message Date
Bill Rich 8a59f4e66a wip 2023-04-20 11:12:31 -07:00
23 changed files with 1002 additions and 1245 deletions
+43 -43
View File
@@ -3,6 +3,7 @@ package decoders
import (
"bytes"
"encoding/base64"
"strings"
"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
)
@@ -14,7 +15,48 @@ var (
b64EndChars = "+/="
)
func getSubstringsOfCharacterSet(data []byte, charset []byte, threshold int) []string {
count := 0
substrings := []string{}
letters := strings.Builder{}
if len(data) == 0 {
return nil
}
for _, char := range string(data) {
if bytes.ContainsRune(charset, char) {
letters.WriteRune(char)
count++
} else {
if count > threshold {
substrings = appendB64Substring(letters, substrings)
}
letters.Reset()
count = 0
}
}
if count > threshold && len(letters.String()) > 0 {
substrings = appendB64Substring(letters, substrings)
}
return substrings
}
func appendB64Substring(letters strings.Builder, substrings []string) []string {
substring := strings.TrimLeft(letters.String(), b64EndChars)
// handle key=value
if strings.Contains(strings.TrimRight(substring, b64EndChars), "=") {
split := strings.SplitN(substring, "=", 2)
substrings = append(substrings, split[len(split)-1])
} else {
substrings = append(substrings, substring)
}
return substrings
}
func (d *Base64) FromChunk(chunk *sources.Chunk) *sources.Chunk {
encodedSubstrings := getSubstringsOfCharacterSet(chunk.Data, b64Charset, 20)
decodedSubstrings := map[string][]byte{}
@@ -27,52 +69,10 @@ func (d *Base64) FromChunk(chunk *sources.Chunk) *sources.Chunk {
if len(decodedSubstrings) > 0 {
for substring, dec := range decodedSubstrings {
chunk.Data = bytes.ReplaceAll(chunk.Data, []byte(substring), dec)
chunk.Data = bytes.Replace(chunk.Data, []byte(substring), dec, 1)
}
return chunk
}
return nil
}
func getSubstringsOfCharacterSet(data []byte, charset []byte, threshold int) []string {
count := 0
var substrings []string
letters := bytes.Buffer{}
if len(data) == 0 {
return nil
}
for _, char := range data {
if bytes.Contains(charset, []byte{char}) {
letters.WriteByte(char)
count++
} else {
if count > threshold {
substrings = appendB64Substring(&letters, substrings)
}
letters.Reset()
count = 0
}
}
if count > threshold && letters.Len() > 0 {
substrings = appendB64Substring(&letters, substrings)
}
return substrings
}
func appendB64Substring(letters *bytes.Buffer, substrings []string) []string {
substring := bytes.TrimLeft(letters.Bytes(), b64EndChars)
// handle key=value
if bytes.Contains(bytes.TrimRight(substring, b64EndChars), []byte("=")) {
split := bytes.SplitN(substring, []byte("="), 2)
substrings = append(substrings, string(split[len(split)-1]))
} else {
substrings = append(substrings, string(substring))
}
return substrings
}
+7 -23
View File
@@ -90,29 +90,13 @@ func TestBase64_FromChunk(t *testing.T) {
}
}
func BenchmarkFromChunkSmall(b *testing.B) {
func BenchmarkFromChunk(benchmark *testing.B) {
d := Base64{}
data := detectors.MustGetBenchmarkData()["small"]
for n := 0; n < b.N; n++ {
d.FromChunk(&sources.Chunk{Data: data})
}
}
func BenchmarkFromChunkMedium(b *testing.B) {
d := Base64{}
data := detectors.MustGetBenchmarkData()["medium"]
for n := 0; n < b.N; n++ {
d.FromChunk(&sources.Chunk{Data: data})
}
}
func BenchmarkFromChunkLarge(b *testing.B) {
d := Base64{}
data := detectors.MustGetBenchmarkData()["big"]
for n := 0; n < b.N; n++ {
d.FromChunk(&sources.Chunk{Data: data})
for name, data := range detectors.MustGetBenchmarkData() {
benchmark.Run(name, func(b *testing.B) {
for n := 0; n < b.N; n++ {
d.FromChunk(&sources.Chunk{Data: data})
}
})
}
}
-1
View File
@@ -8,7 +8,6 @@ func DefaultDecoders() []Decoder {
return []Decoder{
&UTF8{},
&Base64{},
&UTF16{},
}
}
-68
View File
@@ -1,68 +0,0 @@
package decoders
import (
"bytes"
"encoding/binary"
"fmt"
"unicode/utf8"
"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
)
type UTF16 struct{}
func (d *UTF16) FromChunk(chunk *sources.Chunk) *sources.Chunk {
if chunk == nil || len(chunk.Data) == 0 {
return nil
}
if utf16Data, err := utf16ToUTF8(chunk.Data); err == nil {
chunk.Data = utf16Data
return chunk
}
return nil
}
// utf16ToUTF8 converts a byte slice containing UTF-16 encoded data to a UTF-8 encoded byte slice.
func utf16ToUTF8(b []byte) ([]byte, error) {
endianness, err := guessUTF16Endianness(b)
if err != nil {
return nil, err
}
buf := &bytes.Buffer{}
for i := 0; i < len(b); i += 2 {
r := rune(endianness.Uint16(b[i:]))
if utf8.ValidRune(r) {
buf.WriteRune(r)
}
}
return buf.Bytes(), nil
}
func guessUTF16Endianness(b []byte) (binary.ByteOrder, error) {
if len(b) < 2 || len(b)%2 != 0 {
return nil, fmt.Errorf("input length must be even and at least 2 bytes long")
}
var evenNullBytes, oddNullBytes int
for i := 0; i < len(b); i += 2 {
if b[i] == 0 {
oddNullBytes++
}
if b[i+1] == 0 {
evenNullBytes++
}
}
if evenNullBytes > oddNullBytes {
return binary.LittleEndian, nil
}
if oddNullBytes > evenNullBytes {
return binary.BigEndian, nil
}
return nil, fmt.Errorf("could not determine endianness")
}
-73
View File
@@ -1,73 +0,0 @@
package decoders
import (
"bytes"
"testing"
"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
)
func TestUTF16Decoder(t *testing.T) {
testCases := []struct {
name string
input []byte
expected []byte
expectNil bool
}{
{
name: "Valid UTF-16LE input",
input: []byte{72, 0, 101, 0, 108, 0, 108, 0, 111, 0, 32, 0, 87, 0, 111, 0, 114, 0, 108, 0, 100, 0},
expected: []byte("Hello World"),
expectNil: false,
},
{
name: "Valid UTF-16BE input",
input: []byte{0, 72, 0, 101, 0, 108, 0, 108, 0, 111, 0, 32, 0, 87, 0, 111, 0, 114, 0, 108, 0, 100},
expected: []byte("Hello World"),
expectNil: false,
},
{
name: "Invalid UTF-16 input (it's UTF-8)",
input: []byte("Hello World!"),
expected: nil,
expectNil: true,
},
{
name: "Invalid UTF-16 input (odd length)",
input: []byte{72, 0, 101, 0, 108, 0, 108, 0, 111, 0, 32, 0, 87, 0, 111, 0, 114, 0, 108, 0, 0},
expected: nil,
expectNil: true,
},
}
for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
chunk := &sources.Chunk{Data: tc.input}
decoder := &UTF16{}
decodedChunk := decoder.FromChunk(chunk)
if tc.expectNil {
if decodedChunk != nil {
t.Errorf("Expected nil, got chunk with data: %v", decodedChunk.Data)
}
return
}
if decodedChunk == nil {
t.Errorf("Expected chunk with data, got nil")
return
}
if !bytes.Equal(decodedChunk.Data, tc.expected) {
t.Errorf("Expected decoded data: %v, got: %v", tc.expected, decodedChunk.Data)
}
})
}
}
func BenchmarkUtf16ToUtf8(b *testing.B) {
// Example UTF-16LE encoded data
data := []byte{72, 0, 101, 0, 108, 0, 108, 0, 111, 0, 32, 0, 87, 0, 111, 0, 114, 0, 108, 0, 100, 0}
for n := 0; n < b.N; n++ {
_, _ = utf16ToUTF8(data)
}
}
+9 -20
View File
@@ -26,30 +26,19 @@ func (d *UTF8) FromChunk(chunk *sources.Chunk) *sources.Chunk {
// extacting contigous portions of printable characters that we care
// about from some bytes
func extractSubstrings(b []byte) []byte {
isValidByte := func(c byte) bool {
fields := bytes.FieldsFunc(b, func(r rune) bool {
// https://www.rapidtables.com/code/text/ascii-table.html
// split on anything that is not ascii space through tilde
return c > 31 && c < 127
}
return !(r > 31 && r < 127)
})
field := make([]byte, len(b))
fieldLen := 0
buf := &bytes.Buffer{}
for i, c := range b {
if isValidByte(c) {
field[fieldLen] = c
fieldLen++
} else {
if fieldLen > 5 {
buf.Write(field[:fieldLen])
}
fieldLen = 0
}
if i == len(b)-1 && fieldLen > 5 {
buf.Write(field[:fieldLen])
keep := [][]byte{}
for _, field := range fields {
// Remove fields shorter than 6 characters.
if bts := bytes.TrimSpace(field); len(bts) > 5 {
keep = append(keep, bts)
}
}
return buf.Bytes()
return bytes.Join(keep, []byte("\n"))
}
-1
View File
@@ -4,7 +4,6 @@ import (
"testing"
"github.com/kylelemons/godebug/pretty"
"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
)
+20 -20
View File
@@ -56,11 +56,11 @@ func TestAWS_FromChunk(t *testing.T) {
{
DetectorType: detectorspb.DetectorType_AWS,
Verified: true,
Redacted: "AKIASP2TPHJSQH3FJRUX",
Redacted: "AKIAWARWQKZNHMZBLY4I",
ExtraData: map[string]string{
"account": "171436882533",
"arn": "arn:aws:iam::171436882533:user/canarytokens.com@@4dxkh0pdeop3bzu9zx5wob793",
"user_id": "AIDASP2TPHJSUFRSTTZX4",
"account": "413504919130",
"arn": "arn:aws:iam::413504919130:root",
"user_id": "413504919130",
},
},
},
@@ -78,7 +78,7 @@ func TestAWS_FromChunk(t *testing.T) {
{
DetectorType: detectorspb.DetectorType_AWS,
Verified: false,
Redacted: "AKIASP2TPHJSQH3FJRUX",
Redacted: "AKIAWARWQKZNHMZBLY4I",
ExtraData: nil,
},
},
@@ -106,18 +106,18 @@ func TestAWS_FromChunk(t *testing.T) {
want: []detectors.Result{
{
DetectorType: detectorspb.DetectorType_AWS,
Verified: false,
Redacted: "AKIASP2TPHJSQH3FJXYZ",
Verified: true,
Redacted: id,
ExtraData: map[string]string{
"account": "413504919130",
"arn": "arn:aws:iam::413504919130:root",
"user_id": "413504919130",
},
},
{
DetectorType: detectorspb.DetectorType_AWS,
Verified: true,
Redacted: "AKIASP2TPHJSQH3FJRUX",
ExtraData: map[string]string{
"account": "171436882533",
"arn": "arn:aws:iam::171436882533:user/canarytokens.com@@4dxkh0pdeop3bzu9zx5wob793",
"user_id": "AIDASP2TPHJSUFRSTTZX4",
},
Verified: false,
Redacted: inactiveID,
},
},
wantErr: false,
@@ -145,11 +145,11 @@ func TestAWS_FromChunk(t *testing.T) {
{
DetectorType: detectorspb.DetectorType_AWS,
Verified: true,
Redacted: "AKIASP2TPHJSQH3FJRUX",
Redacted: id,
ExtraData: map[string]string{
"account": "171436882533",
"arn": "arn:aws:iam::171436882533:user/canarytokens.com@@4dxkh0pdeop3bzu9zx5wob793",
"user_id": "AIDASP2TPHJSUFRSTTZX4",
"account": "413504919130",
"arn": "arn:aws:iam::413504919130:root",
"user_id": "413504919130",
},
},
{
@@ -172,7 +172,7 @@ func TestAWS_FromChunk(t *testing.T) {
{
DetectorType: detectorspb.DetectorType_AWS,
Verified: false,
Redacted: "AKIASP2TPHJSQH3FJRUX",
Redacted: "AKIAWARWQKZNHMZBLY4I",
},
},
wantErr: false,
@@ -181,7 +181,7 @@ func TestAWS_FromChunk(t *testing.T) {
name: "skipped",
s: scanner{
skipIDs: map[string]struct{}{
"AKIASP2TPHJSQH3FJRUX": {},
"AKIAWARWQKZNHMZBLY4I": {},
},
},
args: args{
-1
View File
@@ -54,7 +54,6 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
s := detectors.Result{
DetectorType: detectorspb.DetectorType_Azure,
Raw: []byte(clientSecret[2]),
RawV2: []byte(clientID[2] + clientSecret[2] + tenantID[2]),
Redacted: clientID[2],
}
@@ -51,7 +51,6 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
s1 := detectors.Result{
DetectorType: detectorspb.DetectorType_DatadogToken,
Raw: []byte(resMatch),
RawV2: []byte(resMatch + resApiMatch),
}
if verify {
+1 -3
View File
@@ -79,16 +79,14 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
raw = []byte(key)
}
credBytes, _ := json.Marshal(creds)
s := detectors.Result{
DetectorType: detectorspb.DetectorType_GCP,
Raw: raw,
RawV2: credBytes,
Redacted: creds.ClientEmail,
}
if verify {
credBytes, _ := json.Marshal(creds)
credentials, err := google.CredentialsFromJSON(ctx, credBytes, "https://www.googleapis.com/auth/cloud-platform")
if err != nil {
continue
+1
View File
@@ -22,6 +22,7 @@ func New(opts ...func(*Scanner)) *Scanner {
ignorePatterns: []regexp.Regexp{},
}
for _, opt := range opts {
opt(scanner)
}
-51
View File
@@ -156,57 +156,6 @@ func TestJdbc_FromChunk(t *testing.T) {
}
}
func TestJdbc_FromDataWithIgnorePattern(t *testing.T) {
type args struct {
ctx context.Context
data []byte
verify bool
}
tests := []struct {
name string
args args
want []detectors.Result
ignorePatterns []string
wantErr bool
}{
{
name: "not found",
args: args{
ctx: context.Background(),
data: []byte("jdbc:sqlite::secretpattern:"),
verify: false,
},
want: nil,
ignorePatterns: []string{
".*secretpattern.*",
},
wantErr: false,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
s := New(WithIgnorePattern(tt.ignorePatterns))
got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
if (err != nil) != tt.wantErr {
t.Errorf("Jdbc.FromDataWithConfig() error = %v, wantErr %v", err, tt.wantErr)
return
}
if os.Getenv("FORCE_PASS_DIFF") == "true" {
return
}
for i := range got {
if len(got[i].Raw) == 0 {
t.Fatal("no raw secret present")
}
got[i].Raw = nil
}
if diff := pretty.Compare(got, tt.want); diff != "" {
t.Errorf("Jdbc.FromDataWithConfig() %s diff: (-got +want)\n%s", tt.name, diff)
}
})
}
}
func BenchmarkFromData(benchmark *testing.B) {
ctx := context.Background()
s := Scanner{}
-1
View File
@@ -60,7 +60,6 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
s1 := detectors.Result{
DetectorType: detectorspb.DetectorType_JiraToken,
Raw: []byte(resToken),
RawV2: []byte(fmt.Sprintf("%s:%s:%s", resEmail, resToken, resDomain)),
}
if verify {
@@ -97,7 +97,6 @@ func TestJiraToken_FromChunk(t *testing.T) {
t.Fatalf("no raw secret present: \n %+v", got[i])
}
got[i].Raw = nil
got[i].RawV2 = nil
}
if diff := pretty.Compare(got, tt.want); diff != "" {
t.Errorf("JiraToken.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
-1
View File
@@ -41,7 +41,6 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
s := detectors.Result{
DetectorType: detectorspb.DetectorType_Okta,
Raw: []byte(token),
RawV2: []byte(fmt.Sprintf("%s:%s", domain, token)),
}
if verify {
-1
View File
@@ -50,7 +50,6 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
s := detectors.Result{
DetectorType: detectorspb.DetectorType_OneLogin,
Raw: []byte(clientID[1]),
RawV2: []byte(fmt.Sprintf("%s:%s", clientID[1], clientSecret[1])),
Redacted: clientID[1],
}
-3
View File
@@ -66,15 +66,12 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
}
rawURL, _ := url.Parse(urlMatch)
rawURLStr := rawURL.String()
// Removing the path causes possible deduplication issues if some paths have basic auth and some do not.
rawURL.Path = ""
redact := strings.TrimSpace(strings.Replace(rawURL.String(), password, "********", -1))
s := detectors.Result{
DetectorType: detectorspb.DetectorType_URI,
Raw: []byte(rawURL.String()),
RawV2: []byte(rawURLStr),
Redacted: redact,
}
+3 -4
View File
@@ -38,7 +38,7 @@ func TestURI_FromChunk(t *testing.T) {
{
DetectorType: detectorspb.DetectorType_URI,
Verified: false,
Redacted: "https://user:********@www.httpwatch.com",
Redacted: "https://user:****@www.httpwatch.com",
},
},
wantErr: false,
@@ -55,7 +55,7 @@ func TestURI_FromChunk(t *testing.T) {
{
DetectorType: detectorspb.DetectorType_URI,
Verified: true,
Redacted: "https://httpwatch:********@www.httpwatch.com",
Redacted: "https://httpwatch:****@www.httpwatch.com",
},
},
wantErr: false,
@@ -72,7 +72,7 @@ func TestURI_FromChunk(t *testing.T) {
{
DetectorType: detectorspb.DetectorType_URI,
Verified: true,
Redacted: "https://httpwatch:********@www.httpwatch.com",
Redacted: "https://httpwatch:****@www.httpwatch.com",
},
},
wantErr: false,
@@ -101,7 +101,6 @@ func TestURI_FromChunk(t *testing.T) {
// }
for i := range got {
got[i].Raw = nil
got[i].RawV2 = nil
}
if diff := pretty.Compare(got, tt.want); diff != "" {
t.Errorf("URI.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
-2
View File
@@ -254,8 +254,6 @@ func (e *Engine) detectorWorker(ctx context.Context) {
decoderType = detectorspb.DecoderType_PLAIN
case *decoders.Base64:
decoderType = detectorspb.DecoderType_BASE64
case *decoders.UTF16:
decoderType = detectorspb.DecoderType_UTF16
default:
ctx.Logger().Info("unknown decoder type", "type", reflect.TypeOf(decoder).String())
decoderType = detectorspb.DecoderType_UNKNOWN
-4
View File
@@ -29,9 +29,6 @@ func PrintJSON(r *detectors.ResultWithMetadata) error {
Verified bool
// Raw contains the raw secret data.
Raw string
// RawV2 contains the raw secret identifier that is a combination of both the ID and the secret.
// This is used for secrets that are multi part and could have the same ID. Ex: AWS credentials
RawV2 string
// Redacted contains the redacted version of the raw secret identification data for display purposes.
// A secret ID should be used if available.
Redacted string
@@ -47,7 +44,6 @@ func PrintJSON(r *detectors.ResultWithMetadata) error {
DecoderName: r.DecoderType.String(),
Verified: r.Verified,
Raw: string(r.Raw),
RawV2: string(r.RawV2),
Redacted: r.Redacted,
ExtraData: r.ExtraData,
StructuredData: r.StructuredData,
File diff suppressed because it is too large Load Diff
-1
View File
@@ -8,7 +8,6 @@ enum DecoderType {
UNKNOWN = 0;
PLAIN = 1;
BASE64 = 2;
UTF16 = 3;
}
enum DetectorType {