Assume ASCII and include valid BE and LE bytes

pkg/decoders/utf16.go

@@ -26,30 +26,31 @@ func (d *UTF16) FromChunk(chunk *sources.Chunk) *sources.Chunk {
 
 // utf16ToUTF8 converts a byte slice containing UTF-16 encoded data to a UTF-8 encoded byte slice.
 func utf16ToUTF8(b []byte) ([]byte, error) {
-	endianness, err := guessUTF16Endianness(b)
-	if err != nil {
-		return nil, err
-	}
-
-	buf := &bytes.Buffer{}
-	for i := 0; i < len(b); i += 2 {
-		r := rune(endianness.Uint16(b[i:]))
-		if utf8.ValidRune(r) {
-			buf.WriteRune(r)
+	var bufBE, bufLE bytes.Buffer
+	for i := 0; i < len(b)-1; i += 2 {
+		if r := rune(binary.BigEndian.Uint16(b[i:])); b[i] == 0 && utf8.ValidRune(r) {
+			bufBE.WriteRune(r)
+		}
+		if r := rune(binary.LittleEndian.Uint16(b[i:])); b[i+1] == 0 && utf8.ValidRune(r) {
+			bufLE.WriteRune(r)
+		}
+		// Guard against index out of bounds for the next check.
+		if i+1 >= len(b)-1 {
+			continue
 		}
 	}
 
-	return buf.Bytes(), nil
+	return append(bufLE.Bytes(), bufBE.Bytes()...), nil
 }
 
 func guessUTF16Endianness(b []byte) (binary.ByteOrder, error) {
-	if len(b) < 2 || len(b)%2 != 0 {
-		return nil, fmt.Errorf("input length must be even and at least 2 bytes long")
+	if len(b) < 2 {
+		return nil, fmt.Errorf("input length must at least 2 bytes long")
 	}
 
 	var evenNullBytes, oddNullBytes int
 
-	for i := 0; i < len(b); i += 2 {
+	for i := 0; i < len(b)-1; i += 2 {
 		if b[i] == 0 {
 			oddNullBytes++
 		}

pkg/decoders/utf16_test.go

@@ -2,6 +2,7 @@ package decoders
 
 import (
 	"bytes"
+	"os"
 	"testing"
 
 	"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
@@ -30,13 +31,13 @@ func TestUTF16Decoder(t *testing.T) {
 			name:      "Invalid UTF-16 input (it's UTF-8)",
 			input:     []byte("Hello World!"),
 			expected:  nil,
-			expectNil: true,
+			expectNil: false,
 		},
 		{
 			name:      "Invalid UTF-16 input (odd length)",
 			input:     []byte{72, 0, 101, 0, 108, 0, 108, 0, 111, 0, 32, 0, 87, 0, 111, 0, 114, 0, 108, 0, 0},
-			expected:  nil,
-			expectNil: true,
+			expected:  []byte("Hello Worl"),
+			expectNil: false,
 		},
 	}
 
@@ -57,12 +58,32 @@ func TestUTF16Decoder(t *testing.T) {
 				return
 			}
 			if !bytes.Equal(decodedChunk.Data, tc.expected) {
-				t.Errorf("Expected decoded data: %v, got: %v", tc.expected, decodedChunk.Data)
+				t.Errorf("Expected decoded data: %s, got: %s", tc.expected, decodedChunk.Data)
 			}
 		})
 	}
 }
 
+func TestDLL(t *testing.T) {
+	data, err := os.ReadFile("utf16_test.dll")
+	if err != nil {
+		t.Errorf("Failed to read test data: %v", err)
+		return
+	}
+
+	chunk := &sources.Chunk{Data: data}
+	decoder := &UTF16{}
+	decodedChunk := decoder.FromChunk(chunk)
+	if decodedChunk == nil {
+		t.Errorf("Expected chunk with data, got nil")
+		return
+	}
+	if !bytes.Contains(decodedChunk.Data, []byte("aws_secret_access_key")) {
+		t.Errorf("Expected chunk to have aws_secret_access_key")
+		return
+	}
+}
+
 func BenchmarkUtf16ToUtf8(b *testing.B) {
 	// Example UTF-16LE encoded data
 	data := []byte{72, 0, 101, 0, 108, 0, 108, 0, 111, 0, 32, 0, 87, 0, 111, 0, 114, 0, 108, 0, 100, 0}