tighten up launchdarkly

* init huggingface detector

* completed test

.github/workflows/lint.yml

@@ -17,7 +17,7 @@ jobs:
     steps:
       - uses: actions/setup-go@v4
         with:
-          go-version: '1.20'
+          go-version: '1.21'
       - uses: actions/checkout@v3
       - name: golangci-lint
         uses: golangci/golangci-lint-action@v3

.github/workflows/performance.yml

@@ -1,4 +1,4 @@
-name: Performance Test 
+name: Performance Test
 
 on: [pull_request]
 
@@ -7,13 +7,13 @@ jobs:
     #   skip if PR is from a fork.
     # TODO: this could probabaly be refactored a bit so that it runs on forks
     if: ${{ ! github.event.pull_request.head.repo.fork }}
-    
+
     runs-on: ubuntu-latest
     steps:
     - name: Install Go
       uses: actions/setup-go@v4
       with:
-        go-version: '1.20'
+        go-version: '1.21'
 
     - name: Checkout code
       uses: actions/checkout@v3

.github/workflows/release.yml

@@ -35,7 +35,7 @@ jobs:
       - name: Set up Go
         uses: actions/setup-go@v4
         with:
-          go-version: '1.20'
+          go-version: '1.21'
       - name: Run GoReleaser
         uses: goreleaser/goreleaser-action@v4
         with:

.github/workflows/secrets.yml

@@ -16,7 +16,7 @@ jobs:
     - name: Install Go
       uses: actions/setup-go@v4
       with:
-        go-version: '1.20'
+        go-version: '1.21'
     - name: Checkout code
       uses: actions/checkout@v3
       with:

.github/workflows/smoke.yml

@@ -11,7 +11,7 @@ jobs:
     - name: Install Go
       uses: actions/setup-go@v4
       with:
-        go-version: '1.20'
+        go-version: '1.21'
     - name: Checkout code
       uses: actions/checkout@v3
     - name: Smoke

.github/workflows/snifftest.yml

@@ -18,7 +18,7 @@ jobs:
     steps:
       - uses: actions/setup-go@v4
         with:
-          go-version: '1.20'
+          go-version: '1.21'
       - uses: actions/checkout@v3
       - name: Run Snifftest
-        run: make snifftest
+        run: make snifftest

.github/workflows/test.yml

@@ -21,7 +21,7 @@ jobs:
     - name: Install Go
       uses: actions/setup-go@v4
       with:
-        go-version: '1.20'
+        go-version: '1.21'
     - name: Checkout code
       uses: actions/checkout@v3
     - id: 'auth'
@@ -42,7 +42,7 @@ jobs:
     - name: Install Go
       uses: actions/setup-go@v4
       with:
-        go-version: '1.20'
+        go-version: '1.21'
     - name: Checkout code
       uses: actions/checkout@v3
     - id: 'auth'
@@ -63,7 +63,7 @@ jobs:
     - name: Install Go
       uses: actions/setup-go@v4
       with:
-        go-version: '1.20'
+        go-version: '1.21'
     - name: Checkout code
       uses: actions/checkout@v3
     - name: Test

pkg/detectors/alchemy/alchemy.go

@@ -65,7 +65,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 				} else if res.StatusCode == 401 {
 					// The secret is determinately not verified (nothing to do)
 				} else {
-					s1.VerificationError = fmt.Errorf("request to %v returned unexpected status %d", res.Request.URL, res.StatusCode)
+					s1.VerificationError = fmt.Errorf("unexpected HTTP response status %d", res.StatusCode)
 				}
 			} else {
 				s1.VerificationError = err

pkg/detectors/alchemy/alchemy_test.go

@@ -133,7 +133,7 @@ func TestAlchemy_FromChunk(t *testing.T) {
 					t.Fatalf("no raw secret present: \n %+v", got[i])
 				}
 				if (got[i].VerificationError != nil) != tt.wantVerificationErr {
-					t.Fatalf(" wantVerificationError = %v, verification error = %v,", tt.wantVerificationErr, got[i].VerificationError)
+					t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError)
 				}
 			}
 			ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "VerificationError")

pkg/detectors/gitlab/gitlab.go

@@ -12,7 +12,11 @@ import (
 	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
 )
 
-type Scanner struct{ detectors.EndpointSetter }
+type Scanner struct {
+	client *http.Client
+
+	detectors.EndpointSetter
+}
 
 // Ensure the Scanner satisfies the interfaces at compile time.
 var _ detectors.Detector = (*Scanner)(nil)
@@ -23,7 +27,8 @@ func (Scanner) Version() int            { return 1 }
 func (Scanner) DefaultEndpoint() string { return "https://gitlab.com" }
 
 var (
-	keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"gitlab"}) + `\b((?:glpat|)[a-zA-Z0-9\-=_]{20,22})\b`)
+	defaultClient = common.SaneHttpClient()
+	keyPat        = regexp.MustCompile(detectors.PrefixRegex([]string{"gitlab"}) + `\b((?:glpat|)[a-zA-Z0-9\-=_]{20,22})\b`)
 )
 
 // Keywords are used for efficiently pre-filtering chunks.
@@ -59,7 +64,10 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 			// one of these scopes has access to, so we just check an example endpoint for each scope. If any
 			// of them contain data, we know we have a valid key, but if they all fail, we don't
 
-			client := common.SaneHttpClient()
+			client := s.client
+			if client == nil {
+				client = defaultClient
+			}
 			for _, baseURL := range s.Endpoints(s.DefaultEndpoint()) {
 				// test `read_user` scope
 				req, err := http.NewRequestWithContext(ctx, "GET", baseURL+"/api/v4/user", nil)
@@ -74,9 +82,19 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 					// 200 means good key and has `read_user` scope
 					// 403 means good key but not the right scope
 					// 401 is bad key
-					if res.StatusCode == http.StatusOK || res.StatusCode == http.StatusForbidden {
+					switch res.StatusCode {
+					case http.StatusOK:
 						secret.Verified = true
+					case http.StatusForbidden:
+						// Good key but not the right scope
+						secret.Verified = true
+					case http.StatusUnauthorized:
+						// Nothing to do; zero values are the ones we want
+					default:
+						secret.VerificationError = fmt.Errorf("unexpected HTTP response status %d", res.StatusCode)
 					}
+				} else {
+					secret.VerificationError = err
 				}
 			}
 		}

pkg/detectors/gitlab/gitlab_test.go

@@ -6,11 +6,11 @@ package gitlab
 import (
 	"context"
 	"fmt"
+	"github.com/google/go-cmp/cmp"
+	"github.com/google/go-cmp/cmp/cmpopts"
 	"testing"
 	"time"
 
-	"github.com/kylelemons/godebug/pretty"
-
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
@@ -31,11 +31,12 @@ func TestGitlab_FromChunk(t *testing.T) {
 		verify bool
 	}
 	tests := []struct {
-		name    string
-		s       Scanner
-		args    args
-		want    []detectors.Result
-		wantErr bool
+		name                string
+		s                   Scanner
+		args                args
+		want                []detectors.Result
+		wantErr             bool
+		wantVerificationErr bool
 	}{
 		{
 			name: "found",
@@ -85,6 +86,56 @@ func TestGitlab_FromChunk(t *testing.T) {
 			},
 			wantErr: false,
 		},
+		{
+			name: "found, would be verified but for timeout",
+			s:    Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte(fmt.Sprintf("You can find a gitlab super secret %s within", secret)),
+				verify: true,
+			},
+			want: []detectors.Result{
+				{
+					DetectorType: detectorspb.DetectorType_Gitlab,
+					Verified:     false,
+				},
+			},
+			wantErr:             false,
+			wantVerificationErr: true,
+		},
+		{
+			name: "found and valid but unexpected api response",
+			s:    Scanner{client: common.ConstantResponseHttpClient(500, "")},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte(fmt.Sprintf("You can find a gitlab super secret %s within", secret)),
+				verify: true,
+			},
+			want: []detectors.Result{
+				{
+					DetectorType: detectorspb.DetectorType_Gitlab,
+					Verified:     false,
+				},
+			},
+			wantErr:             false,
+			wantVerificationErr: true,
+		},
+		{
+			name: "found, good key but wrong scope",
+			s:    Scanner{client: common.ConstantResponseHttpClient(403, "")},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte(fmt.Sprintf("You can find a gitlab super secret %s within", secret)),
+				verify: true,
+			},
+			want: []detectors.Result{
+				{
+					DetectorType: detectorspb.DetectorType_Gitlab,
+					Verified:     true,
+				},
+			},
+			wantErr: false,
+		},
 		{
 			name: "not found",
 			s:    Scanner{},
@@ -99,8 +150,7 @@ func TestGitlab_FromChunk(t *testing.T) {
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			s := Scanner{}
-			got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+			got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
 			if (err != nil) != tt.wantErr {
 				t.Errorf("Gitlab.FromData() error = %v, wantErr %v", err, tt.wantErr)
 				return
@@ -109,9 +159,12 @@ func TestGitlab_FromChunk(t *testing.T) {
 				if len(got[i].Raw) == 0 {
 					t.Fatal("no raw secret present")
 				}
-				got[i].Raw = nil
+				if (got[i].VerificationError != nil) != tt.wantVerificationErr {
+					t.Fatalf(" wantVerificationError = %v, verification error = %v,", tt.wantVerificationErr, got[i].VerificationError)
+				}
 			}
-			if diff := pretty.Compare(got, tt.want); diff != "" {
+			opts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "VerificationError")
+			if diff := cmp.Diff(got, tt.want, opts); diff != "" {
 				t.Errorf("Gitlab.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
 			}
 		})

pkg/detectors/huggingface/huggingface.go

@@ -0,0 +1,91 @@
+package huggingface
+
+import (
+	"context"
+	"fmt"
+	"net/http"
+	"regexp"
+	"strings"
+
+	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+type Scanner struct {
+	client *http.Client
+}
+
+// Ensure the Scanner satisfies the interface at compile time.
+var _ detectors.Detector = (*Scanner)(nil)
+
+var (
+	defaultClient = common.SaneHttpClient()
+	// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
+	keyPat = regexp.MustCompile(`\bhf_[a-zA-Z]{34}\b`)
+)
+
+// Keywords are used for efficiently pre-filtering chunks.
+// Use identifiers in the secret preferably, or the provider name.
+func (s Scanner) Keywords() []string {
+	return []string{"huggingface", "hugging_face", "hf"} // Huggingface docs occasionally use "hf" instead of "huggingface"
+}
+
+// FromData will find and optionally verify Huggingface secrets in a given set of bytes.
+func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
+	dataStr := string(data)
+
+	matches := keyPat.FindAllStringSubmatch(dataStr, -1)
+
+	for _, match := range matches {
+		if len(match) != 1 {
+			continue
+		}
+		resMatch := strings.TrimSpace(match[0])
+
+		s1 := detectors.Result{
+			DetectorType: detectorspb.DetectorType_HuggingFace,
+			Raw:          []byte(resMatch),
+		}
+
+		if verify {
+			client := s.client
+			if client == nil {
+				client = defaultClient
+			}
+			req, err := http.NewRequestWithContext(ctx, "GET", "https://huggingface.co/api/whoami-v2", nil)
+			if err != nil {
+				continue
+			}
+
+			req.Header.Set("Authorization", "Bearer "+resMatch)
+
+			res, err := client.Do(req)
+			if err == nil {
+				defer res.Body.Close()
+				if res.StatusCode >= 200 && res.StatusCode < 300 {
+					s1.Verified = true
+				} else if res.StatusCode == 401 {
+					// The secret is determinately not verified (nothing to do)
+				} else {
+					s1.VerificationError = fmt.Errorf("unexpected HTTP response status %d", res.StatusCode)
+				}
+			} else {
+				s1.VerificationError = err
+			}
+		}
+
+		// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
+		if !s1.Verified && detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
+			continue
+		}
+
+		results = append(results, s1)
+	}
+
+	return results, nil
+}
+
+func (s Scanner) Type() detectorspb.DetectorType {
+	return detectorspb.DetectorType_HuggingFace
+}

pkg/detectors/huggingface/huggingface_test.go

@@ -0,0 +1,160 @@
+//go:build detectors
+// +build detectors
+
+package huggingface
+
+import (
+	"context"
+	"fmt"
+	"github.com/google/go-cmp/cmp"
+	"github.com/google/go-cmp/cmp/cmpopts"
+	"testing"
+	"time"
+
+	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+
+	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestHuggingface_FromChunk(t *testing.T) {
+	ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+	defer cancel()
+	testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")
+	if err != nil {
+		t.Fatalf("could not get test secrets from GCP: %s", err)
+	}
+	secret := testSecrets.MustGetField("HUGGINGFACE")
+	inactiveSecret := testSecrets.MustGetField("HUGGINGFACE_INACTIVE")
+
+	type args struct {
+		ctx    context.Context
+		data   []byte
+		verify bool
+	}
+	tests := []struct {
+		name                string
+		s                   Scanner
+		args                args
+		want                []detectors.Result
+		wantErr             bool
+		wantVerificationErr bool
+	}{
+		{
+			name: "found, verified",
+			s:    Scanner{},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte(fmt.Sprintf("You can find a huggingface secret %s within", secret)),
+				verify: true,
+			},
+			want: []detectors.Result{
+				{
+					DetectorType: detectorspb.DetectorType_HuggingFace,
+					Verified:     true,
+				},
+			},
+			wantErr:             false,
+			wantVerificationErr: false,
+		},
+		{
+			name: "found, unverified",
+			s:    Scanner{},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte(fmt.Sprintf("You can find a huggingface secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+				verify: true,
+			},
+			want: []detectors.Result{
+				{
+					DetectorType: detectorspb.DetectorType_HuggingFace,
+					Verified:     false,
+				},
+			},
+			wantErr:             false,
+			wantVerificationErr: false,
+		},
+		{
+			name: "not found",
+			s:    Scanner{},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte("You cannot find the secret within"),
+				verify: true,
+			},
+			want:                nil,
+			wantErr:             false,
+			wantVerificationErr: false,
+		},
+		{
+			name: "found, would be verified if not for timeout",
+			s:    Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte(fmt.Sprintf("You can find a huggingface secret %s within", secret)),
+				verify: true,
+			},
+			want: []detectors.Result{
+				{
+					DetectorType: detectorspb.DetectorType_HuggingFace,
+					Verified:     false,
+				},
+			},
+			wantErr:             false,
+			wantVerificationErr: true,
+		},
+		{
+			name: "found, verified but unexpected api surface",
+			s:    Scanner{client: common.ConstantResponseHttpClient(404, "")},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte(fmt.Sprintf("You can find a huggingface secret %s within", secret)),
+				verify: true,
+			},
+			want: []detectors.Result{
+				{
+					DetectorType: detectorspb.DetectorType_HuggingFace,
+					Verified:     false,
+				},
+			},
+			wantErr:             false,
+			wantVerificationErr: true,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("Huggingface.FromData() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			for i := range got {
+				if len(got[i].Raw) == 0 {
+					t.Fatalf("no raw secret present: \n %+v", got[i])
+				}
+				if (got[i].VerificationError != nil) != tt.wantVerificationErr {
+					t.Fatalf(" wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError)
+				}
+			}
+			ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "VerificationError")
+			if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {
+				t.Errorf("Huggingface.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+			}
+		})
+	}
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+	ctx := context.Background()
+	s := Scanner{}
+	for name, data := range detectors.MustGetBenchmarkData() {
+		benchmark.Run(name, func(b *testing.B) {
+			for n := 0; n < b.N; n++ {
+				_, err := s.FromData(ctx, false, data)
+				if err != nil {
+					b.Fatal(err)
+				}
+			}
+		})
+	}
+}

pkg/detectors/launchdarkly/launchdarkly.go

@@ -20,13 +20,13 @@ var (
 	client = common.SaneHttpClient()
 
 	// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
-	keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"launchdarkly", "launch_darkly"}) + `\b([a-z0-9-]{40})\b`)
+	keyPat = regexp.MustCompile(`\b((?:api|sdk|mob)-[a-z0-9]{8}-[a-z0-9]{4}-4[a-z0-9]{3}-[a-z0-9]{4}-[a-z0-9]{12})\b`)
 )
 
 // Keywords are used for efficiently pre-filtering chunks.
 // Use identifiers in the secret preferably, or the provider name.
 func (s Scanner) Keywords() []string {
-	return []string{"launchdarkly", "launch_darkly"}
+	return []string{"api", "sdk", "mob"}
 }
 
 // FromData will find and optionally verify LaunchDarkly secrets in a given set of bytes.

pkg/detectors/opsgenie/opsgenie.go

@@ -52,7 +52,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 		}
 
 		if verify {
-			req, err := http.NewRequestWithContext(ctx, "GET", "https://api.opsgenie.com/v2/users", nil)
+			req, err := http.NewRequestWithContext(ctx, "GET", "https://api.opsgenie.com/v2/alerts", nil)
 			if err != nil {
 				continue
 			}

pkg/detectors/pubnubpublishkey/pubnubpublishkey.go

@@ -2,6 +2,7 @@ package pubnubpublishkey
 
 import (
 	"context"
+	"fmt"
 	"net/http"
 	"regexp"
 	"strings"
@@ -11,13 +12,15 @@ import (
 	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
 )
 
-type Scanner struct{}
+type Scanner struct {
+	client *http.Client
+}
 
 // Ensure the Scanner satisfies the interface at compile time.
 var _ detectors.Detector = (*Scanner)(nil)
 
 var (
-	client = common.SaneHttpClient()
+	defaultClient = common.SaneHttpClient()
 
 	// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
 	pubPat = regexp.MustCompile(`\b(pub-c-[0-9a-z]{8}-[0-9a-z]{4}-[a-z0-9]{4}-[a-z0-9]{4}-[a-z0-9]{12})\b`)
@@ -60,19 +63,31 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 				if err != nil {
 					continue
 				}
+				client := s.client
+				if client == nil {
+					client = defaultClient
+				}
 				res, err := client.Do(req)
 				if err == nil {
 					defer res.Body.Close()
 					if res.StatusCode >= 200 && res.StatusCode < 300 {
 						s1.Verified = true
+					} else if res.StatusCode == 400 || res.StatusCode == 403 {
+						// 403 is suggested by the API docs (https://www.pubnub.com/docs/sdks/rest-api/send-signal-to-channel)
+						// 400 is what actually seems to be coming back for invalid credentials
+						// There's nothing to do here, because zero values of Result are what we want
 					} else {
-						// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
-						if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
-							continue
-						}
+						s1.VerificationError = fmt.Errorf("unexpected HTTP response status %d", res.StatusCode)
 					}
+				} else {
+					s1.VerificationError = err
 				}
 			}
+			// This function will check false positives for common test words, but also it will make sure the key
+			// appears 'random' enough to be a real key.
+			if !s1.Verified && detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
+				continue
+			}
 
 			results = append(results, s1)
 		}

pkg/detectors/pubnubpublishkey/pubnubpublishkey_test.go

@@ -6,10 +6,11 @@ package pubnubpublishkey
 import (
 	"context"
 	"fmt"
+	"github.com/google/go-cmp/cmp"
+	"github.com/google/go-cmp/cmp/cmpopts"
 	"testing"
 	"time"
 
-	"github.com/kylelemons/godebug/pretty"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
 
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
@@ -33,11 +34,12 @@ func TestPubNubPublishKey_FromChunk(t *testing.T) {
 		verify bool
 	}
 	tests := []struct {
-		name    string
-		s       Scanner
-		args    args
-		want    []detectors.Result
-		wantErr bool
+		name                string
+		s                   Scanner
+		args                args
+		want                []detectors.Result
+		wantErr             bool
+		wantVerificationErr bool
 	}{
 		{
 			name: "found, verified",
@@ -71,6 +73,40 @@ func TestPubNubPublishKey_FromChunk(t *testing.T) {
 			},
 			wantErr: false,
 		},
+		{
+			name: "found, would be verified but for timeout",
+			s:    Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte(fmt.Sprintf("You can find a pubnub secret %s within pubnub %s", pub, sub)),
+				verify: true,
+			},
+			want: []detectors.Result{
+				{
+					DetectorType: detectorspb.DetectorType_PubNubPublishKey,
+					Verified:     false,
+				},
+			},
+			wantErr:             false,
+			wantVerificationErr: true,
+		},
+		{
+			name: "found, valid but unexpected api response",
+			s:    Scanner{client: common.ConstantResponseHttpClient(500, "")},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte(fmt.Sprintf("You can find a pubnub secret %s within pubnub %s", pub, sub)),
+				verify: true,
+			},
+			want: []detectors.Result{
+				{
+					DetectorType: detectorspb.DetectorType_PubNubPublishKey,
+					Verified:     false,
+				},
+			},
+			wantErr:             false,
+			wantVerificationErr: true,
+		},
 		{
 			name: "not found",
 			s:    Scanner{},
@@ -85,8 +121,7 @@ func TestPubNubPublishKey_FromChunk(t *testing.T) {
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			s := Scanner{}
-			got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+			got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
 			if (err != nil) != tt.wantErr {
 				t.Errorf("PubNubPublishKey.FromData() error = %v, wantErr %v", err, tt.wantErr)
 				return
@@ -95,9 +130,12 @@ func TestPubNubPublishKey_FromChunk(t *testing.T) {
 				if len(got[i].Raw) == 0 {
 					t.Fatalf("no raw secret present: \n %+v", got[i])
 				}
-				got[i].Raw = nil
+				if (got[i].VerificationError != nil) != tt.wantVerificationErr {
+					t.Fatalf(" wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError)
+				}
 			}
-			if diff := pretty.Compare(got, tt.want); diff != "" {
+			opts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "VerificationError")
+			if diff := cmp.Diff(got, tt.want, opts); diff != "" {
 				t.Errorf("PubNubPublishKey.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
 			}
 		})

pkg/engine/engine.go

@@ -595,17 +595,20 @@ func SupportsLineNumbers(sourceType sourcespb.SourceType) bool {
 
 // FragmentLineOffset sets the line number for a provided source chunk with a given detector result.
 func FragmentLineOffset(chunk *sources.Chunk, result *detectors.Result) (int64, bool) {
-	lines := bytes.Split(chunk.Data, []byte("\n"))
-	for i, line := range lines {
-		if bytes.Contains(line, result.Raw) {
-			// if the line contains the ignore tag, we should ignore the result
-			if bytes.Contains(line, []byte(ignoreTag)) {
-				return int64(i), true
-			}
-			return int64(i), false
-		}
+	before, after, found := bytes.Cut(chunk.Data, result.Raw)
+	if !found {
+		return 0, false
 	}
-	return 0, false
+	lineNumber := int64(bytes.Count(before, []byte("\n")))
+	// If the line contains the ignore tag, we should ignore the result.
+	endLine := bytes.Index(after, []byte("\n"))
+	if endLine == -1 {
+		endLine = len(after)
+	}
+	if bytes.Contains(after[:endLine], []byte(ignoreTag)) {
+		return lineNumber, true
+	}
+	return lineNumber, false
 }
 
 // FragmentFirstLine returns the first line number of a fragment along with a pointer to the value to update in the

pkg/engine/engine_test.go

@@ -1,6 +1,7 @@
 package engine
 
 import (
+	"fmt"
 	"testing"
 
 	"github.com/trufflesecurity/trufflehog/v3/pkg/decoders"
@@ -50,6 +51,39 @@ func TestFragmentLineOffset(t *testing.T) {
 			expectedLine: 4,
 			ignore:       false,
 		},
+		{
+			name: "match on consecutive lines",
+			chunk: &sources.Chunk{
+				Data: []byte("line1\nline2\ntrufflehog:ignore\nline4\nsecret\nhere\nline6"),
+			},
+			result: &detectors.Result{
+				Raw: []byte("secret\nhere"),
+			},
+			expectedLine: 4,
+			ignore:       false,
+		},
+		{
+			name: "ignore on last consecutive lines",
+			chunk: &sources.Chunk{
+				Data: []byte("line1\nline2\nline3\nsecret\nhere // trufflehog:ignore\nline5"),
+			},
+			result: &detectors.Result{
+				Raw: []byte("secret\nhere"),
+			},
+			expectedLine: 3,
+			ignore:       true,
+		},
+		{
+			name: "ignore on last line",
+			chunk: &sources.Chunk{
+				Data: []byte("line1\nline2\nline3\nsecret here // trufflehog:ignore"),
+			},
+			result: &detectors.Result{
+				Raw: []byte("secret here"),
+			},
+			expectedLine: 3,
+			ignore:       true,
+		},
 	}
 
 	for _, tt := range tests {
@@ -65,6 +99,43 @@ func TestFragmentLineOffset(t *testing.T) {
 	}
 }
 
+func setupFragmentLineOffsetBench(totalLines, needleLine int) (*sources.Chunk, *detectors.Result) {
+	data := make([]byte, 0, 4096)
+	needle := []byte("needle")
+	for i := 0; i < totalLines; i++ {
+		if i != needleLine {
+			data = append(data, []byte(fmt.Sprintf("line%d\n", i))...)
+			continue
+		}
+		data = append(data, needle...)
+		data = append(data, '\n')
+	}
+	chunk := &sources.Chunk{Data: data}
+	result := &detectors.Result{Raw: needle}
+	return chunk, result
+}
+
+func BenchmarkFragmentLineOffsetStart(b *testing.B) {
+	chunk, result := setupFragmentLineOffsetBench(512, 2)
+	for i := 0; i < b.N; i++ {
+		_, _ = FragmentLineOffset(chunk, result)
+	}
+}
+
+func BenchmarkFragmentLineOffsetMiddle(b *testing.B) {
+	chunk, result := setupFragmentLineOffsetBench(512, 256)
+	for i := 0; i < b.N; i++ {
+		_, _ = FragmentLineOffset(chunk, result)
+	}
+}
+
+func BenchmarkFragmentLineOffsetEnd(b *testing.B) {
+	chunk, result := setupFragmentLineOffsetBench(512, 510)
+	for i := 0; i < b.N; i++ {
+		_, _ = FragmentLineOffset(chunk, result)
+	}
+}
+
 // Test to make sure that DefaultDecoders always returns the UTF8 decoder first.
 // Technically a decoder test but we want this to run and fail in CI
 func TestDefaultDecoders(t *testing.T) {

pkg/pb/detectorspb/detectors.pb.go

@@ -996,6 +996,8 @@ const (
 	DetectorType_TrufflehogEnterprise          DetectorType = 922
 	DetectorType_EnvoyApiKey                   DetectorType = 923
 	DetectorType_GitHubOauth2                  DetectorType = 924
+	// 925 reserved for Salesforce
+	DetectorType_HuggingFace DetectorType = 926
 )
 
 // Enum value maps for DetectorType.
@@ -1922,6 +1924,7 @@ var (
 		922: "TrufflehogEnterprise",
 		923: "EnvoyApiKey",
 		924: "GitHubOauth2",
+		926: "HuggingFace",
 	}
 	DetectorType_value = map[string]int32{
 		"Alibaba":                       0,
@@ -2845,6 +2848,7 @@ var (
 		"TrufflehogEnterprise":          922,
 		"EnvoyApiKey":                   923,
 		"GitHubOauth2":                  924,
+		"HuggingFace":                   926,
 	}
 )
 
@@ -3223,7 +3227,7 @@ var file_detectors_proto_rawDesc = []byte{
 	0x44, 0x65, 0x63, 0x6f, 0x64, 0x65, 0x72, 0x54, 0x79, 0x70, 0x65, 0x12, 0x0b, 0x0a, 0x07, 0x55,
 	0x4e, 0x4b, 0x4e, 0x4f, 0x57, 0x4e, 0x10, 0x00, 0x12, 0x09, 0x0a, 0x05, 0x50, 0x4c, 0x41, 0x49,
 	0x4e, 0x10, 0x01, 0x12, 0x0a, 0x0a, 0x06, 0x42, 0x41, 0x53, 0x45, 0x36, 0x34, 0x10, 0x02, 0x12,
-	0x09, 0x0a, 0x05, 0x55, 0x54, 0x46, 0x31, 0x36, 0x10, 0x03, 0x2a, 0xd9, 0x73, 0x0a, 0x0c, 0x44,
+	0x09, 0x0a, 0x05, 0x55, 0x54, 0x46, 0x31, 0x36, 0x10, 0x03, 0x2a, 0xeb, 0x73, 0x0a, 0x0c, 0x44,
 	0x65, 0x74, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x54, 0x79, 0x70, 0x65, 0x12, 0x0b, 0x0a, 0x07, 0x41,
 	0x6c, 0x69, 0x62, 0x61, 0x62, 0x61, 0x10, 0x00, 0x12, 0x08, 0x0a, 0x04, 0x41, 0x4d, 0x51, 0x50,
 	0x10, 0x01, 0x12, 0x07, 0x0a, 0x03, 0x41, 0x57, 0x53, 0x10, 0x02, 0x12, 0x09, 0x0a, 0x05, 0x41,
@@ -4149,11 +4153,12 @@ var file_detectors_proto_rawDesc = []byte{
 	0x65, 0x68, 0x6f, 0x67, 0x45, 0x6e, 0x74, 0x65, 0x72, 0x70, 0x72, 0x69, 0x73, 0x65, 0x10, 0x9a,
 	0x07, 0x12, 0x10, 0x0a, 0x0b, 0x45, 0x6e, 0x76, 0x6f, 0x79, 0x41, 0x70, 0x69, 0x4b, 0x65, 0x79,
 	0x10, 0x9b, 0x07, 0x12, 0x11, 0x0a, 0x0c, 0x47, 0x69, 0x74, 0x48, 0x75, 0x62, 0x4f, 0x61, 0x75,
-	0x74, 0x68, 0x32, 0x10, 0x9c, 0x07, 0x42, 0x3d, 0x5a, 0x3b, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62,
-	0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x74, 0x72, 0x75, 0x66, 0x66, 0x6c, 0x65, 0x73, 0x65, 0x63, 0x75,
-	0x72, 0x69, 0x74, 0x79, 0x2f, 0x74, 0x72, 0x75, 0x66, 0x66, 0x6c, 0x65, 0x68, 0x6f, 0x67, 0x2f,
-	0x76, 0x33, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x70, 0x62, 0x2f, 0x64, 0x65, 0x74, 0x65, 0x63, 0x74,
-	0x6f, 0x72, 0x73, 0x70, 0x62, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
+	0x74, 0x68, 0x32, 0x10, 0x9c, 0x07, 0x12, 0x10, 0x0a, 0x0b, 0x48, 0x75, 0x67, 0x67, 0x69, 0x6e,
+	0x67, 0x46, 0x61, 0x63, 0x65, 0x10, 0x9e, 0x07, 0x42, 0x3d, 0x5a, 0x3b, 0x67, 0x69, 0x74, 0x68,
+	0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x74, 0x72, 0x75, 0x66, 0x66, 0x6c, 0x65, 0x73, 0x65,
+	0x63, 0x75, 0x72, 0x69, 0x74, 0x79, 0x2f, 0x74, 0x72, 0x75, 0x66, 0x66, 0x6c, 0x65, 0x68, 0x6f,
+	0x67, 0x2f, 0x76, 0x33, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x70, 0x62, 0x2f, 0x64, 0x65, 0x74, 0x65,
+	0x63, 0x74, 0x6f, 0x72, 0x73, 0x70, 0x62, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
 }
 
 var (

pkg/sources/docker/docker.go

@@ -84,6 +84,8 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) err
 		var img v1.Image
 		var err error
 		var base, tag string
+		var hasDigest bool
+		var imageName name.Reference
 
 		if strings.HasPrefix(image, "file://") {
 			image = strings.TrimPrefix(image, "file://")
@@ -93,10 +95,18 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) err
 				return err
 			}
 		} else {
-			base, tag = baseAndTagFromImage(image)
-			imageName, err := name.NewTag(image)
-			if err != nil {
-				return err
+			base, tag, hasDigest = baseAndTagFromImage(image)
+
+			if hasDigest {
+				imageName, err = name.NewDigest(image)
+				if err != nil {
+					return err
+				}
+			} else {
+				imageName, err = name.NewTag(image)
+				if err != nil {
+					return err
+				}
 			}
 
 			img, err = remote.Image(imageName, remoteOpts...)
@@ -194,10 +204,20 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) err
 	return nil
 }
 
-func baseAndTagFromImage(image string) (base, tag string) {
+func baseAndTagFromImage(image string) (base, tag string, hasDigest bool) {
 	regRepoDelimiter := "/"
 	tagDelim := ":"
-	parts := strings.Split(image, tagDelim)
+	digestDelim := "@"
+
+	// Split on digest first, if present.
+	parts := strings.SplitN(image, digestDelim, 2)
+	if len(parts) > 1 {
+		base = parts[0]
+		tag = parts[1]
+		hasDigest = true
+		return
+	}
+	parts = strings.Split(image, tagDelim)
 	// Verify that we aren't confusing a tag for a hostname w/ port for the purposes of weak validation.
 	if len(parts) > 1 && !strings.Contains(parts[len(parts)-1], regRepoDelimiter) {
 		base = strings.Join(parts[:len(parts)-1], tagDelim)

pkg/sources/docker/docker_test.go

@@ -1,5 +1,3 @@
-//go:build integration
-
 package docker
 
 import (
@@ -51,3 +49,64 @@ func TestDockerImageScan(t *testing.T) {
 
 	assert.Equal(t, 1, chunkCounter)
 }
+
+func TestDockerImageScanWithDigest(t *testing.T) {
+	dockerConn := &sourcespb.Docker{
+		Credential: &sourcespb.Docker_Unauthenticated{
+			Unauthenticated: &credentialspb.Unauthenticated{},
+		},
+		Images: []string{"trufflesecurity/secrets@sha256:864f6d41209462d8e37fc302ba1532656e265f7c361f11e29fed6ca1f4208e11"},
+	}
+
+	conn := &anypb.Any{}
+	err := conn.MarshalFrom(dockerConn)
+	assert.NoError(t, err)
+
+	s := &Source{}
+	err = s.Init(context.TODO(), "test source", 0, 0, false, conn, 1)
+	assert.NoError(t, err)
+
+	var wg sync.WaitGroup
+	chunksChan := make(chan *sources.Chunk, 1)
+	chunkCounter := 0
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		for chunk := range chunksChan {
+			assert.NotEmpty(t, chunk)
+			chunkCounter++
+		}
+	}()
+
+	err = s.Chunks(context.TODO(), chunksChan)
+	assert.NoError(t, err)
+
+	close(chunksChan)
+	wg.Wait()
+
+	assert.Equal(t, 1, chunkCounter)
+}
+
+func TestBaseAndTagFromImage(t *testing.T) {
+	tests := []struct {
+		image      string
+		wantBase   string
+		wantTag    string
+		wantDigest bool
+	}{
+		{"golang:1.16", "golang", "1.16", false},
+		{"golang@sha256:abcdef", "golang", "sha256:abcdef", true},
+		{"ghcr.io/golang:1.16", "ghcr.io/golang", "1.16", false},
+		{"ghcr.io/golang:nightly", "ghcr.io/golang", "nightly", false},
+		{"ghcr.io/golang", "ghcr.io/golang", "latest", false},
+		{"ghcr.io/trufflesecurity/secrets", "ghcr.io/trufflesecurity/secrets", "latest", false},
+	}
+
+	for _, tt := range tests {
+		gotBase, gotTag, gotDigest := baseAndTagFromImage(tt.image)
+		if gotBase != tt.wantBase || gotTag != tt.wantTag || gotDigest != tt.wantDigest {
+			t.Errorf("baseAndTagFromImage(%q) = (%q, %q, %v), want (%q, %q, %v)",
+				tt.image, gotBase, gotTag, gotDigest, tt.wantBase, tt.wantTag, tt.wantDigest)
+		}
+	}
+}

pkg/sources/filesystem/filesystem.go

@@ -1,7 +1,6 @@
 package filesystem
 
 import (
-	"bufio"
 	"fmt"
 	"io"
 	"io/fs"
@@ -23,13 +22,6 @@ import (
 	"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
 )
 
-const (
-	// These buffer sizes are mainly driven by our largest credential size, which is GCP @ ~2.25KB.
-	// Having a peek size larger than that ensures that we have complete credential coverage in our chunks.
-	BufferSize = 10 * 1024 // 10KB
-	PeekSize   = 3 * 1024  // 3KB
-)
-
 type Source struct {
 	name     string
 	sourceId int64
@@ -187,34 +179,33 @@ func (s *Source) scanFile(ctx context.Context, path string, chunksChan chan *sou
 	}
 	reReader.Stop()
 
-	for {
-		chunkBytes := make([]byte, BufferSize)
-		reader := bufio.NewReaderSize(reReader, BufferSize)
-		n, err := reader.Read(chunkBytes)
-		if err != nil && !errors.Is(err, io.EOF) {
-			break
+	chunkReader := sources.NewChunkReader()
+	chunkResChan := chunkReader(ctx, reReader)
+	for data := range chunkResChan {
+		if err := data.Error(); err != nil {
+			s.log.Error(err, "error reading chunk.")
+			continue
 		}
-		peekData, _ := reader.Peek(PeekSize)
-		if n > 0 {
-			chunksChan <- &sources.Chunk{
-				SourceType: s.Type(),
-				SourceName: s.name,
-				SourceID:   s.SourceID(),
-				Data:       append(chunkBytes[:n], peekData...),
-				SourceMetadata: &source_metadatapb.MetaData{
-					Data: &source_metadatapb.MetaData_Filesystem{
-						Filesystem: &source_metadatapb.Filesystem{
-							File: sanitizer.UTF8(path),
-						},
+
+		chunk := &sources.Chunk{
+			SourceType: s.Type(),
+			SourceName: s.name,
+			SourceID:   s.SourceID(),
+			Data:       data.Bytes(),
+			SourceMetadata: &source_metadatapb.MetaData{
+				Data: &source_metadatapb.MetaData_Filesystem{
+					Filesystem: &source_metadatapb.Filesystem{
+						File: sanitizer.UTF8(path),
 					},
 				},
-				Verify: s.verify,
-			}
+			},
+			Verify: s.verify,
 		}
-		if errors.Is(err, io.EOF) {
-			break
+		if err := common.CancellableWrite(ctx, chunksChan, chunk); err != nil {
+			return err
 		}
 	}
+
 	return nil
 }
 

pkg/sources/filesystem/filesystem_test.go

@@ -1,10 +1,14 @@
 package filesystem
 
 import (
+	"os"
+	"strings"
 	"testing"
 	"time"
 
+	"github.com/go-logr/logr"
 	"github.com/kylelemons/godebug/pretty"
+	"github.com/stretchr/testify/assert"
 	"google.golang.org/protobuf/types/known/anypb"
 
 	"github.com/trufflesecurity/trufflehog/v3/pkg/context"
@@ -78,3 +82,41 @@ func TestSource_Scan(t *testing.T) {
 		})
 	}
 }
+
+func TestScanFile(t *testing.T) {
+	tmpfile, err := os.CreateTemp("", "example.txt")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.Remove(tmpfile.Name())
+
+	chunkSize := sources.ChunkSize
+	secretPart1 := "SECRET"
+	secretPart2 := "SPLIT"
+	// Split the secret into two parts and pad the rest of the chunk with A's.
+	data := strings.Repeat("A", chunkSize-len(secretPart1)) + secretPart1 + secretPart2 + strings.Repeat("A", chunkSize-len(secretPart2))
+
+	_, err = tmpfile.Write([]byte(data))
+	assert.Nil(t, err)
+
+	err = tmpfile.Close()
+	assert.Nil(t, err)
+
+	source := &Source{}
+	chunksChan := make(chan *sources.Chunk, 2)
+
+	ctx := context.WithLogger(context.Background(), logr.Discard())
+	go func() {
+		defer close(chunksChan)
+		err = source.scanFile(ctx, tmpfile.Name(), chunksChan)
+		assert.Nil(t, err)
+	}()
+
+	// Read from the channel and validate the secrets.
+	foundSecret := ""
+	for chunkCh := range chunksChan {
+		foundSecret += string(chunkCh.Data)
+	}
+
+	assert.Contains(t, foundSecret, secretPart1+secretPart2)
+}

pkg/tui/sources/github/github.go

@@ -21,7 +21,7 @@ func GetFields() githubCmdModel {
 		Key:         "org",
 		Required:    false,
 		Help:        "GitHub organization to scan.",
-		Placeholder: "https://github.com/trufflesecurity",
+		Placeholder: "trufflesecurity",
 	}
 
 	repo := textinputs.InputConfig{

proto/detectors.proto

@@ -933,6 +933,8 @@ enum DetectorType {
   TrufflehogEnterprise = 922;
   EnvoyApiKey = 923;
   GitHubOauth2 = 924;
+  // 925 reserved for Salesforce
+  HuggingFace = 926;
 }
 
 message Result {