updating secret to s1 to match convention

.github/workflows/test.yml

@@ -31,30 +31,6 @@ jobs:
         service_account: 'github-ci-external@trufflehog-testing.iam.gserviceaccount.com'
     - name: Test
       run: make test-integration
-    - name: Set up gotestsum
-      run: |
-        go install gotest.tools/gotestsum@latest
-        mkdir -p tmp/test-results
-    - name: Test 
-      run: |
-        CGO_ENABLED=1 gotestsum --junitfile tmp/test-results/test.xml --raw-command -- go test -json -tags=sources $(go list ./... | grep -v /vendor/ | grep -v pkg/detectors)
-      if: ${{ success() || failure() }} # always run this step, even if there were previous errors
-    - name: Upload test results to BuildPulse for flaky test detection
-      if: '!cancelled()' # Run this step even when the tests fail. Skip if the workflow is cancelled.
-      uses: buildpulse/buildpulse-action@main
-      with:
-        account: 79229934
-        repository: 77726177
-        path: |
-          tmp/test-results/*.xml
-        key: ${{ secrets.BUILDPULSE_ACCESS_KEY_ID }}
-        secret: ${{ secrets.BUILDPULSE_SECRET_ACCESS_KEY }}
-        tags: integration
-    - name: Annotate test results
-      uses: mikepenz/action-junit-report@v3
-      if: success() || failure() # always run even if the previous step fails
-      with:
-        report_paths: 'tmp/test-results/*.xml'
   test-detectors:
     if: ${{ ! github.event.pull_request.head.repo.fork }}
     runs-on: ubuntu-latest

go.mod

@@ -11,7 +11,6 @@ require (
 	cloud.google.com/go/secretmanager v1.11.1
 	cloud.google.com/go/storage v1.31.0
 	github.com/Azure/go-autorest/autorest/azure/auth v0.5.11
-	github.com/AzureAD/microsoft-authentication-library-for-go v0.5.1
 	github.com/BobuSumisu/aho-corasick v1.0.3
 	github.com/TheZeroSlave/zapsentry v1.17.0
 	github.com/aws/aws-sdk-go v1.44.83
@@ -204,7 +203,7 @@ require (
 	github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8 // indirect
 	github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3 // indirect
 	github.com/mitchellh/go-homedir v1.1.0 // indirect
-	github.com/montanaflynn/stats v0.6.6 // indirect
+	github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe // indirect
 	github.com/mtibben/percent v0.2.1 // indirect
 	github.com/muesli/ansi v0.0.0-20211031195517-c9f0611b6c70 // indirect
 	github.com/muesli/cancelreader v0.2.2 // indirect

go.sum

@@ -304,7 +304,6 @@ github.com/goccy/go-json v0.10.0 h1:mXKd9Qw4NuzShiRlOXKews24ufknHO7gx30lsDyokKA=
 github.com/goccy/go-json v0.10.0/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I=
 github.com/godbus/dbus v0.0.0-20190726142602-4481cbc300e2 h1:ZpnhV/YsD2/4cESfV5+Hoeu/iUR3ruzNvZ+yQfO03a0=
 github.com/godbus/dbus v0.0.0-20190726142602-4481cbc300e2/go.mod h1:bBOAhwG1umN6/6ZUMtDFBMQR8jRg9O75tm9K00oMsK4=
-github.com/golang-jwt/jwt v3.2.1+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I=
 github.com/golang-jwt/jwt v3.2.2+incompatible h1:IfV12K8xAKAnZqdXVzCZ+TOjboZ2keLg81eXfW3O+oY=
 github.com/golang-jwt/jwt v3.2.2+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I=
 github.com/golang-jwt/jwt/v4 v4.0.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg=
@@ -383,7 +382,6 @@ github.com/google/pprof v0.0.0-20211214055906-6f57359322fd/go.mod h1:KgnwoLYCZ8I
 github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
 github.com/google/s2a-go v0.1.4 h1:1kZ/sQM3srePvKs3tXAvQzo66XfcReoqFpIpIccE7Oc=
 github.com/google/s2a-go v0.1.4/go.mod h1:Ej+mSEMGRnqRzjc7VtF+jdBwYG5fuJfiZ8ELkjEwM0A=
-github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I=
 github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
@@ -520,9 +518,8 @@ github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3/go.mod h1:RagcQ7I8Ie
 github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y=
 github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
 github.com/modocache/gover v0.0.0-20171022184752-b58185e213c5/go.mod h1:caMODM3PzxT8aQXRPkAt8xlV/e7d7w8GM5g0fa5F0D8=
+github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe h1:iruDEfMl2E6fbMZ9s0scYfZQ84/6SPL6zC8ACM2oIL0=
 github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc=
-github.com/montanaflynn/stats v0.6.6 h1:Duep6KMIDpY4Yo11iFsvyqJDyfzLF9+sndUKT+v64GQ=
-github.com/montanaflynn/stats v0.6.6/go.mod h1:etXPPgVO6n31NxCd9KQUMvCM+ve0ruNzt6R8Bnaayow=
 github.com/mtibben/percent v0.2.1 h1:5gssi8Nqo8QU/r2pynCm+hBQHpkB/uNK7BJCFogWdzs=
 github.com/mtibben/percent v0.2.1/go.mod h1:KG9uO+SZkUp+VkRHsCdYQV3XSZrrSpR3O9ibNBTZrns=
 github.com/muesli/ansi v0.0.0-20211031195517-c9f0611b6c70 h1:kMlmsLSbjkikxQJ1IPwaM+7LJ9ltFu/fi8CRzvSnQmA=
@@ -568,7 +565,6 @@ github.com/pingcap/errors v0.11.4/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTw
 github.com/pjbgf/sha1cd v0.3.0 h1:4D5XXmUUBUl/xQ6IjCkEAbqXskkq/4O7LmGn0AqMDs4=
 github.com/pjbgf/sha1cd v0.3.0/go.mod h1:nZ1rrWOcGJ5uZgEEVL1VUM9iRQiZvWdbZjkKyFzPPsI=
 github.com/pkg/browser v0.0.0-20180916011732-0a3d74bf9ce4/go.mod h1:4OwLy04Bl9Ef3GJJCoec+30X3LQs/0/m4HFRt/2LUSA=
-github.com/pkg/browser v0.0.0-20210115035449-ce105d075bb4/go.mod h1:N6UoU20jOqggOuDwUaBQpluzLNDqif3kq9z2wpdYEfQ=
 github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 h1:KoWmjvw+nsYOo29YJK9vDA65RGE3NrOnUtO7a+RF9HU=
 github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8/go.mod h1:HKlIX3XHQyzLZPlr7++PzdhaXEj94dEiJgZDTsxEqUI=
 github.com/pkg/diff v0.0.0-20200914180035-5b29258ca4f7 h1:+/+DxvQaYifJ+grD4klzrS5y+KJXldn/2YTl5JG+vZ8=

pkg/detectors/detectors.go

@@ -66,7 +66,7 @@ type ResultWithMetadata struct {
 	// SourceMetadata contains source-specific contextual information.
 	SourceMetadata *source_metadatapb.MetaData
 	// SourceID is the ID of the source that the API uses to map secrets to specific sources.
-	SourceID sources.SourceID
+	SourceID int64
 	// SourceType is the type of Source.
 	SourceType sourcespb.SourceType
 	// SourceName is the name of the Source.

pkg/detectors/gitlab/gitlab.go

@@ -53,7 +53,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 			resMatch = keyString[len(keyString)-1]
 		}
 
-		secret := detectors.Result{
+		s1 := detectors.Result{
 			DetectorType: detectorspb.DetectorType_Gitlab,
 			Raw:          []byte(match[1]),
 		}
@@ -84,26 +84,26 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 					// 401 is bad key
 					switch res.StatusCode {
 					case http.StatusOK:
-						secret.Verified = true
+						s1.Verified = true
 					case http.StatusForbidden:
 						// Good key but not the right scope
-						secret.Verified = true
+						s1.Verified = true
 					case http.StatusUnauthorized:
 						// Nothing to do; zero values are the ones we want
 					default:
-						secret.VerificationError = fmt.Errorf("unexpected HTTP response status %d", res.StatusCode)
+						s1.VerificationError = fmt.Errorf("unexpected HTTP response status %d", res.StatusCode)
 					}
 				} else {
-					secret.VerificationError = err
+					s1.VerificationError = err
 				}
 			}
 		}
 
-		if !secret.Verified && detectors.IsKnownFalsePositive(string(secret.Raw), detectors.DefaultFalsePositives, true) {
+		if !s1.Verified && detectors.IsKnownFalsePositive(string(s1.Raw), detectors.DefaultFalsePositives, true) {
 			continue
 		}
 
-		results = append(results, secret)
+		results = append(results, s1)
 	}
 
 	return results, nil

pkg/detectors/gitlab/gitlab_test.go

@@ -6,11 +6,12 @@ package gitlab
 import (
 	"context"
 	"fmt"
-	"github.com/google/go-cmp/cmp"
-	"github.com/google/go-cmp/cmp/cmpopts"
 	"testing"
 	"time"
 
+	"github.com/google/go-cmp/cmp"
+	"github.com/google/go-cmp/cmp/cmpopts"
+
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"

pkg/detectors/gitlab/gitlabv2_test.go

@@ -1,115 +0,0 @@
-package gitlab
-
-import (
-	"context"
-	"fmt"
-	"testing"
-	"time"
-
-	"github.com/kylelemons/godebug/pretty"
-	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
-	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
-	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
-)
-
-func TestGitlabv2_FromChunk(t *testing.T) {
-	ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
-	defer cancel()
-	testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors4")
-	if err != nil {
-		t.Fatalf("could not get test secrets from GCP: %s", err)
-	}
-	secret := testSecrets.MustGetField("GITLABV2")
-	secretInactive := testSecrets.MustGetField("GITLABV2_INACTIVE")
-	type args struct {
-		ctx    context.Context
-		data   []byte
-		verify bool
-	}
-	tests := []struct {
-		name    string
-		s       Scanner
-		args    args
-		want    []detectors.Result
-		wantErr bool
-	}{
-		{
-			name: "found",
-			s:    Scanner{},
-			args: args{
-				ctx:    context.Background(),
-				data:   []byte(fmt.Sprintf("You can find a gitlab secret %s within", secret)),
-				verify: true,
-			},
-			want: []detectors.Result{
-				{
-					DetectorType: detectorspb.DetectorType_Gitlab,
-					Verified:     true,
-				},
-			},
-			wantErr: false,
-		},
-		{
-			name: "found, unverified",
-			s:    Scanner{},
-			args: args{
-				ctx:    context.Background(),
-				data:   []byte(fmt.Sprintf("You can find a gitlab secret %s within", secretInactive)),
-				verify: true,
-			},
-			want: []detectors.Result{
-				{
-					DetectorType: detectorspb.DetectorType_Gitlab,
-					Verified:     false,
-				},
-			},
-			wantErr: false,
-		},
-		{
-			name: "not found",
-			s:    Scanner{},
-			args: args{
-				ctx:    context.Background(),
-				data:   []byte("You cannot find the secret within"),
-				verify: true,
-			},
-			want:    nil,
-			wantErr: false,
-		},
-	}
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			s := Scanner{}
-			got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
-			if (err != nil) != tt.wantErr {
-				t.Errorf("Gitlab.FromData() error = %v, wantErr %v", err, tt.wantErr)
-				return
-			}
-			for i := range got {
-				if len(got[i].Raw) == 0 {
-					t.Fatal("no raw secret present")
-				}
-				got[i].Raw = nil
-			}
-			if diff := pretty.Compare(got, tt.want); diff != "" {
-				t.Errorf("Gitlab.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
-			}
-		})
-	}
-}
-
-func BenchmarkFromData2(benchmark *testing.B) {
-	ctx := context.Background()
-	s := Scanner{}
-	for name, data := range detectors.MustGetBenchmarkData() {
-		benchmark.Run(name, func(b *testing.B) {
-			b.ResetTimer()
-			for n := 0; n < b.N; n++ {
-				_, err := s.FromData(ctx, false, data)
-				if err != nil {
-					b.Fatal(err)
-				}
-			}
-		})
-	}
-}

pkg/detectors/gitlabv2/gitlab.go

@@ -11,7 +11,10 @@ import (
 	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
 )
 
-type Scanner struct{ detectors.EndpointSetter }
+type Scanner struct {
+	client *http.Client
+	detectors.EndpointSetter
+}
 
 // Ensure the Scanner satisfies the interfaces at compile time.
 var _ detectors.Detector = (*Scanner)(nil)
@@ -22,7 +25,8 @@ func (Scanner) Version() int            { return 2 }
 func (Scanner) DefaultEndpoint() string { return "https://gitlab.com" }
 
 var (
-	keyPat = regexp.MustCompile(`\b(glpat-[a-zA-Z0-9\-=_]{20,22})\b`)
+	defaultClient = common.SaneHttpClient()
+	keyPat        = regexp.MustCompile(`\b(glpat-[a-zA-Z0-9\-=_]{20,22})\b`)
 )
 
 // Keywords are used for efficiently pre-filtering chunks.
@@ -42,7 +46,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 			continue
 		}
 
-		secret := detectors.Result{
+		s1 := detectors.Result{
 			DetectorType: detectorspb.DetectorType_Gitlab,
 			Raw:          []byte(match[1]),
 		}
@@ -52,7 +56,10 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 			// they all grant access to different parts of the API. I couldn't find an endpoint that every
 			// one of these scopes has access to, so we just check an example endpoint for each scope. If any
 			// of them contain data, we know we have a valid key, but if they all fail, we don't
-			client := common.SaneHttpClient()
+			client := s.client
+			if client == nil {
+				client = defaultClient
+			}
 			for _, baseURL := range s.Endpoints(s.DefaultEndpoint()) {
 				// test `read_user` scope
 				req, err := http.NewRequestWithContext(ctx, "GET", baseURL+"/api/v4/user", nil)
@@ -67,19 +74,29 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 					// 200 means good key and has `read_user` scope
 					// 403 means good key but not the right scope
 					// 401 is bad key
-					if res.StatusCode == http.StatusOK || res.StatusCode == http.StatusForbidden {
-						secret.Verified = true
+					switch res.StatusCode {
+					case http.StatusOK:
+						s1.Verified = true
+					case http.StatusForbidden:
+						// Good key but not the right scope
+						s1.Verified = true
+					case http.StatusUnauthorized:
+						// Nothing to do; zero values are the ones we want
+					default:
+						s1.VerificationError = fmt.Errorf("unexpected HTTP response status %d", res.StatusCode)
 					}
+				} else {
+					s1.VerificationError = err
 				}
 			}
 
 		}
 
-		if !secret.Verified && detectors.IsKnownFalsePositive(string(secret.Raw), detectors.DefaultFalsePositives, true) {
+		if !s1.Verified && detectors.IsKnownFalsePositive(string(s1.Raw), detectors.DefaultFalsePositives, true) {
 			continue
 		}
 
-		results = append(results, secret)
+		results = append(results, s1)
 	}
 
 	return

pkg/detectors/gitlabv2/gitlab_test.go

@@ -9,7 +9,8 @@ import (
 	"testing"
 	"time"
 
-	"github.com/kylelemons/godebug/pretty"
+	"github.com/google/go-cmp/cmp"
+	"github.com/google/go-cmp/cmp/cmpopts"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
@@ -18,7 +19,7 @@ import (
 func TestGitlab_FromChunk(t *testing.T) {
 	ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
 	defer cancel()
-	testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")
+	testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors4")
 	if err != nil {
 		t.Fatalf("could not get test secrets from GCP: %s", err)
 	}
@@ -31,18 +32,19 @@ func TestGitlab_FromChunk(t *testing.T) {
 		verify bool
 	}
 	tests := []struct {
-		name    string
-		s       Scanner
-		args    args
-		want    []detectors.Result
-		wantErr bool
+		name                string
+		s                   Scanner
+		args                args
+		want                []detectors.Result
+		wantErr             bool
+		wantVerificationErr bool
 	}{
 		{
 			name: "found, verified",
 			s:    Scanner{},
 			args: args{
 				ctx:    context.Background(),
-				data:   []byte(fmt.Sprintf("You can find a aws secret %s within", secret)),
+				data:   []byte(fmt.Sprintf("You can find a gitlab secret %s within", secret)),
 				verify: true,
 			},
 			want: []detectors.Result{
@@ -58,10 +60,65 @@ func TestGitlab_FromChunk(t *testing.T) {
 			s:    Scanner{},
 			args: args{
 				ctx:    context.Background(),
-				data:   []byte(fmt.Sprintf("You can find a aws secret %s within", secretInactive)),
+				data:   []byte(fmt.Sprintf("You can find a gitlab secret %s within", secretInactive)),
 				verify: true,
 			},
-			want:    nil,
+			want: []detectors.Result{
+				{
+					DetectorType: detectorspb.DetectorType_Gitlab,
+					Verified:     false,
+				},
+			},
+			wantErr: false,
+		},
+		{
+			name: "found, would be verified but for timeout",
+			s:    Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte(fmt.Sprintf("You can find a gitlab super secret %s within", secret)),
+				verify: true,
+			},
+			want: []detectors.Result{
+				{
+					DetectorType: detectorspb.DetectorType_Gitlab,
+					Verified:     false,
+				},
+			},
+			wantErr:             false,
+			wantVerificationErr: true,
+		},
+		{
+			name: "found and valid but unexpected api response",
+			s:    Scanner{client: common.ConstantResponseHttpClient(500, "")},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte(fmt.Sprintf("You can find a gitlab super secret %s within", secret)),
+				verify: true,
+			},
+			want: []detectors.Result{
+				{
+					DetectorType: detectorspb.DetectorType_Gitlab,
+					Verified:     false,
+				},
+			},
+			wantErr:             false,
+			wantVerificationErr: true,
+		},
+		{
+			name: "found, good key but wrong scope",
+			s:    Scanner{client: common.ConstantResponseHttpClient(403, "")},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte(fmt.Sprintf("You can find a gitlab super secret %s within", secret)),
+				verify: true,
+			},
+			want: []detectors.Result{
+				{
+					DetectorType: detectorspb.DetectorType_Gitlab,
+					Verified:     true,
+				},
+			},
 			wantErr: false,
 		},
 		{
@@ -78,8 +135,7 @@ func TestGitlab_FromChunk(t *testing.T) {
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			s := Scanner{}
-			got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+			got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
 			if (err != nil) != tt.wantErr {
 				t.Errorf("Gitlab.FromData() error = %v, wantErr %v", err, tt.wantErr)
 				return
@@ -88,9 +144,12 @@ func TestGitlab_FromChunk(t *testing.T) {
 				if len(got[i].Raw) == 0 {
 					t.Fatal("no raw secret present")
 				}
-				got[i].Raw = nil
+				if (got[i].VerificationError != nil) != tt.wantVerificationErr {
+					t.Fatalf(" wantVerificationError = %v, verification error = %v,", tt.wantVerificationErr, got[i].VerificationError)
+				}
 			}
-			if diff := pretty.Compare(got, tt.want); diff != "" {
+			opts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "VerificationError")
+			if diff := cmp.Diff(got, tt.want, opts); diff != "" {
 				t.Errorf("Gitlab.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
 			}
 		})

pkg/detectors/privatekey/cracker.go

@@ -9,7 +9,7 @@ import (
 	"golang.org/x/crypto/ssh"
 )
 
-//go:embed "list.txt"
+//go:embed "rockyou-15.txt"
 var rawCrackList []byte
 var passphrases [][]byte
 
@@ -21,17 +21,17 @@ var (
 	ErrUncrackable = errors.New("unable to crack encryption")
 )
 
-func crack(in []byte) (interface{}, string, error) {
+func crack(in []byte) (interface{}, error) {
 	for _, passphrase := range passphrases {
 		parsed, err := ssh.ParseRawPrivateKeyWithPassphrase(in, passphrase)
 		if err != nil {
 			if errors.Is(err, x509.IncorrectPasswordError) {
 				continue
 			} else {
-				return nil, "", err
+				return nil, err
 			}
 		}
-		return parsed, string(passphrase), nil
+		return parsed, nil
 	}
-	return nil, "", ErrUncrackable
+	return nil, ErrUncrackable
 }

pkg/detectors/privatekey/cracker_test.go

@@ -56,29 +56,23 @@ Hx7UPVlTK8dyvk1Z+Yw0nrfNClI=
 
 func Test_crack(t *testing.T) {
 	tests := []struct {
-		name             string
-		in               []byte
-		wantedPassphrase string
-		wantErr          bool
+		name    string
+		in      []byte
+		wantErr bool
 	}{
 		{
-			name:             "crackable",
-			wantErr:          false,
-			in:               testEncryptedKey,
-			wantedPassphrase: string(testEncryptedKeyCorrectPassword),
+			name:    "crackable",
+			wantErr: false,
+			in:      testEncryptedKey,
 		},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			_, passphrase, err := crack(tt.in)
+			_, err := crack(tt.in)
 			if (err != nil) != tt.wantErr {
 				t.Errorf("crack() error = %v, wantErr %v", err, tt.wantErr)
 				return
 			}
-
-			if passphrase != string(tt.wantedPassphrase) {
-				t.Errorf("crack() passphrase = %v, want %v", passphrase, string(testEncryptedKeyCorrectPassword))
-			}
 		})
 	}
 }

pkg/detectors/privatekey/fingerprint.go

@@ -5,12 +5,10 @@ import (
 	"crypto/ed25519"
 	"crypto/rsa"
 	"crypto/sha1"
-	"crypto/sha256"
 	"crypto/x509"
-	"encoding/base64"
 	"encoding/hex"
 	"errors"
-	"fmt"
+	"strings"
 
 	"golang.org/x/crypto/ssh"
 )
@@ -20,8 +18,18 @@ var (
 	ErrEncryptedKey = errors.New("key is encrypted")
 )
 
-func FingerprintPEMKey(parsedKey any) (string, error) {
-	var pubKey any
+func FingerprintPEMKey(in []byte) (string, error) {
+	parsedKey, err := ssh.ParseRawPrivateKey(in)
+	if err != nil && strings.Contains(err.Error(), "private key is passphrase protected") {
+		parsedKey, err = crack(in)
+		if err != nil {
+			return "", err
+		}
+	} else if err != nil {
+		return "", err
+	}
+
+	var pubKey interface{}
 	switch privateKey := parsedKey.(type) {
 	case *rsa.PrivateKey:
 		pubKey = &privateKey.PublicKey
@@ -36,10 +44,6 @@ func FingerprintPEMKey(parsedKey any) (string, error) {
 		return "", ErrNotSupported
 	}
 
-	return fingerprintPublicKey(pubKey)
-}
-
-func fingerprintPublicKey(pubKey any) (string, error) {
 	publickeyBytes, err := x509.MarshalPKIXPublicKey(pubKey)
 	if err != nil {
 		return "", err
@@ -49,8 +53,3 @@ func fingerprintPublicKey(pubKey any) (string, error) {
 	publicKeyFingerprintInHex := hex.EncodeToString(publicKeyFingerprint[:])
 	return publicKeyFingerprintInHex, nil
 }
-
-func fingerprintSSHPublicKey(pubKey ssh.PublicKey) string {
-	publicKeyFingerprint := sha256.Sum256(pubKey.Marshal())
-	return fmt.Sprintf("SHA256:%s", base64.RawStdEncoding.EncodeToString(publicKeyFingerprint[:]))
-}

pkg/detectors/privatekey/privatekey.go

@@ -6,14 +6,11 @@ import (
 	"fmt"
 	"net/http"
 	"regexp"
-	"strings"
 	"time"
 
-	"github.com/AzureAD/microsoft-authentication-library-for-go/apps/errors"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
-	"golang.org/x/crypto/ssh"
 )
 
 type Scanner struct {
@@ -56,67 +53,19 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) ([]dete
 			Redacted:     token[0:64],
 		}
 
-		secret.ExtraData = make(map[string]string)
-
-		var passphrase string
-		parsedKey, err := ssh.ParseRawPrivateKey([]byte(token))
-		if err != nil && strings.Contains(err.Error(), "private key is passphrase protected") {
-			secret.ExtraData["encrypted"] = "true"
-			parsedKey, passphrase, err = crack([]byte(token))
-			if err != nil {
-				secret.VerificationError = err
-				continue
-			}
-			if passphrase != "" {
-				secret.ExtraData["cracked_encryption_passphrase"] = "true"
-			}
-		} else if err != nil {
-			// couldn't parse key, probably invalid
-			continue
-		}
-
-		fingerprint, err := FingerprintPEMKey(parsedKey)
+		fingerprint, err := FingerprintPEMKey([]byte(token))
 		if err != nil {
 			continue
 		}
 
 		if verify {
-			verificationErrors := []string{}
 			data, err := lookupFingerprint(fingerprint, s.IncludeExpired)
 			if err == nil {
+				secret.StructuredData = data
 				if data != nil {
 					secret.Verified = true
-					secret.ExtraData["certificate_urls"] = strings.Join(data.CertificateURLs, ", ")
 				}
-			} else {
-				verificationErrors = append(verificationErrors, err.Error())
 			}
-
-			user, err := verifyGitHubUser(parsedKey)
-			if err != nil && !errors.Is(err, errPermissionDenied) {
-				verificationErrors = append(verificationErrors, err.Error())
-			}
-			if user != nil {
-				secret.Verified = true
-				secret.ExtraData["github_user"] = *user
-			}
-
-			user, err = verifyGitLabUser(parsedKey)
-			if err != nil && !errors.Is(err, errPermissionDenied) {
-				verificationErrors = append(verificationErrors, err.Error())
-			}
-			if user != nil {
-				secret.Verified = true
-				secret.ExtraData["gitlab_user"] = *user
-			}
-
-			if !secret.Verified && len(verificationErrors) > 0 {
-				secret.VerificationError = fmt.Errorf("verification failures: %s", strings.Join(verificationErrors, ", "))
-			}
-		}
-
-		if len(secret.ExtraData) == 0 {
-			secret.ExtraData = nil
 		}
 
 		results = append(results, secret)
@@ -125,30 +74,23 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) ([]dete
 	return results, nil
 }
 
-type result struct {
-	CertificateURLs []string
-	GitHubUsername  string
-}
-
-func lookupFingerprint(publicKeyFingerprintInHex string, includeExpired bool) (*result, error) {
+func lookupFingerprint(publicKeyFingerprintInHex string, includeExpired bool) (data *detectorspb.StructuredData, err error) {
 	req, err := http.NewRequest("GET", fmt.Sprintf("https://keychecker.trufflesecurity.com/fingerprint/%s", publicKeyFingerprintInHex), nil)
 	if err != nil {
-		return nil, err
+		return
 	}
 	res, err := client.Do(req)
 	if err != nil {
-		return nil, err
+		return
 	}
 	defer res.Body.Close()
 
 	results := DriftwoodResult{}
 	err = json.NewDecoder(res.Body).Decode(&results)
 	if err != nil {
-		return nil, err
+		return
 	}
 
-	var data *result
-
 	seen := map[string]struct{}{}
 	for _, r := range results.CertificateResults {
 		if _, ok := seen[r.CertificateFingerprint]; ok {
@@ -158,13 +100,36 @@ func lookupFingerprint(publicKeyFingerprintInHex string, includeExpired bool) (*
 			continue
 		}
 		if data == nil {
-			data = &result{}
+			data = &detectorspb.StructuredData{}
 		}
-		data.CertificateURLs = append(data.CertificateURLs, fmt.Sprintf("https://crt.sh/?q=%s", r.CertificateFingerprint))
+		if data.TlsPrivateKey == nil {
+			data.TlsPrivateKey = make([]*detectorspb.TlsPrivateKey, 0)
+		}
+		data.TlsPrivateKey = append(data.TlsPrivateKey, &detectorspb.TlsPrivateKey{
+			CertificateFingerprint: r.CertificateFingerprint,
+			ExpirationTimestamp:    r.ExpirationTimestamp.Unix(),
+			VerificationUrl:        fmt.Sprintf("https://crt.sh/?q=%s", r.CertificateFingerprint),
+		})
 		seen[r.CertificateFingerprint] = struct{}{}
 	}
 
-	return data, nil
+	for _, r := range results.GitHubSSHResults {
+		if _, ok := seen[r.Username]; ok {
+			continue
+		}
+		if data == nil {
+			data = &detectorspb.StructuredData{}
+		}
+		if data.GithubSshKey == nil {
+			data.GithubSshKey = make([]*detectorspb.GitHubSSHKey, 0)
+		}
+		data.GithubSshKey = append(data.GithubSshKey, &detectorspb.GitHubSSHKey{
+			User: r.Username,
+		})
+		seen[r.Username] = struct{}{}
+	}
+
+	return
 }
 
 type DriftwoodResult struct {

pkg/detectors/privatekey/privatekey_test.go

@@ -20,15 +20,13 @@ import (
 func TestPrivatekey_FromChunk(t *testing.T) {
 	ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
 	defer cancel()
-	testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors4")
+	testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")
 	if err != nil {
 		t.Fatalf("could not get test secrets from GCP: %s", err)
 	}
 	secretTLS := testSecrets.MustGetField("PRIVATEKEY_TLS")
 	secretGitHub := testSecrets.MustGetField("PRIVATEKEY_GITHUB")
-	secretGitHubEncrypted := testSecrets.MustGetField("PRIVATEKEY_GITHUB_ENCRYPTED")
 	secretInactive := testSecrets.MustGetField("PRIVATEKEY_UNVERIFIED")
-
 	type args struct {
 		ctx    context.Context
 		data   []byte
@@ -77,8 +75,19 @@ func TestPrivatekey_FromChunk(t *testing.T) {
 					DetectorType: detectorspb.DetectorType_PrivateKey,
 					Verified:     true,
 					Redacted:     "-----BEGIN PRIVATE KEY-----\nMIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgw",
-					ExtraData: map[string]string{
-						"certificate_urls": "https://crt.sh/?q=1e20c40deb44a8539dd3ac3e8c53b72750cb19f9, https://crt.sh/?q=0e9de31fb2ee16465a4d5d93b227d54f870326d1",
+					StructuredData: &detectorspb.StructuredData{
+						TlsPrivateKey: []*detectorspb.TlsPrivateKey{
+							{
+								CertificateFingerprint: "1e20c40deb44a8539dd3ac3e8c53b72750cb19f9",
+								VerificationUrl:        "https://crt.sh/?q=1e20c40deb44a8539dd3ac3e8c53b72750cb19f9",
+								ExpirationTimestamp:    1497337731,
+							},
+							{
+								CertificateFingerprint: "0e9de31fb2ee16465a4d5d93b227d54f870326d1",
+								VerificationUrl:        "https://crt.sh/?q=0e9de31fb2ee16465a4d5d93b227d54f870326d1",
+								ExpirationTimestamp:    1528192832,
+							},
+						},
 					},
 				},
 			},
@@ -97,8 +106,12 @@ func TestPrivatekey_FromChunk(t *testing.T) {
 					DetectorType: detectorspb.DetectorType_PrivateKey,
 					Verified:     true,
 					Redacted:     "-----BEGIN OPENSSH PRIVATE KEY-----\nb3BlbnNzaC1rZXktdjEAAAAABG5v",
-					ExtraData: map[string]string{
-						"github_user": "sirdetectsalot",
+					StructuredData: &detectorspb.StructuredData{
+						GithubSshKey: []*detectorspb.GitHubSSHKey{
+							{
+								User: "thisisforgithub0",
+							},
+						},
 					},
 				},
 			},
@@ -108,8 +121,47 @@ func TestPrivatekey_FromChunk(t *testing.T) {
 			name: "found encrypted GitHub SSH private key, verified",
 			s:    Scanner{},
 			args: args{
-				ctx:    context.Background(),
-				data:   []byte(secretGitHubEncrypted),
+				ctx: context.Background(),
+				data: []byte(`-----BEGIN OPENSSH PRIVATE KEY-----
+						b3BlbnNzaC1rZXktdjEAAAAACmFlczI1Ni1jdHIAAAAGYmNyeXB0AAAAGAAAABAjNIZuun
+						xgLkM8KuzfmQuRAAAAEAAAAAEAAAGXAAAAB3NzaC1yc2EAAAADAQABAAABgQDe3Al0EMPz
+						utVNk5DixaYrGMK56RqUoqGBinke6SWVWmqom1lBcJWzor6HlnMRPPr7YCEsJKL4IpuVwu
+						inRa5kdtNTyM7yyQTSR2xXCS0fUItNuq8pUktsH8VUggpMeew8hJv7rFA7tnIg3UXCl6iF
+						OLZKbDA5aa24idpcD8b1I9/RzTOB1fu0of5xd9vgODzGw5JvHQSJ0FaA42aNBMGwrDhDB3
+						sgnRNdWf6NNIh8KpXXMKJADf3klsyn6He8L2bPMp8a4wwys2YB35p5zQ0JURovsdewlOxH
+						NT7eP19eVf4dCreibxUmRUaob5DEoHEk8WrxjKWIYUuLeD6AfcW6oXyRU2Yy8Vrt6SqFl5
+						WAi47VMFTkDZYS/eCvG53q9UBHpCj7Qvb0vSkCZXBvBIhlw193F3PX4WvO1IXsMwvQ1D1X
+						lmomsItbqM0cJyKw6LU18QWiBHvE7BqcphaoL5E08W2ATTSRIMCp6rt4rptM7KyGK8rc6W
+						UYrCnWt6KlCA8AAAWQXk+lVx6bH5itIKKYmQr6cR/5xtZ2GHAxnYtvlW3xnGhU0MHv+lJ2
+						uoWlT2RXE5pdMUQj7rNWAMqkwifSKZs9wBfYeo1TaFDmC3nW7yHSN3XTuO78mPIW5JyvmE
+						Rj5qjsUn7fNmzECoAxnVERhwnF3KqUBEPzIAc6/7v/na9NTiiGaJPco9lvCoPWbVLN08WG
+						SuyU+0x5zc3ebzuPcYqu5/c5nmiGxhALrIhjIS0OV1mtAAFhvdMjMIHOijOzSKVCC7rRk5
+						kG9EMLNvOn/DUVSRHamw5gs2V3V+Zq2g5nYWfgq8aDSTB8XlIzOj1cz3HwfN6pfSNQ/3Qe
+						wOQfWfTWdO+JSL8aoBN5Wg8tDbgmvmbFrINsJfFfSm0wZgcHhC7Ul4U3v4c8PoNdK9HXwi
+						TKKzJ9nxLYb+vDh50cnkseu2gt0KwVpjIorxEqeK755mKPao3JmOMr6uFTQsb+g+ZNgPwl
+						nRHA4Igx+zADFj3twldnKIiRpBQ5J4acur3uQ+saanBTXgul1TiFiUGT2cnz+IiCsdPovg
+						TAMt868W5LmzpfH4Cy54JtaRC4/UuMnkTGbWgutVDnWj2stOAzsQ1YmhH5igUmc94mUL+W
+						8vQDCKpeI8n+quDS9zxTvy4L4H5Iz7OZlh0h6N13BDvCYXKcNF/ugkfxZbu8mZsZQQzXNR
+						wOrEtKoHc4AnXYNzsuHEoEyLyJxGfFRDSTLbyN9wFOS/c0k9Gjte+kQRZjBVGORE5sN6X3
+						akUnTF76RhbEc+LamrwM1h5340bwosRbR8I+UrsQdFfJBEj1ZSyMRJlMkFUNi6blt7bhyx
+						ea+Pm2A614nlYUBjw2KKzzn8N/0H2NpJjIptvDsbrx3BS/rKwOeJwavRrGnIlEzuAag4vx
+						Zb2TPVta45uz7fQP5IBl83b0BJKI5Zv/fniUeLI78W/UsZqb64YQbfRyBzFtI1T/SsCi0B
+						e0EyKMzbxtSceT1Mb8eJiVIq04Xpwez9fIUt5rSedZD8KPq8P6s0cGsR7Qmw6eXZ/dBR/a
+						s5vPhfIUmQawmnwAVuWNRdQQ79jUBSn5M+ZRVVTgEG+vFyvxr/bZqOo1JCoq5BmQhLWGRJ
+						Dk9TolbeFIVFrkuXkcu99a079ux7XSkON64oPzHrcsEzjPA1GPqs9CGBSO16wq/nI3zg+E
+						kcOCaurc9yHJJPwduem0+8WLX3WoGNfQRKurtQze2ppy8KarEtDhDd96sKkhYaqOg3GOX8
+						Yx827L4vuWSJSIqKuO2kH6kOCMUNO16piv0z/8u3CJxOGh9+4FZIop81fiFTKLhV3/gwLm
+						fzFY++KIZrLfZcUjzd80NNEja69F452Eb9HrI5BurN/PznDEi9bzM598Y7beyl4/kd4R2e
+						S7SW9/LOrGw5UgxtiU+kV8nPz1PdgxO4sRlnntSBEwkQBzMkLOpq2h2BuJ2TlMP/TWuwLQ
+						sDkv1Yk1pD0roGmtMzbujnURGxqRJ8gUmuIot4hpfyRSssvnRQQZ3lQCQCwHiE+HJxXWf5
+						c58zOMjW7o21tI8e13uUnbRoQVJM9XYqk1usPXIkYPYL9uOw3AW/Zn+cnDrsXvTK9ZxgGD
+						/90b1BNwVqMlUK+QggHNwl5qD8eoXK5cDvav66te+E+V7FYFQ06w3tytRVz8SjoaiChN02
+						muIjvl6G7Hoj1hObM2t/ZheN1EShS11z868hhS6Mx7GvIdtkXuvdiBYMiBLOshJQxB8Mzx
+						iug9W+Di3upLf0UMC1TqADGphsIHRU7RbmHQ8Rwp7dogswmDfpRSapPt9p0D+6Ad5VBzi3
+						f3BPXj76UBLMEJCrZR1P28vnAA7AyNHaLvMPlWDMG5v3V/UV+ugyFcoBAOyjiQgYST8F3e
+						Hx7UPVlTK8dyvk1Z+Yw0nrfNClI=
+						-----END OPENSSH PRIVATE KEY-----
+		`),
 				verify: true,
 			},
 			want: []detectors.Result{
@@ -117,10 +169,12 @@ func TestPrivatekey_FromChunk(t *testing.T) {
 					DetectorType: detectorspb.DetectorType_PrivateKey,
 					Verified:     true,
 					Redacted:     "-----BEGIN OPENSSH PRIVATE KEY-----\nb3BlbnNzaC1rZXktdjEAAAAACmFl",
-					ExtraData: map[string]string{
-						"github_user":                   "sirdetectsalot",
-						"encrypted":                     "true",
-						"cracked_encryption_passphrase": "true",
+					StructuredData: &detectorspb.StructuredData{
+						GithubSshKey: []*detectorspb.GitHubSSHKey{
+							{
+								User: "thisisforgithub0",
+							},
+						},
 					},
 				},
 			},
@@ -208,7 +262,7 @@ func Test_lookupFingerprint(t *testing.T) {
 				t.Errorf("lookupFingerprint() error = %v, wantErr %v", err, tt.wantErr)
 				return
 			}
-			if !reflect.DeepEqual(gotFingerprints != nil && len(gotFingerprints.CertificateURLs) > 0, tt.wantFingerprints) {
+			if !reflect.DeepEqual(gotFingerprints != nil && len(gotFingerprints.TlsPrivateKey) > 0, tt.wantFingerprints) {
 				t.Errorf("lookupFingerprint() = %v, want %v", gotFingerprints, tt.wantFingerprints)
 			}
 		})

pkg/detectors/privatekey/ssh.go

@@ -1,125 +0,0 @@
-package privatekey
-
-import (
-	"bytes"
-	"errors"
-	"fmt"
-	"net"
-	"strings"
-	"time"
-
-	"golang.org/x/crypto/ssh"
-)
-
-// https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/githubs-ssh-key-fingerprints
-var githubFingerprints = map[string]string{
-	"SHA256:uNiVztksCsDhcc0u9e8BujQXVUpKZIDTMczCvj3tD2s": "RSA",
-	"SHA256:br9IjFspm1vxR3iA35FWE+4VTyz1hYVLIE2t1/CeyWQ": "DSA - deprecated",
-	"SHA256:p2QAMXNIC1TJYWeIOttrVc98/R1BUFWu3/LiyKgUfQM": "ECDSA",
-	"SHA256:+DiY3wvvV6TuJJhbpZisF/zLDA0zPMSvHdkr4UvCOqU": "ED25519",
-}
-
-// https://docs.gitlab.com/ee/user/gitlab_com/index.html#ssh-host-keys-fingerprints
-var gitlabFingerprints = map[string]string{
-	"SHA256:HbW3g8zUjNSksFbqTiUWPWg2Bq1x8xdGUrliXFzSnUw": "ECDSA",
-	"SHA256:eUXGGm1YGsMAS7vkcx6JOJdOGHPem5gQp4taiCfCLB8": "ED25519",
-	"SHA256:ROQFvPThGrW4RuWLoL9tq9I9zJ42fK4XywyRtbOz/EQ": "RSA",
-}
-
-func firstResponseFromSSH(parsedKey any, username, hostport string) (string, error) {
-	signer, err := ssh.NewSignerFromKey(parsedKey)
-	if err != nil {
-		return "", err
-	}
-
-	// Verify the server fingerprint to ensure that there is no MITM replay attack
-	config := &ssh.ClientConfig{
-		Timeout: 3 * time.Second,
-		User:    username,
-		Auth: []ssh.AuthMethod{
-			ssh.PublicKeys(signer),
-		},
-		HostKeyCallback: func(hostname string, _ net.Addr, key ssh.PublicKey) error {
-			switch hostname {
-			case "github.com:22":
-				fingerprint := fingerprintSSHPublicKey(key)
-				if _, ok := githubFingerprints[fingerprint]; !ok {
-					return fmt.Errorf("unknown host fingerprint for github.com, got %s", fingerprint)
-				}
-			case "gitlab.com:22":
-				fingerprint := fingerprintSSHPublicKey(key)
-				if _, ok := gitlabFingerprints[fingerprint]; !ok {
-					return fmt.Errorf("unknown host fingerprint for gitlab.com, got %s", fingerprint)
-				}
-			default:
-				return errors.New("unknown host in fingerprint db")
-			}
-			return nil
-		},
-	}
-
-	client, err := ssh.Dial("tcp", hostport, config)
-	if err != nil {
-		if strings.Contains(err.Error(), "unable to authenticate") {
-			return "", errPermissionDenied
-		}
-		return "", err
-	}
-	defer client.Close()
-
-	session, err := client.NewSession()
-	if err != nil {
-		return "", err
-	}
-	defer session.Close()
-
-	var output bytes.Buffer
-	session.Stderr = &output
-
-	err = session.Shell()
-	if err != nil {
-		return "", err
-	}
-	_ = session.Wait()
-
-	return output.String(), err
-}
-
-var errPermissionDenied = errors.New("permission denied")
-
-func verifyGitHubUser(parsedKey any) (*string, error) {
-	output, err := firstResponseFromSSH(parsedKey, "git", "github.com:22")
-	if err != nil {
-		return nil, err
-	}
-
-	if strings.Contains(output, "Permission denied") {
-		return nil, errPermissionDenied
-	}
-
-	if strings.Contains(output, "successfully authenticated") {
-		username := strings.TrimSuffix(strings.Split(output, " ")[1], "!")
-		return &username, nil
-	}
-
-	return nil, nil
-}
-
-func verifyGitLabUser(parsedKey any) (*string, error) {
-	output, err := firstResponseFromSSH(parsedKey, "git", "gitlab.com:22")
-	if err != nil {
-		return nil, err
-	}
-
-	if strings.Contains(output, "Permission denied") {
-		return nil, errPermissionDenied
-	}
-
-	if strings.Contains(output, "Welcome to GitLab") {
-		split := strings.Split(output, " ")
-		username := strings.TrimPrefix(strings.TrimSuffix(split[len(split)-1], "!"), "@")
-		return &username, nil
-	}
-
-	return nil, nil
-}

pkg/detectors/privatekey/ssh_test.go

@@ -1,34 +0,0 @@
-package privatekey
-
-import (
-	"context"
-	"testing"
-	"time"
-
-	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
-	"golang.org/x/crypto/ssh"
-)
-
-func TestFirstResponseFromSSH(t *testing.T) {
-	ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
-	defer cancel()
-	testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors4")
-	if err != nil {
-		t.Fatalf("could not get test secrets from GCP: %s", err)
-	}
-	secretGitHub := testSecrets.MustGetField("PRIVATEKEY_GITHUB")
-
-	parsedKey, err := ssh.ParseRawPrivateKey([]byte(normalize(secretGitHub)))
-	if err != nil {
-		t.Fatalf("could not parse test secret: %s", err)
-	}
-
-	output, err := firstResponseFromSSH(parsedKey, "git", "github.com:22")
-	if err != nil {
-		t.Fail()
-	}
-
-	if len(output) == 0 {
-		t.Fail()
-	}
-}

pkg/detectors/sendbirdorganizationapi/sendbirdorganizationapi.go

@@ -2,7 +2,6 @@ package sendbirdorganizationapi
 
 import (
 	"context"
-	"fmt"
 	"net/http"
 	"regexp"
 	"strings"
@@ -12,15 +11,13 @@ import (
 	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
 )
 
-type Scanner struct {
-	client *http.Client
-}
+type Scanner struct{}
 
 // Ensure the Scanner satisfies the interface at compile time.
 var _ detectors.Detector = (*Scanner)(nil)
 
 var (
-	defaultClient = common.SaneHttpClient()
+	client = common.SaneHttpClient()
 
 	// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
 	keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"sendbird"}) + `\b([0-9a-f]{24})\b`)
@@ -50,33 +47,25 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 		}
 
 		if verify {
-			client := s.client
-			if client == nil {
-				client = defaultClient
-			}
-
 			req, err := http.NewRequestWithContext(ctx, "GET", "https://gate.sendbird.com/api/v2/applications", nil)
 			if err != nil {
-				s1.VerificationError = err
+				continue
 			}
 			req.Header.Add("SENDBIRDORGANIZATIONAPITOKEN", resMatch)
 			res, err := client.Do(req)
-			if err != nil {
-				s1.VerificationError = err
-			} else {
+			if err == nil {
 				defer res.Body.Close()
 				if res.StatusCode >= 200 && res.StatusCode < 300 {
 					s1.Verified = true
-				} else if res.StatusCode != http.StatusForbidden {
-					s1.VerificationError = fmt.Errorf("unexpected HTTP response status %d", res.StatusCode)
+				} else {
+					// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
+					if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
+						continue
+					}
 				}
 			}
 		}
 
-		// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
-		if !s1.Verified && detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
-			continue
-		}
 		results = append(results, s1)
 	}
 

pkg/detectors/sendbirdorganizationapi/sendbirdorganizationapi_test.go

@@ -9,8 +9,7 @@ import (
 	"testing"
 	"time"
 
-	"github.com/google/go-cmp/cmp"
-	"github.com/google/go-cmp/cmp/cmpopts"
+	"github.com/kylelemons/godebug/pretty"
 
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
@@ -20,7 +19,7 @@ import (
 func TestSendbirdOrganizationAPI_FromChunk(t *testing.T) {
 	ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
 	defer cancel()
-	testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")
+	testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
 	if err != nil {
 		t.Fatalf("could not get test secrets from GCP: %s", err)
 	}
@@ -33,12 +32,11 @@ func TestSendbirdOrganizationAPI_FromChunk(t *testing.T) {
 		verify bool
 	}
 	tests := []struct {
-		name                string
-		s                   Scanner
-		args                args
-		want                []detectors.Result
-		wantErr             bool
-		wantVerificationErr bool
+		name    string
+		s       Scanner
+		args    args
+		want    []detectors.Result
+		wantErr bool
 	}{
 		{
 			name: "found, verified",
@@ -54,8 +52,7 @@ func TestSendbirdOrganizationAPI_FromChunk(t *testing.T) {
 					Verified:     true,
 				},
 			},
-			wantErr:             false,
-			wantVerificationErr: false,
+			wantErr: false,
 		},
 		{
 			name: "found, unverified",
@@ -71,42 +68,7 @@ func TestSendbirdOrganizationAPI_FromChunk(t *testing.T) {
 					Verified:     false,
 				},
 			},
-			wantErr:             false,
-			wantVerificationErr: false,
-		},
-		{
-			name: "found, would be verified if not for timeout",
-			s:    Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},
-			args: args{
-				ctx:    context.Background(),
-				data:   []byte(fmt.Sprintf("You can find a sendbird secret %s within", secret)),
-				verify: true,
-			},
-			want: []detectors.Result{
-				{
-					DetectorType: detectorspb.DetectorType_SendbirdOrganizationAPI,
-					Verified:     false,
-				},
-			},
-			wantErr:             false,
-			wantVerificationErr: true,
-		},
-		{
-			name: "found, verified but unexpected api surface",
-			s:    Scanner{client: common.ConstantResponseHttpClient(404, "")},
-			args: args{
-				ctx:    context.Background(),
-				data:   []byte(fmt.Sprintf("You can find a sendbird secret %s within", secret)),
-				verify: true,
-			},
-			want: []detectors.Result{
-				{
-					DetectorType: detectorspb.DetectorType_SendbirdOrganizationAPI,
-					Verified:     false,
-				},
-			},
-			wantErr:             false,
-			wantVerificationErr: true,
+			wantErr: false,
 		},
 		{
 			name: "not found",
@@ -122,23 +84,20 @@ func TestSendbirdOrganizationAPI_FromChunk(t *testing.T) {
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+			s := Scanner{}
+			got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
 			if (err != nil) != tt.wantErr {
 				t.Errorf("SendbirdOrganizationAPI.FromData() error = %v, wantErr %v", err, tt.wantErr)
 				return
 			}
-
 			for i := range got {
 				if len(got[i].Raw) == 0 {
 					t.Fatalf("no raw secret present: \n %+v", got[i])
 				}
-				if (got[i].VerificationError != nil) != tt.wantVerificationErr {
-					t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError)
-				}
+				got[i].Raw = nil
 			}
-			ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "RawV2", "VerificationError")
-			if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {
-				t.Errorf("Sendbird.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+			if diff := pretty.Compare(got, tt.want); diff != "" {
+				t.Errorf("SendbirdOrganizationAPI.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
 			}
 		})
 	}

pkg/detectors/slackwebhook/slackwebhook.go

@@ -1,7 +1,6 @@
 package slackwebhook
 
 import (
-	"bytes"
 	"context"
 	"fmt"
 	"io"
@@ -70,10 +69,11 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 				if err != nil {
 					continue
 				}
+				body := string(bodyBytes)
 
 				defer res.Body.Close()
 
-				if res.StatusCode >= 200 && res.StatusCode < 300 || (res.StatusCode == 400 && (bytes.Equal(bodyBytes, []byte("no_text")) || bytes.Equal(bodyBytes, []byte("missing_text")))) {
+				if res.StatusCode >= 200 && res.StatusCode < 300 || (res.StatusCode == 400 && (strings.Contains(body, "no_text") || strings.Contains(body, "missing_text"))) {
 					s1.Verified = true
 				} else if res.StatusCode == 401 || res.StatusCode == 403 {
 					// The secret is determinately not verified (nothing to do)

pkg/detectors/slackwebhook/slackwebhook_test.go

@@ -6,14 +6,14 @@ package slackwebhook
 import (
 	"context"
 	"fmt"
+	"github.com/google/go-cmp/cmp"
+	"github.com/google/go-cmp/cmp/cmpopts"
 	"testing"
 	"time"
 
-	"github.com/google/go-cmp/cmp"
-	"github.com/google/go-cmp/cmp/cmpopts"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
 
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
-	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
 )
 
@@ -57,40 +57,6 @@ func TestSlackWebhook_FromChunk(t *testing.T) {
 			wantErr:             false,
 			wantVerificationErr: false,
 		},
-		{
-			name: "found, verified, 400 no_text",
-			s:    Scanner{client: common.ConstantResponseHttpClient(400, "no_text")},
-			args: args{
-				ctx:    context.Background(),
-				data:   []byte(fmt.Sprintf("You can find a slackwebhook secret %s within", secret)),
-				verify: true,
-			},
-			want: []detectors.Result{
-				{
-					DetectorType: detectorspb.DetectorType_SlackWebhook,
-					Verified:     true,
-				},
-			},
-			wantErr:             false,
-			wantVerificationErr: false,
-		},
-		{
-			name: "found, verified, 400 missing_text",
-			s:    Scanner{client: common.ConstantResponseHttpClient(400, "missing_text")},
-			args: args{
-				ctx:    context.Background(),
-				data:   []byte(fmt.Sprintf("You can find a slackwebhook secret %s within", secret)),
-				verify: true,
-			},
-			want: []detectors.Result{
-				{
-					DetectorType: detectorspb.DetectorType_SlackWebhook,
-					Verified:     true,
-				},
-			},
-			wantErr:             false,
-			wantVerificationErr: false,
-		},
 		{
 			name: "found, unverified",
 			s:    Scanner{},
@@ -154,23 +120,6 @@ func TestSlackWebhook_FromChunk(t *testing.T) {
 			wantErr:             false,
 			wantVerificationErr: true,
 		},
-		{
-			name: "found, account disabled",
-			s:    Scanner{client: common.ConstantResponseHttpClient(400, disabledAccountResponse)},
-			args: args{
-				ctx:    context.Background(),
-				data:   []byte(fmt.Sprintf("You can find a slackwebhook secret %s within", secret)),
-				verify: true,
-			},
-			want: []detectors.Result{
-				{
-					DetectorType: detectorspb.DetectorType_SlackWebhook,
-					Verified:     false,
-				},
-			},
-			wantErr:             false,
-			wantVerificationErr: true,
-		},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
@@ -195,8 +144,6 @@ func TestSlackWebhook_FromChunk(t *testing.T) {
 	}
 }
 
-const disabledAccountResponse = `missing_text_or_fallback_or_attachments`
-
 func BenchmarkFromData(benchmark *testing.B) {
 	ctx := context.Background()
 	s := Scanner{}

pkg/engine/circleci.go

@@ -8,6 +8,7 @@ import (
 
 	"github.com/trufflesecurity/trufflehog/v3/pkg/context"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/sources/circleci"
 )
 
@@ -26,13 +27,17 @@ func (e *Engine) ScanCircleCI(ctx context.Context, token string) error {
 		return err
 	}
 
-	sourceName := "trufflehog - Circle CI"
-	sourceID, jobID, _ := e.sourceManager.GetIDs(ctx, sourceName, circleci.SourceType)
-
-	circleSource := &circleci.Source{}
-	if err := circleSource.Init(ctx, "trufflehog - Circle CI", jobID, sourceID, true, &conn, runtime.NumCPU()); err != nil {
+	handle, err := e.sourceManager.Enroll(ctx, "trufflehog - Circle CI", new(circleci.Source).Type(),
+		func(ctx context.Context, jobID, sourceID int64) (sources.Source, error) {
+			circleSource := circleci.Source{}
+			if err := circleSource.Init(ctx, "trufflehog - Circle CI", jobID, sourceID, true, &conn, runtime.NumCPU()); err != nil {
+				return nil, err
+			}
+			return &circleSource, nil
+		})
+	if err != nil {
 		return err
 	}
-	_, err = e.sourceManager.Run(ctx, sourceName, circleSource)
+	_, err = e.sourceManager.ScheduleRun(ctx, handle)
 	return err
 }

pkg/engine/docker.go

@@ -6,18 +6,23 @@ import (
 	"google.golang.org/protobuf/types/known/anypb"
 
 	"github.com/trufflesecurity/trufflehog/v3/pkg/context"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/sources/docker"
 )
 
 // ScanDocker scans a given docker connection.
 func (e *Engine) ScanDocker(ctx context.Context, conn *anypb.Any) error {
-	sourceName := "trufflehog - docker"
-	sourceID, jobID, _ := e.sourceManager.GetIDs(ctx, sourceName, docker.SourceType)
-
-	dockerSource := &docker.Source{}
-	if err := dockerSource.Init(ctx, sourceName, jobID, sourceID, true, conn, runtime.NumCPU()); err != nil {
+	handle, err := e.sourceManager.Enroll(ctx, "trufflehog - docker", new(docker.Source).Type(),
+		func(ctx context.Context, jobID, sourceID int64) (sources.Source, error) {
+			dockerSource := docker.Source{}
+			if err := dockerSource.Init(ctx, "trufflehog - docker", jobID, sourceID, true, conn, runtime.NumCPU()); err != nil {
+				return nil, err
+			}
+			return &dockerSource, nil
+		})
+	if err != nil {
 		return err
 	}
-	_, err := e.sourceManager.Run(ctx, sourceName, dockerSource)
+	_, err = e.sourceManager.ScheduleRun(ctx, handle)
 	return err
 }

pkg/engine/filesystem.go

@@ -24,14 +24,18 @@ func (e *Engine) ScanFileSystem(ctx context.Context, c sources.FilesystemConfig)
 		return err
 	}
 
-	sourceName := "trufflehog - filesystem"
-	sourceID, jobID, _ := e.sourceManager.GetIDs(ctx, sourceName, filesystem.SourceType)
-
-	fileSystemSource := &filesystem.Source{}
-	fileSystemSource.WithFilter(c.Filter)
-	if err := fileSystemSource.Init(ctx, sourceName, jobID, sourceID, true, &conn, runtime.NumCPU()); err != nil {
+	handle, err := e.sourceManager.Enroll(ctx, "trufflehog - filesystem", new(filesystem.Source).Type(),
+		func(ctx context.Context, jobID, sourceID int64) (sources.Source, error) {
+			fileSystemSource := filesystem.Source{}
+			fileSystemSource.WithFilter(c.Filter)
+			if err := fileSystemSource.Init(ctx, "trufflehog - filesystem", jobID, sourceID, true, &conn, runtime.NumCPU()); err != nil {
+				return nil, err
+			}
+			return &fileSystemSource, nil
+		})
+	if err != nil {
 		return err
 	}
-	_, err = e.sourceManager.Run(ctx, sourceName, fileSystemSource)
+	_, err = e.sourceManager.ScheduleRun(ctx, handle)
 	return err
 }

pkg/engine/gcs.go

@@ -44,14 +44,18 @@ func (e *Engine) ScanGCS(ctx context.Context, c sources.GCSConfig) error {
 		return fmt.Errorf("failed to marshal GCS connection: %w", err)
 	}
 
-	sourceName := "trufflehog - gcs"
-	sourceID, jobID, _ := e.sourceManager.GetIDs(ctx, sourceName, gcs.SourceType)
-
-	gcsSource := &gcs.Source{}
-	if err := gcsSource.Init(ctx, sourceName, jobID, sourceID, true, &conn, int(c.Concurrency)); err != nil {
+	handle, err := e.sourceManager.Enroll(ctx, "trufflehog - gcs", new(gcs.Source).Type(),
+		func(ctx context.Context, jobID, sourceID int64) (sources.Source, error) {
+			gcsSource := gcs.Source{}
+			if err := gcsSource.Init(ctx, "trufflehog - gcs", jobID, sourceID, true, &conn, int(c.Concurrency)); err != nil {
+				return nil, err
+			}
+			return &gcsSource, nil
+		})
+	if err != nil {
 		return err
 	}
-	_, err = e.sourceManager.Run(ctx, sourceName, gcsSource)
+	_, err = e.sourceManager.ScheduleRun(ctx, handle)
 	return err
 }
 

pkg/engine/git.go

@@ -51,18 +51,21 @@ func (e *Engine) ScanGit(ctx context.Context, c sources.GitConfig) error {
 		return err
 	}
 
-	sourceName := "trufflehog - git"
-	sourceID, jobID, _ := e.sourceManager.GetIDs(ctx, sourceName, git.SourceType)
-
-	gitSource := &git.Source{}
-	if err := gitSource.Init(ctx, sourceName, jobID, sourceID, true, &conn, runtime.NumCPU()); err != nil {
+	handle, err := e.sourceManager.Enroll(ctx, "trufflehog - git", new(git.Source).Type(),
+		func(ctx context.Context, jobID, sourceID int64) (sources.Source, error) {
+			gitSource := git.Source{}
+			if err := gitSource.Init(ctx, "trufflehog - git", jobID, sourceID, true, &conn, runtime.NumCPU()); err != nil {
+				return nil, err
+			}
+			gitSource.WithScanOptions(scanOptions)
+			// Don't try to clean up the provided directory. That's handled by the
+			// caller of ScanGit.
+			gitSource.WithPreserveTempDirs(true)
+			return &gitSource, nil
+		})
+	if err != nil {
 		return err
 	}
-	gitSource.WithScanOptions(scanOptions)
-	// Don't try to clean up the provided directory. That's handled by the
-	// caller of ScanGit.
-	gitSource.WithPreserveTempDirs(true)
-
-	_, err := e.sourceManager.Run(ctx, sourceName, gitSource)
+	_, err = e.sourceManager.ScheduleRun(ctx, handle)
 	return err
 }

pkg/engine/github.go

@@ -47,14 +47,18 @@ func (e *Engine) ScanGitHub(ctx context.Context, c sources.GithubConfig) error {
 	}
 	scanOptions := git.NewScanOptions(opts...)
 
-	sourceName := "trufflehog - github"
-	sourceID, jobID, _ := e.sourceManager.GetIDs(ctx, sourceName, github.SourceType)
-
-	githubSource := &github.Source{}
-	if err := githubSource.Init(ctx, sourceName, jobID, sourceID, true, &conn, c.Concurrency); err != nil {
+	handle, err := e.sourceManager.Enroll(ctx, "trufflehog - github", new(github.Source).Type(),
+		func(ctx context.Context, jobID, sourceID int64) (sources.Source, error) {
+			githubSource := github.Source{}
+			if err := githubSource.Init(ctx, "trufflehog - github", jobID, sourceID, true, &conn, c.Concurrency); err != nil {
+				return nil, err
+			}
+			githubSource.WithScanOptions(scanOptions)
+			return &githubSource, nil
+		})
+	if err != nil {
 		return err
 	}
-	githubSource.WithScanOptions(scanOptions)
-	_, err = e.sourceManager.Run(ctx, sourceName, githubSource)
+	_, err = e.sourceManager.ScheduleRun(ctx, handle)
 	return err
 }

pkg/engine/gitlab.go

@@ -50,14 +50,18 @@ func (e *Engine) ScanGitLab(ctx context.Context, c sources.GitlabConfig) error {
 		return err
 	}
 
-	sourceName := "trufflehog - gitlab"
-	sourceID, jobID, _ := e.sourceManager.GetIDs(ctx, sourceName, gitlab.SourceType)
-
-	gitlabSource := &gitlab.Source{}
-	if err := gitlabSource.Init(ctx, sourceName, jobID, sourceID, true, &conn, runtime.NumCPU()); err != nil {
+	handle, err := e.sourceManager.Enroll(ctx, "trufflehog - gitlab", new(gitlab.Source).Type(),
+		func(ctx context.Context, jobID, sourceID int64) (sources.Source, error) {
+			gitlabSource := gitlab.Source{}
+			if err := gitlabSource.Init(ctx, "trufflehog - gitlab", jobID, sourceID, true, &conn, runtime.NumCPU()); err != nil {
+				return nil, err
+			}
+			gitlabSource.WithScanOptions(scanOptions)
+			return &gitlabSource, nil
+		})
+	if err != nil {
 		return err
 	}
-	gitlabSource.WithScanOptions(scanOptions)
-	_, err = e.sourceManager.Run(ctx, sourceName, gitlabSource)
+	_, err = e.sourceManager.ScheduleRun(ctx, handle)
 	return err
 }

pkg/engine/s3.go

@@ -58,13 +58,17 @@ func (e *Engine) ScanS3(ctx context.Context, c sources.S3Config) error {
 		return err
 	}
 
-	sourceName := "trufflehog - s3"
-	sourceID, jobID, _ := e.sourceManager.GetIDs(ctx, sourceName, s3.SourceType)
-
-	s3Source := &s3.Source{}
-	if err := s3Source.Init(ctx, sourceName, jobID, sourceID, true, &conn, runtime.NumCPU()); err != nil {
+	handle, err := e.sourceManager.Enroll(ctx, "trufflehog - s3", new(s3.Source).Type(),
+		func(ctx context.Context, jobID, sourceID int64) (sources.Source, error) {
+			s3Source := s3.Source{}
+			if err := s3Source.Init(ctx, "trufflehog - s3", jobID, sourceID, true, &conn, runtime.NumCPU()); err != nil {
+				return nil, err
+			}
+			return &s3Source, nil
+		})
+	if err != nil {
 		return err
 	}
-	_, err = e.sourceManager.Run(ctx, sourceName, s3Source)
+	_, err = e.sourceManager.ScheduleRun(ctx, handle)
 	return err
 }

pkg/engine/syslog.go

@@ -41,14 +41,18 @@ func (e *Engine) ScanSyslog(ctx context.Context, c sources.SyslogConfig) error {
 		return errors.WrapPrefix(err, "error unmarshalling connection", 0)
 	}
 
-	sourceName := "trufflehog - syslog"
-	sourceID, jobID, _ := e.sourceManager.GetIDs(ctx, sourceName, syslog.SourceType)
-	syslogSource := &syslog.Source{}
-	if err := syslogSource.Init(ctx, sourceName, jobID, sourceID, true, &conn, c.Concurrency); err != nil {
+	handle, err := e.sourceManager.Enroll(ctx, "trufflehog - syslog", new(syslog.Source).Type(),
+		func(ctx context.Context, jobID, sourceID int64) (sources.Source, error) {
+			syslogSource := syslog.Source{}
+			if err := syslogSource.Init(ctx, "trufflehog - syslog", jobID, sourceID, true, &conn, c.Concurrency); err != nil {
+				return nil, err
+			}
+			syslogSource.InjectConnection(connection)
+			return &syslogSource, nil
+		})
+	if err != nil {
 		return err
 	}
-	syslogSource.InjectConnection(connection)
-
-	_, err = e.sourceManager.Run(ctx, sourceName, syslogSource)
+	_, err = e.sourceManager.ScheduleRun(ctx, handle)
 	return err
 }

pkg/handlers/archive.go

@@ -102,10 +102,6 @@ func (a *Archive) openArchive(ctx context.Context, depth int, reader io.Reader,
 		return a.handleNonArchiveContent(ctx, arReader, archiveChan)
 	}
 
-	if err != nil {
-		return err
-	}
-
 	switch archive := format.(type) {
 	case archiver.Decompressor:
 		info := decompressorInfo{depth: depth, reader: arReader, archiveChan: archiveChan, archiver: archive}

pkg/handlers/archive_test.go

@@ -236,15 +236,3 @@ func TestExtractRPMContent(t *testing.T) {
 	expectedLength := 1822720
 	assert.Equal(t, expectedLength, len(string(content)))
 }
-
-func TestOpenInvalidArchive(t *testing.T) {
-	reader := strings.NewReader("invalid archive")
-
-	ctx := logContext.AddLogger(context.Background())
-	a := &Archive{}
-
-	archiveChan := make(chan []byte)
-
-	err := a.openArchive(ctx, 0, reader, archiveChan)
-	assert.Error(t, err)
-}

pkg/output/json.go

@@ -10,7 +10,6 @@ import (
 	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/source_metadatapb"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
-	"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
 )
 
 // JSONPrinter is a printer that prints results in JSON format.
@@ -21,7 +20,7 @@ func (p *JSONPrinter) Print(_ context.Context, r *detectors.ResultWithMetadata)
 		// SourceMetadata contains source-specific contextual information.
 		SourceMetadata *source_metadatapb.MetaData
 		// SourceID is the ID of the source that the API uses to map secrets to specific sources.
-		SourceID sources.SourceID
+		SourceID int64
 		// SourceType is the type of Source.
 		SourceType sourcespb.SourceType
 		// SourceName is the name of the Source.

pkg/sources/circleci/circleci.go

@@ -19,17 +19,13 @@ import (
 	"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
 )
 
-const (
-	SourceType = sourcespb.SourceType_SOURCE_TYPE_CIRCLECI
-
-	baseURL = "https://circleci.com/api/v1.1/"
-)
+const baseURL = "https://circleci.com/api/v1.1/"
 
 type Source struct {
 	name     string
 	token    string
-	sourceId sources.SourceID
-	jobId    sources.JobID
+	sourceId int64
+	jobId    int64
 	verify   bool
 	jobPool  *errgroup.Group
 	sources.Progress
@@ -44,19 +40,19 @@ var _ sources.SourceUnitUnmarshaller = (*Source)(nil)
 // Type returns the type of source.
 // It is used for matching source types in configuration and job input.
 func (s *Source) Type() sourcespb.SourceType {
-	return SourceType
+	return sourcespb.SourceType_SOURCE_TYPE_CIRCLECI
 }
 
-func (s *Source) SourceID() sources.SourceID {
+func (s *Source) SourceID() int64 {
 	return s.sourceId
 }
 
-func (s *Source) JobID() sources.JobID {
+func (s *Source) JobID() int64 {
 	return s.jobId
 }
 
 // Init returns an initialized CircleCI source.
-func (s *Source) Init(_ context.Context, name string, jobId sources.JobID, sourceId sources.SourceID, verify bool, connection *anypb.Any, concurrency int) error {
+func (s *Source) Init(_ context.Context, name string, jobId, sourceId int64, verify bool, connection *anypb.Any, concurrency int) error {
 	s.name = name
 	s.sourceId = sourceId
 	s.jobId = jobId

pkg/sources/docker/docker.go

@@ -24,12 +24,10 @@ import (
 	"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
 )
 
-const SourceType = sourcespb.SourceType_SOURCE_TYPE_DOCKER
-
 type Source struct {
 	name        string
-	sourceId    sources.SourceID
-	jobId       sources.JobID
+	sourceId    int64
+	jobId       int64
 	verify      bool
 	concurrency int
 	conn        sourcespb.Docker
@@ -44,19 +42,19 @@ var _ sources.SourceUnitUnmarshaller = (*Source)(nil)
 // Type returns the type of source.
 // It is used for matching source types in configuration and job input.
 func (s *Source) Type() sourcespb.SourceType {
-	return SourceType
+	return sourcespb.SourceType_SOURCE_TYPE_DOCKER
 }
 
-func (s *Source) SourceID() sources.SourceID {
+func (s *Source) SourceID() int64 {
 	return s.sourceId
 }
 
-func (s *Source) JobID() sources.JobID {
+func (s *Source) JobID() int64 {
 	return s.jobId
 }
 
 // Init initializes the source.
-func (s *Source) Init(_ context.Context, name string, jobId sources.JobID, sourceId sources.SourceID, verify bool, connection *anypb.Any, concurrency int) error {
+func (s *Source) Init(_ context.Context, name string, jobId, sourceId int64, verify bool, connection *anypb.Any, concurrency int) error {
 	s.name = name
 	s.sourceId = sourceId
 	s.jobId = jobId

pkg/sources/filesystem/filesystem.go

@@ -22,12 +22,10 @@ import (
 	"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
 )
 
-const SourceType = sourcespb.SourceType_SOURCE_TYPE_FILESYSTEM
-
 type Source struct {
 	name     string
-	sourceId sources.SourceID
-	jobId    sources.JobID
+	sourceId int64
+	jobId    int64
 	verify   bool
 	paths    []string
 	log      logr.Logger
@@ -45,19 +43,19 @@ var _ sources.SourceUnitChunker = (*Source)(nil)
 // Type returns the type of source.
 // It is used for matching source types in configuration and job input.
 func (s *Source) Type() sourcespb.SourceType {
-	return SourceType
+	return sourcespb.SourceType_SOURCE_TYPE_FILESYSTEM
 }
 
-func (s *Source) SourceID() sources.SourceID {
+func (s *Source) SourceID() int64 {
 	return s.sourceId
 }
 
-func (s *Source) JobID() sources.JobID {
+func (s *Source) JobID() int64 {
 	return s.jobId
 }
 
 // Init returns an initialized Filesystem source.
-func (s *Source) Init(aCtx context.Context, name string, jobId sources.JobID, sourceId sources.SourceID, verify bool, connection *anypb.Any, _ int) error {
+func (s *Source) Init(aCtx context.Context, name string, jobId, sourceId int64, verify bool, connection *anypb.Any, _ int) error {
 	s.log = aCtx.Logger()
 
 	s.name = name

pkg/sources/gcs/gcs.go

@@ -28,11 +28,7 @@ import (
 	"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
 )
 
-const (
-	SourceType = sourcespb.SourceType_SOURCE_TYPE_GCS
-
-	defaultCachePersistIncrement = 2500
-)
+const defaultCachePersistIncrement = 2500
 
 // Ensure the Source satisfies the interfaces at compile time.
 var _ sources.Source = (*Source)(nil)
@@ -41,16 +37,16 @@ var _ sources.SourceUnitUnmarshaller = (*Source)(nil)
 // Type returns the type of source.
 // It is used for matching source types in configuration and job input.
 func (s *Source) Type() sourcespb.SourceType {
-	return SourceType
+	return sourcespb.SourceType_SOURCE_TYPE_GCS
 }
 
 // SourceID number for GCS Source.
-func (s *Source) SourceID() sources.SourceID {
+func (s *Source) SourceID() int64 {
 	return s.sourceId
 }
 
 // JobID number for GCS Source.
-func (s *Source) JobID() sources.JobID {
+func (s *Source) JobID() int64 {
 	return s.jobId
 }
 
@@ -62,8 +58,8 @@ type objectManager interface {
 // Source represents a GCS source.
 type Source struct {
 	name        string
-	jobId       sources.JobID
-	sourceId    sources.SourceID
+	jobId       int64
+	sourceId    int64
 	concurrency int
 	verify      bool
 
@@ -111,7 +107,7 @@ func (c *persistableCache) shouldPersist() (bool, string) {
 }
 
 // Init returns an initialized GCS source.
-func (s *Source) Init(aCtx context.Context, name string, id sources.JobID, sourceID sources.SourceID, verify bool, connection *anypb.Any, concurrency int) error {
+func (s *Source) Init(aCtx context.Context, name string, id int64, sourceID int64, verify bool, connection *anypb.Any, concurrency int) error {
 	s.log = aCtx.Logger()
 
 	s.name = name

pkg/sources/git/git.go

@@ -34,12 +34,10 @@ import (
 	"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
 )
 
-const SourceType = sourcespb.SourceType_SOURCE_TYPE_GIT
-
 type Source struct {
 	name     string
-	sourceId sources.SourceID
-	jobId    sources.JobID
+	sourceId int64
+	jobId    int64
 	verify   bool
 	git      *Git
 	sources.Progress
@@ -53,8 +51,8 @@ type Source struct {
 type Git struct {
 	sourceType         sourcespb.SourceType
 	sourceName         string
-	sourceID           sources.SourceID
-	jobID              sources.JobID
+	sourceID           int64
+	jobID              int64
 	sourceMetadataFunc func(file, email, commit, timestamp, repository string, line int64) *source_metadatapb.MetaData
 	verify             bool
 	metrics            metrics
@@ -65,7 +63,7 @@ type metrics struct {
 	commitsScanned uint64
 }
 
-func NewGit(sourceType sourcespb.SourceType, jobID sources.JobID, sourceID sources.SourceID, sourceName string, verify bool, concurrency int,
+func NewGit(sourceType sourcespb.SourceType, jobID, sourceID int64, sourceName string, verify bool, concurrency int,
 	sourceMetadataFunc func(file, email, commit, timestamp, repository string, line int64) *source_metadatapb.MetaData,
 ) *Git {
 	return &Git{
@@ -86,14 +84,14 @@ var _ sources.SourceUnitUnmarshaller = (*Source)(nil)
 // Type returns the type of source.
 // It is used for matching source types in configuration and job input.
 func (s *Source) Type() sourcespb.SourceType {
-	return SourceType
+	return sourcespb.SourceType_SOURCE_TYPE_GIT
 }
 
-func (s *Source) SourceID() sources.SourceID {
+func (s *Source) SourceID() int64 {
 	return s.sourceId
 }
 
-func (s *Source) JobID() sources.JobID {
+func (s *Source) JobID() int64 {
 	return s.jobId
 }
 
@@ -111,7 +109,7 @@ func (s *Source) WithPreserveTempDirs(preserve bool) {
 }
 
 // Init returns an initialized GitHub source.
-func (s *Source) Init(aCtx context.Context, name string, jobId sources.JobID, sourceId sources.SourceID, verify bool, connection *anypb.Any, concurrency int) error {
+func (s *Source) Init(aCtx context.Context, name string, jobId, sourceId int64, verify bool, connection *anypb.Any, concurrency int) error {
 	s.name = name
 	s.sourceId = sourceId
 	s.jobId = jobId
@@ -372,32 +370,6 @@ func CloneRepo(ctx context.Context, userInfo *url.Userinfo, gitUrl string, args
 	return clonePath, repo, nil
 }
 
-// PingRepoUsingToken executes git ls-remote on a repo and returns any error that occurs. It can be used to validate
-// that a repo actually exists and is reachable.
-//
-// Pinging using other authentication methods is only unimplemented because there's been no pressing need for it yet.
-func PingRepoUsingToken(ctx context.Context, token, gitUrl, user string) error {
-	if err := GitCmdCheck(); err != nil {
-		return err
-	}
-	lsUrl, err := GitURLParse(gitUrl)
-	if err != nil {
-		return err
-	}
-	if lsUrl.User == nil {
-		lsUrl.User = url.UserPassword(user, token)
-	}
-
-	// We don't actually care about any refs on the remote, we just care whether can can list them at all. So we query
-	// only for a ref that we know won't exist to minimize the search time on the remote. (By default, ls-remote exits
-	// with 0 even if it doesn't find any matching refs.)
-	fakeRef := "TRUFFLEHOG_CHECK_GIT_REMOTE_URL_REACHABILITY"
-	gitArgs := []string{"ls-remote", lsUrl.String(), "--quiet", fakeRef}
-	cmd := exec.Command("git", gitArgs...)
-	_, err = cmd.CombinedOutput()
-	return err
-}
-
 // CloneRepoUsingToken clones a repo using a provided token.
 func CloneRepoUsingToken(ctx context.Context, token, gitUrl, user string, args ...string) (string, *git.Repository, error) {
 	userInfo := url.UserPassword(user, token)

pkg/sources/github/github.go

@@ -39,8 +39,6 @@ import (
 )
 
 const (
-	SourceType = sourcespb.SourceType_SOURCE_TYPE_GITHUB
-
 	unauthGithubOrgRateLimt = 30
 	defaultPagination       = 100
 	membersAppPagination    = 500
@@ -53,8 +51,8 @@ type Source struct {
 	githubUser  string
 	githubToken string
 
-	sourceID          sources.SourceID
-	jobID             sources.JobID
+	sourceID          int64
+	jobID             int64
 	verify            bool
 	repos             []string
 	members           []string
@@ -106,14 +104,14 @@ var endsWithGithub = regexp.MustCompile(`github\.com/?$`)
 // Type returns the type of source.
 // It is used for matching source types in configuration and job input.
 func (s *Source) Type() sourcespb.SourceType {
-	return SourceType
+	return sourcespb.SourceType_SOURCE_TYPE_GITHUB
 }
 
-func (s *Source) SourceID() sources.SourceID {
+func (s *Source) SourceID() int64 {
 	return s.sourceID
 }
 
-func (s *Source) JobID() sources.JobID {
+func (s *Source) JobID() int64 {
 	return s.jobID
 }
 
@@ -202,7 +200,7 @@ func (c *filteredRepoCache) includeRepo(s string) bool {
 }
 
 // Init returns an initialized GitHub source.
-func (s *Source) Init(aCtx context.Context, name string, jobID sources.JobID, sourceID sources.SourceID, verify bool, connection *anypb.Any, concurrency int) error {
+func (s *Source) Init(aCtx context.Context, name string, jobID, sourceID int64, verify bool, connection *anypb.Any, concurrency int) error {
 	s.log = aCtx.Logger()
 
 	s.name = name

pkg/sources/gitlab/gitlab.go

@@ -29,12 +29,10 @@ import (
 	"google.golang.org/protobuf/types/known/anypb"
 )
 
-const SourceType = sourcespb.SourceType_SOURCE_TYPE_GITLAB
-
 type Source struct {
 	name            string
-	sourceId        sources.SourceID
-	jobId           sources.JobID
+	sourceId        int64
+	jobId           int64
 	verify          bool
 	authMethod      string
 	user            string
@@ -55,24 +53,23 @@ type Source struct {
 // Ensure the Source satisfies the interfaces at compile time.
 var _ sources.Source = (*Source)(nil)
 var _ sources.SourceUnitUnmarshaller = (*Source)(nil)
-var _ sources.Validator = (*Source)(nil)
 
 // Type returns the type of source.
 // It is used for matching source types in configuration and job input.
 func (s *Source) Type() sourcespb.SourceType {
-	return SourceType
+	return sourcespb.SourceType_SOURCE_TYPE_GITLAB
 }
 
-func (s *Source) SourceID() sources.SourceID {
+func (s *Source) SourceID() int64 {
 	return s.sourceId
 }
 
-func (s *Source) JobID() sources.JobID {
+func (s *Source) JobID() int64 {
 	return s.jobId
 }
 
 // Init returns an initialized Gitlab source.
-func (s *Source) Init(_ context.Context, name string, jobId sources.JobID, sourceId sources.SourceID, verify bool, connection *anypb.Any, concurrency int) error {
+func (s *Source) Init(_ context.Context, name string, jobId, sourceId int64, verify bool, connection *anypb.Any, concurrency int) error {
 	s.name = name
 	s.sourceId = sourceId
 	s.jobId = jobId
@@ -149,9 +146,9 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk, _ .
 		return errors.New(err)
 	}
 	// Get repo within target.
-	repos, errs := normalizeRepos(s.repos)
+	repos, errs := s.getRepos()
 	for _, repoErr := range errs {
-		ctx.Logger().Info("error normalizing repo", "error", repoErr)
+		ctx.Logger().Info("error getting repo", "error", repoErr)
 	}
 
 	// End early if we had errors getting specified repos but none were validated.
@@ -161,14 +158,26 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk, _ .
 
 	// Get all repos if not specified.
 	if len(repos) == 0 {
-		ignoreRepo := buildIgnorer(s.ignoreRepos, func(err error, pattern string) {
-			ctx.Logger().Error(err, "could not compile ignore repo glob", "glob", pattern)
-		})
-		gitlabRepos, err2, done := s.getReposFromGitlab(ctx, apiClient, ignoreRepo)
-		if done {
-			return err2
+		projects, err := s.getAllProjects(ctx, apiClient)
+		if err != nil {
+			return fmt.Errorf("error getting all projects: %v", err)
+		}
+		// Turn projects into URLs for Git cloner.
+		for _, prj := range projects {
+			if s.ignoreRepo(ctx, prj.PathWithNamespace) {
+				continue
+			}
+
+			// Ensure the urls are valid before adding them to the repo list.
+			_, err := url.Parse(prj.HTTPURLToRepo)
+			if err != nil {
+				fmt.Printf("could not parse url given by project: %s", prj.HTTPURLToRepo)
+			}
+			repos = append(repos, prj.HTTPURLToRepo)
+		}
+		if len(repos) == 0 {
+			return errors.Errorf("unable to discover any repos")
 		}
-		repos = gitlabRepos
 	}
 
 	s.repos = repos
@@ -178,69 +187,6 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk, _ .
 	return s.scanRepos(ctx, chunksChan)
 }
 
-func (s *Source) Validate(ctx context.Context) []error {
-	// The client is only used to query Gitlab for a repo list - it's not used to actually clone anything. Thus, we
-	// don't use it if there is a list of explicitly configured repos. However, constructing it validates that the
-	// configured authentication method is sensible, so we'll do it here.
-	apiClient, err := s.newClient()
-	if err != nil {
-		return []error{err}
-	}
-
-	_, _, err = apiClient.Users.CurrentUser()
-	if err != nil {
-		msg := fmt.Sprintf("gitlab authentication failed using method %v", s.authMethod)
-		return []error{errors.WrapPrefix(err, msg, 0)}
-	}
-
-	explicitlyConfiguredRepos, errs := normalizeRepos(s.repos)
-
-	if len(explicitlyConfiguredRepos) > 0 {
-		user := s.user
-		if user == "" {
-			user = "placeholder"
-		}
-
-		// We only check reachability for explicitly configured repositories. The purpose of source validation is to
-		// help users validate their configuration files, and Gitlab telling us about repositories that it won't let us
-		// access isn't a local configuration issue.
-		for _, r := range explicitlyConfiguredRepos {
-			if err := git.PingRepoUsingToken(ctx, s.token, r, user); err != nil {
-				msg := fmt.Sprintf("could not reach git repository %q", r)
-				err = errors.WrapPrefix(err, msg, 0)
-				errs = append(errs, err)
-			}
-		}
-
-		if len(s.ignoreRepos) > 0 {
-			errs = append(errs, fmt.Errorf("both repositories and ignore patterns were explicitly configured; ignore patterns will not be used"))
-		}
-	}
-
-	if len(explicitlyConfiguredRepos) > 0 || len(errs) > 0 {
-		return errs
-	}
-
-	ignoreProject := buildIgnorer(s.ignoreRepos, func(err error, pattern string) {
-		msg := fmt.Sprintf("could not compile ignore repo pattern %q", pattern)
-		errs = append(errs, errors.WrapPrefix(err, msg, 0))
-	})
-
-	projects, err := s.getAllProjects(ctx, apiClient)
-	if err != nil {
-		errs = append(errs, err)
-		return errs
-	}
-
-	for _, p := range projects {
-		if !ignoreProject(p.PathWithNamespace) {
-			return errs
-		}
-	}
-
-	return append(errs, fmt.Errorf("ignore patterns excluded all projects"))
-}
-
 func (s *Source) newClient() (*gitlab.Client, error) {
 	// Initialize a new api instance.
 	switch s.authMethod {
@@ -376,30 +322,23 @@ func (s *Source) getAllProjects(ctx context.Context, apiClient *gitlab.Client) (
 	return projectList, nil
 }
 
-func (s *Source) getReposFromGitlab(ctx context.Context, apiClient *gitlab.Client, ignoreRepo func(repo string) bool) ([]string, error, bool) {
-	projects, err := s.getAllProjects(ctx, apiClient)
-	if err != nil {
-		return nil, fmt.Errorf("error getting all projects: %v", err), true
+func (s *Source) getRepos() ([]string, []error) {
+	if len(s.repos) == 0 {
+		return nil, nil
 	}
 
-	// Turn projects into URLs for Git cloner.
-	var repos []string
-	for _, prj := range projects {
-		if ignoreRepo(prj.PathWithNamespace) {
+	var validRepos []string
+	var errs []error
+	for _, prj := range s.repos {
+		repo, err := giturl.NormalizeGitlabRepo(prj)
+		if err != nil {
+			errs = append(errs, errors.WrapPrefix(err, fmt.Sprintf("unable to normalize gitlab repo url %s", prj), 0))
 			continue
 		}
 
-		// Ensure the urls are valid before adding them to the repo list.
-		_, err := url.Parse(prj.HTTPURLToRepo)
-		if err != nil {
-			fmt.Printf("could not parse url given by project: %s", prj.HTTPURLToRepo)
-		}
-		repos = append(repos, prj.HTTPURLToRepo)
+		validRepos = append(validRepos, repo)
 	}
-	if len(repos) == 0 {
-		return nil, errors.Errorf("unable to discover any repos"), true
-	}
-	return repos, nil, false
+	return validRepos, errs
 }
 
 func (s *Source) scanRepos(ctx context.Context, chunksChan chan *sources.Chunk) error {
@@ -472,6 +411,21 @@ func (s *Source) scanRepos(ctx context.Context, chunksChan chan *sources.Chunk)
 	return nil
 }
 
+func (s *Source) ignoreRepo(ctx context.Context, r string) bool {
+	for _, ignore := range s.ignoreRepos {
+		g, err := glob.Compile(ignore)
+		if err != nil {
+			ctx.Logger().Error(err, "could not compile ignore repo glob", "glob", ignore)
+			continue
+		}
+		if g.Match(r) {
+			ctx.Logger().V(2).Info("Ignoring repo", "repo", r)
+			return true
+		}
+	}
+	return false
+}
+
 // setProgressCompleteWithRepo calls the s.SetProgressComplete after safely setting up the encoded resume info string.
 func (s *Source) setProgressCompleteWithRepo(index int, offset int, repoURL string) {
 	s.resumeInfoMutex.Lock()
@@ -491,42 +445,3 @@ func (s *Source) setProgressCompleteWithRepo(index int, offset int, repoURL stri
 func (s *Source) WithScanOptions(scanOptions *git.ScanOptions) {
 	s.scanOptions = scanOptions
 }
-
-func buildIgnorer(patterns []string, onCompileErr func(err error, pattern string)) func(repo string) bool {
-	var globs []glob.Glob
-
-	for _, pattern := range patterns {
-		g, err := glob.Compile(pattern)
-		if err != nil {
-			onCompileErr(err, pattern)
-			continue
-		}
-		globs = append(globs, g)
-	}
-
-	f := func(repo string) bool {
-		for _, g := range globs {
-			if g.Match(repo) {
-				return true
-			}
-		}
-		return false
-	}
-
-	return f
-}
-
-func normalizeRepos(repos []string) ([]string, []error) {
-	var validRepos []string
-	var errs []error
-	for _, prj := range repos {
-		repo, err := giturl.NormalizeGitlabRepo(prj)
-		if err != nil {
-			errs = append(errs, errors.WrapPrefix(err, fmt.Sprintf("unable to normalize gitlab repo url %s", prj), 0))
-			continue
-		}
-
-		validRepos = append(validRepos, repo)
-	}
-	return validRepos, errs
-}

pkg/sources/gitlab/gitlab_test.go

@@ -177,150 +177,6 @@ func TestSource_Scan(t *testing.T) {
 	}
 }
 
-func TestSource_Validate(t *testing.T) {
-	ctx, cancel := context.WithCancel(context.Background())
-	defer cancel()
-
-	secret, err := common.GetTestSecret(ctx)
-	if err != nil {
-		t.Fatal(fmt.Errorf("failed to access secret: %v", err))
-	}
-	token := secret.MustGetField("GITLAB_TOKEN")
-	tokenWrongScope := secret.MustGetField("GITLAB_TOKEN_WRONG_SCOPE")
-
-	tests := []struct {
-		name         string
-		connection   *sourcespb.GitLab
-		wantErrCount int
-		wantErrs     []string
-	}{
-		{
-			name: "basic auth did not authenticate",
-			connection: &sourcespb.GitLab{
-				Credential: &sourcespb.GitLab_BasicAuth{
-					BasicAuth: &credentialspb.BasicAuth{
-						Username: "bad-user",
-						Password: "bad-password",
-					},
-				},
-			},
-			wantErrCount: 1,
-		},
-		{
-			name: "token did not authenticate",
-			connection: &sourcespb.GitLab{
-				Credential: &sourcespb.GitLab_Token{
-					Token: "bad-token",
-				},
-			},
-			wantErrCount: 1,
-		},
-		{
-			name: "bad repo urls",
-			connection: &sourcespb.GitLab{
-				Credential: &sourcespb.GitLab_Token{
-					Token: token,
-				},
-				Repositories: []string{
-					"https://gitlab.com/testermctestface/testy",  // valid
-					"https://gitlab.com/testermctestface/testy/", // trailing slash
-					"ssh:git@gitlab.com/testermctestface/testy",  // bad protocol
-					"https://gitlab.com",                         // no path
-					"https://gitlab.com/",                        // no org name
-					"https://gitlab.com//testy",                  // no org name
-					"https://gitlab.com/testermctestface/",       // no repo name
-				},
-			},
-			wantErrCount: 6,
-		},
-		{
-			name: "token does not have permission to list projects",
-			connection: &sourcespb.GitLab{
-				Credential: &sourcespb.GitLab_Token{
-					Token: tokenWrongScope,
-				},
-			},
-			wantErrCount: 1,
-		},
-		{
-			name: "repositories and ignore globs both configured",
-			connection: &sourcespb.GitLab{
-				Credential: &sourcespb.GitLab_Token{
-					Token: token,
-				},
-				Repositories: []string{
-					"https://gitlab.com/testermctestface/testy", // valid
-				},
-				IgnoreRepos: []string{
-					"tes1188/*-gitlab",
-					"[", // glob doesn't compile, but this won't be checked
-				},
-			},
-			wantErrCount: 1,
-		},
-		{
-			name: "could not compile ignore glob(s)",
-			connection: &sourcespb.GitLab{
-				Credential: &sourcespb.GitLab_Token{
-					Token: token,
-				},
-				IgnoreRepos: []string{
-					"tes1188/*-gitlab",
-					"[",    // glob doesn't compile
-					"[a-]", // glob doesn't compile
-				},
-			},
-			wantErrCount: 2,
-		},
-		{
-			name: "repositories do not exist or are not accessible",
-			connection: &sourcespb.GitLab{
-				Credential: &sourcespb.GitLab_Token{
-					Token: token,
-				},
-				Repositories: []string{
-					"https://gitlab.com/testermctestface/testy",
-					"https://gitlab.com/testermctestface/doesn't-exist",
-					"https://gitlab.com/testermctestface/also-doesn't-exist",
-				},
-			},
-			wantErrCount: 2,
-		},
-		{
-			name: "ignore globs exclude all repos",
-			connection: &sourcespb.GitLab{
-				Credential: &sourcespb.GitLab_Token{
-					Token: token,
-				},
-				IgnoreRepos: []string{
-					"*",
-				},
-			},
-			wantErrCount: 1,
-		},
-	}
-
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			s := Source{}
-
-			conn, err := anypb.New(tt.connection)
-			if err != nil {
-				t.Fatal(err)
-			}
-
-			err = s.Init(ctx, tt.name, 0, 0, false, conn, 1)
-			if err != nil {
-				t.Fatalf("Source.Init() error: %v", err)
-			}
-
-			errs := s.Validate(ctx)
-
-			assert.Equal(t, tt.wantErrCount, len(errs))
-		})
-	}
-}
-
 func Test_setProgressCompleteWithRepo_resumeInfo(t *testing.T) {
 	tests := []struct {
 		startingResumeInfoSlice []string

pkg/sources/job_progress.go

@@ -43,8 +43,8 @@ type JobProgressHook interface {
 // If the job supports it, the reference can also be used to cancel running via
 // CancelRun.
 type JobProgressRef struct {
-	JobID       JobID
-	SourceID    SourceID
+	JobID       int64
+	SourceID    int64
 	SourceName  string
 	jobProgress *JobProgress
 }
@@ -101,8 +101,8 @@ func (f ChunkError) Unwrap() error { return f.err }
 // JobProgress aggregates information about a run of a Source.
 type JobProgress struct {
 	// Unique identifiers for this job.
-	JobID      JobID
-	SourceID   SourceID
+	JobID      int64
+	SourceID   int64
 	SourceName string
 	// Tracks whether the job is finished or not.
 	ctx    context.Context
@@ -155,7 +155,7 @@ func WithCancel(cancel context.CancelCauseFunc) func(*JobProgress) {
 }
 
 // NewJobProgress creates a new job report for the given source and job ID.
-func NewJobProgress(jobID JobID, sourceID SourceID, sourceName string, opts ...func(*JobProgress)) *JobProgress {
+func NewJobProgress(jobID, sourceID int64, sourceName string, opts ...func(*JobProgress)) *JobProgress {
 	ctx, cancel := context.WithCancel(context.Background())
 	jp := &JobProgress{
 		JobID:      jobID,

pkg/sources/job_progress_test.go

@@ -38,8 +38,8 @@ func TestJobProgressFatalErrors(t *testing.T) {
 func TestJobProgressRef(t *testing.T) {
 	jp := NewJobProgress(123, 456, "source name")
 	ref := jp.Ref()
-	assert.Equal(t, JobID(123), ref.JobID)
-	assert.Equal(t, SourceID(456), ref.SourceID)
+	assert.Equal(t, int64(123), ref.JobID)
+	assert.Equal(t, int64(456), ref.SourceID)
 
 	// Test Done() blocks until Finish() is called.
 	select {

pkg/sources/s3/s3.go

@@ -31,8 +31,6 @@ import (
 )
 
 const (
-	SourceType = sourcespb.SourceType_SOURCE_TYPE_S3
-
 	defaultAWSRegion     = "us-east-1"
 	defaultMaxObjectSize = 250 * 1024 * 1024 // 250 MiB
 	maxObjectSizeLimit   = 250 * 1024 * 1024 // 250 MiB
@@ -40,8 +38,8 @@ const (
 
 type Source struct {
 	name        string
-	sourceId    sources.SourceID
-	jobId       sources.JobID
+	sourceId    int64
+	jobId       int64
 	verify      bool
 	concurrency int
 	log         logr.Logger
@@ -60,19 +58,19 @@ var _ sources.Validator = (*Source)(nil)
 
 // Type returns the type of source
 func (s *Source) Type() sourcespb.SourceType {
-	return SourceType
+	return sourcespb.SourceType_SOURCE_TYPE_S3
 }
 
-func (s *Source) SourceID() sources.SourceID {
+func (s *Source) SourceID() int64 {
 	return s.sourceId
 }
 
-func (s *Source) JobID() sources.JobID {
+func (s *Source) JobID() int64 {
 	return s.jobId
 }
 
 // Init returns an initialized AWS source
-func (s *Source) Init(aCtx context.Context, name string, jobId sources.JobID, sourceId sources.SourceID, verify bool, connection *anypb.Any, concurrency int) error {
+func (s *Source) Init(aCtx context.Context, name string, jobId, sourceId int64, verify bool, connection *anypb.Any, concurrency int) error {
 	s.log = context.WithValues(aCtx, "source", s.Type(), "name", name).Logger()
 
 	s.name = name
@@ -200,7 +198,7 @@ func (s *Source) scanBuckets(ctx context.Context, client *s3.S3, role string, bu
 		}
 
 		s.SetProgressComplete(i, len(bucketsToScan), fmt.Sprintf("Bucket: %s", bucket), "")
-		logger.V(3).Info("Scanning bucket")
+		logger.Info("Scanning bucket")
 
 		regionalClient, err := s.getRegionalClientForBucket(ctx, client, role, bucket)
 		if err != nil {
@@ -280,7 +278,7 @@ func (s *Source) pageChunker(ctx context.Context, client *s3.S3, chunksChan chan
 
 		// ignore large files
 		if *obj.Size > s.maxObjectSize {
-			s.log.V(5).Info("Skipping %d byte file (over maxObjectSize limit)", "object", *obj.Key)
+			s.log.V(3).Info("Skipping %d byte file (over maxObjectSize limit)", "object", *obj.Key)
 			continue
 		}
 

pkg/sources/source_manager.go

@@ -13,9 +13,28 @@ import (
 	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
 )
 
+// handle uniquely identifies a Source given to the manager to manage. If the
+// SourceManager is connected to the API, it will be equivalent to the unique
+// source ID, otherwise it behaves as a counter.
+type handle int64
+
+// SourceInitFunc is a function that takes a source and job ID and returns an
+// initialized Source.
+type SourceInitFunc func(ctx context.Context, jobID, sourceID int64) (Source, error)
+
+// sourceInfo is an aggregate struct to store source information provided on
+// initialization.
+type sourceInfo struct {
+	initFunc SourceInitFunc
+	name     string
+}
+
 type SourceManager struct {
 	api   apiClient
 	hooks []JobProgressHook
+	// Map of handle to source initializer.
+	handles     map[handle]sourceInfo
+	handlesLock sync.Mutex
 	// Pool limiting the amount of concurrent sources running.
 	pool                errgroup.Group
 	poolLimit           int
@@ -32,9 +51,10 @@ type SourceManager struct {
 
 // apiClient is an interface for optionally communicating with an external API.
 type apiClient interface {
-	// GetIDs informs the API of the source that's about to run and returns
-	// two identifiers used during source initialization.
-	GetIDs(ctx context.Context, name string, kind sourcespb.SourceType) (SourceID, JobID, error)
+	// RegisterSource lets the API know we have a source and it returns a unique source ID for it.
+	RegisterSource(ctx context.Context, name string, kind sourcespb.SourceType) (int64, error)
+	// GetJobID queries the API for an existing job or new job ID.
+	GetJobID(ctx context.Context, id int64) (int64, error)
 }
 
 // WithAPI adds an API client to the manager for tracking jobs and progress. If
@@ -79,6 +99,7 @@ func NewManager(opts ...func(*SourceManager)) *SourceManager {
 	mgr := SourceManager{
 		// Default to the headless API. Can be overwritten by the WithAPI option.
 		api:          &headlessAPI{},
+		handles:      make(map[handle]sourceInfo),
 		outputChunks: make(chan *Chunk),
 	}
 	for _, opt := range opts {
@@ -87,33 +108,70 @@ func NewManager(opts ...func(*SourceManager)) *SourceManager {
 	return &mgr
 }
 
-func (s *SourceManager) GetIDs(ctx context.Context, sourceName string, kind sourcespb.SourceType) (SourceID, JobID, error) {
-	return s.api.GetIDs(ctx, sourceName, kind)
+// Enroll informs the SourceManager to track and manage a Source.
+func (s *SourceManager) Enroll(ctx context.Context, name string, kind sourcespb.SourceType, f SourceInitFunc) (handle, error) {
+	if s.done {
+		return 0, fmt.Errorf("manager is done")
+	}
+	id, err := s.api.RegisterSource(ctx, name, kind)
+	if err != nil {
+		return 0, err
+	}
+	handleID := handle(id)
+	s.handlesLock.Lock()
+	defer s.handlesLock.Unlock()
+	if _, ok := s.handles[handleID]; ok {
+		// TODO: smartly handle this?
+		return 0, fmt.Errorf("handle ID '%d' already in use", handleID)
+	}
+	s.handles[handleID] = sourceInfo{
+		initFunc: f,
+		name:     name,
+	}
+	return handleID, nil
 }
 
 // Run blocks until a resource is available to run the source, then
+// synchronously runs it. The first fatal error, if any, will be returned.
+func (s *SourceManager) Run(ctx context.Context, handle handle) (JobProgressRef, error) {
+	progress, err := s.asyncRun(ctx, handle)
+	if err != nil {
+		return progress, err
+	}
+	<-progress.Done()
+	return progress, progress.Snapshot().FatalError()
+}
+
+// ScheduleRun blocks until a resource is available to run the source, then
 // asynchronously runs it. Error information is stored and accessible via the
 // JobProgressRef as it becomes available.
-func (s *SourceManager) Run(ctx context.Context, sourceName string, source Source) (JobProgressRef, error) {
-	return s.asyncRun(ctx, sourceName, source)
+func (s *SourceManager) ScheduleRun(ctx context.Context, handle handle) (JobProgressRef, error) {
+	return s.asyncRun(ctx, handle)
 }
 
 // asyncRun is a helper method to asynchronously run the Source. It calls out
 // to the API to get a job ID for this run, creates a JobProgress object, then
 // waits for an available goroutine to asynchronously run it.
-func (s *SourceManager) asyncRun(ctx context.Context, sourceName string, source Source) (JobProgressRef, error) {
-	sourceID, jobID := source.SourceID(), source.JobID()
+func (s *SourceManager) asyncRun(ctx context.Context, handle handle) (JobProgressRef, error) {
 	// Do preflight checks before waiting on the pool.
-	if err := s.preflightChecks(ctx); err != nil {
-		return JobProgressRef{
-			SourceName: sourceName,
-			SourceID:   sourceID,
-			JobID:      jobID,
-		}, err
+	if err := s.preflightChecks(ctx, handle); err != nil {
+		return JobProgressRef{}, err
+	}
+	// Get the name. Should never fail due to preflight checks.
+	sourceInfo, ok := s.getSourceInfo(handle)
+	if !ok {
+		return JobProgressRef{SourceID: int64(handle)}, fmt.Errorf("unrecognized handle")
+	}
+	sourceName := sourceInfo.name
+	// Get a Job ID.
+	ctx = context.WithValue(ctx, "source_id", int64(handle))
+	jobID, err := s.api.GetJobID(ctx, int64(handle))
+	if err != nil {
+		return JobProgressRef{SourceID: int64(handle), SourceName: sourceName}, err
 	}
 	// Create a JobProgress object for tracking progress.
 	ctx, cancel := context.WithCancelCause(ctx)
-	progress := NewJobProgress(jobID, sourceID, sourceName, WithHooks(s.hooks...), WithCancel(cancel))
+	progress := NewJobProgress(jobID, int64(handle), sourceName, WithHooks(s.hooks...), WithCancel(cancel))
 	s.pool.Go(func() error {
 		atomic.AddInt32(&s.currentRunningCount, 1)
 		defer atomic.AddInt32(&s.currentRunningCount, -1)
@@ -123,7 +181,7 @@ func (s *SourceManager) asyncRun(ctx context.Context, sourceName string, source
 		)
 		defer common.Recover(ctx)
 		defer cancel(nil)
-		return s.run(ctx, source, progress)
+		return s.run(ctx, handle, jobID, progress)
 	})
 	return progress.Ref(), nil
 }
@@ -168,19 +226,23 @@ func (s *SourceManager) AvailableCapacity() int {
 }
 
 // preflightChecks is a helper method to check the Manager or the context isn't
-// done.
-func (s *SourceManager) preflightChecks(ctx context.Context) error {
+// done and that the handle is valid.
+func (s *SourceManager) preflightChecks(ctx context.Context, handle handle) error {
 	// Check if the manager has been Waited.
 	if s.done {
 		return fmt.Errorf("manager is done")
 	}
+	// Check the handle is valid.
+	if _, ok := s.getSourceInfo(handle); !ok {
+		return fmt.Errorf("unrecognized handle")
+	}
 	return ctx.Err()
 }
 
 // run is a helper method to sychronously run the source. It does not check for
 // acquired resources. An error is returned if there was a fatal error during
 // the run. This information is also recorded in the JobProgress.
-func (s *SourceManager) run(ctx context.Context, source Source, report *JobProgress) error {
+func (s *SourceManager) run(ctx context.Context, handle handle, jobID int64, report *JobProgress) error {
 	defer report.Finish()
 	report.Start(time.Now())
 	defer func() { report.End(time.Now()) }()
@@ -191,21 +253,34 @@ func (s *SourceManager) run(ctx context.Context, source Source, report *JobProgr
 		}
 	}()
 
+	// Initialize the source.
+	sourceInfo, ok := s.getSourceInfo(handle)
+	if !ok {
+		// Shouldn't happen due to preflight checks.
+		err := fmt.Errorf("unrecognized handle")
+		report.ReportError(Fatal{err})
+		return Fatal{err}
+	}
+	source, err := sourceInfo.initFunc(ctx, jobID, int64(handle))
+	if err != nil {
+		report.ReportError(Fatal{err})
+		return Fatal{err}
+	}
 	report.TrackProgress(source.GetProgress())
 	ctx = context.WithValues(ctx,
 		"source_type", source.Type().String(),
-		"source_name", report.SourceName,
+		"source_name", sourceInfo.name,
 	)
 	// Check for the preferred method of tracking source units.
 	if enumChunker, ok := source.(SourceUnitEnumChunker); ok && s.useSourceUnits {
-		return s.runWithUnits(ctx, enumChunker, report)
+		return s.runWithUnits(ctx, handle, enumChunker, report)
 	}
-	return s.runWithoutUnits(ctx, source, report)
+	return s.runWithoutUnits(ctx, handle, source, report)
 }
 
 // runWithoutUnits is a helper method to run a Source. It has coarse-grained
 // job reporting.
-func (s *SourceManager) runWithoutUnits(ctx context.Context, source Source, report *JobProgress) error {
+func (s *SourceManager) runWithoutUnits(ctx context.Context, handle handle, source Source, report *JobProgress) error {
 	// Introspect on the chunks we get from the Chunks method.
 	ch := make(chan *Chunk, 1)
 	var wg sync.WaitGroup
@@ -235,7 +310,7 @@ func (s *SourceManager) runWithoutUnits(ctx context.Context, source Source, repo
 // runWithUnits is a helper method to run a Source that is also a
 // SourceUnitEnumChunker. This allows better introspection of what is getting
 // scanned and any errors encountered.
-func (s *SourceManager) runWithUnits(ctx context.Context, source SourceUnitEnumChunker, report *JobProgress) error {
+func (s *SourceManager) runWithUnits(ctx context.Context, handle handle, source SourceUnitEnumChunker, report *JobProgress) error {
 	unitReporter := &mgrUnitReporter{
 		unitCh: make(chan SourceUnit, 1),
 		report: report,
@@ -310,15 +385,28 @@ func (s *SourceManager) runWithUnits(ctx context.Context, source SourceUnitEnumC
 	}
 }
 
+// getSourceInfo is a helper method for safe concurrent access to the
+// map[handle]SourceInitFunc map.
+func (s *SourceManager) getSourceInfo(handle handle) (sourceInfo, bool) {
+	s.handlesLock.Lock()
+	defer s.handlesLock.Unlock()
+	f, ok := s.handles[handle]
+	return f, ok
+}
+
 // headlessAPI implements the apiClient interface locally.
 type headlessAPI struct {
-	// Counters for assigning source and job IDs.
+	// Counters for assigning handle and job IDs.
 	sourceIDCounter int64
 	jobIDCounter    int64
 }
 
-func (api *headlessAPI) GetIDs(context.Context, string, sourcespb.SourceType) (SourceID, JobID, error) {
-	return SourceID(atomic.AddInt64(&api.sourceIDCounter, 1)), JobID(atomic.AddInt64(&api.jobIDCounter, 1)), nil
+func (api *headlessAPI) RegisterSource(ctx context.Context, name string, kind sourcespb.SourceType) (int64, error) {
+	return atomic.AddInt64(&api.sourceIDCounter, 1), nil
+}
+
+func (api *headlessAPI) GetJobID(ctx context.Context, id int64) (int64, error) {
+	return atomic.AddInt64(&api.jobIDCounter, 1), nil
 }
 
 // mgrUnitReporter implements the UnitReporter interface.

pkg/sources/source_manager_test.go

@@ -15,15 +15,15 @@ import (
 
 // DummySource implements Source and is used for testing a SourceManager.
 type DummySource struct {
-	sourceID SourceID
-	jobID    JobID
+	sourceID int64
+	jobID    int64
 	chunker
 }
 
 func (d *DummySource) Type() sourcespb.SourceType { return 1337 }
-func (d *DummySource) SourceID() SourceID         { return d.sourceID }
-func (d *DummySource) JobID() JobID               { return d.jobID }
-func (d *DummySource) Init(_ context.Context, _ string, jobID JobID, sourceID SourceID, _ bool, _ *anypb.Any, _ int) error {
+func (d *DummySource) SourceID() int64            { return d.sourceID }
+func (d *DummySource) JobID() int64               { return d.jobID }
+func (d *DummySource) Init(_ context.Context, _ string, jobID, sourceID int64, _ bool, _ *anypb.Any, _ int) error {
 	d.sourceID = sourceID
 	d.jobID = jobID
 	return nil
@@ -80,13 +80,16 @@ func (c errorChunker) Chunks(context.Context, chan *Chunk, ...ChunkingTarget) er
 func (c errorChunker) Enumerate(context.Context, UnitReporter) error                { return c }
 func (c errorChunker) ChunkUnit(context.Context, SourceUnit, ChunkReporter) error   { return c }
 
-// buildDummy is a helper function to enroll a DummySource with a SourceManager.
-func buildDummy(chunkMethod chunker) (Source, error) {
-	source := &DummySource{chunker: chunkMethod}
-	if err := source.Init(context.Background(), "dummy", 123, 456, true, nil, 42); err != nil {
-		return nil, err
-	}
-	return source, nil
+// enrollDummy is a helper function to enroll a DummySource with a SourceManager.
+func enrollDummy(mgr *SourceManager, chunkMethod chunker) (handle, error) {
+	return mgr.Enroll(context.Background(), "dummy", 1337,
+		func(ctx context.Context, jobID, sourceID int64) (Source, error) {
+			source := &DummySource{chunker: chunkMethod}
+			if err := source.Init(ctx, "dummy", jobID, sourceID, true, nil, 42); err != nil {
+				return nil, err
+			}
+			return source, nil
+		})
 }
 
 // tryRead is a helper function that will try to read from a channel and return
@@ -102,13 +105,11 @@ func tryRead(ch <-chan *Chunk) (*Chunk, error) {
 
 func TestSourceManagerRun(t *testing.T) {
 	mgr := NewManager(WithBufferedOutput(8))
-	source, err := buildDummy(&counterChunker{count: 1})
+	handle, err := enrollDummy(mgr, &counterChunker{count: 1})
 	assert.NoError(t, err)
 	for i := 0; i < 3; i++ {
-		ref, err := mgr.Run(context.Background(), "dummy", source)
-		<-ref.Done()
+		_, err = mgr.Run(context.Background(), handle)
 		assert.NoError(t, err)
-		assert.NoError(t, ref.Snapshot().FatalError())
 		chunk, err := tryRead(mgr.Chunks())
 		assert.NoError(t, err)
 		assert.Equal(t, []byte{byte(i)}, chunk.Data)
@@ -120,31 +121,35 @@ func TestSourceManagerRun(t *testing.T) {
 
 func TestSourceManagerWait(t *testing.T) {
 	mgr := NewManager()
-	source, err := buildDummy(&counterChunker{count: 1})
+	handle, err := enrollDummy(mgr, &counterChunker{count: 1})
 	assert.NoError(t, err)
 	// Asynchronously run the source.
-	_, err = mgr.Run(context.Background(), "dummy", source)
+	_, err = mgr.ScheduleRun(context.Background(), handle)
 	assert.NoError(t, err)
 	// Read the 1 chunk we're expecting so Waiting completes.
 	<-mgr.Chunks()
 	// Wait for all resources to complete.
 	assert.NoError(t, mgr.Wait())
-	// Run should return an error now.
-	_, err = buildDummy(&counterChunker{count: 1})
-	assert.NoError(t, err)
-	_, err = mgr.Run(context.Background(), "dummy", source)
+	// Enroll and run should return an error now.
+	_, err = enrollDummy(mgr, &counterChunker{count: 1})
+	assert.Error(t, err)
+	_, err = mgr.ScheduleRun(context.Background(), handle)
 	assert.Error(t, err)
 }
 
 func TestSourceManagerError(t *testing.T) {
 	mgr := NewManager()
-	source, err := buildDummy(errorChunker{fmt.Errorf("oops")})
+	handle, err := enrollDummy(mgr, errorChunker{fmt.Errorf("oops")})
 	assert.NoError(t, err)
-	ref, err := mgr.Run(context.Background(), "dummy", source)
+	// A synchronous run should fail.
+	_, err = mgr.Run(context.Background(), handle)
+	assert.Error(t, err)
+	// Scheduling a run should not fail, but the error should surface in
+	// Wait().
+	ref, err := mgr.ScheduleRun(context.Background(), handle)
 	assert.NoError(t, err)
-	<-ref.Done()
-	assert.Error(t, ref.Snapshot().FatalError())
 	assert.Error(t, mgr.Wait())
+	assert.Error(t, ref.Snapshot().FatalError())
 }
 
 func TestSourceManagerReport(t *testing.T) {
@@ -154,11 +159,11 @@ func TestSourceManagerReport(t *testing.T) {
 		{WithBufferedOutput(8), WithSourceUnits(), WithConcurrentUnits(1)},
 	} {
 		mgr := NewManager(opts...)
-		source, err := buildDummy(&counterChunker{count: 4})
+		handle, err := enrollDummy(mgr, &counterChunker{count: 4})
 		assert.NoError(t, err)
-		ref, err := mgr.Run(context.Background(), "dummy", source)
+		// Synchronously run the source.
+		ref, err := mgr.Run(context.Background(), handle)
 		assert.NoError(t, err)
-		<-ref.Done()
 		assert.Equal(t, 0, len(ref.Snapshot().Errors))
 		assert.Equal(t, uint64(4), ref.Snapshot().TotalChunks)
 	}
@@ -216,11 +221,10 @@ func TestSourceManagerNonFatalError(t *testing.T) {
 		{unit: "three", err: "not again"},
 	}
 	mgr := NewManager(WithBufferedOutput(8), WithSourceUnits())
-	source, err := buildDummy(&unitChunker{input})
+	handle, err := enrollDummy(mgr, &unitChunker{input})
 	assert.NoError(t, err)
-	ref, err := mgr.Run(context.Background(), "dummy", source)
+	ref, err := mgr.Run(context.Background(), handle)
 	assert.NoError(t, err)
-	<-ref.Done()
 	report := ref.Snapshot()
 	assert.Equal(t, len(input), int(report.TotalUnits))
 	assert.Equal(t, len(input), int(report.FinishedUnits))
@@ -231,11 +235,11 @@ func TestSourceManagerNonFatalError(t *testing.T) {
 
 func TestSourceManagerContextCancelled(t *testing.T) {
 	mgr := NewManager(WithBufferedOutput(8))
-	source, err := buildDummy(&counterChunker{count: 100})
+	handle, err := enrollDummy(mgr, &counterChunker{count: 100})
 	assert.NoError(t, err)
 
 	ctx, cancel := context.WithCancel(context.Background())
-	ref, err := mgr.Run(ctx, "dummy", source)
+	ref, err := mgr.ScheduleRun(ctx, handle)
 	assert.NoError(t, err)
 
 	cancel()
@@ -245,18 +249,47 @@ func TestSourceManagerContextCancelled(t *testing.T) {
 }
 
 type DummyAPI struct {
-	registerSource func(context.Context, string, sourcespb.SourceType) (SourceID, error)
-	getJobID       func(context.Context, SourceID) (JobID, error)
+	registerSource func(context.Context, string, sourcespb.SourceType) (int64, error)
+	getJobID       func(context.Context, int64) (int64, error)
 }
 
-func (api DummyAPI) RegisterSource(ctx context.Context, name string, kind sourcespb.SourceType) (SourceID, error) {
+func (api DummyAPI) RegisterSource(ctx context.Context, name string, kind sourcespb.SourceType) (int64, error) {
 	return api.registerSource(ctx, name, kind)
 }
 
-func (api DummyAPI) GetJobID(ctx context.Context, id SourceID) (JobID, error) {
+func (api DummyAPI) GetJobID(ctx context.Context, id int64) (int64, error) {
 	return api.getJobID(ctx, id)
 }
 
+func TestSourceManagerJobAndSourceIDs(t *testing.T) {
+	mgr := NewManager(WithAPI(DummyAPI{
+		registerSource: func(context.Context, string, sourcespb.SourceType) (int64, error) {
+			return 1337, nil
+		},
+		getJobID: func(context.Context, int64) (int64, error) {
+			return 9001, nil
+		},
+	}))
+	var (
+		initializedJobID    int64
+		initializedSourceID int64
+	)
+	handle, err := mgr.Enroll(context.Background(), "dummy", 1337,
+		func(ctx context.Context, jobID, sourceID int64) (Source, error) {
+			initializedJobID = jobID
+			initializedSourceID = sourceID
+			return nil, fmt.Errorf("ignore")
+		})
+	assert.NoError(t, err)
+
+	ref, _ := mgr.Run(context.Background(), handle)
+	assert.Equal(t, int64(1337), initializedSourceID)
+	assert.Equal(t, int64(1337), ref.SourceID)
+	assert.Equal(t, int64(9001), initializedJobID)
+	assert.Equal(t, int64(9001), ref.JobID)
+	assert.Equal(t, "dummy", ref.SourceName)
+}
+
 // Chunk method that has a custom callback for the Chunks method.
 type callbackChunker struct {
 	cb func(context.Context, chan *Chunk) error
@@ -271,7 +304,7 @@ func (c callbackChunker) ChunkUnit(context.Context, SourceUnit, ChunkReporter) e
 func TestSourceManagerCancelRun(t *testing.T) {
 	mgr := NewManager(WithBufferedOutput(8))
 	var returnedErr error
-	source, err := buildDummy(callbackChunker{func(ctx context.Context, _ chan *Chunk) error {
+	handle, err := enrollDummy(mgr, callbackChunker{func(ctx context.Context, _ chan *Chunk) error {
 		// The context passed to Chunks should get cancelled when ref.CancelRun() is called.
 		<-ctx.Done()
 		returnedErr = fmt.Errorf("oh no: %w", ctx.Err())
@@ -279,7 +312,7 @@ func TestSourceManagerCancelRun(t *testing.T) {
 	}})
 	assert.NoError(t, err)
 
-	ref, err := mgr.Run(context.Background(), "dummy", source)
+	ref, err := mgr.ScheduleRun(context.Background(), handle)
 	assert.NoError(t, err)
 
 	cancelErr := fmt.Errorf("abort! abort!")
@@ -293,7 +326,7 @@ func TestSourceManagerCancelRun(t *testing.T) {
 func TestSourceManagerAvailableCapacity(t *testing.T) {
 	mgr := NewManager(WithConcurrentSources(1337))
 	start, end := make(chan struct{}), make(chan struct{})
-	source, err := buildDummy(callbackChunker{func(context.Context, chan *Chunk) error {
+	handle, err := enrollDummy(mgr, callbackChunker{func(context.Context, chan *Chunk) error {
 		start <- struct{}{} // Send start signal.
 		<-end               // Wait for end signal.
 		return nil
@@ -301,7 +334,7 @@ func TestSourceManagerAvailableCapacity(t *testing.T) {
 	assert.NoError(t, err)
 
 	assert.Equal(t, 1337, mgr.AvailableCapacity())
-	ref, err := mgr.Run(context.Background(), "dummy", source)
+	ref, err := mgr.ScheduleRun(context.Background(), handle)
 	assert.NoError(t, err)
 
 	<-start // Wait for start signal.

pkg/sources/sources.go

@@ -11,19 +11,14 @@ import (
 	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
 )
 
-type (
-	SourceID int64
-	JobID    int64
-)
-
 // Chunk contains data to be decoded and scanned along with context on where it came from.
 type Chunk struct {
 	// SourceName is the name of the Source that produced the chunk.
 	SourceName string
 	// SourceID is the ID of the source that the Chunk originated from.
-	SourceID SourceID
+	SourceID int64
 	// JobID is the ID of the job that the Chunk originated from.
-	JobID JobID
+	JobID int64
 	// SourceType is the type of Source that produced the chunk.
 	SourceType sourcespb.SourceType
 	// SourceMetadata holds the context of where the Chunk was found.
@@ -50,11 +45,11 @@ type Source interface {
 	// Type returns the source type, used for matching against configuration and jobs.
 	Type() sourcespb.SourceType
 	// SourceID returns the initialized source ID used for tracking relationships in the DB.
-	SourceID() SourceID
+	SourceID() int64
 	// JobID returns the initialized job ID used for tracking relationships in the DB.
-	JobID() JobID
+	JobID() int64
 	// Init initializes the source.
-	Init(aCtx context.Context, name string, jobId JobID, sourceId SourceID, verify bool, connection *anypb.Any, concurrency int) error
+	Init(aCtx context.Context, name string, jobId, sourceId int64, verify bool, connection *anypb.Any, concurrency int) error
 	// Chunks emits data over a channel which is then decoded and scanned for secrets.
 	// By default, data is obtained indiscriminately. However, by providing one or more
 	// ChunkingTarget parameters, the caller can direct the function to retrieve

pkg/sources/syslog/syslog.go

@@ -23,16 +23,12 @@ import (
 	"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
 )
 
-const (
-	SourceType = sourcespb.SourceType_SOURCE_TYPE_SYSLOG
-
-	nilString = ""
-)
+const nilString = ""
 
 type Source struct {
 	name     string
-	sourceId sources.SourceID
-	jobId    sources.JobID
+	sourceId int64
+	jobId    int64
 	verify   bool
 	syslog   *Syslog
 	sources.Progress
@@ -42,14 +38,14 @@ type Source struct {
 type Syslog struct {
 	sourceType         sourcespb.SourceType
 	sourceName         string
-	sourceID           sources.SourceID
-	jobID              sources.JobID
+	sourceID           int64
+	jobID              int64
 	sourceMetadataFunc func(hostname, appname, procid, timestamp, facility, client string) *source_metadatapb.MetaData
 	verify             bool
 	concurrency        *semaphore.Weighted
 }
 
-func NewSyslog(sourceType sourcespb.SourceType, jobID sources.JobID, sourceID sources.SourceID, sourceName string, verify bool, concurrency int,
+func NewSyslog(sourceType sourcespb.SourceType, jobID, sourceID int64, sourceName string, verify bool, concurrency int,
 	sourceMetadataFunc func(hostname, appname, procid, timestamp, facility, client string) *source_metadatapb.MetaData,
 ) *Syslog {
 	return &Syslog{
@@ -107,14 +103,14 @@ var _ sources.Source = (*Source)(nil)
 // Type returns the type of source.
 // It is used for matching source types in configuration and job input.
 func (s *Source) Type() sourcespb.SourceType {
-	return SourceType
+	return sourcespb.SourceType_SOURCE_TYPE_SYSLOG
 }
 
-func (s *Source) SourceID() sources.SourceID {
+func (s *Source) SourceID() int64 {
 	return s.sourceId
 }
 
-func (s *Source) JobID() sources.JobID {
+func (s *Source) JobID() int64 {
 	return s.jobId
 }
 
@@ -123,7 +119,7 @@ func (s *Source) InjectConnection(conn *sourcespb.Syslog) {
 }
 
 // Init returns an initialized Syslog source.
-func (s *Source) Init(_ context.Context, name string, jobId sources.JobID, sourceId sources.SourceID, verify bool, connection *anypb.Any, concurrency int) error {
+func (s *Source) Init(_ context.Context, name string, jobId, sourceId int64, verify bool, connection *anypb.Any, concurrency int) error {
 
 	s.name = name
 	s.sourceId = sourceId