tighten up launchdarkly

go.mod

@@ -43,7 +43,6 @@ require (
 	github.com/google/go-containerregistry v0.15.2
 	github.com/google/go-github/v42 v42.0.0
 	github.com/googleapis/gax-go/v2 v2.12.0
-	github.com/h2non/filetype v1.1.3
 	github.com/hashicorp/go-retryablehttp v0.7.4
 	github.com/hashicorp/golang-lru v0.5.1
 	github.com/jlaffaye/ftp v0.2.0

go.sum

@@ -315,8 +315,6 @@ github.com/gorilla/css v1.0.0 h1:BQqNyPTi50JCFMTw/b67hByjMVXZRwGha6wxVGkeihY=
 github.com/gorilla/css v1.0.0/go.mod h1:Dn721qIggHpt4+EFCcTLTU/vk5ySda2ReITrtgBl60c=
 github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
 github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw=
-github.com/h2non/filetype v1.1.3 h1:FKkx9QbD7HR/zjK1Ia5XiBsq9zdLi5Kf3zGyFTAFkGg=
-github.com/h2non/filetype v1.1.3/go.mod h1:319b3zT68BvV+WRj7cwy856M2ehB3HqNOt6sy1HndBY=
 github.com/h2non/parth v0.0.0-20190131123155-b4df798d6542 h1:2VTzZjLZBgl62/EtslCrtky5vbi9dd7HrQPQIx6wqiw=
 github.com/h2non/parth v0.0.0-20190131123155-b4df798d6542/go.mod h1:Ow0tF8D4Kplbc8s8sSb3V2oUCygFHVp8gC3Dn6U4MNI=
 github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA=

main.go

@@ -73,20 +73,17 @@ var (
 	_                   = gitScan.Flag("entropy", "No-op flag for backwards compat.").Bool()
 	_                   = gitScan.Flag("regex", "No-op flag for backwards compat.").Bool()
 
-	githubScan              = cli.Command("github", "Find credentials in GitHub repositories.")
-	githubScanEndpoint      = githubScan.Flag("endpoint", "GitHub endpoint.").Default("https://api.github.com").String()
-	githubScanRepos         = githubScan.Flag("repo", `GitHub repository to scan. You can repeat this flag. Example: "https://github.com/dustin-decker/secretsandstuff"`).Strings()
-	githubScanOrgs          = githubScan.Flag("org", `GitHub organization to scan. You can repeat this flag. Example: "trufflesecurity"`).Strings()
-	githubScanToken         = githubScan.Flag("token", "GitHub token. Can be provided with environment variable GITHUB_TOKEN.").Envar("GITHUB_TOKEN").String()
-	githubIncludeForks      = githubScan.Flag("include-forks", "Include forks in scan.").Bool()
-	githubIncludeMembers    = githubScan.Flag("include-members", "Include organization member repositories in scan.").Bool()
-	githubIncludeRepos      = githubScan.Flag("include-repos", `Repositories to include in an org scan. This can also be a glob pattern. You can repeat this flag. Must use Github repo full name. Example: "trufflesecurity/trufflehog", "trufflesecurity/t*"`).Strings()
-	githubExcludeRepos      = githubScan.Flag("exclude-repos", `Repositories to exclude in an org scan. This can also be a glob pattern. You can repeat this flag. Must use Github repo full name. Example: "trufflesecurity/driftwood", "trufflesecurity/d*"`).Strings()
-	githubScanIncludePaths  = githubScan.Flag("include-paths", "Path to file with newline separated regexes for files to include in scan.").Short('i').String()
-	githubScanExcludePaths  = githubScan.Flag("exclude-paths", "Path to file with newline separated regexes for files to exclude in scan.").Short('x').String()
-	githubScanIssueComments = githubScan.Flag("issue-comments", "Include issue comments in scan.").Bool()
-	githubScanPRComments    = githubScan.Flag("pr-comments", "Include pull request comments in scan.").Bool()
-	githubScanGistComments  = githubScan.Flag("gist-comments", "Include gist comments in scan.").Bool()
+	githubScan             = cli.Command("github", "Find credentials in GitHub repositories.")
+	githubScanEndpoint     = githubScan.Flag("endpoint", "GitHub endpoint.").Default("https://api.github.com").String()
+	githubScanRepos        = githubScan.Flag("repo", `GitHub repository to scan. You can repeat this flag. Example: "https://github.com/dustin-decker/secretsandstuff"`).Strings()
+	githubScanOrgs         = githubScan.Flag("org", `GitHub organization to scan. You can repeat this flag. Example: "trufflesecurity"`).Strings()
+	githubScanToken        = githubScan.Flag("token", "GitHub token. Can be provided with environment variable GITHUB_TOKEN.").Envar("GITHUB_TOKEN").String()
+	githubIncludeForks     = githubScan.Flag("include-forks", "Include forks in scan.").Bool()
+	githubIncludeMembers   = githubScan.Flag("include-members", "Include organization member repositories in scan.").Bool()
+	githubIncludeRepos     = githubScan.Flag("include-repos", `Repositories to include in an org scan. This can also be a glob pattern. You can repeat this flag. Must use Github repo full name. Example: "trufflesecurity/trufflehog", "trufflesecurity/t*"`).Strings()
+	githubExcludeRepos     = githubScan.Flag("exclude-repos", `Repositories to exclude in an org scan. This can also be a glob pattern. You can repeat this flag. Must use Github repo full name. Example: "trufflesecurity/driftwood", "trufflesecurity/d*"`).Strings()
+	githubScanIncludePaths = githubScan.Flag("include-paths", "Path to file with newline separated regexes for files to include in scan.").Short('i').String()
+	githubScanExcludePaths = githubScan.Flag("exclude-paths", "Path to file with newline separated regexes for files to exclude in scan.").Short('x').String()
 
 	gitlabScan = cli.Command("gitlab", "Find credentials in GitLab repositories.")
 	// TODO: Add more GitLab options
@@ -412,19 +409,16 @@ func run(state overseer.State) {
 		}
 
 		cfg := sources.GithubConfig{
-			Endpoint:                   *githubScanEndpoint,
-			Token:                      *githubScanToken,
-			IncludeForks:               *githubIncludeForks,
-			IncludeMembers:             *githubIncludeMembers,
-			Concurrency:                *concurrency,
-			ExcludeRepos:               *githubExcludeRepos,
-			IncludeRepos:               *githubIncludeRepos,
-			Repos:                      *githubScanRepos,
-			Orgs:                       *githubScanOrgs,
-			IncludeIssueComments:       *githubScanIssueComments,
-			IncludePullRequestComments: *githubScanPRComments,
-			IncludeGistComments:        *githubScanGistComments,
-			Filter:                     filter,
+			Endpoint:       *githubScanEndpoint,
+			Token:          *githubScanToken,
+			IncludeForks:   *githubIncludeForks,
+			IncludeMembers: *githubIncludeMembers,
+			Concurrency:    *concurrency,
+			ExcludeRepos:   *githubExcludeRepos,
+			IncludeRepos:   *githubIncludeRepos,
+			Repos:          *githubScanRepos,
+			Orgs:           *githubScanOrgs,
+			Filter:         filter,
 		}
 		if err := e.ScanGitHub(ctx, cfg); err != nil {
 			logFatal(err, "Failed to scan Github.")

pkg/detectors/launchdarkly/launchdarkly.go

@@ -20,13 +20,13 @@ var (
 	client = common.SaneHttpClient()
 
 	// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
-	keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"launchdarkly", "launch_darkly"}) + `\b([a-z0-9-]{40})\b`)
+	keyPat = regexp.MustCompile(`\b((?:api|sdk|mob)-[a-z0-9]{8}-[a-z0-9]{4}-4[a-z0-9]{3}-[a-z0-9]{4}-[a-z0-9]{12})\b`)
 )
 
 // Keywords are used for efficiently pre-filtering chunks.
 // Use identifiers in the secret preferably, or the provider name.
 func (s Scanner) Keywords() []string {
-	return []string{"launchdarkly", "launch_darkly"}
+	return []string{"api", "sdk", "mob"}
 }
 
 // FromData will find and optionally verify LaunchDarkly secrets in a given set of bytes.

pkg/detectors/salesforce/salesforce.go

@@ -1,113 +0,0 @@
-package salesforce
-
-import (
-	"context"
-	"fmt"
-	"net/http"
-	"regexp"
-	"strings"
-
-	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
-	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
-	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
-)
-
-const (
-	currentVersion = "58.0" // current Salesforce version
-)
-
-type Scanner struct {
-	client *http.Client
-}
-
-// Ensure the Scanner satisfies the interface at compile time.
-var _ detectors.Detector = (*Scanner)(nil)
-
-var (
-	defaultClient = common.SaneHttpClient()
-	// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
-	accessTokenPat = regexp.MustCompile(`\b00[a-zA-Z0-9]{13}![a-zA-Z0-9_.]{96}\b`)
-	instancePat    = regexp.MustCompile(`\bhttps://[0-9a-zA-Z-\.]{1,100}\.my\.salesforce\.com\b`)
-)
-
-// Keywords are used for efficiently pre-filtering chunks.
-// Use identifiers in the secret preferably, or the provider name.
-func (s Scanner) Keywords() []string {
-	return []string{"salesforce"}
-}
-
-// FromData will find and optionally verify Salesforce secrets in a given set of bytes.
-func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
-	dataStr := string(data)
-
-	instanceMatches := instancePat.FindAllStringSubmatch(dataStr, -1)
-	tokenMatches := accessTokenPat.FindAllStringSubmatch(dataStr, -1)
-
-	fmt.Printf("instanceMatches: %v\n", instanceMatches)
-
-	for _, instance := range instanceMatches {
-		if len(instance) != 1 {
-			continue
-		}
-
-		instanceMatch := strings.TrimSpace(instance[0])
-
-		for _, token := range tokenMatches {
-			if len(token) != 1 {
-				continue
-			}
-
-			tokenMatch := strings.TrimSpace(token[0])
-
-			s1 := detectors.Result{
-				DetectorType: detectorspb.DetectorType_Salesforce,
-				Raw:          []byte(tokenMatch),
-			}
-
-			if verify {
-				client := s.client
-				if client == nil {
-					client = defaultClient
-				}
-
-				req, err := http.NewRequestWithContext(ctx, "GET", instanceMatch+"/services/data/v"+currentVersion+"/query?q=SELECT+name+from+Account", nil)
-				if err != nil {
-					continue
-				}
-				req.Header.Set("Authorization", "Bearer "+tokenMatch)
-
-				res, err := client.Do(req)
-
-				if err != nil {
-					// End execution, append Detector Result if request fails to prevent panic on response body checks
-					s1.VerificationError = err
-					results = append(results, s1)
-					continue
-				}
-
-				verifiedBodyResponse, err := common.ResponseContainsSubstring(res.Body, "records")
-
-				defer res.Body.Close()
-				if res.StatusCode >= 200 && res.StatusCode < 300 && verifiedBodyResponse {
-					s1.Verified = true
-				} else if res.StatusCode >= 200 && res.StatusCode < 300 && !verifiedBodyResponse {
-					s1.Verified = false
-				} else if res.StatusCode == 401 {
-					s1.Verified = false
-				} else {
-					s1.VerificationError = fmt.Errorf("request to %v returned status %d with error %+v", res.Request.URL, res.StatusCode, err)
-				}
-
-			}
-
-			results = append(results, s1)
-
-		}
-	}
-
-	return results, nil
-}
-
-func (s Scanner) Type() detectorspb.DetectorType {
-	return detectorspb.DetectorType_Salesforce
-}

pkg/detectors/salesforce/salesforce_test.go

@@ -1,167 +0,0 @@
-//go:build detectors
-// +build detectors
-
-package salesforce
-
-import (
-	"context"
-	"fmt"
-	"github.com/google/go-cmp/cmp"
-	"github.com/google/go-cmp/cmp/cmpopts"
-	"testing"
-	"time"
-
-	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
-
-	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
-	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
-)
-
-func TestSalesforce_FromChunk(t *testing.T) {
-	ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
-	defer cancel()
-	testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")
-	if err != nil {
-		t.Fatalf("could not get test secrets from GCP: %s", err)
-	}
-	instance := testSecrets.MustGetField("SALESFORCE_INSTANCE")
-	// Salesforce secrets are not valid for long, so we may need to regenerate them.
-	// Steps to regenerate:
-	// 1. Install Salesforce CLI
-	// 2. Authenticate with `sfdx org web login`. This will open a browser window. Use the credentials in the detection test accounts 1Pass vault.
-	// 3. Run `sfdx org display --targetusername zubairkhan@trufflesec.com`to obtain new token.
-	// Source: https://developer.salesforce.com/docs/atlas.en-us.api_rest.meta/api_rest/quickstart_oauth.htm
-	token := testSecrets.MustGetField("SALESFORCE_TOKEN")
-	inactiveToken := testSecrets.MustGetField("SALESFORCE_INACTIVE_TOKEN")
-
-	type args struct {
-		ctx    context.Context
-		data   []byte
-		verify bool
-	}
-	tests := []struct {
-		name                string
-		s                   Scanner
-		args                args
-		want                []detectors.Result
-		wantErr             bool
-		wantVerificationErr bool
-	}{
-		{
-			name: "found, verified",
-			s:    Scanner{},
-			args: args{
-				ctx:    context.Background(),
-				data:   []byte(fmt.Sprintf("You can find a salesforce secret within %s for %s", token, instance)),
-				verify: true,
-			},
-			want: []detectors.Result{
-				{
-					DetectorType: detectorspb.DetectorType_Salesforce,
-					Verified:     true,
-				},
-			},
-			wantErr:             false,
-			wantVerificationErr: false,
-		},
-		{
-			name: "found, unverified",
-			s:    Scanner{},
-			args: args{
-				ctx:    context.Background(),
-				data:   []byte(fmt.Sprintf("You can find a salesforce secret %s within but not valid for %s", inactiveToken, instance)), // the secret would satisfy the regex but not pass validation
-				verify: true,
-			},
-			want: []detectors.Result{
-				{
-					DetectorType: detectorspb.DetectorType_Salesforce,
-					Verified:     false,
-				},
-			},
-			wantErr:             false,
-			wantVerificationErr: false,
-		},
-		{
-			name: "not found",
-			s:    Scanner{},
-			args: args{
-				ctx:    context.Background(),
-				data:   []byte("You cannot find the secret within"),
-				verify: true,
-			},
-			want:                nil,
-			wantErr:             false,
-			wantVerificationErr: false,
-		},
-		{
-			name: "found, would be verified if not for timeout",
-			s:    Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},
-			args: args{
-				ctx:    context.Background(),
-				data:   []byte(fmt.Sprintf("You can find a salesforce secret within %s for %s", token, instance)),
-				verify: true,
-			},
-			want: []detectors.Result{
-				{
-					DetectorType: detectorspb.DetectorType_Salesforce,
-					Verified:     false,
-				},
-			},
-			wantErr:             false,
-			wantVerificationErr: true,
-		},
-		{
-			name: "found, verified but unexpected api surface",
-			s:    Scanner{client: common.ConstantResponseHttpClient(404, "")},
-			args: args{
-				ctx:    context.Background(),
-				data:   []byte(fmt.Sprintf("You can find a salesforce secret within %s for %s", token, instance)),
-				verify: true,
-			},
-			want: []detectors.Result{
-				{
-					DetectorType: detectorspb.DetectorType_Salesforce,
-					Verified:     false,
-				},
-			},
-			wantErr:             false,
-			wantVerificationErr: true,
-		},
-	}
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
-			if (err != nil) != tt.wantErr {
-				t.Errorf("Salesforce.FromData() error = %v, wantErr %v", err, tt.wantErr)
-			}
-			for i := range got {
-				if len(got[i].Raw) == 0 {
-					t.Fatalf("no raw secret present: \n %+v", got[i])
-				}
-
-				if (got[i].VerificationError != nil) != tt.wantVerificationErr {
-					t.Fatalf(" wantVerificationError = %v, verification error = %v,", tt.wantVerificationErr, got[i].VerificationError)
-				}
-			}
-			ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "VerificationError")
-			if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {
-				t.Errorf("Salesforce.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
-			}
-		})
-	}
-}
-
-func BenchmarkFromData(benchmark *testing.B) {
-	ctx := context.Background()
-	s := Scanner{}
-	for name, data := range detectors.MustGetBenchmarkData() {
-		benchmark.Run(name, func(b *testing.B) {
-			for n := 0; n < b.N; n++ {
-				_, err := s.FromData(ctx, false, data)
-				if err != nil {
-					b.Fatal(err)
-				}
-			}
-		})
-	}
-}

pkg/engine/engine.go

@@ -415,13 +415,6 @@ func (e *Engine) ResultsChan() chan detectors.ResultWithMetadata {
 	return e.results
 }
 
-// ScanChunk injects a chunk into the output stream of chunks to be scanned.
-// This method should rarely be used. TODO: Remove when dependencies no longer
-// rely on this functionality.
-func (e *Engine) ScanChunk(chunk *sources.Chunk) {
-	e.sourceManager.ScanChunk(chunk)
-}
-
 // detectableChunk is a decoded chunk that is ready to be scanned by its detector.
 type detectableChunk struct {
 	detector detectors.Detector

pkg/engine/github.go

@@ -15,15 +15,12 @@ import (
 // ScanGitHub scans Github with the provided options.
 func (e *Engine) ScanGitHub(ctx context.Context, c sources.GithubConfig) error {
 	connection := sourcespb.GitHub{
-		Endpoint:                   c.Endpoint,
-		Organizations:              c.Orgs,
-		Repositories:               c.Repos,
-		ScanUsers:                  c.IncludeMembers,
-		IgnoreRepos:                c.ExcludeRepos,
-		IncludeRepos:               c.IncludeRepos,
-		IncludeIssueComments:       c.IncludeIssueComments,
-		IncludePullRequestComments: c.IncludePullRequestComments,
-		IncludeGistComments:        c.IncludeGistComments,
+		Endpoint:      c.Endpoint,
+		Organizations: c.Orgs,
+		Repositories:  c.Repos,
+		ScanUsers:     c.IncludeMembers,
+		IgnoreRepos:   c.ExcludeRepos,
+		IncludeRepos:  c.IncludeRepos,
 	}
 	if len(c.Token) > 0 {
 		connection.Credential = &sourcespb.GitHub_Token{

pkg/handlers/archive.go

@@ -6,13 +6,8 @@ import (
 	"errors"
 	"fmt"
 	"io"
-	"os"
-	"os/exec"
-	"path/filepath"
-	"strings"
 	"time"
 
-	"github.com/h2non/filetype"
 	"github.com/mholt/archiver/v4"
 
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
@@ -31,18 +26,14 @@ var (
 	maxTimeout = time.Duration(30) * time.Second
 )
 
-// Ensure the Archive satisfies the interfaces at compile time.
-var _ SpecializedHandler = (*Archive)(nil)
-
 // Archive is a handler for extracting and decompressing archives.
 type Archive struct {
-	size         int
-	currentDepth int
+	size int
 }
 
 // New sets a default maximum size and current size counter.
-func (a *Archive) New() {
-	a.size = 0
+func (d *Archive) New() {
+	d.size = 0
 }
 
 // SetArchiveMaxSize sets the maximum size of the archive.
@@ -61,14 +52,14 @@ func SetArchiveMaxTimeout(timeout time.Duration) {
 }
 
 // FromFile extracts the files from an archive.
-func (a *Archive) FromFile(originalCtx context.Context, data io.Reader) chan []byte {
-	archiveChan := make(chan []byte, 512)
+func (d *Archive) FromFile(originalCtx context.Context, data io.Reader) chan ([]byte) {
+	archiveChan := make(chan ([]byte), 512)
 	go func() {
 		ctx, cancel := context.WithTimeout(originalCtx, maxTimeout)
 		logger := logContext.AddLogger(ctx).Logger()
 		defer cancel()
 		defer close(archiveChan)
-		err := a.openArchive(ctx, 0, data, archiveChan)
+		err := d.openArchive(ctx, 0, data, archiveChan)
 		if err != nil {
 			if errors.Is(err, archiver.ErrNoMatch) {
 				return
@@ -80,7 +71,7 @@ func (a *Archive) FromFile(originalCtx context.Context, data io.Reader) chan []b
 }
 
 // openArchive takes a reader and extracts the contents up to the maximum depth.
-func (a *Archive) openArchive(ctx context.Context, depth int, reader io.Reader, archiveChan chan []byte) error {
+func (d *Archive) openArchive(ctx context.Context, depth int, reader io.Reader, archiveChan chan ([]byte)) error {
 	if depth >= maxDepth {
 		return fmt.Errorf("max archive depth reached")
 	}
@@ -106,14 +97,14 @@ func (a *Archive) openArchive(ctx context.Context, depth int, reader io.Reader,
 		if err != nil {
 			return err
 		}
-		fileBytes, err := a.ReadToMax(ctx, compReader)
+		fileBytes, err := d.ReadToMax(ctx, compReader)
 		if err != nil {
 			return err
 		}
 		newReader := bytes.NewReader(fileBytes)
-		return a.openArchive(ctx, depth+1, newReader, archiveChan)
+		return d.openArchive(ctx, depth+1, newReader, archiveChan)
 	case archiver.Extractor:
-		err := archive.Extract(context.WithValue(ctx, depthKey, depth+1), reader, nil, a.extractorHandler(archiveChan))
+		err := archive.Extract(context.WithValue(ctx, depthKey, depth+1), reader, nil, d.extractorHandler(archiveChan))
 		if err != nil {
 			return err
 		}
@@ -123,7 +114,7 @@ func (a *Archive) openArchive(ctx context.Context, depth int, reader io.Reader,
 }
 
 // IsFiletype returns true if the provided reader is an archive.
-func (a *Archive) IsFiletype(ctx context.Context, reader io.Reader) (io.Reader, bool) {
+func (d *Archive) IsFiletype(ctx context.Context, reader io.Reader) (io.Reader, bool) {
 	format, readerB, err := archiver.Identify("", reader)
 	if err != nil {
 		return readerB, false
@@ -138,7 +129,7 @@ func (a *Archive) IsFiletype(ctx context.Context, reader io.Reader) (io.Reader,
 }
 
 // extractorHandler is applied to each file in an archiver.Extractor file.
-func (a *Archive) extractorHandler(archiveChan chan []byte) func(context.Context, archiver.File) error {
+func (d *Archive) extractorHandler(archiveChan chan ([]byte)) func(context.Context, archiver.File) error {
 	return func(ctx context.Context, f archiver.File) error {
 		logger := logContext.AddLogger(ctx).Logger()
 		logger.V(5).Info("Handling extracted file.", "filename", f.Name())
@@ -151,13 +142,13 @@ func (a *Archive) extractorHandler(archiveChan chan []byte) func(context.Context
 		if err != nil {
 			return err
 		}
-		fileBytes, err := a.ReadToMax(ctx, fReader)
+		fileBytes, err := d.ReadToMax(ctx, fReader)
 		if err != nil {
 			return err
 		}
 		fileContent := bytes.NewReader(fileBytes)
 
-		err = a.openArchive(ctx, depth, fileContent, archiveChan)
+		err = d.openArchive(ctx, depth, fileContent, archiveChan)
 		if err != nil {
 			return err
 		}
@@ -166,7 +157,7 @@ func (a *Archive) extractorHandler(archiveChan chan []byte) func(context.Context
 }
 
 // ReadToMax reads up to the max size.
-func (a *Archive) ReadToMax(ctx context.Context, reader io.Reader) (data []byte, err error) {
+func (d *Archive) ReadToMax(ctx context.Context, reader io.Reader) (data []byte, err error) {
 	// Archiver v4 is in alpha and using an experimental version of
 	// rardecode. There is a bug somewhere with rar decoder format 29
 	// that can lead to a panic. An issue is open in rardecode repo
@@ -184,7 +175,7 @@ func (a *Archive) ReadToMax(ctx context.Context, reader io.Reader) (data []byte,
 		}
 	}()
 	fileContent := bytes.Buffer{}
-	logger.V(5).Info("Remaining buffer capacity", "bytes", maxSize-a.size)
+	logger.V(5).Info("Remaining buffer capacity", "bytes", maxSize-d.size)
 	for i := 0; i <= maxSize/512; i++ {
 		if common.IsDone(ctx) {
 			return nil, ctx.Err()
@@ -194,285 +185,17 @@ func (a *Archive) ReadToMax(ctx context.Context, reader io.Reader) (data []byte,
 		if err != nil && !errors.Is(err, io.EOF) {
 			return []byte{}, err
 		}
-		a.size += bRead
+		d.size += bRead
 		if len(fileChunk) > 0 {
 			fileContent.Write(fileChunk[0:bRead])
 		}
 		if bRead < 512 {
 			return fileContent.Bytes(), nil
 		}
-		if a.size >= maxSize && bRead == 512 {
+		if d.size >= maxSize && bRead == 512 {
 			logger.V(2).Info("Max archive size reached.")
 			return fileContent.Bytes(), nil
 		}
 	}
 	return fileContent.Bytes(), nil
 }
-
-const (
-	arMimeType  = "application/x-unix-archive"
-	rpmMimeType = "application/x-rpm"
-)
-
-// Define a map of mime types to corresponding command-line tools
-var mimeTools = map[string][]string{
-	arMimeType:  {"ar"},
-	rpmMimeType: {"rpm2cpio", "cpio"},
-}
-
-// Check if the command-line tool is installed.
-func isToolInstalled(tool string) bool {
-	_, err := exec.LookPath(tool)
-	return err == nil
-}
-
-// Ensure all tools are available for given mime type.
-func ensureToolsForMimeType(mimeType string) error {
-	tools, exists := mimeTools[mimeType]
-	if !exists {
-		return fmt.Errorf("unsupported mime type")
-	}
-
-	for _, tool := range tools {
-		if !isToolInstalled(tool) {
-			return fmt.Errorf("Required tool " + tool + " is not installed")
-		}
-	}
-
-	return nil
-}
-
-// HandleSpecialized takes a file path and an io.Reader representing the input file,
-// and processes it based on its extension, such as handling Debian (.deb) and RPM (.rpm) packages.
-// It returns an io.Reader that can be used to read the processed content of the file,
-// and an error if any issues occurred during processing.
-// If the file is specialized, the returned boolean is true with no error.
-// The caller is responsible for closing the returned reader.
-func (a *Archive) HandleSpecialized(ctx logContext.Context, reader io.Reader) (io.Reader, bool, error) {
-	mimeType, reader, err := determineMimeType(reader)
-	if err != nil {
-		return nil, false, err
-	}
-
-	switch mimeType {
-	case arMimeType: // includes .deb files
-		if err := ensureToolsForMimeType(mimeType); err != nil {
-			return nil, false, err
-		}
-		reader, err = a.extractDebContent(ctx, reader)
-	case rpmMimeType:
-		if err := ensureToolsForMimeType(mimeType); err != nil {
-			return nil, false, err
-		}
-		reader, err = a.extractRpmContent(ctx, reader)
-	default:
-		return reader, false, nil
-	}
-
-	if err != nil {
-		return nil, false, fmt.Errorf("unable to extract file with MIME type %s: %w", mimeType, err)
-	}
-	return reader, true, nil
-}
-
-// extractDebContent takes a .deb file as an io.Reader, extracts its contents
-// into a temporary directory, and returns a Reader for the extracted data archive.
-// It handles the extraction process by using the 'ar' command and manages temporary
-// files and directories for the operation.
-// The caller is responsible for closing the returned reader.
-func (a *Archive) extractDebContent(ctx logContext.Context, file io.Reader) (io.ReadCloser, error) {
-	if a.currentDepth >= maxDepth {
-		return nil, fmt.Errorf("max archive depth reached")
-	}
-
-	tmpEnv, err := a.createTempEnv(ctx, file)
-	if err != nil {
-		return nil, err
-	}
-	defer os.Remove(tmpEnv.tempFileName)
-	defer os.RemoveAll(tmpEnv.extractPath)
-
-	cmd := exec.Command("ar", "x", tmpEnv.tempFile.Name())
-	cmd.Dir = tmpEnv.extractPath
-	if err := executeCommand(cmd); err != nil {
-		return nil, err
-	}
-
-	handler := func(ctx logContext.Context, env tempEnv, file string) (string, error) {
-		if strings.HasPrefix(file, "data.tar.") {
-			return file, nil
-		}
-		return a.handleNestedFileMIME(ctx, env, file)
-	}
-
-	dataArchiveName, err := a.handleExtractedFiles(ctx, tmpEnv, handler)
-	if err != nil {
-		return nil, err
-	}
-
-	return openDataArchive(tmpEnv.extractPath, dataArchiveName)
-}
-
-// extractRpmContent takes an .rpm file as an io.Reader, extracts its contents
-// into a temporary directory, and returns a Reader for the extracted data archive.
-// It handles the extraction process by using the 'rpm2cpio' and 'cpio' commands and manages temporary
-// files and directories for the operation.
-// The caller is responsible for closing the returned reader.
-func (a *Archive) extractRpmContent(ctx logContext.Context, file io.Reader) (io.ReadCloser, error) {
-	if a.currentDepth >= maxDepth {
-		return nil, fmt.Errorf("max archive depth reached")
-	}
-
-	tmpEnv, err := a.createTempEnv(ctx, file)
-	if err != nil {
-		return nil, err
-	}
-	defer os.Remove(tmpEnv.tempFileName)
-	defer os.RemoveAll(tmpEnv.extractPath)
-
-	// Use rpm2cpio to convert the RPM file to a cpio archive and then extract it using cpio command.
-	cmd := exec.Command("sh", "-c", "rpm2cpio "+tmpEnv.tempFile.Name()+" | cpio -id")
-	cmd.Dir = tmpEnv.extractPath
-	if err := executeCommand(cmd); err != nil {
-		return nil, err
-	}
-
-	handler := func(ctx logContext.Context, env tempEnv, file string) (string, error) {
-		if strings.HasSuffix(file, ".tar.gz") {
-			return file, nil
-		}
-		return a.handleNestedFileMIME(ctx, env, file)
-	}
-
-	dataArchiveName, err := a.handleExtractedFiles(ctx, tmpEnv, handler)
-	if err != nil {
-		return nil, err
-	}
-
-	return openDataArchive(tmpEnv.extractPath, dataArchiveName)
-}
-
-func (a *Archive) handleNestedFileMIME(ctx logContext.Context, tempEnv tempEnv, fileName string) (string, error) {
-	nestedFile, err := os.Open(filepath.Join(tempEnv.extractPath, fileName))
-	if err != nil {
-		return "", err
-	}
-	defer nestedFile.Close()
-
-	mimeType, reader, err := determineMimeType(nestedFile)
-	if err != nil {
-		return "", fmt.Errorf("unable to determine MIME type of nested filename: %s, %w", nestedFile.Name(), err)
-	}
-
-	switch mimeType {
-	case arMimeType:
-		_, _, err = a.HandleSpecialized(ctx, reader)
-	case rpmMimeType:
-		_, _, err = a.HandleSpecialized(ctx, reader)
-	default:
-		return "", nil
-	}
-
-	if err != nil {
-		return "", fmt.Errorf("unable to extract file with MIME type %s: %w", mimeType, err)
-	}
-
-	return fileName, nil
-}
-
-// determineMimeType reads from the provided reader to detect the MIME type.
-// It returns the detected MIME type and a new reader that includes the read portion.
-func determineMimeType(reader io.Reader) (string, io.Reader, error) {
-	buffer := make([]byte, 512)
-	n, err := reader.Read(buffer)
-	if err != nil {
-		return "", nil, fmt.Errorf("unable to read file for MIME type detection: %w", err)
-	}
-
-	// Create a new reader that starts with the buffer we just read
-	// and continues with the rest of the original reader.
-	reader = io.MultiReader(bytes.NewReader(buffer[:n]), reader)
-
-	kind, err := filetype.Match(buffer)
-	if err != nil {
-		return "", nil, fmt.Errorf("unable to determine file type: %w", err)
-	}
-
-	return kind.MIME.Value, reader, nil
-}
-
-// handleExtractedFiles processes each file in the extracted directory using a provided handler function.
-// The function iterates through the files, applying the handleFile function to each, and returns the name
-// of the data archive it finds. This centralizes the logic for handling specialized files such as .deb and .rpm
-// by using the appropriate handling function passed as an argument. This design allows for flexibility and reuse
-// of this function across various extraction processes in the package.
-func (a *Archive) handleExtractedFiles(ctx logContext.Context, env tempEnv, handleFile func(logContext.Context, tempEnv, string) (string, error)) (string, error) {
-	extractedFiles, err := os.ReadDir(env.extractPath)
-	if err != nil {
-		return "", fmt.Errorf("unable to read extracted directory: %w", err)
-	}
-
-	var dataArchiveName string
-	for _, file := range extractedFiles {
-		name, err := handleFile(ctx, env, file.Name())
-		if err != nil {
-			return "", err
-		}
-		if name != "" {
-			dataArchiveName = name
-			break
-		}
-	}
-
-	return dataArchiveName, nil
-}
-
-type tempEnv struct {
-	tempFile     *os.File
-	tempFileName string
-	extractPath  string
-}
-
-// createTempEnv creates a temporary file and a temporary directory for extracting archives.
-// The caller is responsible for removing these temporary resources
-// (both the file and directory) when they are no longer needed.
-func (a *Archive) createTempEnv(ctx context.Context, file io.Reader) (tempEnv, error) {
-	tempFile, err := os.CreateTemp("", "tmp")
-	if err != nil {
-		return tempEnv{}, fmt.Errorf("unable to create temporary file: %w", err)
-	}
-
-	extractPath, err := os.MkdirTemp("", "tmp_archive")
-	if err != nil {
-		return tempEnv{}, fmt.Errorf("unable to create temporary directory: %w", err)
-	}
-
-	b, err := a.ReadToMax(ctx, file)
-	if err != nil {
-		return tempEnv{}, err
-	}
-
-	if _, err = tempFile.Write(b); err != nil {
-		return tempEnv{}, fmt.Errorf("unable to write to temporary file: %w", err)
-	}
-
-	return tempEnv{tempFile: tempFile, tempFileName: tempFile.Name(), extractPath: extractPath}, nil
-}
-
-func executeCommand(cmd *exec.Cmd) error {
-	var stderr bytes.Buffer
-	cmd.Stderr = &stderr
-	if err := cmd.Run(); err != nil {
-		return fmt.Errorf("unable to execute command %v: %w; error: %s", cmd.String(), err, stderr.String())
-	}
-	return nil
-}
-
-func openDataArchive(extractPath string, dataArchiveName string) (io.ReadCloser, error) {
-	dataArchivePath := filepath.Join(extractPath, dataArchiveName)
-	dataFile, err := os.Open(dataArchivePath)
-	if err != nil {
-		return nil, fmt.Errorf("unable to open file: %w", err)
-	}
-	return dataFile, nil
-}

pkg/handlers/archive_test.go

@@ -2,9 +2,7 @@ package handlers
 
 import (
 	"context"
-	"io"
 	"net/http"
-	"os"
 	"regexp"
 	"strings"
 	"testing"
@@ -12,7 +10,6 @@ import (
 	diskbufferreader "github.com/bill-rich/disk-buffer-reader"
 	"github.com/stretchr/testify/assert"
 
-	logContext "github.com/trufflesecurity/trufflehog/v3/pkg/context"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
 )
 
@@ -125,61 +122,3 @@ func TestHandleFile(t *testing.T) {
 	assert.True(t, HandleFile(context.Background(), reader, &sources.Chunk{}, ch))
 	assert.Equal(t, 1, len(ch))
 }
-
-func TestExtractDebContent(t *testing.T) {
-	// Open the sample .deb file from the testdata folder.
-	file, err := os.Open("testdata/test.deb")
-	assert.Nil(t, err)
-	defer file.Close()
-
-	ctx := logContext.AddLogger(context.Background())
-	a := &Archive{}
-
-	reader, err := a.extractDebContent(ctx, file)
-	assert.Nil(t, err)
-
-	content, err := io.ReadAll(reader)
-	assert.Nil(t, err)
-	expectedLength := 1015582
-	assert.Equal(t, expectedLength, len(string(content)))
-}
-
-func TestExtractTarContent(t *testing.T) {
-	file, err := os.Open("testdata/test.tgz")
-	assert.Nil(t, err)
-	defer file.Close()
-
-	ctx := context.Background()
-
-	chunkCh := make(chan *sources.Chunk)
-	go func() {
-		defer close(chunkCh)
-		ok := HandleFile(ctx, file, &sources.Chunk{}, chunkCh)
-		assert.True(t, ok)
-	}()
-
-	wantCount := 4
-	count := 0
-	for range chunkCh {
-		count++
-	}
-	assert.Equal(t, wantCount, count)
-}
-
-func TestExtractRPMContent(t *testing.T) {
-	// Open the sample .rpm file from the testdata folder.
-	file, err := os.Open("testdata/test.rpm")
-	assert.Nil(t, err)
-	defer file.Close()
-
-	ctx := logContext.AddLogger(context.Background())
-	a := &Archive{}
-
-	reader, err := a.extractRpmContent(ctx, file)
-	assert.Nil(t, err)
-
-	content, err := io.ReadAll(reader)
-	assert.Nil(t, err)
-	expectedLength := 1822720
-	assert.Equal(t, expectedLength, len(string(content)))
-}

pkg/handlers/handlers.go

@@ -4,9 +4,6 @@ import (
 	"context"
 	"io"
 
-	diskbufferreader "github.com/bill-rich/disk-buffer-reader"
-
-	logContext "github.com/trufflesecurity/trufflehog/v3/pkg/context"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
 )
 
@@ -16,82 +13,39 @@ func DefaultHandlers() []Handler {
 	}
 }
 
-// SpecializedHandler defines the interface for handlers that can process specialized archives.
-// It includes a method to handle specialized archives and determine if the file is of a special type.
-type SpecializedHandler interface {
-	// HandleSpecialized examines the provided file reader within the context and determines if it is a specialized archive.
-	// It returns a reader with any necessary modifications, a boolean indicating if the file was specialized,
-	// and an error if something went wrong during processing.
-	HandleSpecialized(logContext.Context, io.Reader) (io.Reader, bool, error)
-}
-
 type Handler interface {
-	FromFile(context.Context, io.Reader) chan []byte
+	FromFile(context.Context, io.Reader) chan ([]byte)
 	IsFiletype(context.Context, io.Reader) (io.Reader, bool)
 	New()
 }
 
-// HandleFile processes a given file by selecting an appropriate handler from DefaultHandlers.
-// It first checks if the handler implements SpecializedHandler for any special processing,
-// then falls back to regular file type handling. If successful, it reads the file in chunks,
-// packages them in the provided chunk skeleton, and sends them to chunksChan.
-// The function returns true if processing was successful and false otherwise.
-// Context is used for cancellation, and the caller is responsible for canceling it if needed.
-func HandleFile(ctx context.Context, file io.Reader, chunkSkel *sources.Chunk, chunksChan chan *sources.Chunk) bool {
-	aCtx := logContext.AddLogger(ctx)
+func HandleFile(ctx context.Context, file io.Reader, chunkSkel *sources.Chunk, chunksChan chan (*sources.Chunk)) bool {
+	// Find a handler for this file.
+	var handler Handler
 	for _, h := range DefaultHandlers() {
 		h.New()
-
-		// The re-reader is used to reset the file reader after checking if the handler implements SpecializedHandler.
-		// This is necessary because the archive pkg doesn't correctly determine the file type when using
-		// an io.MultiReader, which is used by the SpecializedHandler.
-		reReader, err := diskbufferreader.New(file)
-		if err != nil {
-			aCtx.Logger().Error(err, "error creating re-reader reader")
-			return false
+		var isType bool
+		if file, isType = h.IsFiletype(ctx, file); isType {
+			handler = h
+			break
 		}
-		defer reReader.Close()
-
-		// Check if the handler implements SpecializedHandler and process accordingly.
-		if specialHandler, ok := h.(SpecializedHandler); ok {
-			file, isSpecial, err := specialHandler.HandleSpecialized(aCtx, reReader)
-			if isSpecial {
-				return handleChunks(aCtx, h.FromFile(ctx, file), chunkSkel, chunksChan)
-			}
-
-			if err != nil {
-				aCtx.Logger().Error(err, "error handling file")
-			}
-		}
-
-		if err := reReader.Reset(); err != nil {
-			aCtx.Logger().Error(err, "error resetting re-reader")
-			return false
-		}
-		if _, isType := h.IsFiletype(aCtx, reReader); !isType {
-			continue
-		}
-
-		if err := reReader.Reset(); err != nil {
-			aCtx.Logger().Error(err, "error resetting re-reader")
-			return false
-		}
-		reReader.Stop()
-		return handleChunks(aCtx, h.FromFile(ctx, reReader), chunkSkel, chunksChan)
+	}
+	if handler == nil {
+		return false
 	}
 
-	return false
-}
-
-func handleChunks(ctx context.Context, handlerChan chan []byte, chunkSkel *sources.Chunk, chunksChan chan *sources.Chunk) bool {
+	// Process the file and read all []byte chunks from handlerChan.
+	handlerChan := handler.FromFile(ctx, file)
 	for {
 		select {
 		case data, open := <-handlerChan:
 			if !open {
+				// We finished reading everything from handlerChan.
 				return true
 			}
 			chunk := *chunkSkel
 			chunk.Data = data
+			// Send data on chunksChan.
 			select {
 			case chunksChan <- &chunk:
 			case <-ctx.Done():

pkg/pb/detectorspb/detectors.pb.go

@@ -996,8 +996,8 @@ const (
 	DetectorType_TrufflehogEnterprise          DetectorType = 922
 	DetectorType_EnvoyApiKey                   DetectorType = 923
 	DetectorType_GitHubOauth2                  DetectorType = 924
-	DetectorType_Salesforce                    DetectorType = 925
-	DetectorType_HuggingFace                   DetectorType = 926
+	// 925 reserved for Salesforce
+	DetectorType_HuggingFace DetectorType = 926
 )
 
 // Enum value maps for DetectorType.
@@ -1924,7 +1924,6 @@ var (
 		922: "TrufflehogEnterprise",
 		923: "EnvoyApiKey",
 		924: "GitHubOauth2",
-		925: "Salesforce",
 		926: "HuggingFace",
 	}
 	DetectorType_value = map[string]int32{
@@ -2849,7 +2848,6 @@ var (
 		"TrufflehogEnterprise":          922,
 		"EnvoyApiKey":                   923,
 		"GitHubOauth2":                  924,
-		"Salesforce":                    925,
 		"HuggingFace":                   926,
 	}
 )
@@ -3229,7 +3227,7 @@ var file_detectors_proto_rawDesc = []byte{
 	0x44, 0x65, 0x63, 0x6f, 0x64, 0x65, 0x72, 0x54, 0x79, 0x70, 0x65, 0x12, 0x0b, 0x0a, 0x07, 0x55,
 	0x4e, 0x4b, 0x4e, 0x4f, 0x57, 0x4e, 0x10, 0x00, 0x12, 0x09, 0x0a, 0x05, 0x50, 0x4c, 0x41, 0x49,
 	0x4e, 0x10, 0x01, 0x12, 0x0a, 0x0a, 0x06, 0x42, 0x41, 0x53, 0x45, 0x36, 0x34, 0x10, 0x02, 0x12,
-	0x09, 0x0a, 0x05, 0x55, 0x54, 0x46, 0x31, 0x36, 0x10, 0x03, 0x2a, 0xfc, 0x73, 0x0a, 0x0c, 0x44,
+	0x09, 0x0a, 0x05, 0x55, 0x54, 0x46, 0x31, 0x36, 0x10, 0x03, 0x2a, 0xeb, 0x73, 0x0a, 0x0c, 0x44,
 	0x65, 0x74, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x54, 0x79, 0x70, 0x65, 0x12, 0x0b, 0x0a, 0x07, 0x41,
 	0x6c, 0x69, 0x62, 0x61, 0x62, 0x61, 0x10, 0x00, 0x12, 0x08, 0x0a, 0x04, 0x41, 0x4d, 0x51, 0x50,
 	0x10, 0x01, 0x12, 0x07, 0x0a, 0x03, 0x41, 0x57, 0x53, 0x10, 0x02, 0x12, 0x09, 0x0a, 0x05, 0x41,
@@ -4155,13 +4153,12 @@ var file_detectors_proto_rawDesc = []byte{
 	0x65, 0x68, 0x6f, 0x67, 0x45, 0x6e, 0x74, 0x65, 0x72, 0x70, 0x72, 0x69, 0x73, 0x65, 0x10, 0x9a,
 	0x07, 0x12, 0x10, 0x0a, 0x0b, 0x45, 0x6e, 0x76, 0x6f, 0x79, 0x41, 0x70, 0x69, 0x4b, 0x65, 0x79,
 	0x10, 0x9b, 0x07, 0x12, 0x11, 0x0a, 0x0c, 0x47, 0x69, 0x74, 0x48, 0x75, 0x62, 0x4f, 0x61, 0x75,
-	0x74, 0x68, 0x32, 0x10, 0x9c, 0x07, 0x12, 0x0f, 0x0a, 0x0a, 0x53, 0x61, 0x6c, 0x65, 0x73, 0x66,
-	0x6f, 0x72, 0x63, 0x65, 0x10, 0x9d, 0x07, 0x12, 0x10, 0x0a, 0x0b, 0x48, 0x75, 0x67, 0x67, 0x69,
-	0x6e, 0x67, 0x46, 0x61, 0x63, 0x65, 0x10, 0x9e, 0x07, 0x42, 0x3d, 0x5a, 0x3b, 0x67, 0x69, 0x74,
-	0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x74, 0x72, 0x75, 0x66, 0x66, 0x6c, 0x65, 0x73,
-	0x65, 0x63, 0x75, 0x72, 0x69, 0x74, 0x79, 0x2f, 0x74, 0x72, 0x75, 0x66, 0x66, 0x6c, 0x65, 0x68,
-	0x6f, 0x67, 0x2f, 0x76, 0x33, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x70, 0x62, 0x2f, 0x64, 0x65, 0x74,
-	0x65, 0x63, 0x74, 0x6f, 0x72, 0x73, 0x70, 0x62, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
+	0x74, 0x68, 0x32, 0x10, 0x9c, 0x07, 0x12, 0x10, 0x0a, 0x0b, 0x48, 0x75, 0x67, 0x67, 0x69, 0x6e,
+	0x67, 0x46, 0x61, 0x63, 0x65, 0x10, 0x9e, 0x07, 0x42, 0x3d, 0x5a, 0x3b, 0x67, 0x69, 0x74, 0x68,
+	0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x74, 0x72, 0x75, 0x66, 0x66, 0x6c, 0x65, 0x73, 0x65,
+	0x63, 0x75, 0x72, 0x69, 0x74, 0x79, 0x2f, 0x74, 0x72, 0x75, 0x66, 0x66, 0x6c, 0x65, 0x68, 0x6f,
+	0x67, 0x2f, 0x76, 0x33, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x70, 0x62, 0x2f, 0x64, 0x65, 0x74, 0x65,
+	0x63, 0x74, 0x6f, 0x72, 0x73, 0x70, 0x62, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
 }
 
 var (

pkg/sources/github/github.go

@@ -978,6 +978,8 @@ func (s *Source) setProgressCompleteWithRepo(index int, offset int, repoURL stri
 }
 
 func (s *Source) scanComments(ctx context.Context, repoPath string, chunksChan chan *sources.Chunk) error {
+	s.log.Info("scanning comments", "repository", repoPath)
+
 	// Support ssh and https URLs
 	repoURL, err := git.GitURLParse(repoPath)
 	if err != nil {
@@ -986,7 +988,6 @@ func (s *Source) scanComments(ctx context.Context, repoPath string, chunksChan c
 
 	trimmedURL := removeURLAndSplit(repoURL.String())
 	if repoURL.Host == "gist.github.com" && s.includeGistComments {
-		s.log.Info("scanning github gist comments", "repository", repoPath)
 		// GitHub Gist URL.
 		var gistId string
 		if len(trimmedURL) == 2 {
@@ -1035,8 +1036,6 @@ func (s *Source) scanComments(ctx context.Context, repoPath string, chunksChan c
 
 		if s.includeIssueComments {
 
-			s.log.Info("scanning github issue comments", "repository", repoPath)
-
 			issueOpts := &github.IssueListCommentsOptions{
 				Sort:      &sortType,
 				Direction: &directionType,
@@ -1071,8 +1070,6 @@ func (s *Source) scanComments(ctx context.Context, repoPath string, chunksChan c
 		}
 
 		if s.includePRComments {
-			s.log.Info("scanning github pull request comments", "repository", repoPath)
-
 			prOpts := &github.PullRequestListCommentsOptions{
 				Sort:      sortType,
 				Direction: directionType,

pkg/sources/s3/s3.go

@@ -334,15 +334,15 @@ func (s *Source) pageChunker(ctx context.Context, client *s3.S3, chunksChan chan
 			}
 			reader.Stop()
 
+			chunk := *chunkSkel
 			chunkReader := sources.NewChunkReader()
 			chunkResChan := chunkReader(ctx, reader)
 			for data := range chunkResChan {
+				chunk.Data = data.Bytes()
 				if err := data.Error(); err != nil {
 					s.log.Error(err, "error reading chunk.")
 					continue
 				}
-				chunk := *chunkSkel
-				chunk.Data = data.Bytes()
 				if err := common.CancellableWrite(ctx, chunksChan, &chunk); err != nil {
 					return err
 				}

pkg/sources/s3/s3_integration_test.go

@@ -1,47 +0,0 @@
-//go:build integration
-// +build integration
-
-package s3
-
-import (
-	"testing"
-	"time"
-
-	"github.com/stretchr/testify/assert"
-	"google.golang.org/protobuf/types/known/anypb"
-
-	"github.com/trufflesecurity/trufflehog/v3/pkg/context"
-	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
-	"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
-)
-
-func TestSource_ChunksCount(t *testing.T) {
-	ctx, cancel := context.WithTimeout(context.Background(), time.Second*30)
-	defer cancel()
-
-	s := Source{}
-	connection := &sourcespb.S3{
-		Credential: &sourcespb.S3_Unauthenticated{},
-		Buckets:    []string{"truffletestbucket"},
-	}
-	conn, err := anypb.New(connection)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	err = s.Init(ctx, "test name", 0, 0, false, conn, 1)
-	chunksCh := make(chan *sources.Chunk)
-	go func() {
-		defer close(chunksCh)
-		err = s.Chunks(ctx, chunksCh)
-		assert.Nil(t, err)
-	}()
-
-	wantChunkCount := 120
-	got := 0
-
-	for range chunksCh {
-		got++
-	}
-	assert.Greater(t, got, wantChunkCount)
-}

pkg/sources/source_manager.go

@@ -191,13 +191,6 @@ func (s *SourceManager) Wait() error {
 	return s.pool.Wait()
 }
 
-// ScanChunk injects a chunk into the output stream of chunks to be scanned.
-// This method should rarely be used. TODO: Remove when dependencies no longer
-// rely on this functionality.
-func (s *SourceManager) ScanChunk(chunk *Chunk) {
-	s.outputChunks <- chunk
-}
-
 // preflightChecks is a helper method to check the Manager or the context isn't
 // done and that the handle is valid.
 func (s *SourceManager) preflightChecks(ctx context.Context, handle handle) error {

pkg/sources/sources.go

@@ -172,12 +172,6 @@ type GithubConfig struct {
 	IncludeRepos []string
 	// Filter is the filter to use to scan the source.
 	Filter *common.Filter
-	// IncludeIssueComments indicates whether to include GitHub issue comments in the scan.
-	IncludeIssueComments,
-	// IncludePullRequestComments indicates whether to include GitHub pull request comments in the scan.
-	IncludePullRequestComments,
-	// IncludeGistComments indicates whether to include GitHub gist comments in the scan.
-	IncludeGistComments bool
 }
 
 // GitlabConfig defines the optional configuration for a gitlab source.

proto/detectors.proto

@@ -933,7 +933,7 @@ enum DetectorType {
   TrufflehogEnterprise = 922;
   EnvoyApiKey = 923;
   GitHubOauth2 = 924;
-  Salesforce = 925;
+  // 925 reserved for Salesforce
   HuggingFace = 926;
 }