test

README.md

@@ -1,4 +1,4 @@
-<p align="center">
+<p align="center" test="true">
   <img alt="GoReleaser Logo" src="https://storage.googleapis.com/trufflehog-static-sources/pixel_pig.png" height="140" />
   <h2 align="center">TruffleHog</h2>
   <p align="center">Find leaked credentials.</p>
@@ -269,9 +269,9 @@ repos:
     - id: trufflehog
       name: TruffleHog
       description: Detect secrets in your data.
-      entry: bash -c 'trufflehog git file://. --since-commit HEAD --only-verified --fail'
+      entry: bash -c 'trufflehog git file://. --only-verified --fail'
       # For running trufflehog in docker, use the following entry instead:
-      # entry: bash -c 'docker run -v "$(pwd):/workdir" -i --rm trufflesecurity/trufflehog:latest git file:///workdir --since-commit HEAD --only-verified --fail'
+      # entry: bash -c 'docker run -v "$(pwd):/workdir" -i --rm trufflesecurity/trufflehog:latest git file:///workdir --only-verified --fail'
       language: system
       stages: ["commit", "push"]
 ```

go.mod

@@ -29,8 +29,7 @@ require (
 	github.com/go-logr/logr v1.2.3
 	github.com/go-logr/zapr v1.2.3
 	github.com/go-redis/redis v6.15.9+incompatible
-	github.com/go-sql-driver/mysql v1.7.0
-	github.com/gobwas/glob v0.2.3
+	github.com/go-sql-driver/mysql v1.6.0
 	github.com/golang-jwt/jwt v3.2.2+incompatible
 	github.com/google/go-cmp v0.5.9
 	github.com/google/go-github/v42 v42.0.0
@@ -53,7 +52,7 @@ require (
 	github.com/tailscale/depaware v0.0.0-20210622194025-720c4b409502
 	github.com/xanzy/go-gitlab v0.74.0
 	go.mongodb.org/mongo-driver v1.11.0
-	go.uber.org/zap v1.24.0
+	go.uber.org/zap v1.23.0
 	golang.org/x/crypto v0.0.0-20220926161630-eccd6366d1be
 	golang.org/x/exp v0.0.0-20221018205818-5c77f4b2bbd7
 	golang.org/x/oauth2 v0.0.0-20221014153046-6fdb5e3db783

go.sum

@@ -173,11 +173,9 @@ github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY=
 github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0=
 github.com/go-redis/redis v6.15.9+incompatible h1:K0pv1D7EQUjfyoMql+r/jZqCLizCGKFlFgcHWWmHQjg=
 github.com/go-redis/redis v6.15.9+incompatible/go.mod h1:NAIEuMOZ/fxfXJIrKDQDz8wamY7mA7PouImQ2Jvg6kA=
-github.com/go-sql-driver/mysql v1.7.0 h1:ueSltNNllEqE3qcWBTD0iQd3IpL/6U+mJxLkazJ7YPc=
-github.com/go-sql-driver/mysql v1.7.0/go.mod h1:OXbVy3sEdcQ2Doequ6Z5BW6fXNQTmx+9S1MCJN5yJMI=
+github.com/go-sql-driver/mysql v1.6.0 h1:BCTh4TKNUYmOmMUcQ3IipzF5prigylS7XXjEkfCHuOE=
+github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
 github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE=
-github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y=
-github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8=
 github.com/gobwas/httphead v0.0.0-20200921212729-da3d93bc3c58/go.mod h1:L0fX3K22YWvt/FAX9NnzrNzcI4wNYi9Yku4O0LKYflo=
 github.com/gobwas/pool v0.2.1/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6WezmKEw=
 github.com/gobwas/ws v1.0.4/go.mod h1:szmBTxLgaFppYjEmNtny/v3w89xOydFnnZMcgRRu/EM=
@@ -464,8 +462,8 @@ go.uber.org/goleak v1.1.12/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ
 go.uber.org/multierr v1.6.0 h1:y6IPFStTAIT5Ytl7/XYmHvzXQ7S3g/IeZW9hyZ5thw4=
 go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU=
 go.uber.org/zap v1.19.0/go.mod h1:xg/QME4nWcxGxrpdeYfq7UvYrLh66cuVKdrbD1XF/NI=
-go.uber.org/zap v1.24.0 h1:FiJd5l1UOLj0wCgbSE0rwwXHzEdAZS6hiiSnxJN/D60=
-go.uber.org/zap v1.24.0/go.mod h1:2kMP+WWQ8aoFoedH3T2sq6iJ2yDWpHbP0f6MQbS9Gkg=
+go.uber.org/zap v1.23.0 h1:OjGQ5KQDEUawVHxNwQgPpiypGHOxo2mNZsOqTak4fFY=
+go.uber.org/zap v1.23.0/go.mod h1:D+nX8jyLsMHMYrln8A0rJjFt/T/9/bGgIhAqxv5URuY=
 golang.org/x/crypto v0.0.0-20190219172222-a4c6cb3142f2/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=

hack/docs/Adding_Detectors_external.md

@@ -49,12 +49,8 @@ If you think that something should be included outside of these guidelines, plea
    ```bash
    go run hack/generate/generate.go detector <DetectorType enum name>
    ```
-3. Add Secret Scanner
 
-   Add the secret scanner to the `pkg/engine/defaults.go` file like `github.com/trufflesecurity/trufflehog/v3/pkg/detectors/<detector_name>` and 
-   `<detector_name>.Scanner{},`
-
-4. Complete the secret detector.
+3. Complete the secret detector.
 
    The previous step templated a boilerplate + some example code as a package in the `pkg/detectors` folder for you to work on.
    The secret detector can be completed with these general steps:
@@ -108,4 +104,4 @@ Now, the detector test should attempt to load the given env key from that file.
 ### Testing a detector
 ```bash
    go test ./pkg/detectors/<detector> -tags=detectors
-   ```
+   ```

pkg/engine/engine.go

@@ -181,7 +181,7 @@ func (e *Engine) detectorWorker(ctx context.Context) {
 	for originalChunk := range e.chunks {
 		for chunk := range sources.Chunker(originalChunk) {
 			atomic.AddUint64(&e.bytesScanned, uint64(len(chunk.Data)))
-			fragStart, mdLine := FragmentFirstLine(chunk)
+			fragStart, mdLine := fragmentFirstLine(chunk)
 			for _, decoder := range e.decoders {
 				var decoderType detectorspb.DecoderType
 				switch decoder.(type) {
@@ -230,7 +230,10 @@ func (e *Engine) detectorWorker(ctx context.Context) {
 							results = detectors.CleanResults(results)
 						}
 						for _, result := range results {
-							SetResultLineNumber(chunk, &result, fragStart, mdLine)
+							if isGitSource(chunk.SourceType) {
+								offset := FragmentLineOffset(chunk, &result)
+								*mdLine = fragStart + offset
+							}
 							result.DecoderType = decoderType
 							e.results <- detectors.CopyMetadata(chunk, result)
 
@@ -291,9 +294,9 @@ func FragmentLineOffset(chunk *sources.Chunk, result *detectors.Result) int64 {
 	return 0
 }
 
-// FragmentFirstLine returns the first line number of a fragment along with a pointer to the value to update in the
+// fragmentFirstLine returns the first line number of a fragment along with a pointer to the value to update in the
 // chunk metadata.
-func FragmentFirstLine(chunk *sources.Chunk) (int64, *int64) {
+func fragmentFirstLine(chunk *sources.Chunk) (int64, *int64) {
 	var fragmentStart *int64
 	switch metadata := chunk.SourceMetadata.GetData().(type) {
 	case *source_metadatapb.MetaData_Git:
@@ -311,11 +314,3 @@ func FragmentFirstLine(chunk *sources.Chunk) (int64, *int64) {
 	}
 	return *fragmentStart, fragmentStart
 }
-
-// SetResultLineNumber sets the line number in the provided result.
-func SetResultLineNumber(chunk *sources.Chunk, result *detectors.Result, fragStart int64, mdLine *int64) {
-	if isGitSource(chunk.SourceType) {
-		offset := FragmentLineOffset(chunk, result)
-		*mdLine = fragStart + offset
-	}
-}

pkg/sources/chunker.go

@@ -37,9 +37,7 @@ func Chunker(originalChunk *Chunk) chan *Chunk {
 			}
 			peekData, _ := reader.Peek(PeekSize)
 			chunk.Data = append(chunkBytes[:n], peekData...)
-			if n > 0 {
-				chunkChan <- &chunk
-			}
+			chunkChan <- &chunk
 			if errors.Is(err, io.EOF) {
 				break
 			}

pkg/sources/chunker_test.go

@@ -5,14 +5,19 @@ import (
 	"bytes"
 	"errors"
 	"io"
+	"net/http"
 	"testing"
 
 	diskbufferreader "github.com/bill-rich/disk-buffer-reader"
 )
 
 func TestChunker(t *testing.T) {
-	byteBuffer := bytes.NewBuffer(make([]byte, ChunkSize*9))
-	reReader, err := diskbufferreader.New(byteBuffer)
+	resp, err := http.Get("https://raw.githubusercontent.com/bill-rich/bad-secrets/master/FifteenMB.gz")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer resp.Body.Close()
+	reReader, err := diskbufferreader.New(resp.Body)
 	if err != nil {
 		t.Fatal(err)
 	}

pkg/sources/git/git.go

@@ -335,21 +335,25 @@ func (s *Git) ScanCommits(ctx context.Context, repo *git.Repository, path string
 	urlMetadata := getSafeRemoteURL(repo, "origin")
 
 	var depth int64
+	var reachedBase = false
 
 	ctx.Logger().V(1).Info("scanning repo", "repo", urlMetadata, "base", scanOptions.BaseHash, "head", scanOptions.HeadHash)
 	for commit := range commitChan {
-		if len(scanOptions.BaseHash) > 0 {
-			if commit.Hash == scanOptions.BaseHash {
-				ctx.Logger().V(1).Info("reached base commit", "commit", commit.Hash)
-				break
-			}
-		}
+		ctx.Logger().V(5).Info("scanning commit", "commit", commit.Hash, "message", commit.Message)
 		if scanOptions.MaxDepth > 0 && depth >= scanOptions.MaxDepth {
 			ctx.Logger().V(1).Info("reached max depth", "depth", depth)
 			break
 		}
 		depth++
-		ctx.Logger().V(5).Info("scanning commit", "commit", commit.Hash, "message", commit.Message)
+		if reachedBase && commit.Hash != scanOptions.BaseHash {
+			break
+		}
+		if len(scanOptions.BaseHash) > 0 {
+			if commit.Hash == scanOptions.BaseHash {
+				ctx.Logger().V(1).Info("reached base commit", "commit", commit.Hash)
+				reachedBase = true
+			}
+		}
 		for _, diff := range commit.Diffs {
 			if !scanOptions.Filter.Pass(diff.PathB) {
 				continue

pkg/sources/git/git_test.go

@@ -238,13 +238,25 @@ func TestSource_Chunks_Integration(t *testing.T) {
 			name:    "remote repo, limited",
 			repoURL: "https://github.com/dustin-decker/secretsandstuff.git",
 			expectedChunkData: map[string]*byteCompare{
-				"70001020fab32b1fcf2f1f0e5c66424eae649826-aws": {B: []byte("[default]\naws_access_key_id = AKIAXYZDQCEN4B6JSJQI\naws_secret_access_key = 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\noutput = json\nregion = us-east-2\n")},
+				"70001020fab32b1fcf2f1f0e5c66424eae649826-aws":  {B: []byte("[default]\naws_access_key_id = AKIAXYZDQCEN4B6JSJQI\naws_secret_access_key = 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\noutput = json\nregion = us-east-2\n")},
+				"a6f8aa55736d4a85be31a0048a4607396898647a-bump": {B: []byte("\n\nf\n")},
 			},
 			scanOptions: ScanOptions{
 				HeadHash: "70001020fab32b1fcf2f1f0e5c66424eae649826",
 				BaseHash: "a6f8aa55736d4a85be31a0048a4607396898647a",
 			},
 		},
+		{
+			name:    "remote repo, base ahead of head",
+			repoURL: "https://github.com/dustin-decker/secretsandstuff.git",
+			expectedChunkData: map[string]*byteCompare{
+				"a6f8aa55736d4a85be31a0048a4607396898647a-bump": {B: []byte("\n\nf\n")},
+			},
+			scanOptions: ScanOptions{
+				HeadHash: "a6f8aa55736d4a85be31a0048a4607396898647a",
+				BaseHash: "70001020fab32b1fcf2f1f0e5c66424eae649826",
+			},
+		},
 		{
 			name:    "remote repo, main ahead of branch",
 			repoURL: "https://github.com/bill-rich/bad-secrets.git",

pkg/sources/github/github.go

@@ -17,9 +17,9 @@ import (
 	"github.com/bradleyfalzon/ghinstallation/v2"
 	"github.com/go-errors/errors"
 	gogit "github.com/go-git/go-git/v5"
-	"github.com/gobwas/glob"
 	"github.com/google/go-github/v42/github"
 	log "github.com/sirupsen/logrus"
+	"golang.org/x/exp/slices"
 	"golang.org/x/oauth2"
 	"golang.org/x/sync/errgroup"
 	"google.golang.org/protobuf/proto"
@@ -325,6 +325,9 @@ func (s *Source) enumerateWithToken(ctx context.Context, apiEndpoint, token stri
 			if err := s.addRepos(ctx, org, s.getReposByOrg); err != nil {
 				log.WithError(err).Errorf("error fetching repos for org: %s", org)
 			}
+			if err := s.addRepos(ctx, ghUser.GetLogin(), s.getReposByUser); err != nil {
+				log.WithError(err).Errorf("error fetching repos for user: %s", ghUser.GetLogin())
+			}
 
 			if s.conn.ScanUsers {
 				err := s.addMembersByOrg(ctx, org)
@@ -355,15 +358,6 @@ func (s *Source) enumerateWithToken(ctx context.Context, apiEndpoint, token stri
 				log.WithError(err).Error("error fetching repos by org")
 			}
 
-			if err := s.addRepos(ctx, ghUser.GetLogin(), s.getReposByUser); err != nil {
-				log.WithError(err).Errorf("error fetching repos for user: %s", ghUser.GetLogin())
-			}
-
-			// TODO: Test it actually works to list org gists like this.
-			if err := s.addGistsByUser(ctx, org); err != nil {
-				log.WithError(err).Errorf("error fetching gists by org: %s", org)
-			}
-
 			if s.conn.ScanUsers {
 				err := s.addMembersByOrg(ctx, org)
 				if err != nil {
@@ -372,20 +366,24 @@ func (s *Source) enumerateWithToken(ctx context.Context, apiEndpoint, token stri
 				}
 			}
 		}
-
-		// If we enabled ScanUsers above, we've already added the gists for the current user and users from the orgs.
-		// So if we don't have ScanUsers enabled, add the user gists as normal.
-		if err := s.addGistsByUser(ctx, ghUser.GetLogin()); err != nil {
-			log.WithError(err).Errorf("error fetching gists for user %s", ghUser.GetLogin())
-		}
 	}
 
 	if s.conn.ScanUsers {
 		log.Infof("Adding repos from %d members in %d organizations.", len(s.members), len(s.orgs))
 		s.addReposForMembers(ctx)
-		return nil
+	} else {
+		// If we enabled ScanUsers above, we've already added the gists for the current user and users from the orgs.
+		// So if we don't have ScanUsers enabled, add the user gists as normal.
+		if err := s.addGistsByUser(ctx, ghUser.GetLogin()); err != nil {
+			log.WithError(err).Errorf("error fetching gists for user %s", ghUser.GetLogin())
+		}
+		for _, org := range s.orgs {
+			// TODO: Test it actually works to list org gists like this.
+			if err := s.addGistsByUser(ctx, org); err != nil {
+				log.WithError(err).Errorf("error fetching gists by org: %s", org)
+			}
+		}
 	}
-
 	return nil
 }
 
@@ -745,16 +743,9 @@ func (s *Source) getReposByUser(ctx context.Context, user string) ([]string, err
 }
 
 func (s *Source) ignoreRepo(r string) bool {
-	for _, ignore := range s.ignoreRepos {
-		g, err := glob.Compile(ignore)
-		if err != nil {
-			s.log.WithError(err).Errorf("could not compile ignore repo glob %s", ignore)
-			continue
-		}
-		if g.Match(r) {
-			s.log.Debugf("ignoring repo %s", r)
-			return true
-		}
+	if slices.Contains(s.ignoreRepos, r) {
+		s.log.Debugf("ignoring repo %s", r)
+		return true
 	}
 	return false
 }

pkg/sources/github/github_test.go

@@ -71,7 +71,7 @@ func TestAddReposByOrg(t *testing.T) {
 		})
 
 	s := initTestSource(nil)
-	s.ignoreRepos = []string{"secret/super-*-repo2"}
+	s.ignoreRepos = []string{"secret/super-secret-repo2"}
 	// gock works here because github.NewClient is using the default HTTP Transport
 	err := s.addRepos(context.TODO(), "super-secret-org", s.getReposByOrg)
 	assert.Nil(t, err)
@@ -316,13 +316,18 @@ func TestEnumerateWithToken_IncludeRepos(t *testing.T) {
 		Reply(200).
 		JSON(map[string]string{"login": "super-secret-user"})
 
+	gock.New("https://api.github.com").
+		Get("/users/super-secret-user/gists").
+		Reply(200).
+		JSON([]map[string]string{{"clone_url": ""}})
+
 	s := initTestSource(nil)
 	s.repos = []string{"some-special-repo"}
 
 	err := s.enumerateWithToken(context.TODO(), "https://api.github.com", "token")
 	assert.Nil(t, err)
-	assert.Equal(t, 1, len(s.repos))
-	assert.Equal(t, []string{"some-special-repo"}, s.repos)
+	assert.Equal(t, 2, len(s.repos))
+	assert.Equal(t, []string{"some-special-repo", ""}, s.repos)
 	assert.True(t, gock.IsDone())
 }
 

pkg/sources/gitlab/gitlab.go

@@ -20,7 +20,6 @@ import (
 
 	"github.com/go-errors/errors"
 	gogit "github.com/go-git/go-git/v5"
-	"github.com/gobwas/glob"
 	log "github.com/sirupsen/logrus"
 	"github.com/xanzy/go-gitlab"
 	"golang.org/x/exp/slices"
@@ -345,21 +344,6 @@ func (s *Source) scanRepos(ctx context.Context, chunksChan chan *sources.Chunk)
 	return errs
 }
 
-func (s *Source) ignoreRepo(r string) bool {
-	for _, ignore := range s.ignoreRepos {
-		g, err := glob.Compile(ignore)
-		if err != nil {
-			log.WithError(err).Errorf("could not compile ignore repo glob %s", ignore)
-			continue
-		}
-		if g.Match(r) {
-			log.Debugf("Ignoring repo %s", r)
-			return true
-		}
-	}
-	return false
-}
-
 // Chunks emits chunks of bytes over a channel.
 func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) error {
 	// Start client.
@@ -386,10 +370,10 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) err
 		}
 		// Turn projects into URLs for Git cloner.
 		for _, prj := range projects {
-			if s.ignoreRepo(prj.PathWithNamespace) {
+			if slices.Contains(s.ignoreRepos, prj.PathWithNamespace) {
+				log.Debugf("Ignoring repo %s", prj.PathWithNamespace)
 				continue
 			}
-
 			// Ensure the urls are valid before adding them to the repo list.
 			_, err := url.Parse(prj.HTTPURLToRepo)
 			if err != nil {

pkg/sources/gitlab/gitlab_test.go

@@ -46,7 +46,7 @@ func TestSource_Scan(t *testing.T) {
 		wantErr          bool
 	}{
 		{
-			name: "token auth, enumerate repo, with explicit ignore",
+			name: "token auth, enumerate repo",
 			init: init{
 				name: "test source",
 				connection: &sourcespb.GitLab{
@@ -62,23 +62,6 @@ func TestSource_Scan(t *testing.T) {
 			},
 			wantReposScanned: 2,
 		},
-		{
-			name: "token auth, enumerate repo, with glob ignore",
-			init: init{
-				name: "test source",
-				connection: &sourcespb.GitLab{
-					Credential: &sourcespb.GitLab_Token{
-						Token: token,
-					},
-					IgnoreRepos: []string{"tes1188/*-gitlab"},
-				},
-			},
-			wantChunk: &sources.Chunk{
-				SourceType: sourcespb.SourceType_SOURCE_TYPE_GITLAB,
-				SourceName: "test source",
-			},
-			wantReposScanned: 2,
-		},
 		{
 			name: "token auth, scoped repo",
 			init: init{