Create devcontainer.json for codespaces

* Refactor git source to allow ScanOptions and use source in engine

Refactor the Chunks method of the git Source to call out to two helper
methods: scanRepos and scanDirs which scans s.conn.Repositories and
s.conn.Directories respectively. The only notable change in behavior is
that a credential is no longer necessary if there are no
s.conn.Repositories to scan.

* Preserve ScanGit functionality of not cleaning up temporary files

.devcontainer/devcontainer.json

@@ -0,0 +1,5 @@
+{
+  "image": "trufflesecurity/protos:1.18-0",
+  "features": {
+  }
+}

main.go

@@ -38,6 +38,7 @@ var (
 	debug               = cli.Flag("debug", "Run in debug mode.").Bool()
 	trace               = cli.Flag("trace", "Run in trace mode.").Bool()
 	profile             = cli.Flag("profile", "Enables profiling and sets a pprof and fgprof server on :18066.").Bool()
+	localDev            = cli.Flag("local-dev", "Hidden feature to disable overseer for local dev.").Hidden().Bool()
 	jsonOut             = cli.Flag("json", "Output in JSON format.").Short('j').Bool()
 	jsonLegacy          = cli.Flag("json-legacy", "Use the pre-v3.0 JSON format. Only works with git, gitlab, and github sources.").Bool()
 	gitHubActionsFormat = cli.Flag("github-actions", "Output in GitHub Actions format.").Bool()
@@ -161,6 +162,12 @@ func main() {
 	logger, sync := log.New("trufflehog", logFormat(os.Stderr))
 	// make it the default logger for contexts
 	context.SetDefaultLogger(logger)
+
+	if *localDev {
+		run(overseer.State{})
+		os.Exit(0)
+	}
+
 	defer func() { _ = sync() }()
 	logFatal := logFatalFunc(logger)
 

pkg/detectors/mongodb/mongodb.go

@@ -2,6 +2,8 @@ package mongodb
 
 import (
 	"context"
+	"go.mongodb.org/mongo-driver/x/mongo/driver/auth"
+	"go.mongodb.org/mongo-driver/x/mongo/driver/topology"
 	"regexp"
 	"strings"
 	"time"
@@ -14,12 +16,15 @@ import (
 	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
 )
 
-type Scanner struct{}
+type Scanner struct {
+	timeout time.Duration // Zero value means "default timeout"
+}
 
 // Ensure the Scanner satisfies the interface at compile time.
 var _ detectors.Detector = (*Scanner)(nil)
 
 var (
+	defaultTimeout = 2 * time.Second
 	// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
 	keyPat = regexp.MustCompile(`\b(mongodb(\+srv)?://[\S]{3,50}:([\S]{3,88})@[-.%\w\/:]+)\b`)
 	// TODO: Add support for sharded cluster, replica set and Atlas Deployment.
@@ -46,23 +51,15 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 		}
 
 		if verify {
-			func() {
-				ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
-				defer cancel()
-				client, err := mongo.Connect(ctx, options.Client().ApplyURI(resMatch))
-				if err != nil {
-					return
-				}
-				defer func() {
-					if err := client.Disconnect(ctx); err != nil {
-						return
-					}
-				}()
-				if err := client.Ping(ctx, readpref.Primary()); err != nil {
-					return
-				}
-				s1.Verified = true
-			}()
+			timeout := s.timeout
+			if timeout == 0 {
+				timeout = defaultTimeout
+			}
+			err := verifyUri(resMatch, timeout)
+			s1.Verified = err == nil
+			if !isErrDeterminate(err) {
+				s1.VerificationError = err
+			}
 		}
 		results = append(results, s1)
 	}
@@ -70,6 +67,33 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 	return results, nil
 }
 
+func isErrDeterminate(err error) bool {
+	switch e := err.(type) {
+	case topology.ConnectionError:
+		switch e.Unwrap().(type) {
+		case *auth.Error:
+			return true
+		default:
+			return false
+		}
+	default:
+		return false
+	}
+}
+
+func verifyUri(uri string, timeout time.Duration) error {
+	ctx, cancel := context.WithTimeout(context.Background(), timeout)
+	defer cancel()
+	client, err := mongo.Connect(ctx, options.Client().ApplyURI(uri))
+	if err != nil {
+		return err
+	}
+	defer func() {
+		_ = client.Disconnect(ctx)
+	}()
+	return client.Ping(ctx, readpref.Primary())
+}
+
 func (s Scanner) Type() detectorspb.DetectorType {
 	return detectorspb.DetectorType_MongoDB
 }

pkg/detectors/mongodb/mongodb_test.go

@@ -6,6 +6,7 @@ package mongodb
 import (
 	"context"
 	"fmt"
+	"strings"
 	"testing"
 	"time"
 
@@ -34,11 +35,12 @@ func TestMongoDB_FromChunk(t *testing.T) {
 		verify bool
 	}
 	tests := []struct {
-		name    string
-		s       Scanner
-		args    args
-		want    []detectors.Result
-		wantErr bool
+		name                string
+		s                   Scanner
+		args                args
+		want                []detectors.Result
+		wantErr             bool
+		wantVerificationErr bool
 	}{
 		{
 			name: "found, verified",
@@ -72,6 +74,40 @@ func TestMongoDB_FromChunk(t *testing.T) {
 			},
 			wantErr: false,
 		},
+		{
+			name: "found, would be verified but for connection timeout",
+			s:    Scanner{timeout: 1 * time.Microsecond},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte(fmt.Sprintf("You can find a mongodb secret %s within", secret)),
+				verify: true,
+			},
+			want: []detectors.Result{
+				{
+					DetectorType: detectorspb.DetectorType_MongoDB,
+					Verified:     false,
+				},
+			},
+			wantErr:             false,
+			wantVerificationErr: true,
+		},
+		{
+			name: "found, bad host",
+			s:    Scanner{},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte(fmt.Sprintf("You can find a mongodb secret %s within", strings.ReplaceAll(secret, ".mongodb.net", ".mongodb.net.bad"))),
+				verify: true,
+			},
+			want: []detectors.Result{
+				{
+					DetectorType: detectorspb.DetectorType_MongoDB,
+					Verified:     false,
+				},
+			},
+			wantErr:             false,
+			wantVerificationErr: true,
+		},
 		{
 			name: "not found",
 			s:    Scanner{},
@@ -86,8 +122,7 @@ func TestMongoDB_FromChunk(t *testing.T) {
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			s := Scanner{}
-			got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+			got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
 			if (err != nil) != tt.wantErr {
 				t.Errorf("MongoDB.FromData() error = %v, wantErr %v", err, tt.wantErr)
 				return
@@ -97,8 +132,11 @@ func TestMongoDB_FromChunk(t *testing.T) {
 					t.Fatalf("no raw secret present: \n %+v", got[i])
 				}
 				got[i].Raw = nil
+				if (got[i].VerificationError != nil) != tt.wantVerificationErr {
+					t.Fatalf("wantVerificationErr = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError)
+				}
 			}
-			ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "RawV2")
+			ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "RawV2", "VerificationError")
 			if diff := cmp.Diff(tt.want, got, ignoreOpts); diff != "" {
 				t.Errorf("MongoDB.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
 			}

pkg/detectors/virustotal/virustotal.go

@@ -1,10 +1,7 @@
 package virustotal
 
 import (
-	"bytes"
 	"context"
-	"io"
-	"mime/multipart"
 	"net/http"
 	"regexp"
 	"strings"
@@ -50,43 +47,36 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 		}
 
 		if verify {
-			body := &bytes.Buffer{}
-			writer := multipart.NewWriter(body)
-			fw, err := writer.CreateFormField("url")
-			if err != nil {
+			s1.Verified = verifyToken(ctx, client, resMatch)
+			if !s1.Verified && detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
 				continue
 			}
-			_, err = io.Copy(fw, strings.NewReader("https://www.amazon.com"))
-			if err != nil {
-				continue
-			}
-			writer.Close()
-			req, err := http.NewRequestWithContext(ctx, "POST", "https://www.virustotal.com/api/v3/urls", bytes.NewReader(body.Bytes()))
-			if err != nil {
-				continue
-			}
-			req.Header.Add("Content-Type", writer.FormDataContentType())
-			req.Header.Add("x-apikey", resMatch)
-			res, err := client.Do(req)
-			if err == nil {
-				defer res.Body.Close()
-				if res.StatusCode >= 200 && res.StatusCode < 300 {
-					s1.Verified = true
-				} else {
-					// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
-					if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
-						continue
-					}
-				}
-			}
 		}
-
 		results = append(results, s1)
 	}
 
 	return results, nil
 }
 
+func verifyToken(ctx context.Context, client *http.Client, token string) bool {
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, "https://www.virustotal.com/api/v3/metadata", nil)
+	if err != nil {
+		return false
+	}
+	req.Header.Add("x-apikey", token)
+
+	res, err := client.Do(req)
+	if err != nil {
+		return false
+	}
+	defer res.Body.Close()
+
+	if res.StatusCode < 200 || res.StatusCode >= 300 {
+		return false
+	}
+	return true
+}
+
 func (s Scanner) Type() detectorspb.DetectorType {
 	return detectorspb.DetectorType_VirusTotal
 }

pkg/detectors/virustotal/virustotal_test.go

@@ -10,9 +10,9 @@ import (
 	"time"
 
 	"github.com/kylelemons/godebug/pretty"
-	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
 
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
 )
 

pkg/engine/git.go

@@ -4,11 +4,13 @@ import (
 	"fmt"
 	"runtime"
 
+	"github.com/go-errors/errors"
 	gogit "github.com/go-git/go-git/v5"
+	"google.golang.org/protobuf/proto"
+	"google.golang.org/protobuf/types/known/anypb"
 
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/context"
-	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/source_metadatapb"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/sources/git"
@@ -22,16 +24,6 @@ func (e *Engine) ScanGit(ctx context.Context, c sources.GitConfig) error {
 		git.ScanOptionLogOptions(logOptions),
 	}
 
-	options := &gogit.PlainOpenOptions{
-		DetectDotGit:          true,
-		EnableDotGitCommonDir: true,
-	}
-
-	repo, err := gogit.PlainOpenWithOptions(c.RepoPath, options)
-	if err != nil {
-		return fmt.Errorf("could not open repo: %s: %w", c.RepoPath, err)
-	}
-
 	if c.MaxDepth != 0 {
 		opts = append(opts, git.ScanOptionMaxDepth(int64(c.MaxDepth)))
 	}
@@ -46,21 +38,28 @@ func (e *Engine) ScanGit(ctx context.Context, c sources.GitConfig) error {
 	}
 	scanOptions := git.NewScanOptions(opts...)
 
-	gitSource := git.NewGit(sourcespb.SourceType_SOURCE_TYPE_GIT, 0, 0, "trufflehog - git", true, runtime.NumCPU(),
-		func(file, email, commit, timestamp, repository string, line int64) *source_metadatapb.MetaData {
-			return &source_metadatapb.MetaData{
-				Data: &source_metadatapb.MetaData_Git{
-					Git: &source_metadatapb.Git{
-						Commit:     commit,
-						File:       file,
-						Email:      email,
-						Repository: repository,
-						Timestamp:  timestamp,
-						Line:       line,
-					},
-				},
-			}
-		})
+	connection := &sourcespb.Git{
+		// Using Directories here allows us to not pass any
+		// authentication. Also by this point, the c.RepoPath should
+		// still have been prepared and downloaded to a temporary
+		// directory if it was a URL.
+		Directories: []string{c.RepoPath},
+	}
+	var conn anypb.Any
+	err := anypb.MarshalFrom(&conn, connection, proto.MarshalOptions{})
+	if err != nil {
+		ctx.Logger().Error(err, "failed to marshal git connection")
+		return err
+	}
+
+	gitSource := git.Source{}
+	if err := gitSource.Init(ctx, "trufflehog - git", 0, 0, true, &conn, runtime.NumCPU()); err != nil {
+		return errors.WrapPrefix(err, "could not init git source", 0)
+	}
+	gitSource.WithScanOptions(scanOptions)
+	// Don't try to clean up the provided directory. That's handled by the
+	// caller of ScanGit.
+	gitSource.WithPreserveTempDirs(true)
 
 	ctx = context.WithValues(ctx,
 		"source_type", sourcespb.SourceType_SOURCE_TYPE_GIT.String(),
@@ -68,7 +67,7 @@ func (e *Engine) ScanGit(ctx context.Context, c sources.GitConfig) error {
 	)
 	e.sourcesWg.Go(func() error {
 		defer common.RecoverWithExit(ctx)
-		err := gitSource.ScanRepo(ctx, repo, c.RepoPath, scanOptions, e.ChunksChan())
+		err := gitSource.Chunks(ctx, e.ChunksChan())
 		if err != nil {
 			return fmt.Errorf("could not scan repo: %w", err)
 		}

pkg/sources/git/git.go

@@ -42,7 +42,11 @@ type Source struct {
 	verify   bool
 	git      *Git
 	sources.Progress
-	conn *sourcespb.Git
+	conn        *sourcespb.Git
+	scanOptions *ScanOptions
+	// Kludge to preserve engine.ScanGit functionality which doesn't expect
+	// the scanning to clean up the directory.
+	preserveTempDirs bool
 }
 
 type Git struct {
@@ -92,6 +96,19 @@ func (s *Source) JobID() int64 {
 	return s.jobId
 }
 
+// WithScanOptions sets the scan options.
+func (s *Source) WithScanOptions(scanOptions *ScanOptions) {
+	s.scanOptions = scanOptions
+}
+
+// WithPreserveTempDirs sets whether to preserve temp directories when scanning
+// the provided list of s.conn.Directories. NOTE: This is *only* for
+// s.conn.Directories, not all temp directories created. This is also a kludge
+// and should be refactored away.
+func (s *Source) WithPreserveTempDirs(preserve bool) {
+	s.preserveTempDirs = preserve
+}
+
 // Init returns an initialized GitHub source.
 func (s *Source) Init(aCtx context.Context, name string, jobId, sourceId int64, verify bool, connection *anypb.Any, concurrency int) error {
 	s.name = name
@@ -135,8 +152,29 @@ func (s *Source) Init(aCtx context.Context, name string, jobId, sourceId int64,
 
 // Chunks emits chunks of bytes over a channel.
 func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) error {
-	// TODO: refactor to remove duplicate code
+	if err := s.scanRepos(ctx, chunksChan); err != nil {
+		return err
+	}
+	if err := s.scanDirs(ctx, chunksChan); err != nil {
+		return err
+	}
+
 	totalRepos := len(s.conn.Repositories) + len(s.conn.Directories)
+	ctx.Logger().V(1).Info("Git source finished scanning", "repo_count", totalRepos)
+	s.SetProgressComplete(
+		totalRepos, totalRepos,
+		fmt.Sprintf("Completed scanning source %s", s.name), "",
+	)
+	return nil
+}
+
+// scanRepos scans the configured repositories in s.conn.Repositories.
+func (s *Source) scanRepos(ctx context.Context, chunksChan chan *sources.Chunk) error {
+	if len(s.conn.Repositories) == 0 {
+		return nil
+	}
+	totalRepos := len(s.conn.Repositories) + len(s.conn.Directories)
+	// TODO: refactor to remove duplicate code
 	switch cred := s.conn.GetCredential().(type) {
 	case *sourcespb.Git_BasicAuth:
 		user := cred.BasicAuth.Username
@@ -153,7 +191,7 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) err
 				if err != nil {
 					return err
 				}
-				return s.git.ScanRepo(ctx, repo, path, NewScanOptions(), chunksChan)
+				return s.git.ScanRepo(ctx, repo, path, s.scanOptions, chunksChan)
 			}(repoURI)
 			if err != nil {
 				ctx.Logger().Info("error scanning repository", "repo", repoURI, "error", err)
@@ -172,7 +210,7 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) err
 				if err != nil {
 					return err
 				}
-				return s.git.ScanRepo(ctx, repo, path, NewScanOptions(), chunksChan)
+				return s.git.ScanRepo(ctx, repo, path, s.scanOptions, chunksChan)
 			}(repoURI)
 			if err != nil {
 				ctx.Logger().Info("error scanning repository", "repo", repoURI, "error", err)
@@ -191,7 +229,7 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) err
 				if err != nil {
 					return err
 				}
-				return s.git.ScanRepo(ctx, repo, path, NewScanOptions(), chunksChan)
+				return s.git.ScanRepo(ctx, repo, path, s.scanOptions, chunksChan)
 			}(repoURI)
 			if err != nil {
 				ctx.Logger().Info("error scanning repository", "repo", repoURI, "error", err)
@@ -201,41 +239,42 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) err
 	default:
 		return errors.New("invalid connection type for git source")
 	}
+	return nil
+}
 
+// scanDirs scans the configured directories in s.conn.Directories.
+func (s *Source) scanDirs(ctx context.Context, chunksChan chan *sources.Chunk) error {
+	totalRepos := len(s.conn.Repositories) + len(s.conn.Directories)
 	for i, gitDir := range s.conn.Directories {
 		s.SetProgressComplete(len(s.conn.Repositories)+i, totalRepos, fmt.Sprintf("Repo: %s", gitDir), "")
 
 		if len(gitDir) == 0 {
 			continue
 		}
-		if !strings.HasSuffix(gitDir, "git") {
-			// try paths instead of url
-			repo, err := RepoFromPath(gitDir)
-			if err != nil {
-				ctx.Logger().Info("error scanning repository", "repo", gitDir, "error", err)
-				continue
+		if strings.HasSuffix(gitDir, "git") {
+			// TODO: Figure out why we skip directories ending in "git".
+			continue
+		}
+		// try paths instead of url
+		repo, err := RepoFromPath(gitDir)
+		if err != nil {
+			ctx.Logger().Info("error scanning repository", "repo", gitDir, "error", err)
+			continue
+		}
+
+		err = func(repoPath string) error {
+			if !s.preserveTempDirs && strings.HasPrefix(repoPath, filepath.Join(os.TempDir(), "trufflehog")) {
+				defer os.RemoveAll(repoPath)
 			}
 
-			err = func(repoPath string) error {
-				if strings.HasPrefix(repoPath, filepath.Join(os.TempDir(), "trufflehog")) {
-					defer os.RemoveAll(repoPath)
-				}
-
-				return s.git.ScanRepo(ctx, repo, repoPath, NewScanOptions(), chunksChan)
-			}(gitDir)
-			if err != nil {
-				ctx.Logger().Info("error scanning repository", "repo", gitDir, "error", err)
-				continue
-			}
+			return s.git.ScanRepo(ctx, repo, repoPath, s.scanOptions, chunksChan)
+		}(gitDir)
+		if err != nil {
+			ctx.Logger().Info("error scanning repository", "repo", gitDir, "error", err)
+			continue
 		}
 
 	}
-
-	ctx.Logger().V(1).Info("Git source finished scanning", "repo-count", totalRepos)
-	s.SetProgressComplete(
-		totalRepos, totalRepos,
-		fmt.Sprintf("Completed scanning source %s", s.name), "",
-	)
 	return nil
 }