fix broken test.

main.go

@@ -50,8 +50,6 @@ var (
 	archiveMaxSize       = cli.Flag("archive-max-size", "Maximum size of archive to scan.").Bytes()
 	archiveMaxDepth      = cli.Flag("archive-max-depth", "Maximum depth of archive to scan.").Int()
 	archiveTimeout       = cli.Flag("archive-timeout", "Maximum time to spend extracting an archive.").Duration()
-	includeDetectors     = cli.Flag("include-detectors", "Comma separated list of detector types to include. Types must exist in detectors.proto").String()
-	excludeDetectors     = cli.Flag("exclude-detectors", "Comma separated list of detector types to exclude. Types must exist in detectors.proto").String()
 
 	gitScan             = cli.Command("git", "Find credentials in git repositories.")
 	gitScanURI          = gitScan.Arg("uri", "Git repository URL. https://, file://, or ssh:// schema expected.").Required().String()
@@ -213,31 +211,12 @@ func run(state overseer.State) {
 	}
 
 	ctx := context.TODO()
-	var detectorsOption engine.EngineOption
-
-	detectorCfg, err := engine.NewDetectorsConfig(*includeDetectors, *excludeDetectors)
-	if err != nil {
-		logger.Error(err, "failed to create detectors config")
-		os.Exit(1)
-	}
-	dts, err := engine.Detectors(ctx, detectorCfg)
-	if err != nil {
-		logger.Error(err, "failed to create detectors")
-		os.Exit(1)
-	}
-
-	if len(dts) > 0 {
-		detectorsOption = engine.WithDetectors(!*noVerification, dts...)
-	} else {
-		detectorsOption = engine.WithDetectors(!*noVerification, engine.DefaultDetectors()...)
-	}
-
 	e := engine.Start(ctx,
 		engine.WithConcurrency(*concurrency),
 		engine.WithDecoders(decoders.DefaultDecoders()...),
+		engine.WithDetectors(!*noVerification, engine.DefaultDetectors()...),
 		engine.WithDetectors(!*noVerification, conf.Detectors...),
 		engine.WithFilterUnverified(*filterUnverified),
-		detectorsOption,
 	)
 
 	var repoPath string

pkg/cache/simple.go

@@ -0,0 +1,50 @@
+package cache
+
+import (
+	"sync"
+)
+
+// Cache in-memory cache.
+type Cache struct {
+	mu sync.RWMutex
+	m  map[string]string
+}
+
+// Simple constructs a new Simple cache.
+func Simple() *Cache {
+	return &Cache{
+		m: make(map[string]string),
+	}
+}
+
+// Set a value in the cache.
+func (c *Cache) Set(key, value string) error {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	c.m[key] = value
+	return nil
+}
+
+// Get a value and a boolean indicating if the value was found.
+func (c *Cache) Get(key string) (string, bool) {
+	c.mu.RLock()
+	defer c.mu.RUnlock()
+	value, ok := c.m[key]
+	return value, ok
+}
+
+// Delete a value from the cache.
+func (c *Cache) Delete(key string) error {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	delete(c.m, key)
+	return nil
+}
+
+// Clear the cache.
+func (c *Cache) Clear() error {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	c.m = make(map[string]string)
+	return nil
+}

pkg/cache/simple_test.go

@@ -0,0 +1,57 @@
+package cache
+
+import "testing"
+
+func TestCache(t *testing.T) {
+	tests := []struct {
+		name string
+		key  string
+		val  string
+		want string
+		ok   bool
+	}{
+		{"set", "key", "value", "value", true},
+		{"get", "key", "", "value", true},
+		{"get-not-found", "not-found", "", "", false},
+		{"delete", "key", "", "", false},
+		{"clear", "key", "", "", false},
+	}
+
+	c := Simple()
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			switch test.name {
+			case "set":
+				if err := c.Set(test.key, test.val); err != nil {
+					t.Fatalf("unexpected error: %v", err)
+				}
+			case "get":
+				got, ok := c.Get(test.key)
+				if got != test.want || ok != test.ok {
+					t.Errorf("got (%q, %t), want (%q, %t)", got, ok, test.want, test.ok)
+				}
+			case "get-not-found":
+				got, ok := c.Get(test.key)
+				if got != test.want || ok != test.ok {
+					t.Errorf("got (%q, %t), want (%q, %t)", got, ok, test.want, test.ok)
+				}
+			case "delete":
+				if err := c.Delete(test.key); err != nil {
+					t.Fatalf("unexpected error: %v", err)
+				}
+				got, ok := c.Get(test.key)
+				if got != test.want || ok != test.ok {
+					t.Errorf("got (%q, %t), want (%q, %t)", got, ok, test.want, test.ok)
+				}
+			case "clear":
+				if err := c.Clear(); err != nil {
+					t.Fatalf("unexpected error: %v", err)
+				}
+				got, ok := c.Get(test.key)
+				if got != test.want || ok != test.ok {
+					t.Errorf("got (%q, %t), want (%q, %t)", got, ok, test.want, test.ok)
+				}
+			}
+		})
+	}
+}

pkg/engine/circleci.go

@@ -10,6 +10,7 @@ import (
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/context"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/sources/circleci"
 )
 
@@ -29,11 +30,19 @@ func (e *Engine) ScanCircleCI(ctx context.Context, token string) error {
 	}
 
 	circleSource := circleci.Source{}
+	cfg := sources.NewSourceConfig(
+		"trufflehog - Circle CI",
+		0,
+		int64(sourcespb.SourceType_SOURCE_TYPE_CIRCLECI),
+		&conn,
+		sources.WithConcurrency(runtime.NumCPU()),
+		sources.WithVerify(true),
+	)
 	ctx = context.WithValues(ctx,
 		"source_type", circleSource.Type().String(),
 		"source_name", "Circle CI",
 	)
-	err = circleSource.Init(ctx, "trufflehog - Circle CI", 0, int64(sourcespb.SourceType_SOURCE_TYPE_CIRCLECI), true, &conn, runtime.NumCPU())
+	err = circleSource.Init(ctx, cfg)
 	if err != nil {
 		return errors.WrapPrefix(err, "failed to init Circle CI source", 0)
 	}

pkg/engine/circleci_test.go

@@ -0,0 +1,37 @@
+package engine
+
+import (
+	"sync"
+	"testing"
+
+	"github.com/trufflesecurity/trufflehog/v3/pkg/context"
+)
+
+func TestScanCircleCI(t *testing.T) {
+	tests := []struct {
+		name    string
+		token   string
+		wantErr bool
+	}{
+		{
+			name:  "valid token",
+			token: "valid_token",
+		},
+		{
+			name:  "no token",
+			token: "",
+		},
+	}
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			ctx := context.Background()
+			engine := &Engine{}
+			engine.sourcesWg = sync.WaitGroup{}
+
+			err := engine.ScanCircleCI(ctx, test.token)
+			if (err != nil) != test.wantErr {
+				t.Errorf("ScanCircleCI() error = %v, wantErr %v", err, test.wantErr)
+			}
+		})
+	}
+}

pkg/engine/detectors.go

@@ -1,103 +0,0 @@
-package engine
-
-import (
-	"fmt"
-	"strings"
-
-	"github.com/trufflesecurity/trufflehog/v3/pkg/context"
-	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
-)
-
-// DetectorFilter allows for include/exclude detectors.
-type DetectorFilter interface {
-	detectors() []string
-	filter(map[string]struct{}) ([]detectors.Detector, error)
-}
-
-// IncludeDetectorFilter is a detector filter that only includes the specified detectors.
-type IncludeDetectorFilter struct {
-	includeDetectors []string
-}
-
-func (i *IncludeDetectorFilter) detectors() []string {
-	return i.includeDetectors
-}
-
-func (i *IncludeDetectorFilter) filter(include map[string]struct{}) ([]detectors.Detector, error) {
-	ds := make([]detectors.Detector, 0, len(i.includeDetectors))
-	for _, d := range DefaultDetectors() {
-		if _, ok := include[strings.ToLower(d.Type().String())]; ok {
-			ds = append(ds, d)
-		}
-	}
-
-	if len(ds) != len(include) {
-		return nil, fmt.Errorf("1 or more detectors are invalid, please check your detector types")
-	}
-
-	return ds, nil
-}
-
-// ExcludeDetectorFilter is a detector filter that excludes the specified detectors.
-type ExcludeDetectorFilter struct {
-	excludeDetectors []string
-}
-
-func (e *ExcludeDetectorFilter) detectors() []string {
-	return e.excludeDetectors
-}
-
-func (e *ExcludeDetectorFilter) filter(exclude map[string]struct{}) ([]detectors.Detector, error) {
-	defaultDetectors := DefaultDetectors()
-	l := len(defaultDetectors) - len(exclude)
-	result := make([]detectors.Detector, 0, l)
-	for _, detector := range defaultDetectors {
-		if _, ok := exclude[strings.ToLower(detector.Type().String())]; !ok {
-			result = append(result, detector)
-		}
-	}
-
-	if len(result) != l {
-		return nil, fmt.Errorf("1 or more detectors are invalid, please check your detector types")
-	}
-	return result, nil
-}
-
-// NewDetectorsConfig creates a new detector filter based on the include/exclude arguments.
-func NewDetectorsConfig(include, exclude string) (DetectorFilter, error) {
-	if len(include) == 0 && len(exclude) == 0 {
-		return nil, fmt.Errorf("no detectors specified")
-	}
-
-	if len(include) > 0 && len(exclude) > 0 {
-		return nil, fmt.Errorf("cannot specify both include and exclude detectors")
-	}
-
-	// Determine the detector filter type.
-	if len(include) > len(exclude) {
-		return &IncludeDetectorFilter{includeDetectors: strings.Split(include, ",")}, nil
-	}
-	return &ExcludeDetectorFilter{excludeDetectors: strings.Split(exclude, ",")}, nil
-}
-
-// Detectors only returns a specific set of detectors if they are specified in the
-// detector list and are valid. Otherwise, an error is returned.
-func Detectors(ctx context.Context, dt DetectorFilter) ([]detectors.Detector, error) {
-	configured := setDetectors(ctx, dt.detectors())
-
-	if len(configured) == 0 {
-		return nil, fmt.Errorf("no detectors specified")
-	}
-
-	return dt.filter(configured)
-}
-
-func setDetectors(ctx context.Context, dts []string) map[string]struct{} {
-	valid := make(map[string]struct{}, len(dts))
-	for _, d := range dts {
-		ctx.Logger().Info("setting detector", "detector-name", d)
-		valid[strings.ToLower(d)] = struct{}{}
-	}
-
-	return valid
-}

pkg/engine/detectors_test.go

@@ -1,126 +0,0 @@
-package engine
-
-import (
-	"reflect"
-	"strings"
-	"testing"
-
-	"github.com/trufflesecurity/trufflehog/v3/pkg/context"
-	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
-	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/aws"
-	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/azure"
-)
-
-func TestNewDectorsConfig(t *testing.T) {
-	tests := []struct {
-		name     string
-		inc      string
-		exc      string
-		expected DetectorFilter
-		wantErr  bool
-	}{
-		{
-			name:    "No dts specified, returns error",
-			inc:     "",
-			exc:     "",
-			wantErr: true,
-		},
-		{
-			name:     "Include dts specified, returns config with include",
-			inc:      "AWS",
-			expected: &IncludeDetectorFilter{includeDetectors: []string{"AWS"}},
-		},
-		{
-			name:     "Exclude dts specified, returns config with exclude",
-			exc:      "AWS",
-			expected: &ExcludeDetectorFilter{excludeDetectors: []string{"AWS"}},
-		},
-		{
-			name:    "Include and exclude dts specified, returns error",
-			inc:     "AWS",
-			exc:     "Azure",
-			wantErr: true,
-		},
-	}
-
-	for _, test := range tests {
-		result, err := NewDetectorsConfig(test.inc, test.exc)
-		if err != nil && !test.wantErr {
-			t.Errorf("NewDetectorsConfig(%v, %v) got %v, want %v", test.inc, test.exc, err, test.expected)
-		}
-
-		if result != nil {
-			if !reflect.DeepEqual(result, test.expected) {
-				t.Errorf("NewDetectorsConfig(%v, %v) got %v, want %v", test.inc, test.exc, result, test.expected)
-			}
-		}
-	}
-}
-
-func TestDetectors(t *testing.T) {
-	ctx := context.Background()
-	tests := []struct {
-		name     string
-		dt       DetectorFilter
-		expected []detectors.Detector
-		wantErr  bool
-	}{
-		{
-			name:    "No dts specified, returns default set",
-			dt:      &IncludeDetectorFilter{},
-			wantErr: true,
-		},
-		{
-			name:     "Valid include detectors (1) specified, returns valid set",
-			dt:       &IncludeDetectorFilter{includeDetectors: []string{"AWS"}},
-			expected: []detectors.Detector{aws.New()},
-		},
-		{
-			name:     "Valid include detectors (2) specified, returns valid set",
-			dt:       &IncludeDetectorFilter{includeDetectors: []string{"AWS", "Azure"}},
-			expected: []detectors.Detector{aws.New(), &azure.Scanner{}},
-		},
-		{
-			name:    "Invalid include detectors specified, returns error",
-			dt:      &IncludeDetectorFilter{includeDetectors: []string{"AWS", "Azure", "Invalid"}},
-			wantErr: true,
-		},
-		{
-			name:     "Valid exlude detectors (1) specified, returns valid set",
-			dt:       &ExcludeDetectorFilter{excludeDetectors: []string{"AWS"}},
-			expected: excludeDetectors(map[string]struct{}{"aws": {}}),
-		},
-		{
-			name:     "Valid exclude detectors (2) specified, returns valid set",
-			dt:       &ExcludeDetectorFilter{excludeDetectors: []string{"AWS", "Azure"}},
-			expected: excludeDetectors(map[string]struct{}{"aws": {}, "azure": {}}),
-		},
-		{
-			name:    "Invalid exclude detector specified, returns error",
-			dt:      &ExcludeDetectorFilter{excludeDetectors: []string{"AWS", "Azure", "Invalid"}},
-			wantErr: true,
-		},
-	}
-
-	for _, test := range tests {
-		result, err := Detectors(ctx, test.dt)
-		if err != nil && !test.wantErr {
-			t.Errorf("Detectors(%v) got %v, want %v", test.dt, err, test.expected)
-		}
-
-		if !reflect.DeepEqual(result, test.expected) {
-			t.Errorf("Detectors(%v) got %v, want %v", test.dt, result, test.expected)
-		}
-	}
-}
-
-func excludeDetectors(exclude map[string]struct{}) []detectors.Detector {
-	defaultDetectors := DefaultDetectors()
-	result := make([]detectors.Detector, 0, len(defaultDetectors)-len(exclude))
-	for _, detector := range defaultDetectors {
-		if _, ok := exclude[strings.ToLower(detector.Type().String())]; !ok {
-			result = append(result, detector)
-		}
-	}
-	return result
-}

pkg/engine/filesystem.go

@@ -27,12 +27,19 @@ func (e *Engine) ScanFileSystem(ctx context.Context, c sources.FilesystemConfig)
 	}
 
 	fileSystemSource := filesystem.Source{}
-
+	cfg := sources.NewSourceConfig(
+		"trufflehog - filesystem",
+		0,
+		int64(sourcespb.SourceType_SOURCE_TYPE_FILESYSTEM),
+		&conn,
+		sources.WithConcurrency(runtime.NumCPU()),
+		sources.WithVerify(true),
+	)
 	ctx = context.WithValues(ctx,
 		"source_type", fileSystemSource.Type().String(),
 		"source_name", "filesystem",
 	)
-	err = fileSystemSource.Init(ctx, "trufflehog - filesystem", 0, int64(sourcespb.SourceType_SOURCE_TYPE_FILESYSTEM), true, &conn, runtime.NumCPU())
+	err = fileSystemSource.Init(ctx, cfg)
 	if err != nil {
 		return errors.WrapPrefix(err, "could not init filesystem source", 0)
 	}

pkg/engine/filesystem_test.go

@@ -0,0 +1,44 @@
+package engine
+
+import (
+	"sync"
+	"testing"
+
+	"github.com/trufflesecurity/trufflehog/v3/pkg/context"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
+)
+
+func TestEngine_ScanFileSystem(t *testing.T) {
+	tests := []struct {
+		name    string
+		config  sources.FilesystemConfig
+		wantErr bool
+	}{
+		{
+			name: "valid config",
+			config: sources.FilesystemConfig{
+				Directories: []string{"/tmp"},
+			},
+		},
+		{
+			name: "empty directories",
+			config: sources.FilesystemConfig{
+				Directories: []string{},
+			},
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			e := &Engine{
+				sourcesWg: sync.WaitGroup{},
+			}
+
+			ctx := context.Background()
+			err := e.ScanFileSystem(ctx, test.config)
+			if (err != nil) != test.wantErr {
+				t.Errorf("ScanFileSystem(%v) error = %v, wantErr %v", test.config, err, test.wantErr)
+			}
+		})
+	}
+}

pkg/engine/github.go

@@ -37,11 +37,20 @@ func (e *Engine) ScanGitHub(ctx context.Context, c sources.GithubConfig) error {
 		ctx.Logger().Error(err, "failed to marshal github connection")
 		return err
 	}
+
+	cfg := sources.NewSourceConfig(
+		"trufflehog - github",
+		0,
+		int64(sourcespb.SourceType_SOURCE_TYPE_GITHUB),
+		&conn,
+		sources.WithConcurrency(c.Concurrency),
+		sources.WithVerify(true),
+	)
 	ctx = context.WithValues(ctx,
 		"source_type", source.Type().String(),
 		"source_name", "github",
 	)
-	err = source.Init(ctx, "trufflehog - github", 0, 0, false, &conn, c.Concurrency)
+	err = source.Init(ctx, cfg)
 	if err != nil {
 		ctx.Logger().Error(err, "failed to initialize github source")
 		return err

pkg/engine/github_test.go

@@ -0,0 +1,63 @@
+package engine
+
+import (
+	"sync"
+	"testing"
+
+	"github.com/trufflesecurity/trufflehog/v3/pkg/context"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
+)
+
+func TestEngine_ScanGitHub(t *testing.T) {
+	tests := []struct {
+		name    string
+		config  sources.GithubConfig
+		wantErr bool
+	}{
+		{
+			name: "valid config",
+			config: sources.GithubConfig{
+				Endpoint:     "https://api.github.com",
+				Token:        "valid_token",
+				Orgs:         []string{"org1", "org2"},
+				Repos:        []string{"repo1", "repo2"},
+				IncludeForks: true,
+			},
+		},
+		{
+			name: "empty endpoint",
+			config: sources.GithubConfig{
+				Endpoint: "",
+				Token:    "valid_token",
+				Orgs:     []string{"org1", "org2"},
+			},
+		},
+		{
+			name: "empty token",
+			config: sources.GithubConfig{
+				Endpoint: "https://api.github.com",
+				Orgs:     []string{"org1", "org2"},
+			},
+		},
+		{
+			name: "empty orgs",
+			config: sources.GithubConfig{
+				Endpoint: "https://api.github.com",
+				Token:    "valid_token",
+				Orgs:     []string{},
+			},
+		},
+	}
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			e := &Engine{
+				sourcesWg: sync.WaitGroup{},
+			}
+			ctx := context.Background()
+			err := e.ScanGitHub(ctx, test.config)
+			if (err != nil) != test.wantErr {
+				t.Errorf("ScanGitHub(%v) error = %v, wantErr %v", test.config, err, test.wantErr)
+			}
+		})
+	}
+}

pkg/engine/gitlab.go

@@ -53,11 +53,19 @@ func (e *Engine) ScanGitLab(ctx context.Context, c sources.GitlabConfig) error {
 	}
 
 	gitlabSource := gitlab.Source{}
+	cfg := sources.NewSourceConfig(
+		"trufflehog - gitlab",
+		0,
+		int64(sourcespb.SourceType_SOURCE_TYPE_GITLAB),
+		&conn,
+		sources.WithConcurrency(runtime.NumCPU()),
+		sources.WithVerify(true),
+	)
 	ctx = context.WithValues(ctx,
 		"source_type", gitlabSource.Type().String(),
 		"source_name", "gitlab",
 	)
-	err = gitlabSource.Init(ctx, "trufflehog - gitlab", 0, int64(sourcespb.SourceType_SOURCE_TYPE_GITLAB), true, &conn, runtime.NumCPU())
+	err = gitlabSource.Init(ctx, cfg)
 	if err != nil {
 		return errors.WrapPrefix(err, "could not init GitLab source", 0)
 	}

pkg/engine/gitlab_test.go

@@ -0,0 +1,49 @@
+package engine
+
+import (
+	"sync"
+	"testing"
+
+	"github.com/trufflesecurity/trufflehog/v3/pkg/context"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
+)
+
+func TestScanGitLab(t *testing.T) {
+	tests := []struct {
+		name    string
+		c       sources.GitlabConfig
+		wantErr bool
+	}{
+		{
+			name: "Successful scan with valid token and endpoint",
+			c: sources.GitlabConfig{
+				Token:    "valid-token",
+				Endpoint: "https://gitlab.com",
+				Repos:    []string{"repo1", "repo2"},
+			},
+		},
+		{
+			name: "Failed scan with empty token",
+			c: sources.GitlabConfig{
+				Token:    "",
+				Endpoint: "https://gitlab.com",
+				Repos:    []string{"repo1", "repo2"},
+			},
+			wantErr: true,
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			ctx := context.Background()
+			e := &Engine{
+				sourcesWg: sync.WaitGroup{},
+			}
+
+			err := e.ScanGitLab(ctx, test.c)
+			if (err != nil) != test.wantErr {
+				t.Errorf("ScanGitLab(%v) error = %v, wantErr %v", test.c, err, test.wantErr)
+			}
+		})
+	}
+}

pkg/engine/s3.go

@@ -46,11 +46,19 @@ func (e *Engine) ScanS3(ctx context.Context, c sources.S3Config) error {
 	}
 
 	s3Source := s3.Source{}
+	cfg := sources.NewSourceConfig(
+		"trufflehog - s3",
+		0,
+		int64(sourcespb.SourceType_SOURCE_TYPE_S3),
+		&conn,
+		sources.WithConcurrency(runtime.NumCPU()),
+		sources.WithVerify(true),
+	)
 	ctx = context.WithValues(ctx,
 		"source_type", s3Source.Type().String(),
 		"source_name", "s3",
 	)
-	err = s3Source.Init(ctx, "trufflehog - s3", 0, int64(sourcespb.SourceType_SOURCE_TYPE_S3), true, &conn, runtime.NumCPU())
+	err = s3Source.Init(ctx, cfg)
 	if err != nil {
 		return errors.WrapPrefix(err, "failed to init S3 source", 0)
 	}

pkg/engine/s3_test.go

@@ -0,0 +1,66 @@
+package engine
+
+import (
+	"testing"
+
+	"github.com/trufflesecurity/trufflehog/v3/pkg/context"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
+)
+
+func TestScanS3(t *testing.T) {
+	tests := []struct {
+		name      string
+		c         sources.S3Config
+		expectErr bool
+	}{
+		{
+			name: "Test with cloud credentials",
+			c: sources.S3Config{
+				CloudCred: true,
+				Buckets:   []string{"bucket1", "bucket2"},
+			},
+			expectErr: false,
+		},
+		{
+			name: "Test with basic authentication",
+			c: sources.S3Config{
+				CloudCred: false,
+				Key:       "accessKey",
+				Secret:    "secretKey",
+				Buckets:   []string{"bucket1", "bucket2"},
+			},
+			expectErr: false,
+		},
+		{
+			name: "Test with both cloud credentials and basic authentication",
+			c: sources.S3Config{
+				CloudCred: true,
+				Key:       "accessKey",
+				Secret:    "secretKey",
+				Buckets:   []string{"bucket1", "bucket2"},
+			},
+			expectErr: true,
+		},
+		{
+			name: "Test with unauthenticated",
+			c: sources.S3Config{
+				CloudCred: false,
+				Buckets:   []string{"bucket1", "bucket2"},
+			},
+			expectErr: false,
+		},
+	}
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			ctx := context.Background()
+			e := Engine{}
+			err := e.ScanS3(ctx, test.c)
+			if test.expectErr && err == nil {
+				t.Errorf("expected an error but got nil")
+			}
+			if !test.expectErr && err != nil {
+				t.Errorf("expected no error but got %v", err)
+			}
+		})
+	}
+}

pkg/engine/syslog.go

@@ -41,12 +41,21 @@ func (e *Engine) ScanSyslog(ctx context.Context, c sources.SyslogConfig) error {
 	if err != nil {
 		return errors.WrapPrefix(err, "error unmarshalling connection", 0)
 	}
+
 	source := syslog.Source{}
+	cfg := sources.NewSourceConfig(
+		"trufflehog - syslog",
+		0,
+		int64(sourcespb.SourceType_SOURCE_TYPE_SYSLOG),
+		&conn,
+		sources.WithConcurrency(c.Concurrency),
+		sources.WithVerify(true),
+	)
 	ctx = context.WithValues(ctx,
 		"source_type", source.Type().String(),
 		"source_name", "syslog",
 	)
-	err = source.Init(ctx, "trufflehog - syslog", 0, 0, false, &conn, c.Concurrency)
+	err = source.Init(ctx, cfg)
 	source.InjectConnection(connection)
 	if err != nil {
 		ctx.Logger().Error(err, "failed to initialize syslog source")

pkg/engine/syslog_test.go

@@ -0,0 +1,61 @@
+package engine
+
+import (
+	"sync"
+	"testing"
+
+	"github.com/trufflesecurity/trufflehog/v3/pkg/context"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
+)
+
+func TestScanSyslog(t *testing.T) {
+	tests := []struct {
+		name    string
+		c       sources.SyslogConfig
+		wantErr bool
+	}{
+		{
+			name: "Scan syslog successfully",
+			c: sources.SyslogConfig{
+				Protocol:    "tcp",
+				Address:     "127.0.0.1:1514",
+				Format:      "rfc3164",
+				Concurrency: 1,
+			},
+			wantErr: false,
+		},
+		{
+			name: "Scan syslog with invalid cert",
+			c: sources.SyslogConfig{
+				CertPath:    "bad",
+				KeyPath:     "valid",
+				Address:     "127.0.0.1:1514",
+				Format:      "rfc3164",
+				Concurrency: 1,
+			},
+			wantErr: true,
+		},
+		{
+			name: "Scan syslog with invalid keypath",
+			c: sources.SyslogConfig{
+				Protocol:    "tcp",
+				KeyPath:     "invalid",
+				CertPath:    "valid",
+				Format:      "rfc3164",
+				Concurrency: 1,
+			},
+			wantErr: true,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			e := &Engine{
+				sourcesWg: sync.WaitGroup{},
+			}
+			ctx := context.Background()
+			if err := e.ScanSyslog(ctx, tt.c); (err != nil) != tt.wantErr {
+				t.Errorf("ScanSyslog() error = %v, wantErr %v", err, tt.wantErr)
+			}
+		})
+	}
+}

pkg/sources/circleci/circleci.go

@@ -51,17 +51,17 @@ func (s *Source) JobID() int64 {
 }
 
 // Init returns an initialized CircleCI source.
-func (s *Source) Init(_ context.Context, name string, jobId, sourceId int64, verify bool, connection *anypb.Any, concurrency int) error {
-	s.name = name
-	s.sourceId = sourceId
-	s.jobId = jobId
-	s.verify = verify
+func (s *Source) Init(_ context.Context, cfg sources.SourceConfig) error {
+	s.name = cfg.Name
+	s.sourceId = cfg.SourceID
+	s.jobId = cfg.JobID
+	s.verify = cfg.Verify
 	s.jobPool = &errgroup.Group{}
-	s.jobPool.SetLimit(concurrency)
+	s.jobPool.SetLimit(cfg.Concurrency)
 	s.client = common.RetryableHttpClientTimeout(3)
 
 	var conn sourcespb.CircleCI
-	if err := anypb.UnmarshalTo(connection, &conn, proto.UnmarshalOptions{}); err != nil {
+	if err := anypb.UnmarshalTo(cfg.Connection, &conn, proto.UnmarshalOptions{}); err != nil {
 		return errors.WrapPrefix(err, "error unmarshalling connection", 0)
 	}
 

pkg/sources/circleci/circleci_test.go

@@ -71,7 +71,15 @@ func TestSource_Scan(t *testing.T) {
 				t.Fatal(err)
 			}
 
-			err = s.Init(ctx, tt.init.name, 0, 0, tt.init.verify, conn, 5)
+			cfg := sources.NewSourceConfig(
+				"circleci",
+				0,
+				0,
+				conn,
+				sources.WithVerify(tt.init.verify),
+				sources.WithConcurrency(5),
+			)
+			err = s.Init(ctx, cfg)
 			if (err != nil) != tt.wantErr {
 				t.Errorf("Source.Init() error = %v, wantErr %v", err, tt.wantErr)
 				return

pkg/sources/filesystem/filesystem.go

@@ -58,16 +58,16 @@ func (s *Source) JobID() int64 {
 }
 
 // Init returns an initialized Filesystem source.
-func (s *Source) Init(aCtx context.Context, name string, jobId, sourceId int64, verify bool, connection *anypb.Any, _ int) error {
+func (s *Source) Init(aCtx context.Context, cfg sources.SourceConfig) error {
 	s.log = aCtx.Logger()
 
-	s.name = name
-	s.sourceId = sourceId
-	s.jobId = jobId
-	s.verify = verify
+	s.name = cfg.Name
+	s.sourceId = cfg.SourceID
+	s.jobId = cfg.JobID
+	s.verify = cfg.Verify
 
 	var conn sourcespb.Filesystem
-	if err := anypb.UnmarshalTo(connection, &conn, proto.UnmarshalOptions{}); err != nil {
+	if err := anypb.UnmarshalTo(cfg.Connection, &conn, proto.UnmarshalOptions{}); err != nil {
 		return errors.WrapPrefix(err, "error unmarshalling connection", 0)
 	}
 

pkg/sources/filesystem/filesystem_test.go

@@ -56,7 +56,15 @@ func TestSource_Scan(t *testing.T) {
 				t.Fatal(err)
 			}
 
-			err = s.Init(ctx, tt.init.name, 0, 0, tt.init.verify, conn, 5)
+			cfg := sources.NewSourceConfig(
+				tt.init.name,
+				0,
+				0,
+				conn,
+				sources.WithVerify(tt.init.verify),
+				sources.WithConcurrency(5),
+			)
+			err = s.Init(ctx, cfg)
 			if (err != nil) != tt.wantErr {
 				t.Errorf("Source.Init() error = %v, wantErr %v", err, tt.wantErr)
 				return

pkg/sources/git/git.go

@@ -86,25 +86,25 @@ func (s *Source) JobID() int64 {
 }
 
 // Init returns an initialized GitHub source.
-func (s *Source) Init(aCtx context.Context, name string, jobId, sourceId int64, verify bool, connection *anypb.Any, concurrency int) error {
+func (s *Source) Init(aCtx context.Context, cfg sources.SourceConfig) error {
 
-	s.name = name
-	s.sourceId = sourceId
-	s.jobId = jobId
-	s.verify = verify
+	s.name = cfg.Name
+	s.sourceId = cfg.SourceID
+	s.jobId = cfg.JobID
+	s.verify = cfg.Verify
 
 	var conn sourcespb.Git
-	if err := anypb.UnmarshalTo(connection, &conn, proto.UnmarshalOptions{}); err != nil {
+	if err := anypb.UnmarshalTo(cfg.Connection, &conn, proto.UnmarshalOptions{}); err != nil {
 		return errors.WrapPrefix(err, "error unmarshalling connection", 0)
 	}
 
 	s.conn = &conn
 
-	if concurrency == 0 {
-		concurrency = runtime.NumCPU()
+	if cfg.Concurrency == 0 {
+		cfg.Concurrency = runtime.NumCPU()
 	}
 
-	s.git = NewGit(s.Type(), s.jobId, s.sourceId, s.name, s.verify, concurrency,
+	s.git = NewGit(s.Type(), s.jobId, s.sourceId, s.name, s.verify, cfg.Concurrency,
 		func(file, email, commit, timestamp, repository string, line int64) *source_metadatapb.MetaData {
 			return &source_metadatapb.MetaData{
 				Data: &source_metadatapb.MetaData_Git{

pkg/sources/git/git_test.go

@@ -130,7 +130,15 @@ func TestSource_Scan(t *testing.T) {
 				t.Fatal(err)
 			}
 
-			err = s.Init(ctx, tt.init.name, 0, 0, tt.init.verify, conn, tt.init.concurrency)
+			cfg := sources.NewSourceConfig(
+				tt.init.name,
+				0,
+				0,
+				conn,
+				sources.WithVerify(tt.init.verify),
+				sources.WithConcurrency(tt.init.concurrency),
+			)
+			err = s.Init(ctx, cfg)
 			if (err != nil) != tt.wantErr {
 				t.Errorf("Source.Init() error = %v, wantErr %v", err, tt.wantErr)
 				return
@@ -263,7 +271,15 @@ func TestSource_Chunks_Integration(t *testing.T) {
 			if err != nil {
 				t.Fatal(err)
 			}
-			err = s.Init(ctx, tt.init.name, 0, 0, tt.init.verify, conn, 4)
+			cfg := sources.NewSourceConfig(
+				tt.init.name,
+				0,
+				0,
+				conn,
+				sources.WithVerify(tt.init.verify),
+				sources.WithConcurrency(4),
+			)
+			err = s.Init(ctx, cfg)
 			if err != nil {
 				t.Fatal(err)
 			}
@@ -406,7 +422,15 @@ func TestSource_Chunks_Edge_Cases(t *testing.T) {
 				t.Fatal(err)
 			}
 
-			err = s.Init(ctx, tt.init.name, 0, 0, tt.init.verify, conn, 4)
+			cfg := sources.NewSourceConfig(
+				tt.init.name,
+				0,
+				0,
+				conn,
+				sources.WithVerify(tt.init.verify),
+				sources.WithConcurrency(4),
+			)
+			err = s.Init(ctx, cfg)
 			if err != nil {
 				t.Errorf("Source.Init() error = %v", err)
 				return

pkg/sources/github/github.go

@@ -124,21 +124,21 @@ func (s *Source) UserAndToken(ctx context.Context, installationClient *github.Cl
 }
 
 // Init returns an initialized GitHub source.
-func (s *Source) Init(aCtx context.Context, name string, jobID, sourceID int64, verify bool, connection *anypb.Any, concurrency int) error {
+func (s *Source) Init(aCtx context.Context, cfg sources.SourceConfig) error {
 	s.log = aCtx.Logger()
 
-	s.name = name
-	s.sourceID = sourceID
-	s.jobID = jobID
-	s.verify = verify
+	s.name = cfg.Name
+	s.sourceID = cfg.SourceID
+	s.jobID = cfg.JobID
+	s.verify = cfg.Verify
 	s.jobPool = &errgroup.Group{}
-	s.jobPool.SetLimit(concurrency)
+	s.jobPool.SetLimit(cfg.Concurrency)
 
 	s.httpClient = common.RetryableHttpClientTimeout(60)
 	s.apiClient = github.NewClient(s.httpClient)
 
 	var conn sourcespb.GitHub
-	err := anypb.UnmarshalTo(connection, &conn, proto.UnmarshalOptions{})
+	err := anypb.UnmarshalTo(cfg.Connection, &conn, proto.UnmarshalOptions{})
 	if err != nil {
 		return errors.WrapPrefix(err, "error unmarshalling connection", 0)
 	}

pkg/sources/github/github_integration_test.go

@@ -378,7 +378,15 @@ func TestSource_Scan(t *testing.T) {
 				t.Fatal(err)
 			}
 
-			err = s.Init(ctx, tt.init.name, 0, 0, tt.init.verify, conn, 4)
+			cfg := sources.NewSourceConfig(
+				tt.init.name,
+				0,
+				0,
+				conn,
+				sources.WithVerify(tt.init.verify),
+				sources.WithConcurrency(4),
+			)
+			err = s.Init(ctx, cfg)
 			if (err != nil) != tt.wantErr {
 				t.Errorf("Source.Init() error = %v, wantErr %v", err, tt.wantErr)
 				return
@@ -519,7 +527,15 @@ func TestSource_paginateGists(t *testing.T) {
 				t.Fatal(err)
 			}
 
-			err = s.Init(ctx, tt.init.name, 0, 0, tt.init.verify, conn, 4)
+			cfg := sources.NewSourceConfig(
+				tt.init.name,
+				0,
+				0,
+				conn,
+				sources.WithVerify(tt.init.verify),
+				sources.WithConcurrency(4),
+			)
+			err = s.Init(ctx, cfg)
 			if (err != nil) != tt.wantErr {
 				t.Errorf("Source.Init() error = %v, wantErr %v", err, tt.wantErr)
 				return

pkg/sources/github/github_test.go

@@ -23,6 +23,7 @@ import (
 	"github.com/trufflesecurity/trufflehog/v3/pkg/context"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/credentialspb"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
 )
 
 func createTestSource(src *sourcespb.GitHub) (*Source, *anypb.Any) {
@@ -36,7 +37,15 @@ func createTestSource(src *sourcespb.GitHub) (*Source, *anypb.Any) {
 
 func initTestSource(src *sourcespb.GitHub) *Source {
 	s, conn := createTestSource(src)
-	if err := s.Init(context.TODO(), "test - github", 0, 1337, false, conn, 1); err != nil {
+	cfg := sources.NewSourceConfig(
+		"test - github",
+		0,
+		1337,
+		conn,
+		sources.WithVerify(false),
+		sources.WithConcurrency(1),
+	)
+	if err := s.Init(context.TODO(), cfg); err != nil {
 		panic(err)
 	}
 	s.apiClient = github.NewClient(s.httpClient)
@@ -52,7 +61,15 @@ func TestInit(t *testing.T) {
 		},
 	})
 
-	err := source.Init(context.TODO(), "test - github", 0, 1337, false, conn, 1)
+	cfg := sources.NewSourceConfig(
+		"test - github",
+		0,
+		1337,
+		conn,
+		sources.WithVerify(false),
+		sources.WithConcurrency(1),
+	)
+	err := source.Init(context.TODO(), cfg)
 	assert.Nil(t, err)
 
 	// TODO: test error case

pkg/sources/gitlab/gitlab.go

@@ -67,16 +67,16 @@ func (s *Source) JobID() int64 {
 }
 
 // Init returns an initialized Gitlab source.
-func (s *Source) Init(_ context.Context, name string, jobId, sourceId int64, verify bool, connection *anypb.Any, concurrency int) error {
-	s.name = name
-	s.sourceId = sourceId
-	s.jobId = jobId
-	s.verify = verify
+func (s *Source) Init(_ context.Context, cfg sources.SourceConfig) error {
+	s.name = cfg.Name
+	s.sourceId = cfg.SourceID
+	s.jobId = cfg.JobID
+	s.verify = cfg.Verify
 	s.jobPool = &errgroup.Group{}
-	s.jobPool.SetLimit(concurrency)
+	s.jobPool.SetLimit(cfg.Concurrency)
 
 	var conn sourcespb.GitLab
-	err := anypb.UnmarshalTo(connection, &conn, proto.UnmarshalOptions{})
+	err := anypb.UnmarshalTo(cfg.Connection, &conn, proto.UnmarshalOptions{})
 	if err != nil {
 		return errors.WrapPrefix(err, "error unmarshalling connection", 0)
 	}
@@ -103,7 +103,7 @@ func (s *Source) Init(_ context.Context, name string, jobId, sourceId int64, ver
 		// We may need the password as a token if the user is using an access_token with basic auth.
 		s.token = cred.BasicAuth.Password
 	default:
-		return errors.Errorf("Invalid configuration given for source. Name: %s, Type: %s", name, s.Type())
+		return errors.Errorf("Invalid configuration given for source. Name: %s, Type: %s", cfg.Name, s.Type())
 	}
 
 	if len(s.url) == 0 {

pkg/sources/gitlab/gitlab_test.go

@@ -144,7 +144,15 @@ func TestSource_Scan(t *testing.T) {
 				t.Fatal(err)
 			}
 
-			err = s.Init(ctx, tt.init.name, 0, 0, tt.init.verify, conn, 10)
+			cfg := sources.NewSourceConfig(
+				tt.init.name,
+				0,
+				0,
+				conn,
+				sources.WithVerify(tt.init.verify),
+				sources.WithConcurrency(10),
+			)
+			err = s.Init(ctx, cfg)
 			if (err != nil) != tt.wantErr {
 				t.Errorf("Source.Init() error = %v, wantErr %v", err, tt.wantErr)
 				return

pkg/sources/s3/s3.go

@@ -59,21 +59,21 @@ func (s *Source) JobID() int64 {
 }
 
 // Init returns an initialized AWS source
-func (s *Source) Init(aCtx context.Context, name string, jobId, sourceId int64, verify bool, connection *anypb.Any, concurrency int) error {
-	s.log = context.WithValues(aCtx, "source", s.Type(), "name", name).Logger()
+func (s *Source) Init(aCtx context.Context, cfg sources.SourceConfig) error {
+	s.log = context.WithValues(aCtx, "source", s.Type(), "name", cfg.Name).Logger()
 
-	s.name = name
-	s.sourceId = sourceId
-	s.jobId = jobId
-	s.verify = verify
-	s.concurrency = concurrency
+	s.name = cfg.Name
+	s.sourceId = cfg.SourceID
+	s.jobId = cfg.JobID
+	s.verify = cfg.Verify
+	s.concurrency = cfg.Concurrency
 	s.errorCount = &sync.Map{}
 	s.log = aCtx.Logger()
 	s.jobPool = &errgroup.Group{}
-	s.jobPool.SetLimit(concurrency)
+	s.jobPool.SetLimit(s.concurrency)
 
 	var conn sourcespb.S3
-	err := anypb.UnmarshalTo(connection, &conn, proto.UnmarshalOptions{})
+	err := anypb.UnmarshalTo(cfg.Connection, &conn, proto.UnmarshalOptions{})
 	if err != nil {
 		return errors.WrapPrefix(err, "error unmarshalling connection", 0)
 	}

pkg/sources/s3/s3_test.go

@@ -71,7 +71,15 @@ func TestSource_Chunks(t *testing.T) {
 				t.Fatal(err)
 			}
 
-			err = s.Init(ctx, tt.init.name, 0, 0, tt.init.verify, conn, 8)
+			cfg := sources.NewSourceConfig(
+				tt.init.name,
+				0,
+				0,
+				conn,
+				sources.WithVerify(tt.init.verify),
+				sources.WithConcurrency(8),
+			)
+			err = s.Init(ctx, cfg)
 			if (err != nil) != tt.wantErr {
 				t.Errorf("Source.Init() error = %v, wantErr %v", err, tt.wantErr)
 				return

pkg/sources/sources.go

@@ -5,6 +5,7 @@ import (
 
 	"google.golang.org/protobuf/types/known/anypb"
 
+	"github.com/trufflesecurity/trufflehog/v3/pkg/cache"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/context"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/source_metadatapb"
@@ -28,6 +29,71 @@ type Chunk struct {
 	Verify bool
 }
 
+// Cache used by Sources to help reduce duplicate data scans.
+type Cache interface {
+	Set(key, value string) error
+	Get(key string) (string, bool)
+	Delete(key string) error
+	Clear() error
+}
+
+// SourceConfigOption is responsible for configuring a SourceConfig.
+type SourceConfigOption func(*SourceConfig)
+
+// WithConcurrency sets the concurrency for the source.
+func WithConcurrency(concurrency int) SourceConfigOption {
+	return func(c *SourceConfig) {
+		c.Concurrency = concurrency
+	}
+}
+
+// WithVerify sets the verify flag for the source.
+func WithVerify(verify bool) SourceConfigOption {
+	return func(c *SourceConfig) {
+		c.Verify = verify
+	}
+}
+
+// WithSimpleCache sets the cache for the source.
+func WithSimpleCache() SourceConfigOption {
+	return func(c *SourceConfig) {
+		c.Cache = cache.Simple()
+	}
+}
+
+// SourceConfig provides a way to configure a Source.
+type SourceConfig struct {
+	// Name is the name of the source.
+	Name string
+	// JobID is the ID of the job that the source is associated with.
+	JobID int64
+	// SourceID is the ID of the source.
+	SourceID int64
+	// Verify specifies whether any secrets in the Chunk should be verified.
+	Verify bool
+	// Connection is the connection information for the source.
+	Connection *anypb.Any
+	// Concurrency is the number of concurrent workers to use for the source.
+	Concurrency int
+	// Cache used by Sources to help reduce duplicate data scans.
+	Cache Cache
+}
+
+// NewSourceConfig creates a new SourceConfig with the provided options.
+func NewSourceConfig(name string, jobID, sourceID int64, connection *anypb.Any, opts ...SourceConfigOption) SourceConfig {
+	c := &SourceConfig{
+		Name:       name,
+		JobID:      jobID,
+		SourceID:   sourceID,
+		Connection: connection,
+	}
+
+	for _, opt := range opts {
+		opt(c)
+	}
+	return *c
+}
+
 // Source defines the interface required to implement a source chunker.
 type Source interface {
 	// Type returns the source type, used for matching against configuration and jobs.
@@ -37,7 +103,7 @@ type Source interface {
 	// JobID returns the initialized job ID used for tracking relationships in the DB.
 	JobID() int64
 	// Init initializes the source.
-	Init(aCtx context.Context, name string, jobId, sourceId int64, verify bool, connection *anypb.Any, concurrency int) error
+	Init(aCtx context.Context, cfg SourceConfig) error
 	// Chunks emits data over a channel that is decoded and scanned for secrets.
 	Chunks(ctx context.Context, chunksChan chan *Chunk) error
 	// GetProgress is the completion progress (percentage) for Scanned Source.

pkg/sources/syslog/syslog.go

@@ -82,15 +82,15 @@ func (s *Source) InjectConnection(conn *sourcespb.Syslog) {
 }
 
 // Init returns an initialized Syslog source.
-func (s *Source) Init(_ context.Context, name string, jobId, sourceId int64, verify bool, connection *anypb.Any, concurrency int) error {
+func (s *Source) Init(_ context.Context, cfg sources.SourceConfig) error {
 
-	s.name = name
-	s.sourceId = sourceId
-	s.jobId = jobId
-	s.verify = verify
+	s.name = cfg.Name
+	s.sourceId = cfg.SourceID
+	s.jobId = cfg.JobID
+	s.verify = cfg.Verify
 
 	var conn sourcespb.Syslog
-	err := anypb.UnmarshalTo(connection, &conn, proto.UnmarshalOptions{})
+	err := anypb.UnmarshalTo(cfg.Connection, &conn, proto.UnmarshalOptions{})
 	if err != nil {
 		return errors.WrapPrefix(err, "error unmarshalling connection", 0)
 	}