Create devcontainer.json for codespaces

* Refactor git source to allow ScanOptions and use source in engine

Refactor the Chunks method of the git Source to call out to two helper
methods: scanRepos and scanDirs which scans s.conn.Repositories and
s.conn.Directories respectively. The only notable change in behavior is
that a credential is no longer necessary if there are no
s.conn.Repositories to scan.

* Preserve ScanGit functionality of not cleaning up temporary files

.devcontainer/devcontainer.json

@@ -0,0 +1,5 @@
+{
+  "image": "trufflesecurity/protos:1.18-0",
+  "features": {
+  }
+}

go.mod

@@ -34,11 +34,13 @@ require (
 	github.com/go-sql-driver/mysql v1.7.1
 	github.com/gobwas/glob v0.2.3
 	github.com/golang-jwt/jwt v3.2.2+incompatible
+	github.com/golang/mock v1.4.3
 	github.com/google/go-cmp v0.5.9
 	github.com/google/go-containerregistry v0.15.2
 	github.com/google/go-github/v42 v42.0.0
 	github.com/googleapis/gax-go/v2 v2.12.0
 	github.com/hashicorp/go-retryablehttp v0.7.4
+	github.com/hashicorp/golang-lru v0.5.1
 	github.com/jlaffaye/ftp v0.2.0
 	github.com/joho/godotenv v1.5.1
 	github.com/jpillora/overseer v1.1.6
@@ -56,6 +58,7 @@ require (
 	github.com/tailscale/depaware v0.0.0-20210622194025-720c4b409502
 	github.com/xanzy/go-gitlab v0.88.0
 	go.mongodb.org/mongo-driver v1.12.0
+	go.uber.org/mock v0.2.0
 	go.uber.org/zap v1.24.0
 	golang.org/x/crypto v0.11.0
 	golang.org/x/exp v0.0.0-20221018205818-5c77f4b2bbd7
@@ -126,6 +129,7 @@ require (
 	github.com/hashicorp/errwrap v1.0.0 // indirect
 	github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
 	github.com/hashicorp/go-multierror v1.1.1 // indirect
+	github.com/hashicorp/golang-lru/v2 v2.0.4 // indirect
 	github.com/imdario/mergo v0.3.15 // indirect
 	github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect
 	github.com/jmespath/go-jmespath v0.4.0 // indirect

go.sum

@@ -301,7 +301,10 @@ github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9
 github.com/hashicorp/go-retryablehttp v0.7.4 h1:ZQgVdpTdAL7WpMIwLzCfbalOcSUdkDZnpUv3/+BxzFA=
 github.com/hashicorp/go-retryablehttp v0.7.4/go.mod h1:Jy/gPYAdjqffZ/yFGCFV2doI5wjtH1ewM9u8iYVjtX8=
 github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
+github.com/hashicorp/golang-lru v0.5.1 h1:0hERBMJE1eitiLkihrMvRVBYAkpHzc/J3QdDN+dAcgU=
 github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
+github.com/hashicorp/golang-lru/v2 v2.0.4 h1:7GHuZcgid37q8o5i3QI9KMT4nCWQQ3Kx3Ov6bb9MfK0=
+github.com/hashicorp/golang-lru/v2 v2.0.4/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM=
 github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
 github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
 github.com/ianlancetaylor/demangle v0.0.0-20210905161508-09a460cdf81d/go.mod h1:aYm2/VgdVmcIU8iMfdMvDMsRAQjcfZSKFby6HOFvi/w=
@@ -486,6 +489,8 @@ go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
 go.uber.org/goleak v1.1.11/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ=
 go.uber.org/goleak v1.2.1 h1:NBol2c7O1ZokfZ0LEU9K6Whx/KnwvepVetCUhtKja4A=
 go.uber.org/goleak v1.2.1/go.mod h1:qlT2yGI9QafXHhZZLxlSuNsMw3FFLxBr+tBRlmO1xH4=
+go.uber.org/mock v0.2.0 h1:TaP3xedm7JaAgScZO7tlvlKrqT0p7I6OsdGB5YNSMDU=
+go.uber.org/mock v0.2.0/go.mod h1:J0y0rp9L3xiff1+ZBfKxlC1fz2+aO16tw0tsDOixfuM=
 go.uber.org/multierr v1.6.0 h1:y6IPFStTAIT5Ytl7/XYmHvzXQ7S3g/IeZW9hyZ5thw4=
 go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU=
 go.uber.org/zap v1.24.0 h1:FiJd5l1UOLj0wCgbSE0rwwXHzEdAZS6hiiSnxJN/D60=

main.go

@@ -9,7 +9,6 @@ import (
 	"strconv"
 	"strings"
 	"syscall"
-	"time"
 
 	"github.com/felixge/fgprof"
 	"github.com/go-logr/logr"
@@ -39,6 +38,7 @@ var (
 	debug               = cli.Flag("debug", "Run in debug mode.").Bool()
 	trace               = cli.Flag("trace", "Run in trace mode.").Bool()
 	profile             = cli.Flag("profile", "Enables profiling and sets a pprof and fgprof server on :18066.").Bool()
+	localDev            = cli.Flag("local-dev", "Hidden feature to disable overseer for local dev.").Hidden().Bool()
 	jsonOut             = cli.Flag("json", "Output in JSON format.").Short('j').Bool()
 	jsonLegacy          = cli.Flag("json-legacy", "Use the pre-v3.0 JSON format. Only works with git, gitlab, and github sources.").Bool()
 	gitHubActionsFormat = cli.Flag("github-actions", "Output in GitHub Actions format.").Bool()
@@ -162,6 +162,12 @@ func main() {
 	logger, sync := log.New("trufflehog", logFormat(os.Stderr))
 	// make it the default logger for contexts
 	context.SetDefaultLogger(logger)
+
+	if *localDev {
+		run(overseer.State{})
+		os.Exit(0)
+	}
+
 	defer func() { _ = sync() }()
 	logFatal := logFatalFunc(logger)
 
@@ -316,8 +322,21 @@ func run(state overseer.State) {
 		return true
 	}
 
-	e := engine.Start(ctx,
-		engine.WithConcurrency(*concurrency),
+	// Set how the engine will print its results.
+	var printer engine.Printer
+	switch {
+	case *jsonLegacy:
+		printer = new(output.LegacyJSONPrinter)
+	case *jsonOut:
+		printer = new(output.JSONPrinter)
+	case *gitHubActionsFormat:
+		printer = new(output.GitHubActionsPrinter)
+	default:
+		printer = new(output.PlainPrinter)
+	}
+
+	e, err := engine.Start(ctx,
+		engine.WithConcurrency(uint8(*concurrency)),
 		engine.WithDecoders(decoders.DefaultDecoders()...),
 		engine.WithDetectors(!*noVerification, engine.DefaultDetectors()...),
 		engine.WithDetectors(!*noVerification, conf.Detectors...),
@@ -325,7 +344,13 @@ func run(state overseer.State) {
 		engine.WithFilterDetectors(excludeFilter),
 		engine.WithFilterDetectors(endpointCustomizer),
 		engine.WithFilterUnverified(*filterUnverified),
+		engine.WithOnlyVerified(*onlyVerified),
+		engine.WithPrintAvgDetectorTime(*printAvgDetectorTime),
+		engine.WithPrinter(printer),
 	)
+	if err != nil {
+		logFatal(err, "error initializing engine")
+	}
 
 	var repoPath string
 	var remote bool
@@ -475,52 +500,48 @@ func run(state overseer.State) {
 			logFatal(err, "Failed to scan Docker.")
 		}
 	}
-	// asynchronously wait for scanning to finish and cleanup
-	go e.Finish(ctx, logFatal)
 
 	if !*jsonLegacy && !*jsonOut {
 		fmt.Fprintf(os.Stderr, "🐷🔑🐷  TruffleHog. Unearth your secrets. 🐷🔑🐷\n\n")
 	}
 
-	// NOTE: this loop will terminate when the results channel is closed in
-	// e.Finish()
-	foundResults := false
-	for r := range e.ResultsChan() {
-		if *onlyVerified && !r.Verified {
-			continue
+	// Wait for all workers to finish.
+	if err = e.Finish(ctx); err != nil {
+		logFatal(err, "engine failed to finish execution")
 	}
-		foundResults = true
 
-		var err error
-		switch {
-		case *jsonLegacy:
-			err = output.PrintLegacyJSON(ctx, &r)
-		case *jsonOut:
-			err = output.PrintJSON(&r)
-		case *gitHubActionsFormat:
-			err = output.PrintGitHubActionsOutput(&r)
-		default:
-			err = output.PrintPlainOutput(&r)
-		}
-		if err != nil {
-			logFatal(err, "error printing results")
-		}
-	}
-	logger.V(2).Info("finished scanning",
-		"chunks", e.ChunksScanned(),
-		"bytes", e.BytesScanned(),
+	metrics := e.GetMetrics()
+	// Print results.
+	logger.Info("finished scanning",
+		"chunks", metrics.ChunksScanned,
+		"bytes", metrics.BytesScanned,
+		"verified_secrets", metrics.VerifiedSecretsFound,
+		"unverified_secrets", metrics.UnverifiedSecretsFound,
 	)
 
 	if *printAvgDetectorTime {
 		printAverageDetectorTime(e)
 	}
 
-	if foundResults && *fail {
+	if e.HasFoundResults() && *fail {
 		logger.V(2).Info("exiting with code 183 because results were found")
 		os.Exit(183)
 	}
 }
 
+// logFatalFunc returns a log.Fatal style function. Calling the returned
+// function will terminate the program without cleanup.
+func logFatalFunc(logger logr.Logger) func(error, string, ...any) {
+	return func(err error, message string, keyAndVals ...any) {
+		logger.Error(err, message, keyAndVals...)
+		if err != nil {
+			os.Exit(1)
+			return
+		}
+		os.Exit(0)
+	}
+}
+
 func commaSeparatedToSlice(s []string) []string {
 	var result []string
 	for _, items := range s {
@@ -537,26 +558,8 @@ func commaSeparatedToSlice(s []string) []string {
 
 func printAverageDetectorTime(e *engine.Engine) {
 	fmt.Fprintln(os.Stderr, "Average detector time is the measurement of average time spent on each detector when results are returned.")
-	for detectorName, durations := range e.DetectorAvgTime() {
-		var total time.Duration
-		for _, d := range durations {
-			total += d
-		}
-		avgDuration := total / time.Duration(len(durations))
-		fmt.Fprintf(os.Stderr, "%s: %s\n", detectorName, avgDuration)
-	}
-}
-
-// logFatalFunc returns a log.Fatal style function. Calling the returned
-// function will terminate the program without cleanup.
-func logFatalFunc(logger logr.Logger) func(error, string, ...any) {
-	return func(err error, message string, keyAndVals ...any) {
-		logger.Error(err, message, keyAndVals...)
-		if err != nil {
-			os.Exit(1)
-			return
-		}
-		os.Exit(0)
+	for detectorName, duration := range e.GetDetectorsMetrics() {
+		fmt.Fprintf(os.Stderr, "%s: %s\n", detectorName, duration)
 	}
 }
 

pkg/common/http.go

@@ -100,14 +100,14 @@ func NewCustomTransport(T http.RoundTripper) *CustomTransport {
 	return &CustomTransport{T}
 }
 
-func ConstantStatusHttpClient(statusCode int) *http.Client {
+func ConstantResponseHttpClient(statusCode int, body string) *http.Client {
 	return &http.Client{
 		Timeout: DefaultResponseTimeout,
 		Transport: FakeTransport{
 			CreateResponse: func(req *http.Request) (*http.Response, error) {
 				return &http.Response{
 					Request:    req,
-					Body:       io.NopCloser(strings.NewReader("")),
+					Body:       io.NopCloser(strings.NewReader(body)),
 					StatusCode: statusCode,
 				}, nil
 			},

pkg/common/utils.go

@@ -3,7 +3,9 @@ package common
 import (
 	"bufio"
 	"bytes"
+	"crypto/rand"
 	"io"
+	"math/big"
 	"strings"
 )
 
@@ -49,3 +51,16 @@ func ResponseContainsSubstring(reader io.ReadCloser, target string) (bool, error
 	}
 	return false, nil
 }
+
+var letters = []rune("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789")
+
+// RandomID returns a random string of the given length.
+func RandomID(length int) string {
+	b := make([]rune, length)
+	for i := range b {
+		randInt, _ := rand.Int(rand.Reader, big.NewInt(int64(len(letters))))
+		b[i] = letters[randInt.Int64()]
+	}
+
+	return string(b)
+}

pkg/context/context.go

@@ -3,18 +3,26 @@ package context
 import (
 	"context"
 	"fmt"
+	"os"
 	"runtime/debug"
 	"sync"
 	"time"
 
 	"github.com/go-logr/logr"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/log"
 )
 
 var (
 	// defaultLogger can be set via SetDefaultLogger.
-	defaultLogger logr.Logger = logr.Discard()
+	// It is initialized to write to stderr. To disable, you can call
+	// SetDefaultLogger with logr.Discard().
+	defaultLogger logr.Logger
 )
 
+func init() {
+	defaultLogger, _ = log.New("context", log.WithConsoleSink(os.Stderr))
+}
+
 // Context wraps context.Context and includes an additional Logger() method.
 type Context interface {
 	context.Context
@@ -136,7 +144,9 @@ func AddLogger(parent context.Context) Context {
 }
 
 // SetupDefaultLogger sets the package-level global default logger that will be
-// used for Background and TODO contexts.
+// used for Background and TODO contexts. On startup, the default logger will
+// be configured to output logs to stderr. Use logr.Discard() to disable all
+// logs from Contexts.
 func SetDefaultLogger(l logr.Logger) {
 	defaultLogger = l
 }

pkg/context/context_test.go

@@ -158,7 +158,7 @@ func TestWithValues(t *testing.T) {
 	assert.NotContains(t, logs[6], `what does this do?`)
 }
 
-func TestDiscardLogger(t *testing.T) {
+func TestDefaultLogger(t *testing.T) {
 	var panicked bool
 	defer func() {
 		if r := recover(); r != nil {

pkg/detectors/alchemy/alchemy.go

@@ -62,21 +62,21 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 				defer res.Body.Close()
 				if res.StatusCode >= 200 && res.StatusCode < 300 {
 					s1.Verified = true
+				} else if res.StatusCode == 401 {
+					// The secret is determinately not verified (nothing to do)
 				} else {
-					// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
-					if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
-						continue
-					}
-
-					if res.StatusCode != 401 {
 					s1.VerificationError = fmt.Errorf("request to %v returned unexpected status %d", res.Request.URL, res.StatusCode)
 				}
-				}
 			} else {
 				s1.VerificationError = err
 			}
 		}
 
+		// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
+		if !s1.Verified && detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
+			continue
+		}
+
 		results = append(results, s1)
 	}
 

pkg/detectors/alchemy/alchemy_test.go

@@ -104,7 +104,7 @@ func TestAlchemy_FromChunk(t *testing.T) {
 		},
 		{
 			name: "found, verified but unexpected api surface",
-			s:    Scanner{client: common.ConstantStatusHttpClient(404)},
+			s:    Scanner{client: common.ConstantResponseHttpClient(404, "")},
 			args: args{
 				ctx:    context.Background(),
 				data:   []byte(fmt.Sprintf("You can find a alchemy secret %s within", secret)),

pkg/detectors/aws/aws.go

@@ -18,6 +18,7 @@ import (
 )
 
 type scanner struct {
+	verificationClient *http.Client
 	skipIDs            map[string]struct{}
 }
 
@@ -48,7 +49,7 @@ func WithSkipIDs(skipIDs []string) func(*scanner) {
 var _ detectors.Detector = (*scanner)(nil)
 
 var (
-	client = common.SaneHttpClient()
+	defaultVerificationClient = common.SaneHttpClient()
 
 	// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
 	// Key types are from this list https://docs.aws.amazon.com/IAM/latest/UserGuide/reference_identifiers.html#identifiers-unique-ids
@@ -123,8 +124,9 @@ func (s scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 				host := "sts.amazonaws.com"
 				region := "us-east-1"
 				endpoint := "https://sts.amazonaws.com"
-				datestamp := time.Now().UTC().Format("20060102")
-				amzDate := time.Now().UTC().Format("20060102T150405Z0700")
+				now := time.Now().UTC()
+				datestamp := now.Format("20060102")
+				amzDate := now.Format("20060102T150405Z0700")
 
 				req, err := http.NewRequestWithContext(ctx, method, endpoint, nil)
 				if err != nil {
@@ -171,6 +173,10 @@ func (s scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 				req.Header.Add("Content-type", "application/x-www-form-urlencoded; charset=utf-8")
 				req.URL.RawQuery = params.Encode()
 
+				client := s.verificationClient
+				if client == nil {
+					client = defaultVerificationClient
+				}
 				res, err := client.Do(req)
 				if err == nil {
 
@@ -194,7 +200,25 @@ func (s scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 							continue
 						}
 
-						if res.StatusCode != 403 {
+						if res.StatusCode == 403 {
+							var body awsErrorResponseBody
+							err = json.NewDecoder(res.Body).Decode(&body)
+							res.Body.Close()
+							if err == nil {
+								// All instances of the code I've seen in the wild are PascalCased but this check is
+								// case-insensitive out of an abundance of caution
+								if strings.EqualFold(body.Error.Code, "InvalidClientTokenId") {
+									// determinate failure - nothing to do
+								} else {
+									// We see a surprising number of false-negative signature mismatch errors here
+									// (The official SDK somehow elicits even more than just making the request
+									// ourselves)
+									s1.VerificationError = fmt.Errorf("request to %v returned status %d with an unexpected reason (%s: %s)", res.Request.URL, res.StatusCode, body.Error.Code, body.Error.Message)
+								}
+							} else {
+								s1.VerificationError = fmt.Errorf("couldn't parse the sts response body (%v)", err)
+							}
+						} else {
 							s1.VerificationError = fmt.Errorf("request to %v returned unexpected status %d", res.Request.URL, res.StatusCode)
 						}
 					}
@@ -245,6 +269,15 @@ func awsCustomCleanResults(results []detectors.Result) []detectors.Result {
 	return out
 }
 
+type awsError struct {
+	Code    string `json:"Code"`
+	Message string `json:"Message"`
+}
+
+type awsErrorResponseBody struct {
+	Error awsError `json:"Error"`
+}
+
 type identityRes struct {
 	GetCallerIdentityResponse struct {
 		GetCallerIdentityResult struct {

pkg/detectors/aws/aws_test.go

@@ -17,6 +17,8 @@ import (
 	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
 )
 
+var unverifiedSecretClient = common.ConstantResponseHttpClient(403, `{"Error": {"Code": "InvalidClientTokenId"} }`)
+
 func TestAWS_FromChunk(t *testing.T) {
 	ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
 	defer cancel()
@@ -69,7 +71,7 @@ func TestAWS_FromChunk(t *testing.T) {
 		},
 		{
 			name: "found, unverified",
-			s:    scanner{},
+			s:    scanner{verificationClient: unverifiedSecretClient},
 			args: args{
 				ctx:    context.Background(),
 				data:   []byte(fmt.Sprintf("You can find a aws secret %s within aws %s but not valid", inactiveSecret, id)), // the secret would satisfy the regex but not pass validation
@@ -163,7 +165,7 @@ func TestAWS_FromChunk(t *testing.T) {
 		},
 		{
 			name: "found, unverified, with leading +",
-			s:    scanner{},
+			s:    scanner{verificationClient: unverifiedSecretClient},
 			args: args{
 				ctx:    context.Background(),
 				data:   []byte(fmt.Sprintf("You can find a aws secret %s within aws %s but not valid", "+HaNv9cTwheDKGJaws/+BMF2GgybQgBWdhcOOdfF", id)), // the secret would satisfy the regex but not pass validation
@@ -194,9 +196,45 @@ func TestAWS_FromChunk(t *testing.T) {
 		},
 		{
 			name: "found, would be verified if not for http timeout",
-			s:    scanner{},
+			s:    scanner{verificationClient: common.SaneHttpClientTimeOut(1 * time.Microsecond)},
 			args: args{
-				ctx:    timeoutContext(1 * time.Microsecond),
+				ctx:    context.Background(),
+				data:   []byte(fmt.Sprintf("You can find a aws secret %s within aws %s", secret, id)),
+				verify: true,
+			},
+			want: []detectors.Result{
+				{
+					DetectorType: detectorspb.DetectorType_AWS,
+					Verified:     false,
+					Redacted:     "AKIASP2TPHJSQH3FJRUX",
+				},
+			},
+			wantErr:               false,
+			wantVerificationError: true,
+		},
+		{
+			name: "found, unverified due to unexpected http response status",
+			s:    scanner{verificationClient: common.ConstantResponseHttpClient(500, "internal server error")},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte(fmt.Sprintf("You can find a aws secret %s within aws %s", secret, id)),
+				verify: true,
+			},
+			want: []detectors.Result{
+				{
+					DetectorType: detectorspb.DetectorType_AWS,
+					Verified:     false,
+					Redacted:     "AKIASP2TPHJSQH3FJRUX",
+				},
+			},
+			wantErr:               false,
+			wantVerificationError: true,
+		},
+		{
+			name: "found, unverified due to unexpected 403 response reason",
+			s:    scanner{verificationClient: common.ConstantResponseHttpClient(403, `{"Error": {"Code": "SignatureDoesNotMatch"} }`)},
+			args: args{
+				ctx:    context.Background(),
 				data:   []byte(fmt.Sprintf("You can find a aws secret %s within aws %s", secret, id)),
 				verify: true,
 			},
@@ -224,7 +262,7 @@ func TestAWS_FromChunk(t *testing.T) {
 					t.Fatalf("no raw secret present: \n %+v", got[i])
 				}
 				if (got[i].VerificationError != nil) != tt.wantVerificationError {
-					t.Fatalf("verification error = %v, wantVerificationError %v", got[i].VerificationError, tt.wantVerificationError)
+					t.Fatalf("wantVerificationError %v, verification error = %v", tt.wantVerificationError, got[i].VerificationError)
 				}
 			}
 			ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "RawV2", "Raw", "VerificationError")
@@ -249,8 +287,3 @@ func BenchmarkFromData(benchmark *testing.B) {
 		})
 	}
 }
-
-func timeoutContext(timeout time.Duration) context.Context {
-	c, _ := context.WithTimeout(context.Background(), timeout)
-	return c
-}

pkg/detectors/mongodb/mongodb.go

@@ -2,6 +2,8 @@ package mongodb
 
 import (
 	"context"
+	"go.mongodb.org/mongo-driver/x/mongo/driver/auth"
+	"go.mongodb.org/mongo-driver/x/mongo/driver/topology"
 	"regexp"
 	"strings"
 	"time"
@@ -14,12 +16,15 @@ import (
 	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
 )
 
-type Scanner struct{}
+type Scanner struct {
+	timeout time.Duration // Zero value means "default timeout"
+}
 
 // Ensure the Scanner satisfies the interface at compile time.
 var _ detectors.Detector = (*Scanner)(nil)
 
 var (
+	defaultTimeout = 2 * time.Second
 	// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
 	keyPat = regexp.MustCompile(`\b(mongodb(\+srv)?://[\S]{3,50}:([\S]{3,88})@[-.%\w\/:]+)\b`)
 	// TODO: Add support for sharded cluster, replica set and Atlas Deployment.
@@ -46,23 +51,15 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 		}
 
 		if verify {
-			func() {
-				ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
-				defer cancel()
-				client, err := mongo.Connect(ctx, options.Client().ApplyURI(resMatch))
-				if err != nil {
-					return
+			timeout := s.timeout
+			if timeout == 0 {
+				timeout = defaultTimeout
 			}
-				defer func() {
-					if err := client.Disconnect(ctx); err != nil {
-						return
+			err := verifyUri(resMatch, timeout)
+			s1.Verified = err == nil
+			if !isErrDeterminate(err) {
+				s1.VerificationError = err
 			}
-				}()
-				if err := client.Ping(ctx, readpref.Primary()); err != nil {
-					return
-				}
-				s1.Verified = true
-			}()
 		}
 		results = append(results, s1)
 	}
@@ -70,6 +67,33 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 	return results, nil
 }
 
+func isErrDeterminate(err error) bool {
+	switch e := err.(type) {
+	case topology.ConnectionError:
+		switch e.Unwrap().(type) {
+		case *auth.Error:
+			return true
+		default:
+			return false
+		}
+	default:
+		return false
+	}
+}
+
+func verifyUri(uri string, timeout time.Duration) error {
+	ctx, cancel := context.WithTimeout(context.Background(), timeout)
+	defer cancel()
+	client, err := mongo.Connect(ctx, options.Client().ApplyURI(uri))
+	if err != nil {
+		return err
+	}
+	defer func() {
+		_ = client.Disconnect(ctx)
+	}()
+	return client.Ping(ctx, readpref.Primary())
+}
+
 func (s Scanner) Type() detectorspb.DetectorType {
 	return detectorspb.DetectorType_MongoDB
 }

pkg/detectors/mongodb/mongodb_test.go

@@ -6,6 +6,7 @@ package mongodb
 import (
 	"context"
 	"fmt"
+	"strings"
 	"testing"
 	"time"
 
@@ -39,6 +40,7 @@ func TestMongoDB_FromChunk(t *testing.T) {
 		args                args
 		want                []detectors.Result
 		wantErr             bool
+		wantVerificationErr bool
 	}{
 		{
 			name: "found, verified",
@@ -72,6 +74,40 @@ func TestMongoDB_FromChunk(t *testing.T) {
 			},
 			wantErr: false,
 		},
+		{
+			name: "found, would be verified but for connection timeout",
+			s:    Scanner{timeout: 1 * time.Microsecond},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte(fmt.Sprintf("You can find a mongodb secret %s within", secret)),
+				verify: true,
+			},
+			want: []detectors.Result{
+				{
+					DetectorType: detectorspb.DetectorType_MongoDB,
+					Verified:     false,
+				},
+			},
+			wantErr:             false,
+			wantVerificationErr: true,
+		},
+		{
+			name: "found, bad host",
+			s:    Scanner{},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte(fmt.Sprintf("You can find a mongodb secret %s within", strings.ReplaceAll(secret, ".mongodb.net", ".mongodb.net.bad"))),
+				verify: true,
+			},
+			want: []detectors.Result{
+				{
+					DetectorType: detectorspb.DetectorType_MongoDB,
+					Verified:     false,
+				},
+			},
+			wantErr:             false,
+			wantVerificationErr: true,
+		},
 		{
 			name: "not found",
 			s:    Scanner{},
@@ -86,8 +122,7 @@ func TestMongoDB_FromChunk(t *testing.T) {
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			s := Scanner{}
-			got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+			got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
 			if (err != nil) != tt.wantErr {
 				t.Errorf("MongoDB.FromData() error = %v, wantErr %v", err, tt.wantErr)
 				return
@@ -97,8 +132,11 @@ func TestMongoDB_FromChunk(t *testing.T) {
 					t.Fatalf("no raw secret present: \n %+v", got[i])
 				}
 				got[i].Raw = nil
+				if (got[i].VerificationError != nil) != tt.wantVerificationErr {
+					t.Fatalf("wantVerificationErr = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError)
 				}
-			ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "RawV2")
+			}
+			ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "RawV2", "VerificationError")
 			if diff := cmp.Diff(tt.want, got, ignoreOpts); diff != "" {
 				t.Errorf("MongoDB.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
 			}

pkg/detectors/pubnubpublishkey/pubnubpublishkey.go

@@ -21,7 +21,7 @@ var (
 
 	// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
 	pubPat = regexp.MustCompile(`\b(pub-c-[0-9a-z]{8}-[0-9a-z]{4}-[a-z0-9]{4}-[a-z0-9]{4}-[a-z0-9]{12})\b`)
-	subPat = regexp.MustCompile(`\b(sub-c-[0-9a-z]{8}-[a-z]{4}-[a-z0-9]{4}-[a-z0-9]{4}-[a-z0-9]{12})\b`)
+	subPat = regexp.MustCompile(`\b(sub-c-[0-9a-z]{8}-[a-z0-9]{4}-[a-z0-9]{4}-[a-z0-9]{4}-[a-z0-9]{12})\b`)
 )
 
 // Keywords are used for efficiently pre-filtering chunks.

pkg/detectors/pubnubsubscriptionkey/pubnubsubscriptionkey.go

@@ -20,7 +20,7 @@ var (
 	client = common.SaneHttpClient()
 
 	// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
-	keyPat = regexp.MustCompile(`\b(sub-c-[0-9a-z]{8}-[a-z]{4}-[a-z0-9]{4}-[a-z0-9]{4}-[a-z0-9]{12})\b`)
+	keyPat = regexp.MustCompile(`\b(sub-c-[0-9a-z]{8}-[a-z0-9]{4}-[a-z0-9]{4}-[a-z0-9]{4}-[a-z0-9]{12})\b`)
 )
 
 // Keywords are used for efficiently pre-filtering chunks.

pkg/detectors/shodankey/shodankey.go

@@ -2,6 +2,8 @@ package shodankey
 
 import (
 	"context"
+	"encoding/json"
+	"io"
 	"net/http"
 	"regexp"
 	"strings"
@@ -48,31 +50,58 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 		}
 
 		if verify {
-			req, err := http.NewRequestWithContext(ctx, "GET", "https://api.shodan.io/shodan/host/count?key="+resMatch+"&query=port:22&facets=org,os", nil)
-			if err != nil {
-				continue
-			}
-
-			res, err := client.Do(req)
-			if err == nil {
-				defer res.Body.Close()
-				if res.StatusCode >= 200 && res.StatusCode < 300 {
-					s1.Verified = true
-				} else {
-					// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
-					if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
+			s1.Verified = verifyToken(ctx, client, resMatch)
+			if !s1.Verified && detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
 				continue
 			}
 		}
-			}
-		}
-
 		results = append(results, s1)
 	}
 
 	return results, nil
 }
 
+type shodanInfoRes struct {
+	ScanCredits int `json:"scan_credits"`
+	UsageLimits struct {
+		ScanCredits  int `json:"scan_credits"`
+		QueryCredits int `json:"query_credits"`
+		MonitoredIps int `json:"monitored_ips"`
+	} `json:"usage_limits"`
+	Plan         string `json:"plan"`
+	HTTPS        bool   `json:"https"`
+	Unlocked     bool   `json:"unlocked"`
+	QueryCredits int    `json:"query_credits"`
+	MonitoredIps int    `json:"monitored_ips"`
+	UnlockedLeft int    `json:"unlocked_left"`
+	Telnet       bool   `json:"telnet"`
+}
+
+func verifyToken(ctx context.Context, client *http.Client, token string) bool {
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, "https://api.shodan.io/api-info?key="+token, nil)
+	if err != nil {
+		return false
+	}
+
+	res, err := client.Do(req)
+	if err != nil {
+		return false
+	}
+	defer res.Body.Close()
+
+	if res.StatusCode < 200 || res.StatusCode >= 300 {
+		return false
+	}
+
+	bytes, err := io.ReadAll(res.Body)
+	if err != nil {
+		return false
+	}
+
+	var info shodanInfoRes
+	return json.Unmarshal(bytes, &info) == nil
+}
+
 func (s Scanner) Type() detectorspb.DetectorType {
 	return detectorspb.DetectorType_ShodanKey
 }

pkg/detectors/shodankey/shodankey_test.go

@@ -10,9 +10,9 @@ import (
 	"time"
 
 	"github.com/kylelemons/godebug/pretty"
-	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
 
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
 )
 

pkg/detectors/virustotal/virustotal.go

@@ -1,10 +1,7 @@
 package virustotal
 
 import (
-	"bytes"
 	"context"
-	"io"
-	"mime/multipart"
 	"net/http"
 	"regexp"
 	"strings"
@@ -50,43 +47,36 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 		}
 
 		if verify {
-			body := &bytes.Buffer{}
-			writer := multipart.NewWriter(body)
-			fw, err := writer.CreateFormField("url")
-			if err != nil {
-				continue
-			}
-			_, err = io.Copy(fw, strings.NewReader("https://www.amazon.com"))
-			if err != nil {
-				continue
-			}
-			writer.Close()
-			req, err := http.NewRequestWithContext(ctx, "POST", "https://www.virustotal.com/api/v3/urls", bytes.NewReader(body.Bytes()))
-			if err != nil {
-				continue
-			}
-			req.Header.Add("Content-Type", writer.FormDataContentType())
-			req.Header.Add("x-apikey", resMatch)
-			res, err := client.Do(req)
-			if err == nil {
-				defer res.Body.Close()
-				if res.StatusCode >= 200 && res.StatusCode < 300 {
-					s1.Verified = true
-				} else {
-					// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
-					if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
+			s1.Verified = verifyToken(ctx, client, resMatch)
+			if !s1.Verified && detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
 				continue
 			}
 		}
-			}
-		}
-
 		results = append(results, s1)
 	}
 
 	return results, nil
 }
 
+func verifyToken(ctx context.Context, client *http.Client, token string) bool {
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, "https://www.virustotal.com/api/v3/metadata", nil)
+	if err != nil {
+		return false
+	}
+	req.Header.Add("x-apikey", token)
+
+	res, err := client.Do(req)
+	if err != nil {
+		return false
+	}
+	defer res.Body.Close()
+
+	if res.StatusCode < 200 || res.StatusCode >= 300 {
+		return false
+	}
+	return true
+}
+
 func (s Scanner) Type() detectorspb.DetectorType {
 	return detectorspb.DetectorType_VirusTotal
 }

pkg/detectors/virustotal/virustotal_test.go

@@ -10,9 +10,9 @@ import (
 	"time"
 
 	"github.com/kylelemons/godebug/pretty"
-	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
 
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
 )
 

pkg/engine/engine.go

@@ -11,6 +11,7 @@ import (
 	"time"
 
 	ahocorasick "github.com/BobuSumisu/aho-corasick"
+	lru "github.com/hashicorp/golang-lru"
 	"golang.org/x/sync/errgroup"
 	"google.golang.org/protobuf/proto"
 
@@ -19,36 +20,76 @@ import (
 	"github.com/trufflesecurity/trufflehog/v3/pkg/context"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/decoders"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/output"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/source_metadatapb"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
 )
 
+// Metrics for the scan engine for external consumption.
+type Metrics struct {
+	BytesScanned           uint64
+	ChunksScanned          uint64
+	VerifiedSecretsFound   uint64
+	UnverifiedSecretsFound uint64
+	AvgDetectorTime        map[string]time.Duration
+}
+
+// runtimeMetrics for the scan engine for internal use by the engine.
+type runtimeMetrics struct {
+	mu sync.RWMutex
+	Metrics
+	detectorAvgTime sync.Map
+}
+
+// Printer is used to format found results and output them to the user. Ex JSON, plain text, etc.
+type Printer interface {
+	Print(ctx context.Context, r *detectors.ResultWithMetadata) error
+}
+
 type Engine struct {
-	concurrency     int
+	concurrency uint8
 	chunks      chan *sources.Chunk
 	results     chan detectors.ResultWithMetadata
 	decoders    []decoders.Decoder
 	detectors   map[bool][]detectors.Detector
-	chunksScanned   uint64
-	bytesScanned    uint64
-	detectorAvgTime sync.Map
 	sourcesWg   *errgroup.Group
 	workersWg   sync.WaitGroup
 	// filterUnverified is used to reduce the number of unverified results.
 	// If there are multiple unverified results for the same chunk for the same detector,
 	// only the first one will be kept.
 	filterUnverified     bool
+	onlyVerified         bool
+	printAvgDetectorTime bool
 
 	// prefilter is a ahocorasick struct used for doing efficient string
 	// matching given a set of words (keywords from the rules in the config)
 	prefilter ahocorasick.Trie
+
+	detectableChunksChan chan detectableChunk
+	wgDetectorWorkers    sync.WaitGroup
+	WgNotifier           sync.WaitGroup
+
+	// Runtime metrics.
+	metrics runtimeMetrics
+
+	// numFoundResults is used to keep track of the number of results found.
+	numFoundResults uint32
+
+	// printer provides a method for formatting and outputting search results.
+	// The specific implementation (e.g., JSON, plain text)
+	// should be set during initialization based on user preference or program requirements.
+	printer Printer
+
+	// dedupeCache is used to deduplicate results by comparing the
+	// detector type, raw result, and source metadata
+	dedupeCache *lru.Cache
 }
 
 type EngineOption func(*Engine)
 
-func WithConcurrency(concurrency int) EngineOption {
+func WithConcurrency(concurrency uint8) EngineOption {
 	return func(e *Engine) {
 		e.concurrency = concurrency
 	}
@@ -83,6 +124,26 @@ func WithFilterUnverified(filter bool) EngineOption {
 	}
 }
 
+// WithOnlyVerified sets the onlyVerified flag on the engine. If set to true,
+// the engine will only print verified results.
+func WithOnlyVerified(onlyVerified bool) EngineOption {
+	return func(e *Engine) {
+		e.onlyVerified = onlyVerified
+	}
+}
+
+// WithPrintAvgDetectorTime sets the printAvgDetectorTime flag on the engine. If set to
+// true, the engine will print the average time taken by each detector.
+// This option allows us to measure the time taken for each detector ONLY if
+// the engine is configured to print the results.
+// Calculating the average time taken by each detector is an expensive operation
+// and should be avoided unless specified by the user.
+func WithPrintAvgDetectorTime(printAvgDetectorTime bool) EngineOption {
+	return func(e *Engine) {
+		e.printAvgDetectorTime = printAvgDetectorTime
+	}
+}
+
 // WithFilterDetectors applies a filter to the configured list of detectors. If
 // the filterFunc returns true, the detector will be included for scanning.
 // This option applies to the existing list of detectors configured, so the
@@ -98,6 +159,13 @@ func WithFilterDetectors(filterFunc func(detectors.Detector) bool) EngineOption
 	}
 }
 
+// WithPrinter sets the Printer on the engine.
+func WithPrinter(printer Printer) EngineOption {
+	return func(e *Engine) {
+		e.printer = printer
+	}
+}
+
 func filterDetectors(filterFunc func(detectors.Detector) bool, input []detectors.Detector) []detectors.Detector {
 	var output []detectors.Detector
 	for _, detector := range input {
@@ -108,12 +176,92 @@ func filterDetectors(filterFunc func(detectors.Detector) bool, input []detectors
 	return output
 }
 
-func Start(ctx context.Context, options ...EngineOption) *Engine {
+// HasFoundResults returns true if any results are found.
+func (e *Engine) HasFoundResults() bool {
+	return atomic.LoadUint32(&e.numFoundResults) > 0
+}
+
+// GetMetrics returns a copy of Metrics.
+// It's safe for concurrent use, and the caller can't modify the original data.
+func (e *Engine) GetMetrics() Metrics {
+	e.metrics.mu.RLock()
+	defer e.metrics.mu.RUnlock()
+
+	result := e.metrics.Metrics
+	result.AvgDetectorTime = make(map[string]time.Duration, len(e.metrics.AvgDetectorTime))
+
+	for detectorName, durations := range e.DetectorAvgTime() {
+		var total time.Duration
+		for _, d := range durations {
+			total += d
+		}
+		avgDuration := total / time.Duration(len(durations))
+		result.AvgDetectorTime[detectorName] = avgDuration
+	}
+
+	return result
+}
+
+// GetDetectorsMetrics returns a copy of the average time taken by each detector.
+func (e *Engine) GetDetectorsMetrics() map[string]time.Duration {
+	e.metrics.mu.RLock()
+	defer e.metrics.mu.RUnlock()
+
+	result := make(map[string]time.Duration, len(DefaultDetectors()))
+	for detectorName, durations := range e.DetectorAvgTime() {
+		var total time.Duration
+		for _, d := range durations {
+			total += d
+		}
+		avgDuration := total / time.Duration(len(durations))
+		result[detectorName] = avgDuration
+	}
+
+	return result
+}
+
+// DetectorAvgTime returns the average time taken by each detector.
+func (e *Engine) DetectorAvgTime() map[string][]time.Duration {
+	logger := context.Background().Logger()
+	avgTime := map[string][]time.Duration{}
+	e.metrics.detectorAvgTime.Range(func(k, v interface{}) bool {
+		key, ok := k.(string)
+		if !ok {
+			logger.Info("expected detectorAvgTime key to be a string")
+			return true
+		}
+
+		value, ok := v.([]time.Duration)
+		if !ok {
+			logger.Info("expected detectorAvgTime value to be []time.Duration")
+			return true
+		}
+		avgTime[key] = value
+		return true
+	})
+	return avgTime
+}
+
+// Start the engine with options.
+func Start(ctx context.Context, options ...EngineOption) (*Engine, error) {
+	const (
+		defaultChannelBuffer = 1
+		// TODO (ahrav): Determine the optimal cache size.
+		cacheSize = 512 // number of entries in the LRU cache
+	)
+
+	cache, err := lru.New(cacheSize)
+	if err != nil {
+		return nil, fmt.Errorf("failed to initialize LRU cache: %w", err)
+	}
+
 	e := &Engine{
-		chunks:          make(chan *sources.Chunk),
-		results:         make(chan detectors.ResultWithMetadata),
-		detectorAvgTime: sync.Map{},
+		chunks:               make(chan *sources.Chunk, defaultChannelBuffer),
+		detectableChunksChan: make(chan detectableChunk, defaultChannelBuffer),
+		results:              make(chan detectors.ResultWithMetadata, defaultChannelBuffer),
 		sourcesWg:            &errgroup.Group{},
+		dedupeCache:          cache,
+		printer:              new(output.PlainPrinter), // default printer
 	}
 
 	for _, option := range options {
@@ -124,12 +272,12 @@ func Start(ctx context.Context, options ...EngineOption) *Engine {
 	if e.concurrency == 0 {
 		numCPU := runtime.NumCPU()
 		ctx.Logger().Info("No concurrency specified, defaulting to max", "cpu", numCPU)
-		e.concurrency = numCPU
+		e.concurrency = uint8(numCPU)
 	}
-	ctx.Logger().V(2).Info("engine started", "workers", e.concurrency)
+	ctx.Logger().V(3).Info("engine started", "workers", e.concurrency)
 
 	// Limit number of concurrent goroutines dedicated to chunking a source.
-	e.sourcesWg.SetLimit(e.concurrency)
+	e.sourcesWg.SetLimit(int(e.concurrency))
 
 	if len(e.decoders) == 0 {
 		e.decoders = decoders.DefaultDecoders()
@@ -156,8 +304,8 @@ func Start(ctx context.Context, options ...EngineOption) *Engine {
 	}
 	e.prefilter = *ahocorasick.NewTrieBuilder().AddStrings(keywords).Build()
 
-	ctx.Logger().Info("loaded decoders", "count", len(e.decoders))
-	ctx.Logger().Info("loaded detectors",
+	ctx.Logger().V(3).Info("loaded decoders", "count", len(e.decoders))
+	ctx.Logger().V(3).Info("loaded detectors",
 		"total", len(e.detectors[true])+len(e.detectors[false]),
 		"verification_enabled", len(e.detectors[true]),
 		"verification_disabled", len(e.detectors[false]),
@@ -178,41 +326,68 @@ func Start(ctx context.Context, options ...EngineOption) *Engine {
 		}
 	}
 
-	// Start the workers.
-	for i := 0; i < e.concurrency; i++ {
+	ctx.Logger().V(2).Info("starting scanner workers", "count", e.concurrency)
+	// Run the Secret scanner workers and Notifier pipelines.
+	for worker := uint64(0); worker < uint64(e.concurrency); worker++ {
 		e.workersWg.Add(1)
 		go func() {
+			ctx := context.WithValue(ctx, "secret_worker_id", common.RandomID(5))
 			defer common.Recover(ctx)
 			defer e.workersWg.Done()
 			e.detectorWorker(ctx)
 		}()
 	}
 
-	return e
+	const detectorWorkerMultiplier = 50
+	ctx.Logger().V(2).Info("starting detector workers", "count", e.concurrency*detectorWorkerMultiplier)
+	for worker := uint64(0); worker < uint64(e.concurrency*detectorWorkerMultiplier); worker++ {
+		e.wgDetectorWorkers.Add(1)
+		go func() {
+			ctx := context.WithValue(ctx, "detector_worker_id", common.RandomID(5))
+			defer common.Recover(ctx)
+			defer e.wgDetectorWorkers.Done()
+			e.detectChunks(ctx)
+		}()
+	}
+
+	// We want 1/4th of the notifier workers as the number of scanner workers.
+	const notifierWorkerRatio = 4
+	maxNotifierWorkers := 1
+	if numWorkers := e.concurrency / notifierWorkerRatio; numWorkers > 0 {
+		maxNotifierWorkers = int(numWorkers)
+	}
+	ctx.Logger().V(2).Info("starting notifier workers", "count", maxNotifierWorkers)
+	for worker := 0; worker < maxNotifierWorkers; worker++ {
+		e.WgNotifier.Add(1)
+		go func() {
+			ctx := context.WithValue(ctx, "notifier_worker_id", common.RandomID(5))
+			defer common.Recover(ctx)
+			defer e.WgNotifier.Done()
+			e.notifyResults(ctx)
+		}()
+	}
+
+	return e, nil
 }
 
 // Finish waits for running sources to complete and workers to finish scanning
 // chunks before closing their respective channels. Once Finish is called, no
 // more sources may be scanned by the engine.
-func (e *Engine) Finish(ctx context.Context, logFunc func(error, string, ...any)) {
+func (e *Engine) Finish(ctx context.Context) error {
 	defer common.RecoverWithExit(ctx)
-	// wait for the sources to finish putting chunks onto the chunks channel
-	sourceErr := e.sourcesWg.Wait()
-	if sourceErr != nil {
-		logFunc(sourceErr, "error occurred while collecting chunks")
-	}
+	// Wait for the sources to finish putting chunks onto the chunks channel.
+	err := e.sourcesWg.Wait()
 
-	close(e.chunks)
-	// wait for the workers to finish processing all of the chunks and putting
-	// results onto the results channel
-	e.workersWg.Wait()
+	close(e.chunks) // Source workers are done.
 
-	// TODO: re-evaluate whether this is needed and investigate why if so
-	//
-	// not entirely sure why results don't get processed without this pause
-	// since we've put all results on the channel at this point.
-	time.Sleep(time.Second)
-	close(e.results)
+	e.workersWg.Wait() // Wait for the workers to finish scanning chunks.
+	close(e.detectableChunksChan)
+	e.wgDetectorWorkers.Wait() // Wait for the detector workers to finish detecting chunks.
+
+	close(e.results)    // Detector workers are done, close the results channel and call it a day.
+	e.WgNotifier.Wait() // Wait for the notifier workers to finish notifying results.
+
+	return err
 }
 
 func (e *Engine) ChunksChan() chan *sources.Chunk {
@@ -223,57 +398,21 @@ func (e *Engine) ResultsChan() chan detectors.ResultWithMetadata {
 	return e.results
 }
 
-func (e *Engine) ChunksScanned() uint64 {
-	return e.chunksScanned
-}
-
-func (e *Engine) BytesScanned() uint64 {
-	return e.bytesScanned
-}
-
-func (e *Engine) dedupeAndSend(chunkResults []detectors.ResultWithMetadata) {
-	dedupeMap := make(map[string]struct{})
-	for _, result := range chunkResults {
-		// dedupe by comparing the detector type, raw result, and source metadata
-		// NOTE: in order for the PLAIN decoder to maintain precedence, make sure UTF8 is the first decoder in the
-		// default decoders list
-		key := fmt.Sprintf("%s%s%s%+v", result.DetectorType.String(), result.Raw, result.RawV2, result.SourceMetadata)
-		if _, ok := dedupeMap[key]; ok {
-			continue
-		}
-		dedupeMap[key] = struct{}{}
-		e.results <- result
-	}
-
-}
-
-func (e *Engine) DetectorAvgTime() map[string][]time.Duration {
-	logger := context.Background().Logger()
-	avgTime := map[string][]time.Duration{}
-	e.detectorAvgTime.Range(func(k, v interface{}) bool {
-		key, ok := k.(string)
-		if !ok {
-			logger.Info("expected DetectorAvgTime key to be a string")
-			return true
-		}
-
-		value, ok := v.([]time.Duration)
-		if !ok {
-			logger.Info("expected DetectorAvgTime value to be []time.Duration")
-			return true
-		}
-		avgTime[key] = value
-		return true
-	})
-	return avgTime
+// detectableChunk is a decoded chunk that is ready to be scanned by its detector.
+type detectableChunk struct {
+	detector detectors.Detector
+	chunk    sources.Chunk
+	decoder  detectorspb.DecoderType
+	wgDoneFn func()
 }
 
 func (e *Engine) detectorWorker(ctx context.Context) {
+	var wgDetect sync.WaitGroup
+
 	for originalChunk := range e.chunks {
 		for chunk := range sources.Chunker(originalChunk) {
-			var chunkResults []detectors.ResultWithMetadata
 			matchedKeywords := make(map[string]struct{})
-			atomic.AddUint64(&e.bytesScanned, uint64(len(chunk.Data)))
+			atomic.AddUint64(&e.metrics.BytesScanned, uint64(len(chunk.Data)))
 			for _, decoder := range e.decoders {
 				var decoderType detectorspb.DecoderType
 				switch decoder.(type) {
@@ -313,67 +452,112 @@ func (e *Engine) detectorWorker(ctx context.Context) {
 							continue
 						}
 
-						start := time.Now()
+						decoded.Verify = verify
+						wgDetect.Add(1)
+						e.detectableChunksChan <- detectableChunk{
+							chunk:    *decoded,
+							detector: detector,
+							decoder:  decoderType,
+							wgDoneFn: wgDetect.Done,
+						}
+					}
+				}
+			}
+		}
+		atomic.AddUint64(&e.metrics.ChunksScanned, 1)
+	}
+	wgDetect.Wait()
+}
 
-						results, err := func() ([]detectors.Result, error) {
+func (e *Engine) detectChunks(ctx context.Context) {
+	for data := range e.detectableChunksChan {
+		e.detectChunk(ctx, data)
+	}
+}
+
+func (e *Engine) detectChunk(ctx context.Context, data detectableChunk) {
+	var start time.Time
+	if e.printAvgDetectorTime {
+		start = time.Now()
+	}
 	ctx, cancel := context.WithTimeout(ctx, time.Second*10)
-							defer cancel()
 	defer common.Recover(ctx)
-							return detector.FromData(ctx, verify, decoded.Data)
-						}()
+	defer cancel()
+
+	results, err := data.detector.FromData(ctx, data.chunk.Verify, data.chunk.Data)
 	if err != nil {
-							ctx.Logger().Error(err, "could not scan chunk",
-								"source_type", decoded.SourceType.String(),
-								"metadata", decoded.SourceMetadata,
-							)
-							continue
+		ctx.Logger().Error(err, "error scanning chunk")
+	}
+	if e.printAvgDetectorTime && len(results) > 0 {
+		elapsed := time.Since(start)
+		detectorName := results[0].DetectorType.String()
+		avgTimeI, ok := e.metrics.detectorAvgTime.Load(detectorName)
+		var avgTime []time.Duration
+		if ok {
+			avgTime, ok = avgTimeI.([]time.Duration)
+			if !ok {
+				return
+			}
+		}
+		avgTime = append(avgTime, elapsed)
+		e.metrics.detectorAvgTime.Store(detectorName, avgTime)
 	}
 
 	if e.filterUnverified {
 		results = detectors.CleanResults(results)
 	}
-						for _, result := range results {
-							resultChunk := chunk
+
+	for _, res := range results {
+		e.processResult(data, res)
+	}
+	data.wgDoneFn()
+}
+
+func (e *Engine) processResult(data detectableChunk, res detectors.Result) {
 	ignoreLinePresent := false
-							if SupportsLineNumbers(chunk.SourceType) {
-								copyChunk := *chunk
-								copyMetaDataClone := proto.Clone(chunk.SourceMetadata)
+	if SupportsLineNumbers(data.chunk.SourceType) {
+		copyChunk := data.chunk
+		copyMetaDataClone := proto.Clone(data.chunk.SourceMetadata)
 		if copyMetaData, ok := copyMetaDataClone.(*source_metadatapb.MetaData); ok {
 			copyChunk.SourceMetadata = copyMetaData
 		}
 		fragStart, mdLine := FragmentFirstLine(&copyChunk)
-								ignoreLinePresent = SetResultLineNumber(&copyChunk, &result, fragStart, mdLine)
-								resultChunk = &copyChunk
+		ignoreLinePresent = SetResultLineNumber(&copyChunk, &res, fragStart, mdLine)
+		data.chunk = copyChunk
 	}
 	if ignoreLinePresent {
-								continue
+		return
 	}
-							result.DecoderType = decoderType
-							chunkResults = append(chunkResults, detectors.CopyMetadata(resultChunk, result))
 
+	secret := detectors.CopyMetadata(&data.chunk, res)
+	secret.DecoderType = data.decoder
+	e.results <- secret
 }
-						if len(results) > 0 {
-							elapsed := time.Since(start)
-							detectorName := results[0].DetectorType.String()
-							avgTimeI, ok := e.detectorAvgTime.Load(detectorName)
-							var avgTime []time.Duration
-							if ok {
-								avgTime, ok = avgTimeI.([]time.Duration)
-								if !ok {
+
+func (e *Engine) notifyResults(ctx context.Context) {
+	for r := range e.ResultsChan() {
+		if e.onlyVerified && !r.Verified {
 			continue
 		}
+		atomic.AddUint32(&e.numFoundResults, 1)
+
+		key := fmt.Sprintf("%s%s%s%+v", r.DetectorType.String(), r.Raw, r.RawV2, r.SourceMetadata)
+		if _, ok := e.dedupeCache.Get(key); ok {
+			continue
 		}
-							avgTime = append(avgTime, elapsed)
-							e.detectorAvgTime.Store(detectorName, avgTime)
+		e.dedupeCache.Add(key, struct{}{})
+
+		if r.Verified {
+			atomic.AddUint64(&e.metrics.VerifiedSecretsFound, 1)
+		} else {
+			atomic.AddUint64(&e.metrics.UnverifiedSecretsFound, 1)
 		}
+
+		if err := e.printer.Print(ctx, &r); err != nil {
+			ctx.Logger().Error(err, "error printing result")
 		}
 	}
 }
-			e.dedupeAndSend(chunkResults)
-		}
-		atomic.AddUint64(&e.chunksScanned, 1)
-	}
-}
 
 // SupportsLineNumbers determines if a line number can be found for a source type.
 func SupportsLineNumbers(sourceType sourcespb.SourceType) bool {

pkg/engine/gcs_test.go

@@ -4,6 +4,8 @@ import (
 	"strings"
 	"testing"
 
+	"github.com/stretchr/testify/assert"
+
 	"github.com/trufflesecurity/trufflehog/v3/pkg/context"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/decoders"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
@@ -59,11 +61,13 @@ func TestScanGCS(t *testing.T) {
 			ctx, cancel := context.WithCancel(context.TODO())
 			defer cancel()
 
-			e := Start(ctx,
+			e, err := Start(ctx,
 				WithConcurrency(1),
 				WithDecoders(decoders.DefaultDecoders()...),
 				WithDetectors(false, DefaultDetectors()...),
 			)
+			assert.Nil(t, err)
+
 			go func() {
 				resultCount := 0
 				for range e.ResultsChan() {
@@ -71,15 +75,12 @@ func TestScanGCS(t *testing.T) {
 				}
 			}()
 
-			err := e.ScanGCS(ctx, test.gcsConfig)
+			err = e.ScanGCS(ctx, test.gcsConfig)
 			if err != nil && !test.wantErr && !strings.Contains(err.Error(), "googleapi: Error 400: Bad Request") {
 				t.Errorf("ScanGCS() got: %v, want: %v", err, nil)
 				return
 			}
-			logFatalFunc := func(_ error, _ string, _ ...any) {
-				t.Fatalf("error logging function should not have been called")
-			}
-			e.Finish(ctx, logFatalFunc)
+			assert.Nil(t, e.Finish(ctx))
 
 			if err == nil && test.wantErr {
 				t.Errorf("ScanGCS() got: %v, want: %v", err, "error")

pkg/engine/git.go

@@ -4,11 +4,13 @@ import (
 	"fmt"
 	"runtime"
 
+	"github.com/go-errors/errors"
 	gogit "github.com/go-git/go-git/v5"
+	"google.golang.org/protobuf/proto"
+	"google.golang.org/protobuf/types/known/anypb"
 
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/context"
-	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/source_metadatapb"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/sources/git"
@@ -22,16 +24,6 @@ func (e *Engine) ScanGit(ctx context.Context, c sources.GitConfig) error {
 		git.ScanOptionLogOptions(logOptions),
 	}
 
-	options := &gogit.PlainOpenOptions{
-		DetectDotGit:          true,
-		EnableDotGitCommonDir: true,
-	}
-
-	repo, err := gogit.PlainOpenWithOptions(c.RepoPath, options)
-	if err != nil {
-		return fmt.Errorf("could not open repo: %s: %w", c.RepoPath, err)
-	}
-
 	if c.MaxDepth != 0 {
 		opts = append(opts, git.ScanOptionMaxDepth(int64(c.MaxDepth)))
 	}
@@ -46,21 +38,28 @@ func (e *Engine) ScanGit(ctx context.Context, c sources.GitConfig) error {
 	}
 	scanOptions := git.NewScanOptions(opts...)
 
-	gitSource := git.NewGit(sourcespb.SourceType_SOURCE_TYPE_GIT, 0, 0, "trufflehog - git", true, runtime.NumCPU(),
-		func(file, email, commit, timestamp, repository string, line int64) *source_metadatapb.MetaData {
-			return &source_metadatapb.MetaData{
-				Data: &source_metadatapb.MetaData_Git{
-					Git: &source_metadatapb.Git{
-						Commit:     commit,
-						File:       file,
-						Email:      email,
-						Repository: repository,
-						Timestamp:  timestamp,
-						Line:       line,
-					},
-				},
+	connection := &sourcespb.Git{
+		// Using Directories here allows us to not pass any
+		// authentication. Also by this point, the c.RepoPath should
+		// still have been prepared and downloaded to a temporary
+		// directory if it was a URL.
+		Directories: []string{c.RepoPath},
 	}
-		})
+	var conn anypb.Any
+	err := anypb.MarshalFrom(&conn, connection, proto.MarshalOptions{})
+	if err != nil {
+		ctx.Logger().Error(err, "failed to marshal git connection")
+		return err
+	}
+
+	gitSource := git.Source{}
+	if err := gitSource.Init(ctx, "trufflehog - git", 0, 0, true, &conn, runtime.NumCPU()); err != nil {
+		return errors.WrapPrefix(err, "could not init git source", 0)
+	}
+	gitSource.WithScanOptions(scanOptions)
+	// Don't try to clean up the provided directory. That's handled by the
+	// caller of ScanGit.
+	gitSource.WithPreserveTempDirs(true)
 
 	ctx = context.WithValues(ctx,
 		"source_type", sourcespb.SourceType_SOURCE_TYPE_GIT.String(),
@@ -68,7 +67,7 @@ func (e *Engine) ScanGit(ctx context.Context, c sources.GitConfig) error {
 	)
 	e.sourcesWg.Go(func() error {
 		defer common.RecoverWithExit(ctx)
-		err := gitSource.ScanRepo(ctx, repo, c.RepoPath, scanOptions, e.ChunksChan())
+		err := gitSource.Chunks(ctx, e.ChunksChan())
 		if err != nil {
 			return fmt.Errorf("could not scan repo: %w", err)
 		}

pkg/engine/git_test.go

@@ -4,6 +4,8 @@ import (
 	"os"
 	"testing"
 
+	"github.com/stretchr/testify/assert"
+
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/context"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/decoders"
@@ -19,6 +21,13 @@ type expResult struct {
 	Verified   bool
 }
 
+type discardPrinter struct{}
+
+func (p *discardPrinter) Print(context.Context, *detectors.ResultWithMetadata) error {
+	// This method intentionally does nothing.
+	return nil
+}
+
 func TestGitEngine(t *testing.T) {
 	ctx := context.Background()
 	repoUrl := "https://github.com/dustin-decker/secretsandstuff.git"
@@ -56,14 +65,13 @@ func TestGitEngine(t *testing.T) {
 		},
 	} {
 		t.Run(tName, func(t *testing.T) {
-			e := Start(ctx,
+			e, err := Start(ctx,
 				WithConcurrency(1),
 				WithDecoders(decoders.DefaultDecoders()...),
 				WithDetectors(true, DefaultDetectors()...),
+				WithPrinter(new(discardPrinter)),
 			)
-			// Make the channels buffered so Finish returns.
-			e.chunks = make(chan *sources.Chunk, 10)
-			e.results = make(chan detectors.ResultWithMetadata, 10)
+			assert.Nil(t, err)
 
 			cfg := sources.GitConfig{
 				RepoPath: path,
@@ -76,12 +84,8 @@ func TestGitEngine(t *testing.T) {
 				return
 			}
 
-			logFatalFunc := func(_ error, _ string, _ ...any) {
-				t.Fatalf("error logging function should not have been called")
-			}
 			// Wait for all the chunks to be processed.
-			e.Finish(ctx, logFatalFunc)
-			resultCount := 0
+			assert.Nil(t, e.Finish(ctx))
 			for result := range e.ResultsChan() {
 				switch meta := result.SourceMetadata.GetData().(type) {
 				case *source_metadatapb.MetaData_Git:
@@ -95,12 +99,10 @@ func TestGitEngine(t *testing.T) {
 						t.Errorf("%s: unexpected verification. Got: %v, Expected: %v", tName, result.Verified, tTest.expected[meta.Git.Commit].Verified)
 					}
 				}
-				resultCount++
 
 			}
-			if resultCount != len(tTest.expected) {
-				t.Errorf("%s: unexpected number of results. Got: %d, Expected: %d", tName, resultCount, len(tTest.expected))
-			}
+			metrics := e.GetMetrics()
+			assert.Equal(t, len(tTest.expected), int(metrics.VerifiedSecretsFound)+int(metrics.UnverifiedSecretsFound))
 		})
 	}
 }
@@ -117,11 +119,13 @@ func BenchmarkGitEngine(b *testing.B) {
 	ctx, cancel := context.WithCancel(ctx)
 	defer cancel()
 
-	e := Start(ctx,
+	e, err := Start(ctx,
 		WithConcurrency(1),
 		WithDecoders(decoders.DefaultDecoders()...),
 		WithDetectors(false, DefaultDetectors()...),
 	)
+	assert.Nil(b, err)
+
 	go func() {
 		resultCount := 0
 		for range e.ResultsChan() {
@@ -140,8 +144,5 @@ func BenchmarkGitEngine(b *testing.B) {
 			return
 		}
 	}
-	logFatalFunc := func(_ error, _ string, _ ...any) {
-		b.Fatalf("error logging function should not have been called")
-	}
-	e.Finish(ctx, logFatalFunc)
+	assert.Nil(b, e.Finish(ctx))
 }

pkg/gitparse/gitparse.go

@@ -237,7 +237,7 @@ func (c *Parser) FromReader(ctx context.Context, stdOut io.Reader, commitChan ch
 	outReader := bufio.NewReader(stdOut)
 	var (
 		currentCommit *Commit
-		currentDiff   *Diff
+		currentDiff   Diff
 
 		totalLogSize int
 	)
@@ -260,8 +260,8 @@ func (c *Parser) FromReader(ctx context.Context, stdOut io.Reader, commitChan ch
 			latestState = CommitLine
 
 			// If there is a currentDiff, add it to currentCommit.
-			if currentDiff != nil && currentDiff.Content.Len() > 0 {
-				currentCommit.Diffs = append(currentCommit.Diffs, *currentDiff)
+			if currentDiff.Content.Len() > 0 {
+				currentCommit.Diffs = append(currentCommit.Diffs, currentDiff)
 				currentCommit.Size += currentDiff.Content.Len()
 			}
 			// If there is a currentCommit, send it to the channel.
@@ -270,7 +270,7 @@ func (c *Parser) FromReader(ctx context.Context, stdOut io.Reader, commitChan ch
 				totalLogSize += currentCommit.Size
 			}
 			// Create a new currentDiff and currentCommit
-			currentDiff = &Diff{}
+			currentDiff = Diff{}
 			currentCommit = &Commit{
 				Message: strings.Builder{},
 			}
@@ -309,8 +309,8 @@ func (c *Parser) FromReader(ctx context.Context, stdOut io.Reader, commitChan ch
 			if currentCommit == nil {
 				currentCommit = &Commit{}
 			}
-			if currentDiff != nil && currentDiff.Content.Len() > 0 {
-				currentCommit.Diffs = append(currentCommit.Diffs, *currentDiff)
+			if currentDiff.Content.Len() > 0 {
+				currentCommit.Diffs = append(currentCommit.Diffs, currentDiff)
 				// If the currentDiff is over 1GB, drop it into the channel so it isn't held in memory waiting for more commits.
 				totalSize := 0
 				for _, diff := range currentCommit.Diffs {
@@ -331,7 +331,7 @@ func (c *Parser) FromReader(ctx context.Context, stdOut io.Reader, commitChan ch
 					currentCommit.Message.WriteString(oldCommit.Message.String())
 				}
 			}
-			currentDiff = &Diff{}
+			currentDiff = Diff{}
 		case isModeLine(isStaged, latestState, line):
 			latestState = ModeLine
 			// NoOp
@@ -354,16 +354,13 @@ func (c *Parser) FromReader(ctx context.Context, stdOut io.Reader, commitChan ch
 		case isHunkLineNumberLine(isStaged, latestState, line):
 			latestState = HunkLineNumberLine
 
-			// TODO: Is it still necessary to check whether the currentDiff is nil?
-			if currentDiff != nil && currentDiff.Content.Len() > 0 {
-				currentCommit.Diffs = append(currentCommit.Diffs, *currentDiff)
+			if currentDiff.Content.Len() > 0 {
+				currentCommit.Diffs = append(currentCommit.Diffs, currentDiff)
 			}
-			newDiff := &Diff{
+			currentDiff = Diff{
 				PathB: currentDiff.PathB,
 			}
 
-			currentDiff = newDiff
-
 			words := bytes.Split(line, []byte(" "))
 			if len(words) >= 3 {
 				startSlice := bytes.Split(words[2], []byte(","))
@@ -427,7 +424,7 @@ func (c *Parser) FromReader(ctx context.Context, stdOut io.Reader, commitChan ch
 			break
 		}
 	}
-	cleanupParse(currentCommit, currentDiff, commitChan, &totalLogSize)
+	cleanupParse(currentCommit, &currentDiff, commitChan, &totalLogSize)
 
 	ctx.Logger().V(2).Info("finished parsing git log.", "total_log_size", totalLogSize)
 }

pkg/giturl/giturl.go

@@ -7,12 +7,20 @@ import (
 	"github.com/pkg/errors"
 )
 
+type provider string
+
+const (
+	providerGithub    provider = "Github"
+	providerGitlab    provider = "Gitlab"
+	providerBitbucket provider = "Bitbucket"
+)
+
 func NormalizeBitbucketRepo(repoURL string) (string, error) {
 	if !strings.HasPrefix(repoURL, "https") {
 		return "", errors.New("Bitbucket requires https repo urls: e.g. https://bitbucket.org/org/repo.git")
 	}
 
-	return NormalizeOrgRepoURL("Bitbucket", repoURL)
+	return NormalizeOrgRepoURL(providerBitbucket, repoURL)
 }
 
 func NormalizeGerritProject(project string) (string, error) {
@@ -20,7 +28,7 @@ func NormalizeGerritProject(project string) (string, error) {
 }
 
 func NormalizeGithubRepo(repoURL string) (string, error) {
-	return NormalizeOrgRepoURL("Github", repoURL)
+	return NormalizeOrgRepoURL(providerGithub, repoURL)
 }
 
 func NormalizeGitlabRepo(repoURL string) (string, error) {
@@ -28,12 +36,12 @@ func NormalizeGitlabRepo(repoURL string) (string, error) {
 		return "", errors.New("Gitlab requires http/https repo urls: e.g. https://gitlab.com/org/repo.git")
 	}
 
-	return NormalizeOrgRepoURL("Gitlab", repoURL)
+	return NormalizeOrgRepoURL(providerGitlab, repoURL)
 }
 
 // NormalizeOrgRepoURL attempts to normalize repos for any provider using the example.com/org/repo style.
 // e.g. %s, Gitlab and Bitbucket
-func NormalizeOrgRepoURL(provider, repoURL string) (string, error) {
+func NormalizeOrgRepoURL(provider provider, repoURL string) (string, error) {
 	if strings.HasSuffix(repoURL, ".git") {
 		return repoURL, nil
 	}

pkg/giturl/giturl_test.go

@@ -8,22 +8,22 @@ import (
 
 func Test_NormalizeOrgRepoURL(t *testing.T) {
 	tests := map[string]struct {
-		Provider string
+		Provider provider
 		Repo     string
 		Out      string
 		Err      error
 	}{
-		"github is good":             {Provider: "Github", Repo: "https://github.com/org/repo", Out: "https://github.com/org/repo.git", Err: nil},
-		"gitlab is good":             {Provider: "Gitlab", Repo: "https://gitlab.com/org/repo", Out: "https://gitlab.com/org/repo.git", Err: nil},
-		"bitbucket is good":          {Provider: "Bitbucket", Repo: "https://bitbucket.com/org/repo", Out: "https://bitbucket.com/org/repo.git", Err: nil},
+		"github is good":             {Provider: providerGithub, Repo: "https://github.com/org/repo", Out: "https://github.com/org/repo.git", Err: nil},
+		"gitlab is good":             {Provider: providerGitlab, Repo: "https://gitlab.com/org/repo", Out: "https://gitlab.com/org/repo.git", Err: nil},
+		"bitbucket is good":          {Provider: providerBitbucket, Repo: "https://bitbucket.com/org/repo", Out: "https://bitbucket.com/org/repo.git", Err: nil},
 		"example provider is good":   {Provider: "example", Repo: "https://example.com/org/repo", Out: "https://example.com/org/repo.git", Err: nil},
 		"example provider problem":   {Provider: "example", Repo: "https://example.com/org", Out: "", Err: errors.Errorf("example repo appears to be missing the repo name. Org: %q Repo url: %q", "org", "https://example.com/org")},
-		"no path":                    {Provider: "Github", Repo: "https://github.com", Out: "", Err: errors.Errorf("Github repo appears to be missing the path. Repo url: %q", "https://github.com")},
-		"org but no repo":            {Provider: "Github", Repo: "https://github.com/org", Out: "", Err: errors.Errorf("Github repo appears to be missing the repo name. Org: %q Repo url: %q", "org", "https://github.com/org")},
-		"org but no repo with slash": {Provider: "Github", Repo: "https://github.com/org/", Out: "", Err: errors.Errorf("Github repo appears to be missing the repo name. Org: %q Repo url: %q", "org", "https://github.com/org/")},
-		"two slashes":                {Provider: "Github", Repo: "https://github.com//", Out: "", Err: errors.Errorf("Github repo appears to be missing the org name. Repo url: %q", "https://github.com//")},
-		"repo with trailing slash":   {Provider: "Github", Repo: "https://github.com/org/repo/", Out: "", Err: errors.Errorf("Github repo contains a trailing slash. Repo url: %q", "https://github.com/org/repo/")},
-		"too many url path parts":    {Provider: "Github", Repo: "https://github.com/org/repo/unknown/", Out: "", Err: errors.Errorf("Github repo contains a trailing slash. Repo url: %q", "https://github.com/org/repo/unknown/")},
+		"no path":                    {Provider: providerGithub, Repo: "https://github.com", Out: "", Err: errors.Errorf("Github repo appears to be missing the path. Repo url: %q", "https://github.com")},
+		"org but no repo":            {Provider: providerGithub, Repo: "https://github.com/org", Out: "", Err: errors.Errorf("Github repo appears to be missing the repo name. Org: %q Repo url: %q", "org", "https://github.com/org")},
+		"org but no repo with slash": {Provider: providerGithub, Repo: "https://github.com/org/", Out: "", Err: errors.Errorf("Github repo appears to be missing the repo name. Org: %q Repo url: %q", "org", "https://github.com/org/")},
+		"two slashes":                {Provider: providerGithub, Repo: "https://github.com//", Out: "", Err: errors.Errorf("Github repo appears to be missing the org name. Repo url: %q", "https://github.com//")},
+		"repo with trailing slash":   {Provider: providerGithub, Repo: "https://github.com/org/repo/", Out: "", Err: errors.Errorf("Github repo contains a trailing slash. Repo url: %q", "https://github.com/org/repo/")},
+		"too many url path parts":    {Provider: providerGithub, Repo: "https://github.com/org/repo/unknown/", Out: "", Err: errors.Errorf("Github repo contains a trailing slash. Repo url: %q", "https://github.com/org/repo/unknown/")},
 	}
 
 	for name, tt := range tests {

pkg/output/github_actions.go

@@ -5,13 +5,17 @@ import (
 	"encoding/hex"
 	"fmt"
 
+	"github.com/trufflesecurity/trufflehog/v3/pkg/context"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
 )
 
 var dedupeCache = make(map[string]struct{})
 
-func PrintGitHubActionsOutput(r *detectors.ResultWithMetadata) error {
+// GitHubActionsPrinter is a printer that prints results in GitHub Actions format.
+type GitHubActionsPrinter struct{}
+
+func (p *GitHubActionsPrinter) Print(_ context.Context, r *detectors.ResultWithMetadata) error {
 	out := gitHubActionsOutputFormat{
 		DetectorType: r.Result.DetectorType.String(),
 		DecoderType:  r.Result.DecoderType.String(),

pkg/output/json.go

@@ -4,13 +4,17 @@ import (
 	"encoding/json"
 	"fmt"
 
+	"github.com/trufflesecurity/trufflehog/v3/pkg/context"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/source_metadatapb"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
 )
 
-func PrintJSON(r *detectors.ResultWithMetadata) error {
+// JSONPrinter is a printer that prints results in JSON format.
+type JSONPrinter struct{}
+
+func (p *JSONPrinter) Print(_ context.Context, r *detectors.ResultWithMetadata) error {
 	v := &struct {
 		// SourceMetadata contains source-specific contextual information.
 		SourceMetadata *source_metadatapb.MetaData

pkg/output/legacy_json.go

@@ -19,7 +19,10 @@ import (
 	"github.com/trufflesecurity/trufflehog/v3/pkg/sources/git"
 )
 
-func PrintLegacyJSON(ctx context.Context, r *detectors.ResultWithMetadata) error {
+// LegacyJSONPrinter is a printer that prints results in legacy JSON format for backwards compatibility.
+type LegacyJSONPrinter struct{}
+
+func (p *LegacyJSONPrinter) Print(ctx context.Context, r *detectors.ResultWithMetadata) error {
 	var repo string
 	switch r.SourceType {
 	case sourcespb.SourceType_SOURCE_TYPE_GIT:
@@ -41,7 +44,7 @@ func PrintLegacyJSON(ctx context.Context, r *detectors.ResultWithMetadata) error
 		defer os.RemoveAll(repoPath)
 	}
 
-	legacy, err := ConvertToLegacyJSON(r, repoPath)
+	legacy, err := convertToLegacyJSON(r, repoPath)
 	if err != nil {
 		return fmt.Errorf("could not convert to legacy JSON: %w", err)
 	}
@@ -53,7 +56,7 @@ func PrintLegacyJSON(ctx context.Context, r *detectors.ResultWithMetadata) error
 	return nil
 }
 
-func ConvertToLegacyJSON(r *detectors.ResultWithMetadata, repoPath string) (*LegacyJSONOutput, error) {
+func convertToLegacyJSON(r *detectors.ResultWithMetadata, repoPath string) (*LegacyJSONOutput, error) {
 	var source LegacyJSONCompatibleSource
 	switch r.SourceType {
 	case sourcespb.SourceType_SOURCE_TYPE_GIT:

pkg/output/plain.go

@@ -10,6 +10,7 @@ import (
 	"golang.org/x/text/cases"
 	"golang.org/x/text/language"
 
+	"github.com/trufflesecurity/trufflehog/v3/pkg/context"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/source_metadatapb"
 )
@@ -20,7 +21,10 @@ var (
 	whitePrinter  = color.New(color.FgWhite)
 )
 
-func PrintPlainOutput(r *detectors.ResultWithMetadata) error {
+// PlainPrinter is a printer that prints results in plain text format.
+type PlainPrinter struct{}
+
+func (p *PlainPrinter) Print(_ context.Context, r *detectors.ResultWithMetadata) error {
 	out := outputFormat{
 		DetectorType: r.Result.DetectorType.String(),
 		DecoderType:  r.Result.DecoderType.String(),

pkg/pb/sourcespb/sources.pb.go

@@ -1428,6 +1428,7 @@ type GitHub struct {
 	IncludeRepos               []string            `protobuf:"bytes,12,rep,name=includeRepos,proto3" json:"includeRepos,omitempty"`
 	IncludePullRequestComments bool                `protobuf:"varint,14,opt,name=includePullRequestComments,proto3" json:"includePullRequestComments,omitempty"`
 	IncludeIssueComments       bool                `protobuf:"varint,15,opt,name=includeIssueComments,proto3" json:"includeIssueComments,omitempty"`
+	IncludeGistComments        bool                `protobuf:"varint,16,opt,name=includeGistComments,proto3" json:"includeGistComments,omitempty"`
 }
 
 func (x *GitHub) Reset() {
@@ -1574,6 +1575,13 @@ func (x *GitHub) GetIncludeIssueComments() bool {
 	return false
 }
 
+func (x *GitHub) GetIncludeGistComments() bool {
+	if x != nil {
+		return x.IncludeGistComments
+	}
+	return false
+}
+
 type isGitHub_Credential interface {
 	isGitHub_Credential()
 }
@@ -3328,7 +3336,7 @@ var file_sources_proto_rawDesc = []byte{
 	0x65, 0x70, 0x6f, 0x73, 0x69, 0x74, 0x6f, 0x72, 0x69, 0x65, 0x73, 0x12, 0x21, 0x0a, 0x0c, 0x69,
 	0x67, 0x6e, 0x6f, 0x72, 0x65, 0x5f, 0x72, 0x65, 0x70, 0x6f, 0x73, 0x18, 0x06, 0x20, 0x03, 0x28,
 	0x09, 0x52, 0x0b, 0x69, 0x67, 0x6e, 0x6f, 0x72, 0x65, 0x52, 0x65, 0x70, 0x6f, 0x73, 0x42, 0x0c,
-	0x0a, 0x0a, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x22, 0xfe, 0x04, 0x0a,
+	0x0a, 0x0a, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x22, 0xb0, 0x05, 0x0a,
 	0x06, 0x47, 0x69, 0x74, 0x48, 0x75, 0x62, 0x12, 0x24, 0x0a, 0x08, 0x65, 0x6e, 0x64, 0x70, 0x6f,
 	0x69, 0x6e, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x42, 0x08, 0xfa, 0x42, 0x05, 0x72, 0x03,
 	0x90, 0x01, 0x01, 0x52, 0x08, 0x65, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x12, 0x37, 0x0a,
@@ -3367,264 +3375,267 @@ var file_sources_proto_rawDesc = []byte{
 	0x65, 0x73, 0x74, 0x43, 0x6f, 0x6d, 0x6d, 0x65, 0x6e, 0x74, 0x73, 0x12, 0x32, 0x0a, 0x14, 0x69,
 	0x6e, 0x63, 0x6c, 0x75, 0x64, 0x65, 0x49, 0x73, 0x73, 0x75, 0x65, 0x43, 0x6f, 0x6d, 0x6d, 0x65,
 	0x6e, 0x74, 0x73, 0x18, 0x0f, 0x20, 0x01, 0x28, 0x08, 0x52, 0x14, 0x69, 0x6e, 0x63, 0x6c, 0x75,
-	0x64, 0x65, 0x49, 0x73, 0x73, 0x75, 0x65, 0x43, 0x6f, 0x6d, 0x6d, 0x65, 0x6e, 0x74, 0x73, 0x42,
-	0x0c, 0x0a, 0x0a, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x22, 0x42, 0x0a,
-	0x0b, 0x47, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x44, 0x72, 0x69, 0x76, 0x65, 0x12, 0x25, 0x0a, 0x0d,
-	0x72, 0x65, 0x66, 0x72, 0x65, 0x73, 0x68, 0x5f, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x18, 0x01, 0x20,
-	0x01, 0x28, 0x09, 0x48, 0x00, 0x52, 0x0c, 0x72, 0x65, 0x66, 0x72, 0x65, 0x73, 0x68, 0x54, 0x6f,
-	0x6b, 0x65, 0x6e, 0x42, 0x0c, 0x0a, 0x0a, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61,
-	0x6c, 0x22, 0xc7, 0x02, 0x0a, 0x04, 0x4a, 0x49, 0x52, 0x41, 0x12, 0x24, 0x0a, 0x08, 0x65, 0x6e,
-	0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x42, 0x08, 0xfa, 0x42,
-	0x05, 0x72, 0x03, 0x90, 0x01, 0x01, 0x52, 0x08, 0x65, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74,
-	0x12, 0x37, 0x0a, 0x0a, 0x62, 0x61, 0x73, 0x69, 0x63, 0x5f, 0x61, 0x75, 0x74, 0x68, 0x18, 0x02,
-	0x20, 0x01, 0x28, 0x0b, 0x32, 0x16, 0x2e, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61,
-	0x6c, 0x73, 0x2e, 0x42, 0x61, 0x73, 0x69, 0x63, 0x41, 0x75, 0x74, 0x68, 0x48, 0x00, 0x52, 0x09,
-	0x62, 0x61, 0x73, 0x69, 0x63, 0x41, 0x75, 0x74, 0x68, 0x12, 0x48, 0x0a, 0x0f, 0x75, 0x6e, 0x61,
-	0x75, 0x74, 0x68, 0x65, 0x6e, 0x74, 0x69, 0x63, 0x61, 0x74, 0x65, 0x64, 0x18, 0x03, 0x20, 0x01,
-	0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x73,
-	0x2e, 0x55, 0x6e, 0x61, 0x75, 0x74, 0x68, 0x65, 0x6e, 0x74, 0x69, 0x63, 0x61, 0x74, 0x65, 0x64,
-	0x48, 0x00, 0x52, 0x0f, 0x75, 0x6e, 0x61, 0x75, 0x74, 0x68, 0x65, 0x6e, 0x74, 0x69, 0x63, 0x61,
-	0x74, 0x65, 0x64, 0x12, 0x2b, 0x0a, 0x05, 0x6f, 0x61, 0x75, 0x74, 0x68, 0x18, 0x04, 0x20, 0x01,
-	0x28, 0x0b, 0x32, 0x13, 0x2e, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x73,
-	0x2e, 0x4f, 0x61, 0x75, 0x74, 0x68, 0x32, 0x48, 0x00, 0x52, 0x05, 0x6f, 0x61, 0x75, 0x74, 0x68,
-	0x12, 0x16, 0x0a, 0x05, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x48,
-	0x00, 0x52, 0x05, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x12, 0x1a, 0x0a, 0x08, 0x70, 0x72, 0x6f, 0x6a,
-	0x65, 0x63, 0x74, 0x73, 0x18, 0x05, 0x20, 0x03, 0x28, 0x09, 0x52, 0x08, 0x70, 0x72, 0x6f, 0x6a,
-	0x65, 0x63, 0x74, 0x73, 0x12, 0x27, 0x0a, 0x0f, 0x69, 0x67, 0x6e, 0x6f, 0x72, 0x65, 0x5f, 0x70,
-	0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x18, 0x07, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0e, 0x69,
-	0x67, 0x6e, 0x6f, 0x72, 0x65, 0x50, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x42, 0x0c, 0x0a,
-	0x0a, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x22, 0x73, 0x0a, 0x19, 0x4e,
-	0x50, 0x4d, 0x55, 0x6e, 0x61, 0x75, 0x74, 0x68, 0x65, 0x6e, 0x74, 0x69, 0x63, 0x61, 0x74, 0x65,
-	0x64, 0x50, 0x61, 0x63, 0x6b, 0x61, 0x67, 0x65, 0x12, 0x48, 0x0a, 0x0f, 0x75, 0x6e, 0x61, 0x75,
-	0x74, 0x68, 0x65, 0x6e, 0x74, 0x69, 0x63, 0x61, 0x74, 0x65, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28,
+	0x64, 0x65, 0x49, 0x73, 0x73, 0x75, 0x65, 0x43, 0x6f, 0x6d, 0x6d, 0x65, 0x6e, 0x74, 0x73, 0x12,
+	0x30, 0x0a, 0x13, 0x69, 0x6e, 0x63, 0x6c, 0x75, 0x64, 0x65, 0x47, 0x69, 0x73, 0x74, 0x43, 0x6f,
+	0x6d, 0x6d, 0x65, 0x6e, 0x74, 0x73, 0x18, 0x10, 0x20, 0x01, 0x28, 0x08, 0x52, 0x13, 0x69, 0x6e,
+	0x63, 0x6c, 0x75, 0x64, 0x65, 0x47, 0x69, 0x73, 0x74, 0x43, 0x6f, 0x6d, 0x6d, 0x65, 0x6e, 0x74,
+	0x73, 0x42, 0x0c, 0x0a, 0x0a, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x22,
+	0x42, 0x0a, 0x0b, 0x47, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x44, 0x72, 0x69, 0x76, 0x65, 0x12, 0x25,
+	0x0a, 0x0d, 0x72, 0x65, 0x66, 0x72, 0x65, 0x73, 0x68, 0x5f, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x18,
+	0x01, 0x20, 0x01, 0x28, 0x09, 0x48, 0x00, 0x52, 0x0c, 0x72, 0x65, 0x66, 0x72, 0x65, 0x73, 0x68,
+	0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x42, 0x0c, 0x0a, 0x0a, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74,
+	0x69, 0x61, 0x6c, 0x22, 0xc7, 0x02, 0x0a, 0x04, 0x4a, 0x49, 0x52, 0x41, 0x12, 0x24, 0x0a, 0x08,
+	0x65, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x42, 0x08,
+	0xfa, 0x42, 0x05, 0x72, 0x03, 0x90, 0x01, 0x01, 0x52, 0x08, 0x65, 0x6e, 0x64, 0x70, 0x6f, 0x69,
+	0x6e, 0x74, 0x12, 0x37, 0x0a, 0x0a, 0x62, 0x61, 0x73, 0x69, 0x63, 0x5f, 0x61, 0x75, 0x74, 0x68,
+	0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x16, 0x2e, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74,
+	0x69, 0x61, 0x6c, 0x73, 0x2e, 0x42, 0x61, 0x73, 0x69, 0x63, 0x41, 0x75, 0x74, 0x68, 0x48, 0x00,
+	0x52, 0x09, 0x62, 0x61, 0x73, 0x69, 0x63, 0x41, 0x75, 0x74, 0x68, 0x12, 0x48, 0x0a, 0x0f, 0x75,
+	0x6e, 0x61, 0x75, 0x74, 0x68, 0x65, 0x6e, 0x74, 0x69, 0x63, 0x61, 0x74, 0x65, 0x64, 0x18, 0x03,
+	0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61,
+	0x6c, 0x73, 0x2e, 0x55, 0x6e, 0x61, 0x75, 0x74, 0x68, 0x65, 0x6e, 0x74, 0x69, 0x63, 0x61, 0x74,
+	0x65, 0x64, 0x48, 0x00, 0x52, 0x0f, 0x75, 0x6e, 0x61, 0x75, 0x74, 0x68, 0x65, 0x6e, 0x74, 0x69,
+	0x63, 0x61, 0x74, 0x65, 0x64, 0x12, 0x2b, 0x0a, 0x05, 0x6f, 0x61, 0x75, 0x74, 0x68, 0x18, 0x04,
+	0x20, 0x01, 0x28, 0x0b, 0x32, 0x13, 0x2e, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61,
+	0x6c, 0x73, 0x2e, 0x4f, 0x61, 0x75, 0x74, 0x68, 0x32, 0x48, 0x00, 0x52, 0x05, 0x6f, 0x61, 0x75,
+	0x74, 0x68, 0x12, 0x16, 0x0a, 0x05, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x18, 0x06, 0x20, 0x01, 0x28,
+	0x09, 0x48, 0x00, 0x52, 0x05, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x12, 0x1a, 0x0a, 0x08, 0x70, 0x72,
+	0x6f, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x18, 0x05, 0x20, 0x03, 0x28, 0x09, 0x52, 0x08, 0x70, 0x72,
+	0x6f, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x12, 0x27, 0x0a, 0x0f, 0x69, 0x67, 0x6e, 0x6f, 0x72, 0x65,
+	0x5f, 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x18, 0x07, 0x20, 0x03, 0x28, 0x09, 0x52,
+	0x0e, 0x69, 0x67, 0x6e, 0x6f, 0x72, 0x65, 0x50, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x42,
+	0x0c, 0x0a, 0x0a, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x22, 0x73, 0x0a,
+	0x19, 0x4e, 0x50, 0x4d, 0x55, 0x6e, 0x61, 0x75, 0x74, 0x68, 0x65, 0x6e, 0x74, 0x69, 0x63, 0x61,
+	0x74, 0x65, 0x64, 0x50, 0x61, 0x63, 0x6b, 0x61, 0x67, 0x65, 0x12, 0x48, 0x0a, 0x0f, 0x75, 0x6e,
+	0x61, 0x75, 0x74, 0x68, 0x65, 0x6e, 0x74, 0x69, 0x63, 0x61, 0x74, 0x65, 0x64, 0x18, 0x01, 0x20,
+	0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c,
+	0x73, 0x2e, 0x55, 0x6e, 0x61, 0x75, 0x74, 0x68, 0x65, 0x6e, 0x74, 0x69, 0x63, 0x61, 0x74, 0x65,
+	0x64, 0x48, 0x00, 0x52, 0x0f, 0x75, 0x6e, 0x61, 0x75, 0x74, 0x68, 0x65, 0x6e, 0x74, 0x69, 0x63,
+	0x61, 0x74, 0x65, 0x64, 0x42, 0x0c, 0x0a, 0x0a, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69,
+	0x61, 0x6c, 0x22, 0x74, 0x0a, 0x1a, 0x50, 0x79, 0x50, 0x49, 0x55, 0x6e, 0x61, 0x75, 0x74, 0x68,
+	0x65, 0x6e, 0x74, 0x69, 0x63, 0x61, 0x74, 0x65, 0x64, 0x50, 0x61, 0x63, 0x6b, 0x61, 0x67, 0x65,
+	0x12, 0x48, 0x0a, 0x0f, 0x75, 0x6e, 0x61, 0x75, 0x74, 0x68, 0x65, 0x6e, 0x74, 0x69, 0x63, 0x61,
+	0x74, 0x65, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x63, 0x72, 0x65, 0x64,
+	0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x73, 0x2e, 0x55, 0x6e, 0x61, 0x75, 0x74, 0x68, 0x65, 0x6e,
+	0x74, 0x69, 0x63, 0x61, 0x74, 0x65, 0x64, 0x48, 0x00, 0x52, 0x0f, 0x75, 0x6e, 0x61, 0x75, 0x74,
+	0x68, 0x65, 0x6e, 0x74, 0x69, 0x63, 0x61, 0x74, 0x65, 0x64, 0x42, 0x0c, 0x0a, 0x0a, 0x63, 0x72,
+	0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x22, 0xf0, 0x02, 0x0a, 0x02, 0x53, 0x33, 0x12,
+	0x37, 0x0a, 0x0a, 0x61, 0x63, 0x63, 0x65, 0x73, 0x73, 0x5f, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20,
+	0x01, 0x28, 0x0b, 0x32, 0x16, 0x2e, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c,
+	0x73, 0x2e, 0x4b, 0x65, 0x79, 0x53, 0x65, 0x63, 0x72, 0x65, 0x74, 0x48, 0x00, 0x52, 0x09, 0x61,
+	0x63, 0x63, 0x65, 0x73, 0x73, 0x4b, 0x65, 0x79, 0x12, 0x48, 0x0a, 0x0f, 0x75, 0x6e, 0x61, 0x75,
+	0x74, 0x68, 0x65, 0x6e, 0x74, 0x69, 0x63, 0x61, 0x74, 0x65, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28,
 	0x0b, 0x32, 0x1c, 0x2e, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x73, 0x2e,
 	0x55, 0x6e, 0x61, 0x75, 0x74, 0x68, 0x65, 0x6e, 0x74, 0x69, 0x63, 0x61, 0x74, 0x65, 0x64, 0x48,
 	0x00, 0x52, 0x0f, 0x75, 0x6e, 0x61, 0x75, 0x74, 0x68, 0x65, 0x6e, 0x74, 0x69, 0x63, 0x61, 0x74,
-	0x65, 0x64, 0x42, 0x0c, 0x0a, 0x0a, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c,
-	0x22, 0x74, 0x0a, 0x1a, 0x50, 0x79, 0x50, 0x49, 0x55, 0x6e, 0x61, 0x75, 0x74, 0x68, 0x65, 0x6e,
-	0x74, 0x69, 0x63, 0x61, 0x74, 0x65, 0x64, 0x50, 0x61, 0x63, 0x6b, 0x61, 0x67, 0x65, 0x12, 0x48,
-	0x0a, 0x0f, 0x75, 0x6e, 0x61, 0x75, 0x74, 0x68, 0x65, 0x6e, 0x74, 0x69, 0x63, 0x61, 0x74, 0x65,
-	0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e,
-	0x74, 0x69, 0x61, 0x6c, 0x73, 0x2e, 0x55, 0x6e, 0x61, 0x75, 0x74, 0x68, 0x65, 0x6e, 0x74, 0x69,
-	0x63, 0x61, 0x74, 0x65, 0x64, 0x48, 0x00, 0x52, 0x0f, 0x75, 0x6e, 0x61, 0x75, 0x74, 0x68, 0x65,
-	0x6e, 0x74, 0x69, 0x63, 0x61, 0x74, 0x65, 0x64, 0x42, 0x0c, 0x0a, 0x0a, 0x63, 0x72, 0x65, 0x64,
-	0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x22, 0xf0, 0x02, 0x0a, 0x02, 0x53, 0x33, 0x12, 0x37, 0x0a,
-	0x0a, 0x61, 0x63, 0x63, 0x65, 0x73, 0x73, 0x5f, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28,
+	0x65, 0x64, 0x12, 0x4c, 0x0a, 0x11, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x5f, 0x65, 0x6e, 0x76, 0x69,
+	0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1d, 0x2e,
+	0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x73, 0x2e, 0x43, 0x6c, 0x6f, 0x75,
+	0x64, 0x45, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x48, 0x00, 0x52, 0x10,
+	0x63, 0x6c, 0x6f, 0x75, 0x64, 0x45, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74,
+	0x12, 0x49, 0x0a, 0x0d, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x5f, 0x74, 0x6f, 0x6b, 0x65,
+	0x6e, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x22, 0x2e, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e,
+	0x74, 0x69, 0x61, 0x6c, 0x73, 0x2e, 0x41, 0x57, 0x53, 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e,
+	0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x53, 0x65, 0x63, 0x72, 0x65, 0x74, 0x48, 0x00, 0x52, 0x0c, 0x73,
+	0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x12, 0x18, 0x0a, 0x07, 0x62,
+	0x75, 0x63, 0x6b, 0x65, 0x74, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x09, 0x52, 0x07, 0x62, 0x75,
+	0x63, 0x6b, 0x65, 0x74, 0x73, 0x12, 0x26, 0x0a, 0x0f, 0x6d, 0x61, 0x78, 0x5f, 0x6f, 0x62, 0x6a,
+	0x65, 0x63, 0x74, 0x5f, 0x73, 0x69, 0x7a, 0x65, 0x18, 0x06, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0d,
+	0x6d, 0x61, 0x78, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x53, 0x69, 0x7a, 0x65, 0x42, 0x0c, 0x0a,
+	0x0a, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x22, 0xc3, 0x01, 0x0a, 0x05,
+	0x53, 0x6c, 0x61, 0x63, 0x6b, 0x12, 0x24, 0x0a, 0x08, 0x65, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e,
+	0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x42, 0x08, 0xfa, 0x42, 0x05, 0x72, 0x03, 0x90, 0x01,
+	0x01, 0x52, 0x08, 0x65, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x12, 0x16, 0x0a, 0x05, 0x74,
+	0x6f, 0x6b, 0x65, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x48, 0x00, 0x52, 0x05, 0x74, 0x6f,
+	0x6b, 0x65, 0x6e, 0x12, 0x32, 0x0a, 0x06, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x73, 0x18, 0x05, 0x20,
+	0x01, 0x28, 0x0b, 0x32, 0x18, 0x2e, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c,
+	0x73, 0x2e, 0x53, 0x6c, 0x61, 0x63, 0x6b, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x73, 0x48, 0x00, 0x52,
+	0x06, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x73, 0x12, 0x1a, 0x0a, 0x08, 0x63, 0x68, 0x61, 0x6e, 0x6e,
+	0x65, 0x6c, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x09, 0x52, 0x08, 0x63, 0x68, 0x61, 0x6e, 0x6e,
+	0x65, 0x6c, 0x73, 0x12, 0x1e, 0x0a, 0x0a, 0x69, 0x67, 0x6e, 0x6f, 0x72, 0x65, 0x4c, 0x69, 0x73,
+	0x74, 0x18, 0x04, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0a, 0x69, 0x67, 0x6e, 0x6f, 0x72, 0x65, 0x4c,
+	0x69, 0x73, 0x74, 0x42, 0x0c, 0x0a, 0x0a, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61,
+	0x6c, 0x22, 0x06, 0x0a, 0x04, 0x54, 0x65, 0x73, 0x74, 0x22, 0x31, 0x0a, 0x09, 0x42, 0x75, 0x69,
+	0x6c, 0x64, 0x6b, 0x69, 0x74, 0x65, 0x12, 0x16, 0x0a, 0x05, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x18,
+	0x01, 0x20, 0x01, 0x28, 0x09, 0x48, 0x00, 0x52, 0x05, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x42, 0x0c,
+	0x0a, 0x0a, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x22, 0xdb, 0x01, 0x0a,
+	0x06, 0x47, 0x65, 0x72, 0x72, 0x69, 0x74, 0x12, 0x24, 0x0a, 0x08, 0x65, 0x6e, 0x64, 0x70, 0x6f,
+	0x69, 0x6e, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x42, 0x08, 0xfa, 0x42, 0x05, 0x72, 0x03,
+	0x90, 0x01, 0x01, 0x52, 0x08, 0x65, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x12, 0x37, 0x0a,
+	0x0a, 0x62, 0x61, 0x73, 0x69, 0x63, 0x5f, 0x61, 0x75, 0x74, 0x68, 0x18, 0x02, 0x20, 0x01, 0x28,
 	0x0b, 0x32, 0x16, 0x2e, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x73, 0x2e,
-	0x4b, 0x65, 0x79, 0x53, 0x65, 0x63, 0x72, 0x65, 0x74, 0x48, 0x00, 0x52, 0x09, 0x61, 0x63, 0x63,
-	0x65, 0x73, 0x73, 0x4b, 0x65, 0x79, 0x12, 0x48, 0x0a, 0x0f, 0x75, 0x6e, 0x61, 0x75, 0x74, 0x68,
-	0x65, 0x6e, 0x74, 0x69, 0x63, 0x61, 0x74, 0x65, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32,
+	0x42, 0x61, 0x73, 0x69, 0x63, 0x41, 0x75, 0x74, 0x68, 0x48, 0x00, 0x52, 0x09, 0x62, 0x61, 0x73,
+	0x69, 0x63, 0x41, 0x75, 0x74, 0x68, 0x12, 0x48, 0x0a, 0x0f, 0x75, 0x6e, 0x61, 0x75, 0x74, 0x68,
+	0x65, 0x6e, 0x74, 0x69, 0x63, 0x61, 0x74, 0x65, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32,
 	0x1c, 0x2e, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x73, 0x2e, 0x55, 0x6e,
 	0x61, 0x75, 0x74, 0x68, 0x65, 0x6e, 0x74, 0x69, 0x63, 0x61, 0x74, 0x65, 0x64, 0x48, 0x00, 0x52,
 	0x0f, 0x75, 0x6e, 0x61, 0x75, 0x74, 0x68, 0x65, 0x6e, 0x74, 0x69, 0x63, 0x61, 0x74, 0x65, 0x64,
-	0x12, 0x4c, 0x0a, 0x11, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x5f, 0x65, 0x6e, 0x76, 0x69, 0x72, 0x6f,
-	0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1d, 0x2e, 0x63, 0x72,
-	0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x73, 0x2e, 0x43, 0x6c, 0x6f, 0x75, 0x64, 0x45,
-	0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x48, 0x00, 0x52, 0x10, 0x63, 0x6c,
-	0x6f, 0x75, 0x64, 0x45, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x12, 0x49,
-	0x0a, 0x0d, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x5f, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x18,
-	0x05, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x22, 0x2e, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69,
-	0x61, 0x6c, 0x73, 0x2e, 0x41, 0x57, 0x53, 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x54, 0x6f,
-	0x6b, 0x65, 0x6e, 0x53, 0x65, 0x63, 0x72, 0x65, 0x74, 0x48, 0x00, 0x52, 0x0c, 0x73, 0x65, 0x73,
-	0x73, 0x69, 0x6f, 0x6e, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x12, 0x18, 0x0a, 0x07, 0x62, 0x75, 0x63,
-	0x6b, 0x65, 0x74, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x09, 0x52, 0x07, 0x62, 0x75, 0x63, 0x6b,
-	0x65, 0x74, 0x73, 0x12, 0x26, 0x0a, 0x0f, 0x6d, 0x61, 0x78, 0x5f, 0x6f, 0x62, 0x6a, 0x65, 0x63,
-	0x74, 0x5f, 0x73, 0x69, 0x7a, 0x65, 0x18, 0x06, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0d, 0x6d, 0x61,
-	0x78, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x53, 0x69, 0x7a, 0x65, 0x42, 0x0c, 0x0a, 0x0a, 0x63,
-	0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x22, 0xc3, 0x01, 0x0a, 0x05, 0x53, 0x6c,
-	0x61, 0x63, 0x6b, 0x12, 0x24, 0x0a, 0x08, 0x65, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x18,
-	0x01, 0x20, 0x01, 0x28, 0x09, 0x42, 0x08, 0xfa, 0x42, 0x05, 0x72, 0x03, 0x90, 0x01, 0x01, 0x52,
-	0x08, 0x65, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x12, 0x16, 0x0a, 0x05, 0x74, 0x6f, 0x6b,
-	0x65, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x48, 0x00, 0x52, 0x05, 0x74, 0x6f, 0x6b, 0x65,
-	0x6e, 0x12, 0x32, 0x0a, 0x06, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x73, 0x18, 0x05, 0x20, 0x01, 0x28,
-	0x0b, 0x32, 0x18, 0x2e, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x73, 0x2e,
-	0x53, 0x6c, 0x61, 0x63, 0x6b, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x73, 0x48, 0x00, 0x52, 0x06, 0x74,
-	0x6f, 0x6b, 0x65, 0x6e, 0x73, 0x12, 0x1a, 0x0a, 0x08, 0x63, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c,
-	0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x09, 0x52, 0x08, 0x63, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c,
-	0x73, 0x12, 0x1e, 0x0a, 0x0a, 0x69, 0x67, 0x6e, 0x6f, 0x72, 0x65, 0x4c, 0x69, 0x73, 0x74, 0x18,
-	0x04, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0a, 0x69, 0x67, 0x6e, 0x6f, 0x72, 0x65, 0x4c, 0x69, 0x73,
-	0x74, 0x42, 0x0c, 0x0a, 0x0a, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x22,
-	0x06, 0x0a, 0x04, 0x54, 0x65, 0x73, 0x74, 0x22, 0x31, 0x0a, 0x09, 0x42, 0x75, 0x69, 0x6c, 0x64,
-	0x6b, 0x69, 0x74, 0x65, 0x12, 0x16, 0x0a, 0x05, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x18, 0x01, 0x20,
-	0x01, 0x28, 0x09, 0x48, 0x00, 0x52, 0x05, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x42, 0x0c, 0x0a, 0x0a,
-	0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x22, 0xdb, 0x01, 0x0a, 0x06, 0x47,
-	0x65, 0x72, 0x72, 0x69, 0x74, 0x12, 0x24, 0x0a, 0x08, 0x65, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e,
-	0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x42, 0x08, 0xfa, 0x42, 0x05, 0x72, 0x03, 0x90, 0x01,
-	0x01, 0x52, 0x08, 0x65, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x12, 0x37, 0x0a, 0x0a, 0x62,
-	0x61, 0x73, 0x69, 0x63, 0x5f, 0x61, 0x75, 0x74, 0x68, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32,
-	0x16, 0x2e, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x73, 0x2e, 0x42, 0x61,
-	0x73, 0x69, 0x63, 0x41, 0x75, 0x74, 0x68, 0x48, 0x00, 0x52, 0x09, 0x62, 0x61, 0x73, 0x69, 0x63,
-	0x41, 0x75, 0x74, 0x68, 0x12, 0x48, 0x0a, 0x0f, 0x75, 0x6e, 0x61, 0x75, 0x74, 0x68, 0x65, 0x6e,
-	0x74, 0x69, 0x63, 0x61, 0x74, 0x65, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e,
-	0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x73, 0x2e, 0x55, 0x6e, 0x61, 0x75,
-	0x74, 0x68, 0x65, 0x6e, 0x74, 0x69, 0x63, 0x61, 0x74, 0x65, 0x64, 0x48, 0x00, 0x52, 0x0f, 0x75,
-	0x6e, 0x61, 0x75, 0x74, 0x68, 0x65, 0x6e, 0x74, 0x69, 0x63, 0x61, 0x74, 0x65, 0x64, 0x12, 0x1a,
-	0x0a, 0x08, 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x09,
-	0x52, 0x08, 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x42, 0x0c, 0x0a, 0x0a, 0x63, 0x72,
-	0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x22, 0xa5, 0x01, 0x0a, 0x07, 0x4a, 0x65, 0x6e,
-	0x6b, 0x69, 0x6e, 0x73, 0x12, 0x24, 0x0a, 0x08, 0x65, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74,
+	0x12, 0x1a, 0x0a, 0x08, 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x18, 0x04, 0x20, 0x03,
+	0x28, 0x09, 0x52, 0x08, 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x42, 0x0c, 0x0a, 0x0a,
+	0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x22, 0xa5, 0x01, 0x0a, 0x07, 0x4a,
+	0x65, 0x6e, 0x6b, 0x69, 0x6e, 0x73, 0x12, 0x24, 0x0a, 0x08, 0x65, 0x6e, 0x64, 0x70, 0x6f, 0x69,
+	0x6e, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x42, 0x08, 0xfa, 0x42, 0x05, 0x72, 0x03, 0x90,
+	0x01, 0x01, 0x52, 0x08, 0x65, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x12, 0x37, 0x0a, 0x0a,
+	0x62, 0x61, 0x73, 0x69, 0x63, 0x5f, 0x61, 0x75, 0x74, 0x68, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b,
+	0x32, 0x16, 0x2e, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x73, 0x2e, 0x42,
+	0x61, 0x73, 0x69, 0x63, 0x41, 0x75, 0x74, 0x68, 0x48, 0x00, 0x52, 0x09, 0x62, 0x61, 0x73, 0x69,
+	0x63, 0x41, 0x75, 0x74, 0x68, 0x12, 0x2d, 0x0a, 0x06, 0x68, 0x65, 0x61, 0x64, 0x65, 0x72, 0x18,
+	0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x13, 0x2e, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69,
+	0x61, 0x6c, 0x73, 0x2e, 0x48, 0x65, 0x61, 0x64, 0x65, 0x72, 0x48, 0x00, 0x52, 0x06, 0x68, 0x65,
+	0x61, 0x64, 0x65, 0x72, 0x42, 0x0c, 0x0a, 0x0a, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69,
+	0x61, 0x6c, 0x22, 0x9f, 0x02, 0x0a, 0x05, 0x54, 0x65, 0x61, 0x6d, 0x73, 0x12, 0x24, 0x0a, 0x08,
+	0x65, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x42, 0x08,
+	0xfa, 0x42, 0x05, 0x72, 0x03, 0x90, 0x01, 0x01, 0x52, 0x08, 0x65, 0x6e, 0x64, 0x70, 0x6f, 0x69,
+	0x6e, 0x74, 0x12, 0x16, 0x0a, 0x05, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28,
+	0x09, 0x48, 0x00, 0x52, 0x05, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x12, 0x46, 0x0a, 0x0d, 0x61, 0x75,
+	0x74, 0x68, 0x65, 0x6e, 0x74, 0x69, 0x63, 0x61, 0x74, 0x65, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28,
+	0x0b, 0x32, 0x1e, 0x2e, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x73, 0x2e,
+	0x43, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x43, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c,
+	0x73, 0x48, 0x00, 0x52, 0x0d, 0x61, 0x75, 0x74, 0x68, 0x65, 0x6e, 0x74, 0x69, 0x63, 0x61, 0x74,
+	0x65, 0x64, 0x12, 0x2b, 0x0a, 0x05, 0x6f, 0x61, 0x75, 0x74, 0x68, 0x18, 0x07, 0x20, 0x01, 0x28,
+	0x0b, 0x32, 0x13, 0x2e, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x73, 0x2e,
+	0x4f, 0x61, 0x75, 0x74, 0x68, 0x32, 0x48, 0x00, 0x52, 0x05, 0x6f, 0x61, 0x75, 0x74, 0x68, 0x12,
+	0x1a, 0x0a, 0x08, 0x63, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28,
+	0x09, 0x52, 0x08, 0x63, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x73, 0x12, 0x1e, 0x0a, 0x0a, 0x69,
+	0x67, 0x6e, 0x6f, 0x72, 0x65, 0x4c, 0x69, 0x73, 0x74, 0x18, 0x05, 0x20, 0x03, 0x28, 0x09, 0x52,
+	0x0a, 0x69, 0x67, 0x6e, 0x6f, 0x72, 0x65, 0x4c, 0x69, 0x73, 0x74, 0x12, 0x19, 0x0a, 0x08, 0x74,
+	0x65, 0x61, 0x6d, 0x5f, 0x69, 0x64, 0x73, 0x18, 0x06, 0x20, 0x03, 0x28, 0x09, 0x52, 0x07, 0x74,
+	0x65, 0x61, 0x6d, 0x49, 0x64, 0x73, 0x42, 0x0c, 0x0a, 0x0a, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e,
+	0x74, 0x69, 0x61, 0x6c, 0x22, 0xc3, 0x01, 0x0a, 0x0b, 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63,
+	0x74, 0x6f, 0x72, 0x79, 0x12, 0x24, 0x0a, 0x08, 0x65, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74,
 	0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x42, 0x08, 0xfa, 0x42, 0x05, 0x72, 0x03, 0x90, 0x01, 0x01,
 	0x52, 0x08, 0x65, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x12, 0x37, 0x0a, 0x0a, 0x62, 0x61,
 	0x73, 0x69, 0x63, 0x5f, 0x61, 0x75, 0x74, 0x68, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x16,
 	0x2e, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x73, 0x2e, 0x42, 0x61, 0x73,
 	0x69, 0x63, 0x41, 0x75, 0x74, 0x68, 0x48, 0x00, 0x52, 0x09, 0x62, 0x61, 0x73, 0x69, 0x63, 0x41,
-	0x75, 0x74, 0x68, 0x12, 0x2d, 0x0a, 0x06, 0x68, 0x65, 0x61, 0x64, 0x65, 0x72, 0x18, 0x03, 0x20,
-	0x01, 0x28, 0x0b, 0x32, 0x13, 0x2e, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c,
-	0x73, 0x2e, 0x48, 0x65, 0x61, 0x64, 0x65, 0x72, 0x48, 0x00, 0x52, 0x06, 0x68, 0x65, 0x61, 0x64,
-	0x65, 0x72, 0x42, 0x0c, 0x0a, 0x0a, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c,
-	0x22, 0x9f, 0x02, 0x0a, 0x05, 0x54, 0x65, 0x61, 0x6d, 0x73, 0x12, 0x24, 0x0a, 0x08, 0x65, 0x6e,
-	0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x42, 0x08, 0xfa, 0x42,
-	0x05, 0x72, 0x03, 0x90, 0x01, 0x01, 0x52, 0x08, 0x65, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74,
-	0x12, 0x16, 0x0a, 0x05, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x48,
-	0x00, 0x52, 0x05, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x12, 0x46, 0x0a, 0x0d, 0x61, 0x75, 0x74, 0x68,
-	0x65, 0x6e, 0x74, 0x69, 0x63, 0x61, 0x74, 0x65, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32,
-	0x1e, 0x2e, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x73, 0x2e, 0x43, 0x6c,
-	0x69, 0x65, 0x6e, 0x74, 0x43, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x73, 0x48,
-	0x00, 0x52, 0x0d, 0x61, 0x75, 0x74, 0x68, 0x65, 0x6e, 0x74, 0x69, 0x63, 0x61, 0x74, 0x65, 0x64,
-	0x12, 0x2b, 0x0a, 0x05, 0x6f, 0x61, 0x75, 0x74, 0x68, 0x18, 0x07, 0x20, 0x01, 0x28, 0x0b, 0x32,
-	0x13, 0x2e, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x73, 0x2e, 0x4f, 0x61,
-	0x75, 0x74, 0x68, 0x32, 0x48, 0x00, 0x52, 0x05, 0x6f, 0x61, 0x75, 0x74, 0x68, 0x12, 0x1a, 0x0a,
-	0x08, 0x63, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x09, 0x52,
-	0x08, 0x63, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x73, 0x12, 0x1e, 0x0a, 0x0a, 0x69, 0x67, 0x6e,
-	0x6f, 0x72, 0x65, 0x4c, 0x69, 0x73, 0x74, 0x18, 0x05, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0a, 0x69,
-	0x67, 0x6e, 0x6f, 0x72, 0x65, 0x4c, 0x69, 0x73, 0x74, 0x12, 0x19, 0x0a, 0x08, 0x74, 0x65, 0x61,
-	0x6d, 0x5f, 0x69, 0x64, 0x73, 0x18, 0x06, 0x20, 0x03, 0x28, 0x09, 0x52, 0x07, 0x74, 0x65, 0x61,
-	0x6d, 0x49, 0x64, 0x73, 0x42, 0x0c, 0x0a, 0x0a, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69,
-	0x61, 0x6c, 0x22, 0xc3, 0x01, 0x0a, 0x0b, 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x6f,
-	0x72, 0x79, 0x12, 0x24, 0x0a, 0x08, 0x65, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x18, 0x01,
-	0x20, 0x01, 0x28, 0x09, 0x42, 0x08, 0xfa, 0x42, 0x05, 0x72, 0x03, 0x90, 0x01, 0x01, 0x52, 0x08,
-	0x65, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x12, 0x37, 0x0a, 0x0a, 0x62, 0x61, 0x73, 0x69,
-	0x63, 0x5f, 0x61, 0x75, 0x74, 0x68, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x16, 0x2e, 0x63,
-	0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x73, 0x2e, 0x42, 0x61, 0x73, 0x69, 0x63,
-	0x41, 0x75, 0x74, 0x68, 0x48, 0x00, 0x52, 0x09, 0x62, 0x61, 0x73, 0x69, 0x63, 0x41, 0x75, 0x74,
-	0x68, 0x12, 0x23, 0x0a, 0x0c, 0x61, 0x63, 0x63, 0x65, 0x73, 0x73, 0x5f, 0x74, 0x6f, 0x6b, 0x65,
-	0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x48, 0x00, 0x52, 0x0b, 0x61, 0x63, 0x63, 0x65, 0x73,
-	0x73, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x12, 0x22, 0x0a, 0x0c, 0x72, 0x65, 0x70, 0x6f, 0x73, 0x69,
-	0x74, 0x6f, 0x72, 0x69, 0x65, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0c, 0x72, 0x65,
-	0x70, 0x6f, 0x73, 0x69, 0x74, 0x6f, 0x72, 0x69, 0x65, 0x73, 0x42, 0x0c, 0x0a, 0x0a, 0x63, 0x72,
-	0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x22, 0x94, 0x01, 0x0a, 0x06, 0x53, 0x79, 0x73,
-	0x6c, 0x6f, 0x67, 0x12, 0x1a, 0x0a, 0x08, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x18,
-	0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x12,
-	0x24, 0x0a, 0x0d, 0x6c, 0x69, 0x73, 0x74, 0x65, 0x6e, 0x41, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73,
-	0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0d, 0x6c, 0x69, 0x73, 0x74, 0x65, 0x6e, 0x41, 0x64,
-	0x64, 0x72, 0x65, 0x73, 0x73, 0x12, 0x18, 0x0a, 0x07, 0x74, 0x6c, 0x73, 0x43, 0x65, 0x72, 0x74,
-	0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x74, 0x6c, 0x73, 0x43, 0x65, 0x72, 0x74, 0x12,
-	0x16, 0x0a, 0x06, 0x74, 0x6c, 0x73, 0x4b, 0x65, 0x79, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52,
-	0x06, 0x74, 0x6c, 0x73, 0x4b, 0x65, 0x79, 0x12, 0x16, 0x0a, 0x06, 0x66, 0x6f, 0x72, 0x6d, 0x61,
-	0x74, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x66, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x22,
-	0xd6, 0x01, 0x0a, 0x15, 0x50, 0x75, 0x62, 0x6c, 0x69, 0x63, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x4d,
-	0x6f, 0x6e, 0x69, 0x74, 0x6f, 0x72, 0x69, 0x6e, 0x67, 0x12, 0x48, 0x0a, 0x0f, 0x75, 0x6e, 0x61,
-	0x75, 0x74, 0x68, 0x65, 0x6e, 0x74, 0x69, 0x63, 0x61, 0x74, 0x65, 0x64, 0x18, 0x01, 0x20, 0x01,
-	0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x73,
-	0x2e, 0x55, 0x6e, 0x61, 0x75, 0x74, 0x68, 0x65, 0x6e, 0x74, 0x69, 0x63, 0x61, 0x74, 0x65, 0x64,
-	0x48, 0x00, 0x52, 0x0f, 0x75, 0x6e, 0x61, 0x75, 0x74, 0x68, 0x65, 0x6e, 0x74, 0x69, 0x63, 0x61,
-	0x74, 0x65, 0x64, 0x12, 0x16, 0x0a, 0x06, 0x64, 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x18, 0x02, 0x20,
-	0x01, 0x28, 0x09, 0x52, 0x06, 0x64, 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x12, 0x1b, 0x0a, 0x09, 0x6d,
-	0x61, 0x78, 0x5f, 0x64, 0x65, 0x70, 0x74, 0x68, 0x18, 0x03, 0x20, 0x01, 0x28, 0x03, 0x52, 0x08,
-	0x6d, 0x61, 0x78, 0x44, 0x65, 0x70, 0x74, 0x68, 0x12, 0x30, 0x0a, 0x05, 0x73, 0x69, 0x6e, 0x63,
-	0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65,
-	0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74,
-	0x61, 0x6d, 0x70, 0x52, 0x05, 0x73, 0x69, 0x6e, 0x63, 0x65, 0x42, 0x0c, 0x0a, 0x0a, 0x63, 0x72,
-	0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x22, 0x51, 0x0a, 0x0d, 0x53, 0x6c, 0x61, 0x63,
-	0x6b, 0x52, 0x65, 0x61, 0x6c, 0x74, 0x69, 0x6d, 0x65, 0x12, 0x32, 0x0a, 0x06, 0x74, 0x6f, 0x6b,
-	0x65, 0x6e, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x18, 0x2e, 0x63, 0x72, 0x65, 0x64,
-	0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x73, 0x2e, 0x53, 0x6c, 0x61, 0x63, 0x6b, 0x54, 0x6f, 0x6b,
-	0x65, 0x6e, 0x73, 0x48, 0x00, 0x52, 0x06, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x73, 0x42, 0x0c, 0x0a,
-	0x0a, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x22, 0x62, 0x0a, 0x0a, 0x53,
-	0x68, 0x61, 0x72, 0x65, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x12, 0x2b, 0x0a, 0x05, 0x6f, 0x61, 0x75,
-	0x74, 0x68, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x13, 0x2e, 0x63, 0x72, 0x65, 0x64, 0x65,
-	0x6e, 0x74, 0x69, 0x61, 0x6c, 0x73, 0x2e, 0x4f, 0x61, 0x75, 0x74, 0x68, 0x32, 0x48, 0x00, 0x52,
-	0x05, 0x6f, 0x61, 0x75, 0x74, 0x68, 0x12, 0x19, 0x0a, 0x08, 0x73, 0x69, 0x74, 0x65, 0x5f, 0x75,
-	0x72, 0x6c, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x73, 0x69, 0x74, 0x65, 0x55, 0x72,
-	0x6c, 0x42, 0x0c, 0x0a, 0x0a, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x22,
-	0xa7, 0x03, 0x0a, 0x0a, 0x41, 0x7a, 0x75, 0x72, 0x65, 0x52, 0x65, 0x70, 0x6f, 0x73, 0x12, 0x24,
-	0x0a, 0x08, 0x65, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09,
-	0x42, 0x08, 0xfa, 0x42, 0x05, 0x72, 0x03, 0x90, 0x01, 0x01, 0x52, 0x08, 0x65, 0x6e, 0x64, 0x70,
-	0x6f, 0x69, 0x6e, 0x74, 0x12, 0x16, 0x0a, 0x05, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x18, 0x02, 0x20,
-	0x01, 0x28, 0x09, 0x48, 0x00, 0x52, 0x05, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x12, 0x2b, 0x0a, 0x05,
-	0x6f, 0x61, 0x75, 0x74, 0x68, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x13, 0x2e, 0x63, 0x72,
-	0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x73, 0x2e, 0x4f, 0x61, 0x75, 0x74, 0x68, 0x32,
-	0x48, 0x00, 0x52, 0x05, 0x6f, 0x61, 0x75, 0x74, 0x68, 0x12, 0x22, 0x0a, 0x0c, 0x72, 0x65, 0x70,
-	0x6f, 0x73, 0x69, 0x74, 0x6f, 0x72, 0x69, 0x65, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x09, 0x52,
-	0x0c, 0x72, 0x65, 0x70, 0x6f, 0x73, 0x69, 0x74, 0x6f, 0x72, 0x69, 0x65, 0x73, 0x12, 0x24, 0x0a,
-	0x0d, 0x6f, 0x72, 0x67, 0x61, 0x6e, 0x69, 0x7a, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x05,
-	0x20, 0x03, 0x28, 0x09, 0x52, 0x0d, 0x6f, 0x72, 0x67, 0x61, 0x6e, 0x69, 0x7a, 0x61, 0x74, 0x69,
-	0x6f, 0x6e, 0x73, 0x12, 0x1a, 0x0a, 0x08, 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x18,
-	0x06, 0x20, 0x03, 0x28, 0x09, 0x52, 0x08, 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x12,
-	0x22, 0x0a, 0x0c, 0x69, 0x6e, 0x63, 0x6c, 0x75, 0x64, 0x65, 0x46, 0x6f, 0x72, 0x6b, 0x73, 0x18,
-	0x07, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0c, 0x69, 0x6e, 0x63, 0x6c, 0x75, 0x64, 0x65, 0x46, 0x6f,
-	0x72, 0x6b, 0x73, 0x12, 0x20, 0x0a, 0x0b, 0x69, 0x67, 0x6e, 0x6f, 0x72, 0x65, 0x52, 0x65, 0x70,
-	0x6f, 0x73, 0x18, 0x08, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0b, 0x69, 0x67, 0x6e, 0x6f, 0x72, 0x65,
-	0x52, 0x65, 0x70, 0x6f, 0x73, 0x12, 0x22, 0x0a, 0x0c, 0x69, 0x6e, 0x63, 0x6c, 0x75, 0x64, 0x65,
-	0x52, 0x65, 0x70, 0x6f, 0x73, 0x18, 0x09, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0c, 0x69, 0x6e, 0x63,
-	0x6c, 0x75, 0x64, 0x65, 0x52, 0x65, 0x70, 0x6f, 0x73, 0x12, 0x28, 0x0a, 0x0f, 0x69, 0x6e, 0x63,
-	0x6c, 0x75, 0x64, 0x65, 0x50, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x18, 0x0a, 0x20, 0x03,
-	0x28, 0x09, 0x52, 0x0f, 0x69, 0x6e, 0x63, 0x6c, 0x75, 0x64, 0x65, 0x50, 0x72, 0x6f, 0x6a, 0x65,
-	0x63, 0x74, 0x73, 0x12, 0x26, 0x0a, 0x0e, 0x69, 0x67, 0x6e, 0x6f, 0x72, 0x65, 0x50, 0x72, 0x6f,
-	0x6a, 0x65, 0x63, 0x74, 0x73, 0x18, 0x0b, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0e, 0x69, 0x67, 0x6e,
-	0x6f, 0x72, 0x65, 0x50, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x42, 0x0c, 0x0a, 0x0a, 0x63,
-	0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x2a, 0x84, 0x07, 0x0a, 0x0a, 0x53, 0x6f,
-	0x75, 0x72, 0x63, 0x65, 0x54, 0x79, 0x70, 0x65, 0x12, 0x1d, 0x0a, 0x19, 0x53, 0x4f, 0x55, 0x52,
-	0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x41, 0x5a, 0x55, 0x52, 0x45, 0x5f, 0x53, 0x54,
-	0x4f, 0x52, 0x41, 0x47, 0x45, 0x10, 0x00, 0x12, 0x19, 0x0a, 0x15, 0x53, 0x4f, 0x55, 0x52, 0x43,
-	0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x42, 0x49, 0x54, 0x42, 0x55, 0x43, 0x4b, 0x45, 0x54,
-	0x10, 0x01, 0x12, 0x18, 0x0a, 0x14, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50,
-	0x45, 0x5f, 0x43, 0x49, 0x52, 0x43, 0x4c, 0x45, 0x43, 0x49, 0x10, 0x02, 0x12, 0x1a, 0x0a, 0x16,
-	0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x43, 0x4f, 0x4e, 0x46,
-	0x4c, 0x55, 0x45, 0x4e, 0x43, 0x45, 0x10, 0x03, 0x12, 0x16, 0x0a, 0x12, 0x53, 0x4f, 0x55, 0x52,
-	0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x44, 0x4f, 0x43, 0x4b, 0x45, 0x52, 0x10, 0x04,
-	0x12, 0x13, 0x0a, 0x0f, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f,
-	0x45, 0x43, 0x52, 0x10, 0x05, 0x12, 0x13, 0x0a, 0x0f, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f,
-	0x54, 0x59, 0x50, 0x45, 0x5f, 0x47, 0x43, 0x53, 0x10, 0x06, 0x12, 0x16, 0x0a, 0x12, 0x53, 0x4f,
+	0x75, 0x74, 0x68, 0x12, 0x23, 0x0a, 0x0c, 0x61, 0x63, 0x63, 0x65, 0x73, 0x73, 0x5f, 0x74, 0x6f,
+	0x6b, 0x65, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x48, 0x00, 0x52, 0x0b, 0x61, 0x63, 0x63,
+	0x65, 0x73, 0x73, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x12, 0x22, 0x0a, 0x0c, 0x72, 0x65, 0x70, 0x6f,
+	0x73, 0x69, 0x74, 0x6f, 0x72, 0x69, 0x65, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0c,
+	0x72, 0x65, 0x70, 0x6f, 0x73, 0x69, 0x74, 0x6f, 0x72, 0x69, 0x65, 0x73, 0x42, 0x0c, 0x0a, 0x0a,
+	0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x22, 0x94, 0x01, 0x0a, 0x06, 0x53,
+	0x79, 0x73, 0x6c, 0x6f, 0x67, 0x12, 0x1a, 0x0a, 0x08, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f,
+	0x6c, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f,
+	0x6c, 0x12, 0x24, 0x0a, 0x0d, 0x6c, 0x69, 0x73, 0x74, 0x65, 0x6e, 0x41, 0x64, 0x64, 0x72, 0x65,
+	0x73, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0d, 0x6c, 0x69, 0x73, 0x74, 0x65, 0x6e,
+	0x41, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x12, 0x18, 0x0a, 0x07, 0x74, 0x6c, 0x73, 0x43, 0x65,
+	0x72, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x74, 0x6c, 0x73, 0x43, 0x65, 0x72,
+	0x74, 0x12, 0x16, 0x0a, 0x06, 0x74, 0x6c, 0x73, 0x4b, 0x65, 0x79, 0x18, 0x04, 0x20, 0x01, 0x28,
+	0x09, 0x52, 0x06, 0x74, 0x6c, 0x73, 0x4b, 0x65, 0x79, 0x12, 0x16, 0x0a, 0x06, 0x66, 0x6f, 0x72,
+	0x6d, 0x61, 0x74, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x66, 0x6f, 0x72, 0x6d, 0x61,
+	0x74, 0x22, 0xd6, 0x01, 0x0a, 0x15, 0x50, 0x75, 0x62, 0x6c, 0x69, 0x63, 0x45, 0x76, 0x65, 0x6e,
+	0x74, 0x4d, 0x6f, 0x6e, 0x69, 0x74, 0x6f, 0x72, 0x69, 0x6e, 0x67, 0x12, 0x48, 0x0a, 0x0f, 0x75,
+	0x6e, 0x61, 0x75, 0x74, 0x68, 0x65, 0x6e, 0x74, 0x69, 0x63, 0x61, 0x74, 0x65, 0x64, 0x18, 0x01,
+	0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61,
+	0x6c, 0x73, 0x2e, 0x55, 0x6e, 0x61, 0x75, 0x74, 0x68, 0x65, 0x6e, 0x74, 0x69, 0x63, 0x61, 0x74,
+	0x65, 0x64, 0x48, 0x00, 0x52, 0x0f, 0x75, 0x6e, 0x61, 0x75, 0x74, 0x68, 0x65, 0x6e, 0x74, 0x69,
+	0x63, 0x61, 0x74, 0x65, 0x64, 0x12, 0x16, 0x0a, 0x06, 0x64, 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x18,
+	0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x64, 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x12, 0x1b, 0x0a,
+	0x09, 0x6d, 0x61, 0x78, 0x5f, 0x64, 0x65, 0x70, 0x74, 0x68, 0x18, 0x03, 0x20, 0x01, 0x28, 0x03,
+	0x52, 0x08, 0x6d, 0x61, 0x78, 0x44, 0x65, 0x70, 0x74, 0x68, 0x12, 0x30, 0x0a, 0x05, 0x73, 0x69,
+	0x6e, 0x63, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67,
+	0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65,
+	0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x05, 0x73, 0x69, 0x6e, 0x63, 0x65, 0x42, 0x0c, 0x0a, 0x0a,
+	0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x22, 0x51, 0x0a, 0x0d, 0x53, 0x6c,
+	0x61, 0x63, 0x6b, 0x52, 0x65, 0x61, 0x6c, 0x74, 0x69, 0x6d, 0x65, 0x12, 0x32, 0x0a, 0x06, 0x74,
+	0x6f, 0x6b, 0x65, 0x6e, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x18, 0x2e, 0x63, 0x72,
+	0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x73, 0x2e, 0x53, 0x6c, 0x61, 0x63, 0x6b, 0x54,
+	0x6f, 0x6b, 0x65, 0x6e, 0x73, 0x48, 0x00, 0x52, 0x06, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x73, 0x42,
+	0x0c, 0x0a, 0x0a, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x22, 0x62, 0x0a,
+	0x0a, 0x53, 0x68, 0x61, 0x72, 0x65, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x12, 0x2b, 0x0a, 0x05, 0x6f,
+	0x61, 0x75, 0x74, 0x68, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x13, 0x2e, 0x63, 0x72, 0x65,
+	0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x73, 0x2e, 0x4f, 0x61, 0x75, 0x74, 0x68, 0x32, 0x48,
+	0x00, 0x52, 0x05, 0x6f, 0x61, 0x75, 0x74, 0x68, 0x12, 0x19, 0x0a, 0x08, 0x73, 0x69, 0x74, 0x65,
+	0x5f, 0x75, 0x72, 0x6c, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x73, 0x69, 0x74, 0x65,
+	0x55, 0x72, 0x6c, 0x42, 0x0c, 0x0a, 0x0a, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61,
+	0x6c, 0x22, 0xa7, 0x03, 0x0a, 0x0a, 0x41, 0x7a, 0x75, 0x72, 0x65, 0x52, 0x65, 0x70, 0x6f, 0x73,
+	0x12, 0x24, 0x0a, 0x08, 0x65, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x18, 0x01, 0x20, 0x01,
+	0x28, 0x09, 0x42, 0x08, 0xfa, 0x42, 0x05, 0x72, 0x03, 0x90, 0x01, 0x01, 0x52, 0x08, 0x65, 0x6e,
+	0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x12, 0x16, 0x0a, 0x05, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x18,
+	0x02, 0x20, 0x01, 0x28, 0x09, 0x48, 0x00, 0x52, 0x05, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x12, 0x2b,
+	0x0a, 0x05, 0x6f, 0x61, 0x75, 0x74, 0x68, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x13, 0x2e,
+	0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x73, 0x2e, 0x4f, 0x61, 0x75, 0x74,
+	0x68, 0x32, 0x48, 0x00, 0x52, 0x05, 0x6f, 0x61, 0x75, 0x74, 0x68, 0x12, 0x22, 0x0a, 0x0c, 0x72,
+	0x65, 0x70, 0x6f, 0x73, 0x69, 0x74, 0x6f, 0x72, 0x69, 0x65, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28,
+	0x09, 0x52, 0x0c, 0x72, 0x65, 0x70, 0x6f, 0x73, 0x69, 0x74, 0x6f, 0x72, 0x69, 0x65, 0x73, 0x12,
+	0x24, 0x0a, 0x0d, 0x6f, 0x72, 0x67, 0x61, 0x6e, 0x69, 0x7a, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73,
+	0x18, 0x05, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0d, 0x6f, 0x72, 0x67, 0x61, 0x6e, 0x69, 0x7a, 0x61,
+	0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x1a, 0x0a, 0x08, 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74,
+	0x73, 0x18, 0x06, 0x20, 0x03, 0x28, 0x09, 0x52, 0x08, 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74,
+	0x73, 0x12, 0x22, 0x0a, 0x0c, 0x69, 0x6e, 0x63, 0x6c, 0x75, 0x64, 0x65, 0x46, 0x6f, 0x72, 0x6b,
+	0x73, 0x18, 0x07, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0c, 0x69, 0x6e, 0x63, 0x6c, 0x75, 0x64, 0x65,
+	0x46, 0x6f, 0x72, 0x6b, 0x73, 0x12, 0x20, 0x0a, 0x0b, 0x69, 0x67, 0x6e, 0x6f, 0x72, 0x65, 0x52,
+	0x65, 0x70, 0x6f, 0x73, 0x18, 0x08, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0b, 0x69, 0x67, 0x6e, 0x6f,
+	0x72, 0x65, 0x52, 0x65, 0x70, 0x6f, 0x73, 0x12, 0x22, 0x0a, 0x0c, 0x69, 0x6e, 0x63, 0x6c, 0x75,
+	0x64, 0x65, 0x52, 0x65, 0x70, 0x6f, 0x73, 0x18, 0x09, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0c, 0x69,
+	0x6e, 0x63, 0x6c, 0x75, 0x64, 0x65, 0x52, 0x65, 0x70, 0x6f, 0x73, 0x12, 0x28, 0x0a, 0x0f, 0x69,
+	0x6e, 0x63, 0x6c, 0x75, 0x64, 0x65, 0x50, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x18, 0x0a,
+	0x20, 0x03, 0x28, 0x09, 0x52, 0x0f, 0x69, 0x6e, 0x63, 0x6c, 0x75, 0x64, 0x65, 0x50, 0x72, 0x6f,
+	0x6a, 0x65, 0x63, 0x74, 0x73, 0x12, 0x26, 0x0a, 0x0e, 0x69, 0x67, 0x6e, 0x6f, 0x72, 0x65, 0x50,
+	0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x18, 0x0b, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0e, 0x69,
+	0x67, 0x6e, 0x6f, 0x72, 0x65, 0x50, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x42, 0x0c, 0x0a,
+	0x0a, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x2a, 0x84, 0x07, 0x0a, 0x0a,
+	0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x54, 0x79, 0x70, 0x65, 0x12, 0x1d, 0x0a, 0x19, 0x53, 0x4f,
+	0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x41, 0x5a, 0x55, 0x52, 0x45, 0x5f,
+	0x53, 0x54, 0x4f, 0x52, 0x41, 0x47, 0x45, 0x10, 0x00, 0x12, 0x19, 0x0a, 0x15, 0x53, 0x4f, 0x55,
+	0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x42, 0x49, 0x54, 0x42, 0x55, 0x43, 0x4b,
+	0x45, 0x54, 0x10, 0x01, 0x12, 0x18, 0x0a, 0x14, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54,
+	0x59, 0x50, 0x45, 0x5f, 0x43, 0x49, 0x52, 0x43, 0x4c, 0x45, 0x43, 0x49, 0x10, 0x02, 0x12, 0x1a,
+	0x0a, 0x16, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x43, 0x4f,
+	0x4e, 0x46, 0x4c, 0x55, 0x45, 0x4e, 0x43, 0x45, 0x10, 0x03, 0x12, 0x16, 0x0a, 0x12, 0x53, 0x4f,
+	0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x44, 0x4f, 0x43, 0x4b, 0x45, 0x52,
+	0x10, 0x04, 0x12, 0x13, 0x0a, 0x0f, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50,
+	0x45, 0x5f, 0x45, 0x43, 0x52, 0x10, 0x05, 0x12, 0x13, 0x0a, 0x0f, 0x53, 0x4f, 0x55, 0x52, 0x43,
+	0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x47, 0x43, 0x53, 0x10, 0x06, 0x12, 0x16, 0x0a, 0x12,
+	0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x47, 0x49, 0x54, 0x48,
+	0x55, 0x42, 0x10, 0x07, 0x12, 0x1a, 0x0a, 0x16, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54,
+	0x59, 0x50, 0x45, 0x5f, 0x50, 0x55, 0x42, 0x4c, 0x49, 0x43, 0x5f, 0x47, 0x49, 0x54, 0x10, 0x08,
+	0x12, 0x16, 0x0a, 0x12, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f,
+	0x47, 0x49, 0x54, 0x4c, 0x41, 0x42, 0x10, 0x09, 0x12, 0x14, 0x0a, 0x10, 0x53, 0x4f, 0x55, 0x52,
+	0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x4a, 0x49, 0x52, 0x41, 0x10, 0x0a, 0x12, 0x24,
+	0x0a, 0x20, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x4e, 0x50,
+	0x4d, 0x5f, 0x55, 0x4e, 0x41, 0x55, 0x54, 0x48, 0x44, 0x5f, 0x50, 0x41, 0x43, 0x4b, 0x41, 0x47,
+	0x45, 0x53, 0x10, 0x0b, 0x12, 0x25, 0x0a, 0x21, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54,
+	0x59, 0x50, 0x45, 0x5f, 0x50, 0x59, 0x50, 0x49, 0x5f, 0x55, 0x4e, 0x41, 0x55, 0x54, 0x48, 0x44,
+	0x5f, 0x50, 0x41, 0x43, 0x4b, 0x41, 0x47, 0x45, 0x53, 0x10, 0x0c, 0x12, 0x12, 0x0a, 0x0e, 0x53,
+	0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x53, 0x33, 0x10, 0x0d, 0x12,
+	0x15, 0x0a, 0x11, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x53,
+	0x4c, 0x41, 0x43, 0x4b, 0x10, 0x0e, 0x12, 0x1a, 0x0a, 0x16, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45,
+	0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x46, 0x49, 0x4c, 0x45, 0x53, 0x59, 0x53, 0x54, 0x45, 0x4d,
+	0x10, 0x0f, 0x12, 0x13, 0x0a, 0x0f, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50,
+	0x45, 0x5f, 0x47, 0x49, 0x54, 0x10, 0x10, 0x12, 0x14, 0x0a, 0x10, 0x53, 0x4f, 0x55, 0x52, 0x43,
+	0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x54, 0x45, 0x53, 0x54, 0x10, 0x11, 0x12, 0x1b, 0x0a,
+	0x17, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x53, 0x33, 0x5f,
+	0x55, 0x4e, 0x41, 0x55, 0x54, 0x48, 0x45, 0x44, 0x10, 0x12, 0x12, 0x2a, 0x0a, 0x26, 0x53, 0x4f,
 	0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x47, 0x49, 0x54, 0x48, 0x55, 0x42,
-	0x10, 0x07, 0x12, 0x1a, 0x0a, 0x16, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50,
-	0x45, 0x5f, 0x50, 0x55, 0x42, 0x4c, 0x49, 0x43, 0x5f, 0x47, 0x49, 0x54, 0x10, 0x08, 0x12, 0x16,
-	0x0a, 0x12, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x47, 0x49,
-	0x54, 0x4c, 0x41, 0x42, 0x10, 0x09, 0x12, 0x14, 0x0a, 0x10, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45,
-	0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x4a, 0x49, 0x52, 0x41, 0x10, 0x0a, 0x12, 0x24, 0x0a, 0x20,
-	0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x4e, 0x50, 0x4d, 0x5f,
-	0x55, 0x4e, 0x41, 0x55, 0x54, 0x48, 0x44, 0x5f, 0x50, 0x41, 0x43, 0x4b, 0x41, 0x47, 0x45, 0x53,
-	0x10, 0x0b, 0x12, 0x25, 0x0a, 0x21, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50,
-	0x45, 0x5f, 0x50, 0x59, 0x50, 0x49, 0x5f, 0x55, 0x4e, 0x41, 0x55, 0x54, 0x48, 0x44, 0x5f, 0x50,
-	0x41, 0x43, 0x4b, 0x41, 0x47, 0x45, 0x53, 0x10, 0x0c, 0x12, 0x12, 0x0a, 0x0e, 0x53, 0x4f, 0x55,
-	0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x53, 0x33, 0x10, 0x0d, 0x12, 0x15, 0x0a,
-	0x11, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x53, 0x4c, 0x41,
-	0x43, 0x4b, 0x10, 0x0e, 0x12, 0x1a, 0x0a, 0x16, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54,
-	0x59, 0x50, 0x45, 0x5f, 0x46, 0x49, 0x4c, 0x45, 0x53, 0x59, 0x53, 0x54, 0x45, 0x4d, 0x10, 0x0f,
-	0x12, 0x13, 0x0a, 0x0f, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f,
-	0x47, 0x49, 0x54, 0x10, 0x10, 0x12, 0x14, 0x0a, 0x10, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f,
-	0x54, 0x59, 0x50, 0x45, 0x5f, 0x54, 0x45, 0x53, 0x54, 0x10, 0x11, 0x12, 0x1b, 0x0a, 0x17, 0x53,
-	0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x53, 0x33, 0x5f, 0x55, 0x4e,
-	0x41, 0x55, 0x54, 0x48, 0x45, 0x44, 0x10, 0x12, 0x12, 0x2a, 0x0a, 0x26, 0x53, 0x4f, 0x55, 0x52,
-	0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x47, 0x49, 0x54, 0x48, 0x55, 0x42, 0x5f, 0x55,
-	0x4e, 0x41, 0x55, 0x54, 0x48, 0x45, 0x4e, 0x54, 0x49, 0x43, 0x41, 0x54, 0x45, 0x44, 0x5f, 0x4f,
-	0x52, 0x47, 0x10, 0x13, 0x12, 0x19, 0x0a, 0x15, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54,
-	0x59, 0x50, 0x45, 0x5f, 0x42, 0x55, 0x49, 0x4c, 0x44, 0x4b, 0x49, 0x54, 0x45, 0x10, 0x14, 0x12,
-	0x16, 0x0a, 0x12, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x47,
-	0x45, 0x52, 0x52, 0x49, 0x54, 0x10, 0x15, 0x12, 0x17, 0x0a, 0x13, 0x53, 0x4f, 0x55, 0x52, 0x43,
-	0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x4a, 0x45, 0x4e, 0x4b, 0x49, 0x4e, 0x53, 0x10, 0x16,
-	0x12, 0x15, 0x0a, 0x11, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f,
-	0x54, 0x45, 0x41, 0x4d, 0x53, 0x10, 0x17, 0x12, 0x21, 0x0a, 0x1d, 0x53, 0x4f, 0x55, 0x52, 0x43,
-	0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x4a, 0x46, 0x52, 0x4f, 0x47, 0x5f, 0x41, 0x52, 0x54,
-	0x49, 0x46, 0x41, 0x43, 0x54, 0x4f, 0x52, 0x59, 0x10, 0x18, 0x12, 0x16, 0x0a, 0x12, 0x53, 0x4f,
-	0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x53, 0x59, 0x53, 0x4c, 0x4f, 0x47,
-	0x10, 0x19, 0x12, 0x27, 0x0a, 0x23, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50,
-	0x45, 0x5f, 0x50, 0x55, 0x42, 0x4c, 0x49, 0x43, 0x5f, 0x45, 0x56, 0x45, 0x4e, 0x54, 0x5f, 0x4d,
-	0x4f, 0x4e, 0x49, 0x54, 0x4f, 0x52, 0x49, 0x4e, 0x47, 0x10, 0x1a, 0x12, 0x1e, 0x0a, 0x1a, 0x53,
-	0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x53, 0x4c, 0x41, 0x43, 0x4b,
-	0x5f, 0x52, 0x45, 0x41, 0x4c, 0x54, 0x49, 0x4d, 0x45, 0x10, 0x1b, 0x12, 0x1c, 0x0a, 0x18, 0x53,
-	0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x47, 0x4f, 0x4f, 0x47, 0x4c,
-	0x45, 0x5f, 0x44, 0x52, 0x49, 0x56, 0x45, 0x10, 0x1c, 0x12, 0x1a, 0x0a, 0x16, 0x53, 0x4f, 0x55,
-	0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x53, 0x48, 0x41, 0x52, 0x45, 0x50, 0x4f,
-	0x49, 0x4e, 0x54, 0x10, 0x1d, 0x12, 0x1c, 0x0a, 0x18, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f,
-	0x54, 0x59, 0x50, 0x45, 0x5f, 0x47, 0x43, 0x53, 0x5f, 0x55, 0x4e, 0x41, 0x55, 0x54, 0x48, 0x45,
-	0x44, 0x10, 0x1e, 0x12, 0x1b, 0x0a, 0x17, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59,
-	0x50, 0x45, 0x5f, 0x41, 0x5a, 0x55, 0x52, 0x45, 0x5f, 0x52, 0x45, 0x50, 0x4f, 0x53, 0x10, 0x1f,
-	0x42, 0x3b, 0x5a, 0x39, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x74,
-	0x72, 0x75, 0x66, 0x66, 0x6c, 0x65, 0x73, 0x65, 0x63, 0x75, 0x72, 0x69, 0x74, 0x79, 0x2f, 0x74,
-	0x72, 0x75, 0x66, 0x66, 0x6c, 0x65, 0x68, 0x6f, 0x67, 0x2f, 0x76, 0x33, 0x2f, 0x70, 0x6b, 0x67,
-	0x2f, 0x70, 0x62, 0x2f, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x73, 0x70, 0x62, 0x62, 0x06, 0x70,
-	0x72, 0x6f, 0x74, 0x6f, 0x33,
+	0x5f, 0x55, 0x4e, 0x41, 0x55, 0x54, 0x48, 0x45, 0x4e, 0x54, 0x49, 0x43, 0x41, 0x54, 0x45, 0x44,
+	0x5f, 0x4f, 0x52, 0x47, 0x10, 0x13, 0x12, 0x19, 0x0a, 0x15, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45,
+	0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x42, 0x55, 0x49, 0x4c, 0x44, 0x4b, 0x49, 0x54, 0x45, 0x10,
+	0x14, 0x12, 0x16, 0x0a, 0x12, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45,
+	0x5f, 0x47, 0x45, 0x52, 0x52, 0x49, 0x54, 0x10, 0x15, 0x12, 0x17, 0x0a, 0x13, 0x53, 0x4f, 0x55,
+	0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x4a, 0x45, 0x4e, 0x4b, 0x49, 0x4e, 0x53,
+	0x10, 0x16, 0x12, 0x15, 0x0a, 0x11, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50,
+	0x45, 0x5f, 0x54, 0x45, 0x41, 0x4d, 0x53, 0x10, 0x17, 0x12, 0x21, 0x0a, 0x1d, 0x53, 0x4f, 0x55,
+	0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x4a, 0x46, 0x52, 0x4f, 0x47, 0x5f, 0x41,
+	0x52, 0x54, 0x49, 0x46, 0x41, 0x43, 0x54, 0x4f, 0x52, 0x59, 0x10, 0x18, 0x12, 0x16, 0x0a, 0x12,
+	0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x53, 0x59, 0x53, 0x4c,
+	0x4f, 0x47, 0x10, 0x19, 0x12, 0x27, 0x0a, 0x23, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54,
+	0x59, 0x50, 0x45, 0x5f, 0x50, 0x55, 0x42, 0x4c, 0x49, 0x43, 0x5f, 0x45, 0x56, 0x45, 0x4e, 0x54,
+	0x5f, 0x4d, 0x4f, 0x4e, 0x49, 0x54, 0x4f, 0x52, 0x49, 0x4e, 0x47, 0x10, 0x1a, 0x12, 0x1e, 0x0a,
+	0x1a, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x53, 0x4c, 0x41,
+	0x43, 0x4b, 0x5f, 0x52, 0x45, 0x41, 0x4c, 0x54, 0x49, 0x4d, 0x45, 0x10, 0x1b, 0x12, 0x1c, 0x0a,
+	0x18, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x47, 0x4f, 0x4f,
+	0x47, 0x4c, 0x45, 0x5f, 0x44, 0x52, 0x49, 0x56, 0x45, 0x10, 0x1c, 0x12, 0x1a, 0x0a, 0x16, 0x53,
+	0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x53, 0x48, 0x41, 0x52, 0x45,
+	0x50, 0x4f, 0x49, 0x4e, 0x54, 0x10, 0x1d, 0x12, 0x1c, 0x0a, 0x18, 0x53, 0x4f, 0x55, 0x52, 0x43,
+	0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x47, 0x43, 0x53, 0x5f, 0x55, 0x4e, 0x41, 0x55, 0x54,
+	0x48, 0x45, 0x44, 0x10, 0x1e, 0x12, 0x1b, 0x0a, 0x17, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f,
+	0x54, 0x59, 0x50, 0x45, 0x5f, 0x41, 0x5a, 0x55, 0x52, 0x45, 0x5f, 0x52, 0x45, 0x50, 0x4f, 0x53,
+	0x10, 0x1f, 0x42, 0x3b, 0x5a, 0x39, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d,
+	0x2f, 0x74, 0x72, 0x75, 0x66, 0x66, 0x6c, 0x65, 0x73, 0x65, 0x63, 0x75, 0x72, 0x69, 0x74, 0x79,
+	0x2f, 0x74, 0x72, 0x75, 0x66, 0x66, 0x6c, 0x65, 0x68, 0x6f, 0x67, 0x2f, 0x76, 0x33, 0x2f, 0x70,
+	0x6b, 0x67, 0x2f, 0x70, 0x62, 0x2f, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x73, 0x70, 0x62, 0x62,
+	0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
 }
 
 var (

pkg/pb/sourcespb/sources.pb.validate.go

@@ -1884,6 +1884,8 @@ func (m *GitHub) validate(all bool) error {
 
 	// no validation rules for IncludeIssueComments
 
+	// no validation rules for IncludeGistComments
+
 	switch m.Credential.(type) {
 
 	case *GitHub_GithubApp:

pkg/sources/errors.go

@@ -1,6 +1,7 @@
 package sources
 
 import (
+	"errors"
 	"fmt"
 	"sync"
 )
@@ -32,5 +33,13 @@ func (s *ScanErrors) Count() uint64 {
 }
 
 func (s *ScanErrors) String() string {
+	s.mu.RLock()
+	defer s.mu.RUnlock()
 	return fmt.Sprintf("%v", s.errors)
 }
+
+func (s *ScanErrors) Errors() error {
+	s.mu.RLock()
+	defer s.mu.RUnlock()
+	return errors.Join(s.errors...)
+}

pkg/sources/git/git.go

@@ -12,6 +12,7 @@ import (
 	"runtime"
 	"strconv"
 	"strings"
+	"sync/atomic"
 	"time"
 
 	diskbufferreader "github.com/bill-rich/disk-buffer-reader"
@@ -42,6 +43,10 @@ type Source struct {
 	git      *Git
 	sources.Progress
 	conn        *sourcespb.Git
+	scanOptions *ScanOptions
+	// Kludge to preserve engine.ScanGit functionality which doesn't expect
+	// the scanning to clean up the directory.
+	preserveTempDirs bool
 }
 
 type Git struct {
@@ -56,7 +61,7 @@ type Git struct {
 }
 
 type metrics struct {
-	commitsScanned int
+	commitsScanned uint64
 }
 
 func NewGit(sourceType sourcespb.SourceType, jobID, sourceID int64, sourceName string, verify bool, concurrency int,
@@ -91,6 +96,19 @@ func (s *Source) JobID() int64 {
 	return s.jobId
 }
 
+// WithScanOptions sets the scan options.
+func (s *Source) WithScanOptions(scanOptions *ScanOptions) {
+	s.scanOptions = scanOptions
+}
+
+// WithPreserveTempDirs sets whether to preserve temp directories when scanning
+// the provided list of s.conn.Directories. NOTE: This is *only* for
+// s.conn.Directories, not all temp directories created. This is also a kludge
+// and should be refactored away.
+func (s *Source) WithPreserveTempDirs(preserve bool) {
+	s.preserveTempDirs = preserve
+}
+
 // Init returns an initialized GitHub source.
 func (s *Source) Init(aCtx context.Context, name string, jobId, sourceId int64, verify bool, connection *anypb.Any, concurrency int) error {
 	s.name = name
@@ -134,8 +152,29 @@ func (s *Source) Init(aCtx context.Context, name string, jobId, sourceId int64,
 
 // Chunks emits chunks of bytes over a channel.
 func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) error {
-	// TODO: refactor to remove duplicate code
+	if err := s.scanRepos(ctx, chunksChan); err != nil {
+		return err
+	}
+	if err := s.scanDirs(ctx, chunksChan); err != nil {
+		return err
+	}
+
 	totalRepos := len(s.conn.Repositories) + len(s.conn.Directories)
+	ctx.Logger().V(1).Info("Git source finished scanning", "repo_count", totalRepos)
+	s.SetProgressComplete(
+		totalRepos, totalRepos,
+		fmt.Sprintf("Completed scanning source %s", s.name), "",
+	)
+	return nil
+}
+
+// scanRepos scans the configured repositories in s.conn.Repositories.
+func (s *Source) scanRepos(ctx context.Context, chunksChan chan *sources.Chunk) error {
+	if len(s.conn.Repositories) == 0 {
+		return nil
+	}
+	totalRepos := len(s.conn.Repositories) + len(s.conn.Directories)
+	// TODO: refactor to remove duplicate code
 	switch cred := s.conn.GetCredential().(type) {
 	case *sourcespb.Git_BasicAuth:
 		user := cred.BasicAuth.Username
@@ -152,7 +191,7 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) err
 				if err != nil {
 					return err
 				}
-				return s.git.ScanRepo(ctx, repo, path, NewScanOptions(), chunksChan)
+				return s.git.ScanRepo(ctx, repo, path, s.scanOptions, chunksChan)
 			}(repoURI)
 			if err != nil {
 				ctx.Logger().Info("error scanning repository", "repo", repoURI, "error", err)
@@ -171,7 +210,7 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) err
 				if err != nil {
 					return err
 				}
-				return s.git.ScanRepo(ctx, repo, path, NewScanOptions(), chunksChan)
+				return s.git.ScanRepo(ctx, repo, path, s.scanOptions, chunksChan)
 			}(repoURI)
 			if err != nil {
 				ctx.Logger().Info("error scanning repository", "repo", repoURI, "error", err)
@@ -190,7 +229,7 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) err
 				if err != nil {
 					return err
 				}
-				return s.git.ScanRepo(ctx, repo, path, NewScanOptions(), chunksChan)
+				return s.git.ScanRepo(ctx, repo, path, s.scanOptions, chunksChan)
 			}(repoURI)
 			if err != nil {
 				ctx.Logger().Info("error scanning repository", "repo", repoURI, "error", err)
@@ -200,14 +239,22 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) err
 	default:
 		return errors.New("invalid connection type for git source")
 	}
+	return nil
+}
 
+// scanDirs scans the configured directories in s.conn.Directories.
+func (s *Source) scanDirs(ctx context.Context, chunksChan chan *sources.Chunk) error {
+	totalRepos := len(s.conn.Repositories) + len(s.conn.Directories)
 	for i, gitDir := range s.conn.Directories {
 		s.SetProgressComplete(len(s.conn.Repositories)+i, totalRepos, fmt.Sprintf("Repo: %s", gitDir), "")
 
 		if len(gitDir) == 0 {
 			continue
 		}
-		if !strings.HasSuffix(gitDir, "git") {
+		if strings.HasSuffix(gitDir, "git") {
+			// TODO: Figure out why we skip directories ending in "git".
+			continue
+		}
 		// try paths instead of url
 		repo, err := RepoFromPath(gitDir)
 		if err != nil {
@@ -216,25 +263,18 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) err
 		}
 
 		err = func(repoPath string) error {
-				if strings.HasPrefix(repoPath, filepath.Join(os.TempDir(), "trufflehog")) {
+			if !s.preserveTempDirs && strings.HasPrefix(repoPath, filepath.Join(os.TempDir(), "trufflehog")) {
 				defer os.RemoveAll(repoPath)
 			}
 
-				return s.git.ScanRepo(ctx, repo, repoPath, NewScanOptions(), chunksChan)
+			return s.git.ScanRepo(ctx, repo, repoPath, s.scanOptions, chunksChan)
 		}(gitDir)
 		if err != nil {
 			ctx.Logger().Info("error scanning repository", "repo", gitDir, "error", err)
 			continue
 		}
-		}
 
 	}
-
-	ctx.Logger().V(1).Info("Git source finished scanning", "repo-count", totalRepos)
-	s.SetProgressComplete(
-		totalRepos, totalRepos,
-		fmt.Sprintf("Completed scanning source %s", s.name), "",
-	)
 	return nil
 }
 
@@ -344,8 +384,8 @@ func CloneRepoUsingSSH(ctx context.Context, gitUrl string, args ...string) (stri
 	return CloneRepo(ctx, userInfo, gitUrl, args...)
 }
 
-func (s *Git) CommitsScanned() int {
-	return s.metrics.commitsScanned
+func (s *Git) CommitsScanned() uint64 {
+	return atomic.LoadUint64(&s.metrics.commitsScanned)
 }
 
 func (s *Git) ScanCommits(ctx context.Context, repo *git.Repository, path string, scanOptions *ScanOptions, chunksChan chan *sources.Chunk) error {
@@ -380,7 +420,7 @@ func (s *Git) ScanCommits(ctx context.Context, repo *git.Repository, path string
 			break
 		}
 		depth++
-		s.metrics.commitsScanned++
+		atomic.AddUint64(&s.metrics.commitsScanned, 1)
 		logger.V(5).Info("scanning commit", "commit", commit.Hash)
 		for _, diff := range commit.Diffs {
 			if !scanOptions.Filter.Pass(diff.PathB) {

pkg/sources/github/github.go

@@ -45,8 +45,11 @@ const (
 
 type Source struct {
 	name string
+	// Protects the user and token.
+	userMu      sync.Mutex
 	githubUser  string
 	githubToken string
+
 	sourceID          int64
 	jobID             int64
 	verify            bool
@@ -58,7 +61,10 @@ type Source struct {
 	repoSizes         repoSize
 	totalRepoSize     int // total size in bytes of all repos
 	git               *git.Git
+
+	scanOptMu   sync.Mutex // protects the scanOptions
 	scanOptions *git.ScanOptions
+
 	httpClient      *http.Client
 	log             logr.Logger
 	conn            *sourcespb.GitHub
@@ -66,10 +72,13 @@ type Source struct {
 	resumeInfoMutex sync.Mutex
 	resumeInfoSlice []string
 	apiClient       *github.Client
-	mu                   sync.Mutex
+
+	mu        sync.Mutex // protects the visibility maps
 	publicMap map[string]source_metadatapb.Visibility
+
 	includePRComments    bool
 	includeIssueComments bool
+	includeGistComments  bool
 	sources.Progress
 	sources.CommonSourceUnitUnmarshaller
 }
@@ -78,6 +87,13 @@ func (s *Source) WithScanOptions(scanOptions *git.ScanOptions) {
 	s.scanOptions = scanOptions
 }
 
+func (s *Source) setScanOptions(base, head string) {
+	s.scanOptMu.Lock()
+	defer s.scanOptMu.Unlock()
+	s.scanOptions.BaseHash = base
+	s.scanOptions.HeadHash = head
+}
+
 // Ensure the Source satisfies the interfaces at compile time
 var _ sources.Source = (*Source)(nil)
 var _ sources.SourceUnitUnmarshaller = (*Source)(nil)
@@ -217,6 +233,7 @@ func (s *Source) Init(aCtx context.Context, name string, jobID, sourceID int64,
 
 	s.includeIssueComments = s.conn.IncludeIssueComments
 	s.includePRComments = s.conn.IncludePullRequestComments
+	s.includeGistComments = s.conn.IncludeGistComments
 
 	s.orgsCache = memory.New()
 	for _, org := range s.conn.Organizations {
@@ -554,25 +571,11 @@ func (s *Source) enumerateWithApp(ctx context.Context, apiEndpoint string, app *
 		return nil, errors.New(err)
 	}
 
-	// This client is used for most APIs.
-	itr, err := ghinstallation.New(
-		s.httpClient.Transport,
-		appID,
-		installationID,
-		[]byte(app.PrivateKey))
-	if err != nil {
-		return nil, errors.New(err)
-	}
-	itr.BaseURL = apiEndpoint
-
-	s.httpClient.Transport = itr
-	s.apiClient, err = github.NewEnterpriseClient(apiEndpoint, apiEndpoint, s.httpClient)
-	if err != nil {
-		return nil, errors.New(err)
-	}
-
 	// This client is required to create installation tokens for cloning.
 	// Otherwise, the required JWT is not in the request for the token :/
+	// This client uses the source's original HTTP transport, so make sure
+	// to build it before modifying that transport (such as is done during
+	// the creation of the other API client below).
 	appItr, err := ghinstallation.NewAppsTransport(
 		s.httpClient.Transport,
 		appID,
@@ -590,6 +593,23 @@ func (s *Source) enumerateWithApp(ctx context.Context, apiEndpoint string, app *
 		return nil, errors.New(err)
 	}
 
+	// This client is used for most APIs.
+	itr, err := ghinstallation.New(
+		s.httpClient.Transport,
+		appID,
+		installationID,
+		[]byte(app.PrivateKey))
+	if err != nil {
+		return nil, errors.New(err)
+	}
+	itr.BaseURL = apiEndpoint
+
+	s.httpClient.Transport = itr
+	s.apiClient, err = github.NewEnterpriseClient(apiEndpoint, apiEndpoint, s.httpClient)
+	if err != nil {
+		return nil, errors.New(err)
+	}
+
 	// If no repos were provided, enumerate them.
 	if len(s.repos) == 0 {
 		if err = s.getReposByApp(ctx); err != nil {
@@ -678,6 +698,7 @@ func (s *Source) scan(ctx context.Context, installationClient *github.Client, ch
 			path, repo, err = s.cloneRepo(ctx, repoURL, installationClient)
 			if err != nil {
 				scanErrs.Add(err)
+				return nil
 			}
 
 			defer os.RemoveAll(path)
@@ -686,8 +707,7 @@ func (s *Source) scan(ctx context.Context, installationClient *github.Client, ch
 				return nil
 			}
 
-			s.scanOptions.BaseHash = s.conn.Base
-			s.scanOptions.HeadHash = s.conn.Head
+			s.setScanOptions(s.conn.Base, s.conn.Head)
 
 			repoSize := s.repoSizes.getRepo(repoURL)
 			logger.V(2).Info(fmt.Sprintf("scanning repo %d/%d", i, len(s.repos)), "repo_size", repoSize)
@@ -697,15 +717,16 @@ func (s *Source) scan(ctx context.Context, installationClient *github.Client, ch
 				logger.V(2).Info(fmt.Sprintf("scanned %d/%d repos", scanned, len(s.repos)), "repo_size", repoSize, "duration_seconds", time.Since(start).Seconds())
 			}(now)
 
+			if err = s.git.ScanRepo(ctx, repo, path, s.scanOptions, chunksChan); err != nil {
+				scanErrs.Add(fmt.Errorf("error scanning repo %s: %w", repoURL, err))
+				return nil
+			}
+
 			if err = s.scanComments(ctx, repoURL, chunksChan); err != nil {
 				scanErrs.Add(fmt.Errorf("error scanning comments in repo %s: %w", repoURL, err))
 				return nil
 			}
 
-			if err = s.git.ScanRepo(ctx, repo, path, s.scanOptions, chunksChan); err != nil {
-				scanErrs.Add(fmt.Errorf("error scanning repo %s: %w", repoURL, err))
-				return nil
-			}
 			atomic.AddUint64(&scanned, 1)
 
 			return nil
@@ -966,6 +987,44 @@ func (s *Source) scanComments(ctx context.Context, repoPath string, chunksChan c
 	}
 
 	trimmedURL := removeURLAndSplit(repoURL.String())
+	if repoURL.Host == "gist.github.com" && s.includeGistComments {
+		// GitHub Gist URL.
+		var gistId string
+		if len(trimmedURL) == 2 {
+			// https://gist.github.com/<id>
+			gistId = trimmedURL[1]
+		} else if len(trimmedURL) == 3 {
+			// https://gist.github.com/<owner>/<id>
+			gistId = trimmedURL[2]
+		} else {
+			return fmt.Errorf("failed to parse Gist URL: '%s'", repoURL.String())
+		}
+
+		options := &github.ListOptions{
+			PerPage: defaultPagination,
+			Page:    1,
+		}
+		for {
+			comments, resp, err := s.apiClient.Gists.ListComments(ctx, gistId, options)
+			if s.handleRateLimit(err, resp) {
+				break
+			}
+			if err != nil {
+				return err
+			}
+
+			err = s.chunkGistComments(ctx, repoURL.String(), comments, chunksChan)
+			if err != nil {
+				return err
+			}
+
+			options.Page++
+			if len(comments) < options.PerPage {
+				break
+			}
+		}
+	} else {
+		// Normal repository URL (https://github.com/<owner>/<repo>).
 		owner := trimmedURL[1]
 		repo := trimmedURL[2]
 
@@ -1042,6 +1101,7 @@ func (s *Source) scanComments(ctx context.Context, repoPath string, chunksChan c
 				}
 			}
 		}
+	}
 
 	return nil
 }
@@ -1109,6 +1169,40 @@ func (s *Source) chunkPullRequestComments(ctx context.Context, repo string, comm
 	return nil
 }
 
+func (s *Source) chunkGistComments(ctx context.Context, gistUrl string, comments []*github.GistComment, chunksChan chan *sources.Chunk) error {
+	for _, comment := range comments {
+		// Create chunk and send it to the channel.
+		chunk := &sources.Chunk{
+			SourceName: s.name,
+			SourceID:   s.SourceID(),
+			SourceType: s.Type(),
+			SourceMetadata: &source_metadatapb.MetaData{
+				Data: &source_metadatapb.MetaData_Github{
+					Github: &source_metadatapb.Github{
+						Link:       sanitizer.UTF8(comment.GetURL()),
+						Username:   sanitizer.UTF8(comment.GetUser().GetLogin()),
+						Email:      sanitizer.UTF8(comment.GetUser().GetEmail()),
+						Repository: sanitizer.UTF8(gistUrl),
+						Timestamp:  sanitizer.UTF8(comment.GetCreatedAt().String()),
+						// Fetching this information requires making an additional API call.
+						// We may want to include this in the future.
+						//Visibility: s.visibilityOf(ctx, repoPath),
+					},
+				},
+			},
+			Data:   []byte(sanitizer.UTF8(comment.GetBody())),
+			Verify: s.verify,
+		}
+
+		select {
+		case <-ctx.Done():
+			return ctx.Err()
+		case chunksChan <- chunk:
+		}
+	}
+	return nil
+}
+
 func removeURLAndSplit(url string) []string {
 	trimmedURL := strings.TrimPrefix(url, "https://")
 	trimmedURL = strings.TrimSuffix(trimmedURL, ".git")

pkg/sources/github/github_integration_test.go

@@ -223,6 +223,77 @@ func TestSource_ScanComments(t *testing.T) {
 	}
 }
 
+func TestSource_ScanChunks(t *testing.T) {
+	ctx, cancel := context.WithTimeout(context.Background(), time.Second*30)
+	defer cancel()
+
+	secret, err := common.GetTestSecret(ctx)
+	if err != nil {
+		t.Fatal(fmt.Errorf("failed to access secret: %v", err))
+	}
+
+	// For the personal access token test.
+	githubToken := secret.MustGetField("GITHUB_TOKEN")
+
+	type init struct {
+		name       string
+		verify     bool
+		connection *sourcespb.GitHub
+	}
+	tests := []struct {
+		name       string
+		init       init
+		wantChunks int
+	}{
+		{
+			name: "token authenticated, 4 repos",
+			init: init{
+				name: "test source",
+				connection: &sourcespb.GitHub{
+					Repositories: []string{
+						"https://github.com/truffle-test-integration-org/another-test-repo.git",
+						"https://github.com/trufflesecurity/trufflehog.git",
+						"https://github.com/Akash-goyal-github/Inventory-Management-System.git",
+						"https://github.com/R1ck404/Crypto-Exchange-Example.git",
+						"https://github.com/Stability-AI/generative-models.git",
+						"https://github.com/bloomberg/blazingmq.git",
+						"https://github.com/Kong/kong.git",
+					},
+					Credential: &sourcespb.GitHub_Token{Token: githubToken},
+				},
+			},
+			wantChunks: 20000,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			s := Source{}
+
+			conn, err := anypb.New(tt.init.connection)
+			if err != nil {
+				t.Fatal(err)
+			}
+
+			err = s.Init(ctx, tt.init.name, 0, 0, tt.init.verify, conn, 8)
+			assert.Nil(t, err)
+
+			chunksCh := make(chan *sources.Chunk, 1)
+			go func() {
+				defer close(chunksCh)
+				err = s.Chunks(ctx, chunksCh)
+				assert.Nil(t, err)
+			}()
+
+			i := 0
+			for range chunksCh {
+				i++
+			}
+			assert.GreaterOrEqual(t, i, tt.wantChunks)
+		})
+	}
+}
+
 func TestSource_Scan(t *testing.T) {
 	ctx, cancel := context.WithTimeout(context.Background(), time.Second*300)
 	defer cancel()

pkg/sources/github/repo.go

@@ -49,13 +49,9 @@ func (s *Source) cloneRepo(
 		}
 
 	case *sourcespb.GitHub_Token:
-		// We never refresh user provided tokens, so if we already have them, we never need to try and fetch them again.
-		if s.githubUser == "" || s.githubToken == "" {
-			s.githubUser, s.githubToken, err = s.userAndToken(ctx, installationClient)
-			if err != nil {
+		if err := s.getUserAndToken(ctx, repoURL, installationClient); err != nil {
 			return "", nil, fmt.Errorf("error getting token for repo %s: %w", repoURL, err)
 		}
-		}
 		path, repo, err = git.CloneRepoUsingToken(ctx, s.githubToken, repoURL, s.githubUser)
 		if err != nil {
 			return "", nil, fmt.Errorf("error cloning repo %s: %w", repoURL, err)
@@ -66,6 +62,20 @@ func (s *Source) cloneRepo(
 	return path, repo, nil
 }
 
+func (s *Source) getUserAndToken(ctx context.Context, repoURL string, installationClient *github.Client) error {
+	// We never refresh user provided tokens, so if we already have them, we never need to try and fetch them again.
+	s.userMu.Lock()
+	defer s.userMu.Unlock()
+	if s.githubUser == "" || s.githubToken == "" {
+		var err error
+		s.githubUser, s.githubToken, err = s.userAndToken(ctx, installationClient)
+		if err != nil {
+			return fmt.Errorf("error getting token for repo %s: %w", repoURL, err)
+		}
+	}
+	return nil
+}
+
 func (s *Source) userAndToken(ctx context.Context, installationClient *github.Client) (string, string, error) {
 	switch cred := s.conn.GetCredential().(type) {
 	case *sourcespb.GitHub_BasicAuth:

pkg/sources/job_progress.go

@@ -0,0 +1,272 @@
+package sources
+
+//go:generate mockgen --source=./job_progress.go --destination=mock_job_progress_test.go --package=sources
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"sync"
+	"sync/atomic"
+	"time"
+)
+
+type JobProgressHook interface {
+	// Start and End marks the overall start and end time for this job.
+	Start(JobProgressRef, time.Time)
+	End(JobProgressRef, time.Time)
+	// StartEnumerating and EndEnumerating marks the start and end time for
+	// calling the source's Enumerate method. If the source does not
+	// support enumeration these methods will never be called.
+	StartEnumerating(JobProgressRef, time.Time)
+	EndEnumerating(JobProgressRef, time.Time)
+	// StartUnitChunking and EndUnitChunking marks the start and end time
+	// for calling the source's ChunkUnit method for a given unit. If the
+	// source does not support enumeration these methods will never be
+	// called.
+	StartUnitChunking(JobProgressRef, SourceUnit, time.Time)
+	EndUnitChunking(JobProgressRef, SourceUnit, time.Time)
+	// ReportError is called when any general error is encountered, usually
+	// from enumeration.
+	ReportError(JobProgressRef, error)
+	// ReportUnit is called when a unit has been enumerated. If the source
+	// does not support enumeration this method will never be called.
+	ReportUnit(JobProgressRef, SourceUnit)
+	// ReportChunk is called when a chunk has been produced for the given
+	// unit. The unit will be nil if the source does not support
+	// enumeration.
+	ReportChunk(JobProgressRef, SourceUnit, *Chunk)
+	// Finish marks the job as done.
+	Finish(JobProgressRef)
+}
+
+// JobProgressRef is a wrapper of a JobProgress for read-only access to its state.
+type JobProgressRef struct {
+	SourceID    int64
+	JobID       int64
+	jobProgress *JobProgress
+}
+
+// Snapshot returns a snapshot of the job's current metrics.
+func (r *JobProgressRef) Snapshot() JobProgressMetrics {
+	if r.jobProgress == nil {
+		return JobProgressMetrics{}
+	}
+	return r.jobProgress.Snapshot()
+}
+
+// Done returns a channel that will block until the job has completed.
+func (r *JobProgressRef) Done() <-chan struct{} {
+	if r.jobProgress == nil {
+		// Return a closed channel so it does not block.
+		ch := make(chan struct{})
+		close(ch)
+		return ch
+	}
+	return r.jobProgress.Done()
+}
+
+// Fatal is a wrapper around error to differentiate non-fatal errors from fatal
+// ones. A fatal error is typically from a finished context or any error
+// returned from a source's Init, Chunks, Enumerate, or ChunkUnit methods.
+type Fatal struct{ error }
+
+func (f Fatal) Error() string { return fmt.Sprintf("fatal: %s", f.error.Error()) }
+func (f Fatal) Unwrap() error { return f.error }
+
+// ChunkError is a custom error type for errors encountered during chunking of
+// a specific unit.
+type ChunkError struct {
+	unit SourceUnit
+	err  error
+}
+
+func (f ChunkError) Error() string {
+	return fmt.Sprintf("error chunking unit %q: %s", f.unit.SourceUnitID(), f.err.Error())
+}
+func (f ChunkError) Unwrap() error { return f.err }
+
+// JobProgress aggregates information about a run of a Source.
+type JobProgress struct {
+	// Unique identifiers for this job.
+	SourceID int64
+	JobID    int64
+	// Tracks whether the job is finished or not.
+	ctx    context.Context
+	cancel context.CancelFunc
+	// Metrics.
+	metrics     JobProgressMetrics
+	metricsLock sync.Mutex
+	// Coarse grained hooks for adding extra functionality when events trigger.
+	hooks []JobProgressHook
+}
+
+// JobProgressMetrics tracks the metrics of a job.
+type JobProgressMetrics struct {
+	StartTime       time.Time
+	EndTime         time.Time
+	TotalUnits      uint64
+	FinishedUnits   uint64
+	TotalChunks     uint64
+	Errors          []error
+	DoneEnumerating bool
+}
+
+// WithHooks adds hooks to be called when an event triggers.
+func WithHooks(hooks ...JobProgressHook) func(*JobProgress) {
+	return func(jp *JobProgress) { jp.hooks = append(jp.hooks, hooks...) }
+}
+
+// NewJobProgress creates a new job report for the given source and job ID.
+func NewJobProgress(sourceID, jobID int64, opts ...func(*JobProgress)) *JobProgress {
+	ctx, cancel := context.WithCancel(context.Background())
+	jp := &JobProgress{
+		SourceID: sourceID,
+		JobID:    jobID,
+		ctx:      ctx,
+		cancel:   cancel,
+	}
+	for _, opt := range opts {
+		opt(jp)
+	}
+	return jp
+}
+
+// executeHooks is a helper method to execute all the hooks for the given
+// closure.
+func (jp *JobProgress) executeHooks(todo func(hook JobProgressHook)) {
+	for _, hook := range jp.hooks {
+		// TODO: Non-blocking?
+		todo(hook)
+	}
+}
+
+// TODO: Comment all this mess. They are mostly implementing JobProgressHook but
+// without the JobProgressRef parameter.
+func (jp *JobProgress) Start(start time.Time) {
+	jp.metricsLock.Lock()
+	jp.metrics.StartTime = start
+	jp.metricsLock.Unlock()
+
+	jp.executeHooks(func(hook JobProgressHook) { hook.Start(jp.Ref(), start) })
+}
+func (jp *JobProgress) End(end time.Time) {
+	jp.metricsLock.Lock()
+	jp.metrics.EndTime = end
+	jp.metricsLock.Unlock()
+
+	jp.executeHooks(func(hook JobProgressHook) { hook.End(jp.Ref(), end) })
+}
+func (jp *JobProgress) Finish() {
+	jp.cancel()
+	jp.executeHooks(func(hook JobProgressHook) { hook.Finish(jp.Ref()) })
+}
+func (jp *JobProgress) Done() <-chan struct{} { return jp.ctx.Done() }
+func (jp *JobProgress) ReportUnit(unit SourceUnit) {
+	atomic.AddUint64(&jp.metrics.TotalUnits, 1)
+	jp.executeHooks(func(hook JobProgressHook) { hook.ReportUnit(jp.Ref(), unit) })
+}
+func (jp *JobProgress) ReportChunk(unit SourceUnit, chunk *Chunk) {
+	atomic.AddUint64(&jp.metrics.TotalChunks, 1)
+	jp.executeHooks(func(hook JobProgressHook) { hook.ReportChunk(jp.Ref(), unit, chunk) })
+}
+func (jp *JobProgress) StartUnitChunking(unit SourceUnit, start time.Time) {
+	// TODO: Record time.
+	jp.executeHooks(func(hook JobProgressHook) { hook.StartUnitChunking(jp.Ref(), unit, start) })
+}
+func (jp *JobProgress) EndUnitChunking(unit SourceUnit, end time.Time) {
+	// TODO: Record time.
+	atomic.AddUint64(&jp.metrics.FinishedUnits, 1)
+	jp.executeHooks(func(hook JobProgressHook) { hook.EndUnitChunking(jp.Ref(), unit, end) })
+}
+func (jp *JobProgress) StartEnumerating(start time.Time) {
+	// TODO: Record time.
+	jp.executeHooks(func(hook JobProgressHook) { hook.StartEnumerating(jp.Ref(), start) })
+}
+func (jp *JobProgress) EndEnumerating(end time.Time) {
+	jp.metricsLock.Lock()
+	// TODO: Record time.
+	jp.metrics.DoneEnumerating = true
+	jp.metricsLock.Unlock()
+
+	jp.executeHooks(func(hook JobProgressHook) { hook.EndEnumerating(jp.Ref(), end) })
+}
+
+// Snapshot safely gets the job's current metrics.
+func (jp *JobProgress) Snapshot() JobProgressMetrics {
+	jp.metricsLock.Lock()
+	defer jp.metricsLock.Unlock()
+	return jp.metrics
+}
+
+// ReportError adds a non-nil error to the aggregate of errors
+// encountered during scanning.
+func (jp *JobProgress) ReportError(err error) {
+	if err == nil {
+		return
+	}
+	jp.metricsLock.Lock()
+	jp.metrics.Errors = append(jp.metrics.Errors, err)
+	jp.metricsLock.Unlock()
+
+	jp.executeHooks(func(hook JobProgressHook) { hook.ReportError(jp.Ref(), err) })
+}
+
+// Ref provides a read-only reference to the JobProgress.
+func (jp *JobProgress) Ref() JobProgressRef {
+	return JobProgressRef{
+		SourceID:    jp.SourceID,
+		JobID:       jp.JobID,
+		jobProgress: jp,
+	}
+}
+
+// EnumerationErrors joins all errors encountered during initialization or
+// enumeration.
+func (m JobProgressMetrics) EnumerationError() error {
+	return errors.Join(m.Errors...)
+}
+
+// ChunkErrors joins all errors encountered during chunking.
+func (m JobProgressMetrics) ChunkError() error {
+	var aggregate []error
+	for _, err := range m.Errors {
+		var chunkErr ChunkError
+		if ok := errors.As(err, &chunkErr); ok {
+			aggregate = append(aggregate, err)
+		}
+	}
+	return errors.Join(aggregate...)
+}
+
+// FatalError returns the first Fatal error, if any, encountered in the scan.
+func (m JobProgressMetrics) FatalError() error {
+	for _, err := range m.Errors {
+		var fatalErr Fatal
+		if found := errors.As(err, &fatalErr); found {
+			return fatalErr
+		}
+	}
+	return nil
+}
+
+// FatalErrors returns all of the encountered fatal errors joined together.
+func (m JobProgressMetrics) FatalErrors() error {
+	var aggregate []error
+	for _, err := range m.Errors {
+		var fatalErr Fatal
+		if found := errors.As(err, &fatalErr); found {
+			aggregate = append(aggregate, fatalErr)
+		}
+	}
+	return errors.Join(aggregate...)
+}
+
+func (m JobProgressMetrics) PercentComplete() int {
+	num := m.FinishedUnits
+	den := m.TotalUnits
+	if num == 0 && den == 0 {
+		return 0
+	}
+	return int(num * 100 / den)
+}

pkg/sources/job_progress_test.go

@@ -0,0 +1,117 @@
+package sources
+
+import (
+	"errors"
+	"fmt"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"go.uber.org/mock/gomock"
+)
+
+func TestJobProgressFatalErrors(t *testing.T) {
+	var jp JobProgress
+
+	// Add a non-fatal error.
+	jp.ReportError(fmt.Errorf("oh no"))
+	assert.Greater(t, len(jp.Snapshot().Errors), 0)
+	assert.NoError(t, jp.Snapshot().FatalError())
+	assert.NoError(t, jp.Snapshot().ChunkError())
+
+	// Add a fatal error and make sure we can test comparison.
+	err := fmt.Errorf("fatal error")
+	jp.ReportError(Fatal{err})
+	assert.Greater(t, len(jp.Snapshot().Errors), 0)
+	assert.Error(t, jp.Snapshot().FatalError())
+	assert.NoError(t, jp.Snapshot().ChunkError())
+	assert.True(t, errors.Is(jp.Snapshot().FatalError(), err))
+
+	// Add another fatal error and test we still return the first.
+	jp.ReportError(Fatal{fmt.Errorf("second fatal error")})
+	assert.Greater(t, len(jp.Snapshot().Errors), 0)
+	assert.Error(t, jp.Snapshot().FatalError())
+	assert.NoError(t, jp.Snapshot().ChunkError())
+	assert.True(t, errors.Is(jp.Snapshot().FatalError(), err))
+}
+
+func TestJobProgressRef(t *testing.T) {
+	jp := NewJobProgress(123, 456)
+	ref := jp.Ref()
+	assert.Equal(t, int64(123), ref.SourceID)
+	assert.Equal(t, int64(456), ref.JobID)
+
+	// Test Done() blocks until Finish() is called.
+	select {
+	case <-jp.Done():
+		assert.FailNow(t, "job should not be finished")
+	default:
+	}
+
+	jp.Finish()
+	select {
+	case <-jp.Done():
+	default:
+		assert.FailNow(t, "job should be finished")
+	}
+}
+
+func TestJobProgressHook(t *testing.T) {
+	ctrl := gomock.NewController(t)
+	defer ctrl.Finish()
+
+	hook := NewMockJobProgressHook(ctrl)
+	jp := NewJobProgress(123, 456, WithHooks(hook))
+
+	// Start(JobProgressRef, time.Time)
+	// End(JobProgressRef, time.Time)
+	// StartEnumerating(JobProgressRef, time.Time)
+	// EndEnumerating(JobProgressRef, time.Time)
+	// StartUnitChunking(JobProgressRef, SourceUnit, time.Time)
+	// EndUnitChunking(JobProgressRef, SourceUnit, time.Time)
+	// ReportError(JobProgressRef, error)
+	// ReportUnit(JobProgressRef, SourceUnit)
+	// ReportChunk(JobProgressRef, SourceUnit, *Chunk)
+	// Finish(JobProgressRef)
+
+	startTime := time.Now()
+	endTime := time.Now().Add(10 * time.Second)
+	startEnum := time.Now().Add(20 * time.Second)
+	endEnum := time.Now().Add(30 * time.Second)
+	startChunk := time.Now().Add(40 * time.Second)
+	endChunk := time.Now().Add(50 * time.Second)
+	reportErr := fmt.Errorf("reporting error")
+	reportUnit := CommonSourceUnit{"reporting unit"}
+	reportChunk := &Chunk{Data: []byte("reporting chunk")}
+
+	hook.EXPECT().Start(gomock.Any(), startTime)
+	hook.EXPECT().End(gomock.Any(), endTime)
+	hook.EXPECT().StartEnumerating(gomock.Any(), startEnum)
+	hook.EXPECT().EndEnumerating(gomock.Any(), endEnum)
+	hook.EXPECT().StartUnitChunking(gomock.Any(), reportUnit, startChunk)
+	hook.EXPECT().EndUnitChunking(gomock.Any(), reportUnit, endChunk)
+	hook.EXPECT().ReportError(gomock.Any(), reportErr)
+	hook.EXPECT().ReportUnit(gomock.Any(), reportUnit)
+	hook.EXPECT().ReportChunk(gomock.Any(), reportUnit, reportChunk)
+	hook.EXPECT().Finish(gomock.Any())
+
+	jp.Start(startTime)
+	jp.End(endTime)
+	jp.StartEnumerating(startEnum)
+	jp.EndEnumerating(endEnum)
+	jp.StartUnitChunking(reportUnit, startChunk)
+	jp.EndUnitChunking(reportUnit, endChunk)
+	jp.ReportError(reportErr)
+	jp.ReportUnit(reportUnit)
+	jp.ReportChunk(reportUnit, reportChunk)
+	jp.Finish()
+}
+
+func TestJobProgressDone(t *testing.T) {
+	ref := JobProgressRef{}
+	select {
+	case <-ref.Done():
+	default:
+		assert.FailNow(t, "done should not block for a nil job")
+	}
+}

pkg/sources/job_report.go

@@ -1,37 +0,0 @@
-package sources
-
-import (
-	"errors"
-	"sync"
-	"time"
-)
-
-// JobReport aggregates information about a run of a Source.
-type JobReport struct {
-	SourceID    int64
-	JobID       int64
-	StartTime   time.Time
-	EndTime     time.Time
-	TotalChunks uint64
-	errors      []error
-	errorsLock  sync.Mutex
-}
-
-// AddError adds a non-nil error to the aggregate of errors encountered during
-// scanning.
-func (jr *JobReport) AddError(err error) {
-	if err == nil {
-		return
-	}
-	jr.errorsLock.Lock()
-	defer jr.errorsLock.Unlock()
-	jr.errors = append(jr.errors, err)
-}
-
-// Errors joins all aggregated errors into one. If there were no errors, nil is
-// returned. errors.Is can be used to check for specific errors.
-func (jr *JobReport) Errors() error {
-	jr.errorsLock.Lock()
-	defer jr.errorsLock.Unlock()
-	return errors.Join(jr.errors...)
-}

pkg/sources/mock_job_progress_test.go

@@ -0,0 +1,155 @@
+// Code generated by MockGen. DO NOT EDIT.
+// Source: ./job_progress.go
+
+// Package sources is a generated GoMock package.
+package sources
+
+import (
+	reflect "reflect"
+	time "time"
+
+	gomock "go.uber.org/mock/gomock"
+)
+
+// MockJobProgressHook is a mock of JobProgressHook interface.
+type MockJobProgressHook struct {
+	ctrl     *gomock.Controller
+	recorder *MockJobProgressHookMockRecorder
+}
+
+// MockJobProgressHookMockRecorder is the mock recorder for MockJobProgressHook.
+type MockJobProgressHookMockRecorder struct {
+	mock *MockJobProgressHook
+}
+
+// NewMockJobProgressHook creates a new mock instance.
+func NewMockJobProgressHook(ctrl *gomock.Controller) *MockJobProgressHook {
+	mock := &MockJobProgressHook{ctrl: ctrl}
+	mock.recorder = &MockJobProgressHookMockRecorder{mock}
+	return mock
+}
+
+// EXPECT returns an object that allows the caller to indicate expected use.
+func (m *MockJobProgressHook) EXPECT() *MockJobProgressHookMockRecorder {
+	return m.recorder
+}
+
+// End mocks base method.
+func (m *MockJobProgressHook) End(arg0 JobProgressRef, arg1 time.Time) {
+	m.ctrl.T.Helper()
+	m.ctrl.Call(m, "End", arg0, arg1)
+}
+
+// End indicates an expected call of End.
+func (mr *MockJobProgressHookMockRecorder) End(arg0, arg1 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "End", reflect.TypeOf((*MockJobProgressHook)(nil).End), arg0, arg1)
+}
+
+// EndEnumerating mocks base method.
+func (m *MockJobProgressHook) EndEnumerating(arg0 JobProgressRef, arg1 time.Time) {
+	m.ctrl.T.Helper()
+	m.ctrl.Call(m, "EndEnumerating", arg0, arg1)
+}
+
+// EndEnumerating indicates an expected call of EndEnumerating.
+func (mr *MockJobProgressHookMockRecorder) EndEnumerating(arg0, arg1 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "EndEnumerating", reflect.TypeOf((*MockJobProgressHook)(nil).EndEnumerating), arg0, arg1)
+}
+
+// EndUnitChunking mocks base method.
+func (m *MockJobProgressHook) EndUnitChunking(arg0 JobProgressRef, arg1 SourceUnit, arg2 time.Time) {
+	m.ctrl.T.Helper()
+	m.ctrl.Call(m, "EndUnitChunking", arg0, arg1, arg2)
+}
+
+// EndUnitChunking indicates an expected call of EndUnitChunking.
+func (mr *MockJobProgressHookMockRecorder) EndUnitChunking(arg0, arg1, arg2 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "EndUnitChunking", reflect.TypeOf((*MockJobProgressHook)(nil).EndUnitChunking), arg0, arg1, arg2)
+}
+
+// Finish mocks base method.
+func (m *MockJobProgressHook) Finish(arg0 JobProgressRef) {
+	m.ctrl.T.Helper()
+	m.ctrl.Call(m, "Finish", arg0)
+}
+
+// Finish indicates an expected call of Finish.
+func (mr *MockJobProgressHookMockRecorder) Finish(arg0 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Finish", reflect.TypeOf((*MockJobProgressHook)(nil).Finish), arg0)
+}
+
+// ReportChunk mocks base method.
+func (m *MockJobProgressHook) ReportChunk(arg0 JobProgressRef, arg1 SourceUnit, arg2 *Chunk) {
+	m.ctrl.T.Helper()
+	m.ctrl.Call(m, "ReportChunk", arg0, arg1, arg2)
+}
+
+// ReportChunk indicates an expected call of ReportChunk.
+func (mr *MockJobProgressHookMockRecorder) ReportChunk(arg0, arg1, arg2 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ReportChunk", reflect.TypeOf((*MockJobProgressHook)(nil).ReportChunk), arg0, arg1, arg2)
+}
+
+// ReportError mocks base method.
+func (m *MockJobProgressHook) ReportError(arg0 JobProgressRef, arg1 error) {
+	m.ctrl.T.Helper()
+	m.ctrl.Call(m, "ReportError", arg0, arg1)
+}
+
+// ReportError indicates an expected call of ReportError.
+func (mr *MockJobProgressHookMockRecorder) ReportError(arg0, arg1 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ReportError", reflect.TypeOf((*MockJobProgressHook)(nil).ReportError), arg0, arg1)
+}
+
+// ReportUnit mocks base method.
+func (m *MockJobProgressHook) ReportUnit(arg0 JobProgressRef, arg1 SourceUnit) {
+	m.ctrl.T.Helper()
+	m.ctrl.Call(m, "ReportUnit", arg0, arg1)
+}
+
+// ReportUnit indicates an expected call of ReportUnit.
+func (mr *MockJobProgressHookMockRecorder) ReportUnit(arg0, arg1 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ReportUnit", reflect.TypeOf((*MockJobProgressHook)(nil).ReportUnit), arg0, arg1)
+}
+
+// Start mocks base method.
+func (m *MockJobProgressHook) Start(arg0 JobProgressRef, arg1 time.Time) {
+	m.ctrl.T.Helper()
+	m.ctrl.Call(m, "Start", arg0, arg1)
+}
+
+// Start indicates an expected call of Start.
+func (mr *MockJobProgressHookMockRecorder) Start(arg0, arg1 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Start", reflect.TypeOf((*MockJobProgressHook)(nil).Start), arg0, arg1)
+}
+
+// StartEnumerating mocks base method.
+func (m *MockJobProgressHook) StartEnumerating(arg0 JobProgressRef, arg1 time.Time) {
+	m.ctrl.T.Helper()
+	m.ctrl.Call(m, "StartEnumerating", arg0, arg1)
+}
+
+// StartEnumerating indicates an expected call of StartEnumerating.
+func (mr *MockJobProgressHookMockRecorder) StartEnumerating(arg0, arg1 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "StartEnumerating", reflect.TypeOf((*MockJobProgressHook)(nil).StartEnumerating), arg0, arg1)
+}
+
+// StartUnitChunking mocks base method.
+func (m *MockJobProgressHook) StartUnitChunking(arg0 JobProgressRef, arg1 SourceUnit, arg2 time.Time) {
+	m.ctrl.T.Helper()
+	m.ctrl.Call(m, "StartUnitChunking", arg0, arg1, arg2)
+}
+
+// StartUnitChunking indicates an expected call of StartUnitChunking.
+func (mr *MockJobProgressHookMockRecorder) StartUnitChunking(arg0, arg1, arg2 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "StartUnitChunking", reflect.TypeOf((*MockJobProgressHook)(nil).StartUnitChunking), arg0, arg1, arg2)
+}

pkg/sources/source_manager.go

@@ -1,7 +1,6 @@
 package sources
 
 import (
-	"errors"
 	"fmt"
 	"sync"
 	"sync/atomic"
@@ -24,13 +23,10 @@ type SourceInitFunc func(ctx context.Context, sourceID int64, jobID int64) (Sour
 
 type SourceManager struct {
 	api   apiClient
+	hooks []JobProgressHook
 	// Map of handle to source initializer.
 	handles     map[handle]SourceInitFunc
 	handlesLock sync.Mutex
-	// Map of handle to job reports.
-	// TODO: Manage culling and flushing to the API.
-	report     map[handle]*JobReport
-	reportLock sync.Mutex
 	// Pool limiting the amount of concurrent sources running.
 	pool            errgroup.Group
 	concurrentUnits int
@@ -40,7 +36,6 @@ type SourceManager struct {
 	outputChunks chan *Chunk
 	// Set when Wait() returns.
 	done bool
-	doneErr error
 }
 
 // apiClient is an interface for optionally communicating with an external API.
@@ -51,11 +46,18 @@ type apiClient interface {
 	GetJobID(ctx context.Context, id int64) (int64, error)
 }
 
-// WithAPI adds an API client to the manager for tracking jobs and progress.
+// WithAPI adds an API client to the manager for tracking jobs and progress. If
+// the API is also a JobProgressHook, it will be added to the list of event hooks.
 func WithAPI(api apiClient) func(*SourceManager) {
 	return func(mgr *SourceManager) { mgr.api = api }
 }
 
+func WithReportHook(hook JobProgressHook) func(*SourceManager) {
+	return func(mgr *SourceManager) {
+		mgr.hooks = append(mgr.hooks, hook)
+	}
+}
+
 // WithConcurrency limits the concurrent number of sources a manager can run.
 func WithConcurrency(concurrency int) func(*SourceManager) {
 	return func(mgr *SourceManager) { mgr.pool.SetLimit(concurrency) }
@@ -84,7 +86,6 @@ func NewManager(opts ...func(*SourceManager)) *SourceManager {
 		// Default to the headless API. Can be overwritten by the WithAPI option.
 		api:          &headlessAPI{},
 		handles:      make(map[handle]SourceInitFunc),
-		report:       make(map[handle]*JobReport),
 		outputChunks: make(chan *Chunk),
 	}
 	for _, opt := range opts {
@@ -114,52 +115,44 @@ func (s *SourceManager) Enroll(ctx context.Context, name string, kind sourcespb.
 }
 
 // Run blocks until a resource is available to run the source, then
-// synchronously runs it.
-func (s *SourceManager) Run(ctx context.Context, handle handle) error {
-	// Do preflight checks before waiting on the pool.
-	if err := s.preflightChecks(ctx, handle); err != nil {
-		return err
-	}
-	ch := make(chan error)
-	s.pool.Go(func() error {
-		defer common.Recover(ctx)
-		report, err := s.run(ctx, handle)
-		if report != nil {
-			s.reportLock.Lock()
-			s.report[handle] = report
-			s.reportLock.Unlock()
-		}
+// synchronously runs it. The first fatal error, if any, will be returned.
+func (s *SourceManager) Run(ctx context.Context, handle handle) (JobProgressRef, error) {
+	report, err := s.asyncRun(ctx, handle)
 	if err != nil {
-			ch <- err
-			return nil
+		return report, err
 	}
-		ch <- report.Errors()
-		return nil
-	})
-	return <-ch
+	<-report.Done()
+	return report, report.Snapshot().FatalError()
 }
 
 // ScheduleRun blocks until a resource is available to run the source, then
-// asynchronously runs it. Error information is stored and returned by Wait().
-func (s *SourceManager) ScheduleRun(ctx context.Context, handle handle) error {
+// asynchronously runs it. Error information is stored and accessible via the
+// JobProgressRef as it becomes available.
+func (s *SourceManager) ScheduleRun(ctx context.Context, handle handle) (JobProgressRef, error) {
+	return s.asyncRun(ctx, handle)
+}
+
+// asyncRun is a helper method to asynchronously run the Source. It calls out
+// to the API to get a job ID for this run, creates a report, then waits for an
+// available goroutine to asynchronously run it.
+func (s *SourceManager) asyncRun(ctx context.Context, handle handle) (JobProgressRef, error) {
 	// Do preflight checks before waiting on the pool.
 	if err := s.preflightChecks(ctx, handle); err != nil {
-		return err
+		return JobProgressRef{}, err
 	}
+	// Get a Job ID.
+	jobID, err := s.api.GetJobID(ctx, int64(handle))
+	if err != nil {
+		return JobProgressRef{SourceID: int64(handle)}, err
+	}
+	// Start a report for this job.
+	report := NewJobProgress(int64(handle), jobID, WithHooks(s.hooks...))
 	s.pool.Go(func() error {
 		defer common.Recover(ctx)
-		// The error is already saved in the report, so we can ignore
-		// it here.
-		report, _ := s.run(ctx, handle)
-		if report != nil {
-			s.reportLock.Lock()
-			s.report[handle] = report
-			s.reportLock.Unlock()
-		}
+		_ = s.run(ctx, handle, jobID, report)
 		return nil
 	})
-	// TODO: Maybe wait for a signal here that initialization was successful?
-	return nil
+	return report.Ref(), nil
 }
 
 // Chunks returns the read only channel of all the chunks produced by all of
@@ -168,13 +161,14 @@ func (s *SourceManager) Chunks() <-chan *Chunk {
 	return s.outputChunks
 }
 
-// Wait blocks until all running sources are completed and returns an error if
-// any of the sources had fatal errors. It also closes the channel returned by
-// Chunks(). The manager should not be reused after calling this method.
-func (s *SourceManager) Wait() error {
+// Wait blocks until all running sources are completed and closes the channel
+// returned by Chunks(). The manager should not be reused after calling this
+// method. This current implementation is not thread safe and should only be
+// called by one thread.
+func (s *SourceManager) Wait() {
 	// Check if the manager has been Waited.
 	if s.done {
-		return s.doneErr
+		return
 	}
 	defer close(s.outputChunks)
 	defer func() { s.done = true }()
@@ -182,26 +176,6 @@ func (s *SourceManager) Wait() error {
 	// We are only using the errgroup for limiting concurrency.
 	// TODO: Maybe switch to using a semaphore.Weighted.
 	_ = s.pool.Wait()
-
-	// Aggregate all errors from all job reports.
-	// TODO: This should probably only be the fatal errors. We'll also need
-	//       to rewrite this for when the reports start getting culled.
-	s.reportLock.Lock()
-	defer s.reportLock.Unlock()
-	errs := make([]error, 0, len(s.report))
-	for _, report := range s.report {
-		errs = append(errs, report.Errors())
-	}
-	s.doneErr = errors.Join(errs...)
-	return s.doneErr
-}
-
-// Report retrieves a scan report for a given handle. If no report exists or
-// the Source has not finished, nil will be returned.
-func (s *SourceManager) Report(handle handle) *JobReport {
-	s.reportLock.Lock()
-	defer s.reportLock.Unlock()
-	return s.report[handle]
 }
 
 // preflightChecks is a helper method to check the Manager or the context isn't
@@ -219,40 +193,25 @@ func (s *SourceManager) preflightChecks(ctx context.Context, handle handle) erro
 }
 
 // run is a helper method to sychronously run the source. It does not check for
-// acquired resources. Possible return values are:
-//
-//   - *JobReport, nil
-//     Successfully ran the source, but the report could have errors.
-//
-//   - *JobReport, error
-//     There was an error calling Init or Chunks. This sort of error indicates
-//     a fatal error and is also recorded in the report.
-//
-//   - nil, error:
-//     There was an error from the API or the handle is invalid. The latter of
-//     which should never happen due to the preflightChecks.
-func (s *SourceManager) run(ctx context.Context, handle handle) (*JobReport, error) {
-	jobID, err := s.api.GetJobID(ctx, int64(handle))
-	if err != nil {
-		return nil, err
-	}
-	initFunc, ok := s.getInitFunc(handle)
-	if !ok {
-		return nil, fmt.Errorf("unrecognized handle")
-	}
-	// Create a report for this run.
-	report := &JobReport{
-		SourceID:  int64(handle),
-		JobID:     jobID,
-		StartTime: time.Now(),
-	}
-	defer func() { report.EndTime = time.Now() }()
+// acquired resources. An error is returned if there was a fatal error during
+// the run. This information is also recorded in the JobProgress.
+func (s *SourceManager) run(ctx context.Context, handle handle, jobID int64, report *JobProgress) error {
+	defer report.Finish()
+	report.Start(time.Now())
+	defer func() { report.End(time.Now()) }()
 
 	// Initialize the source.
-	source, err := initFunc(ctx, jobID, int64(handle))
+	initFunc, ok := s.getInitFunc(handle)
+	if !ok {
+		// Shouldn't happen due to preflight checks.
+		err := fmt.Errorf("unrecognized handle")
+		report.ReportError(Fatal{err})
+		return Fatal{err}
+	}
+	source, err := initFunc(ctx, int64(handle), jobID)
 	if err != nil {
-		report.AddError(err)
-		return report, err
+		report.ReportError(Fatal{err})
+		return Fatal{err}
 	}
 	// Check for the preferred method of tracking source units.
 	if enumChunker, ok := source.(SourceUnitEnumChunker); ok && s.useSourceUnits {
@@ -263,7 +222,7 @@ func (s *SourceManager) run(ctx context.Context, handle handle) (*JobReport, err
 
 // runWithoutUnits is a helper method to run a Source. It has coarse-grained
 // job reporting.
-func (s *SourceManager) runWithoutUnits(ctx context.Context, handle handle, source Source, report *JobReport) (*JobReport, error) {
+func (s *SourceManager) runWithoutUnits(ctx context.Context, handle handle, source Source, report *JobProgress) error {
 	// Introspect on the chunks we get from the Chunks method.
 	ch := make(chan *Chunk)
 	var wg sync.WaitGroup
@@ -272,7 +231,7 @@ func (s *SourceManager) runWithoutUnits(ctx context.Context, handle handle, sour
 	go func() {
 		defer wg.Done()
 		for chunk := range ch {
-			atomic.AddUint64(&report.TotalChunks, 1)
+			report.ReportChunk(nil, chunk)
 			_ = common.CancellableWrite(ctx, s.outputChunks, chunk)
 		}
 	}()
@@ -283,25 +242,38 @@ func (s *SourceManager) runWithoutUnits(ctx context.Context, handle handle, sour
 	defer wg.Wait()
 	defer close(ch)
 	if err := source.Chunks(ctx, ch); err != nil {
-		report.AddError(err)
-		return report, err
+		report.ReportError(Fatal{err})
+		return Fatal{err}
 	}
-	return report, nil
+	return nil
 }
 
 // runWithUnits is a helper method to run a Source that is also a
 // SourceUnitEnumChunker. This allows better introspection of what is getting
 // scanned and any errors encountered.
-func (s *SourceManager) runWithUnits(ctx context.Context, handle handle, source SourceUnitEnumChunker, report *JobReport) (*JobReport, error) {
-	reporter := &mgrUnitReporter{
+func (s *SourceManager) runWithUnits(ctx context.Context, handle handle, source SourceUnitEnumChunker, report *JobProgress) error {
+	unitReporter := &mgrUnitReporter{
 		unitCh: make(chan SourceUnit),
+		report: report,
+	}
+	// Create a function that will save the first error encountered (if
+	// any) and discard the rest.
+	fatalErr := make(chan error, 1)
+	catchFirstFatal := func(err error) {
+		select {
+		case fatalErr <- err:
+		default:
+		}
 	}
 	// Produce units.
 	go func() {
 		// TODO: Catch panics and add to report.
-		defer close(reporter.unitCh)
-		if err := source.Enumerate(ctx, reporter); err != nil {
-			report.AddError(err)
+		report.StartEnumerating(time.Now())
+		defer func() { report.EndEnumerating(time.Now()) }()
+		defer close(unitReporter.unitCh)
+		if err := source.Enumerate(ctx, unitReporter); err != nil {
+			report.ReportError(Fatal{err})
+			catchFirstFatal(Fatal{err})
 		}
 	}()
 	var wg sync.WaitGroup
@@ -311,18 +283,21 @@ func (s *SourceManager) runWithUnits(ctx context.Context, handle handle, source
 		// Negative values indicated no limit.
 		unitPool.SetLimit(s.concurrentUnits)
 	}
-	for unit := range reporter.unitCh {
-		reporter := &mgrChunkReporter{
-			unitID:  unit.SourceUnitID(),
-			chunkCh: make(chan *Chunk),
-		}
+	for unit := range unitReporter.unitCh {
 		unit := unit
+		chunkReporter := &mgrChunkReporter{
+			unit:    unit,
+			chunkCh: make(chan *Chunk),
+			report:  report,
+		}
 		// Consume units and produce chunks.
 		unitPool.Go(func() error {
+			report.StartUnitChunking(unit, time.Now())
 			// TODO: Catch panics and add to report.
-			defer close(reporter.chunkCh)
-			if err := source.ChunkUnit(ctx, unit, reporter); err != nil {
-				report.AddError(err)
+			defer close(chunkReporter.chunkCh)
+			if err := source.ChunkUnit(ctx, unit, chunkReporter); err != nil {
+				report.ReportError(Fatal{err})
+				catchFirstFatal(Fatal{err})
 			}
 			return nil
 		})
@@ -330,16 +305,20 @@ func (s *SourceManager) runWithUnits(ctx context.Context, handle handle, source
 		wg.Add(1)
 		go func() {
 			defer wg.Done()
-			for chunk := range reporter.chunkCh {
-				// TODO: Introspect on the chunks we got from this unit.
-				atomic.AddUint64(&report.TotalChunks, 1)
+			defer func() { report.EndUnitChunking(unit, time.Now()) }()
+			for chunk := range chunkReporter.chunkCh {
+				report.ReportChunk(chunkReporter.unit, chunk)
 				_ = common.CancellableWrite(ctx, s.outputChunks, chunk)
 			}
 		}()
 	}
 	wg.Wait()
-	// TODO: Return fatal errors.
-	return report, nil
+	select {
+	case err := <-fatalErr:
+		return err
+	default:
+		return nil
+	}
 }
 
 // getInitFunc is a helper method for safe concurrent access to the
@@ -369,8 +348,7 @@ func (api *headlessAPI) GetJobID(ctx context.Context, id int64) (int64, error) {
 // mgrUnitReporter implements the UnitReporter interface.
 type mgrUnitReporter struct {
 	unitCh chan SourceUnit
-	unitErrs     []error
-	unitErrsLock sync.Mutex
+	report *JobProgress
 }
 
 func (s *mgrUnitReporter) UnitOk(ctx context.Context, unit SourceUnit) error {
@@ -378,18 +356,15 @@ func (s *mgrUnitReporter) UnitOk(ctx context.Context, unit SourceUnit) error {
 }
 
 func (s *mgrUnitReporter) UnitErr(ctx context.Context, err error) error {
-	s.unitErrsLock.Lock()
-	defer s.unitErrsLock.Unlock()
-	s.unitErrs = append(s.unitErrs, err)
+	s.report.ReportError(err)
 	return nil
 }
 
 // mgrChunkReporter implements the ChunkReporter interface.
 type mgrChunkReporter struct {
-	unitID        string
+	unit    SourceUnit
 	chunkCh chan *Chunk
-	chunkErrs     []error
-	chunkErrsLock sync.Mutex
+	report  *JobProgress
 }
 
 func (s *mgrChunkReporter) ChunkOk(ctx context.Context, chunk Chunk) error {
@@ -397,8 +372,6 @@ func (s *mgrChunkReporter) ChunkOk(ctx context.Context, chunk Chunk) error {
 }
 
 func (s *mgrChunkReporter) ChunkErr(ctx context.Context, err error) error {
-	s.chunkErrsLock.Lock()
-	defer s.chunkErrsLock.Unlock()
-	s.chunkErrs = append(s.chunkErrs, err)
+	s.report.ReportError(ChunkError{s.unit, err})
 	return nil
 }

pkg/sources/source_manager_test.go

@@ -102,7 +102,7 @@ func TestSourceManagerRun(t *testing.T) {
 		t.Fatalf("unexpected error enrolling source: %v", err)
 	}
 	for i := 0; i < 3; i++ {
-		if err := mgr.Run(context.Background(), handle); err != nil {
+		if _, err := mgr.Run(context.Background(), handle); err != nil {
 			t.Fatalf("unexpected error running source: %v", err)
 		}
 		chunk, err := tryRead(mgr.Chunks())
@@ -127,20 +127,18 @@ func TestSourceManagerWait(t *testing.T) {
 		t.Fatalf("unexpected error enrolling source: %v", err)
 	}
 	// Asynchronously run the source.
-	if err := mgr.ScheduleRun(context.Background(), handle); err != nil {
+	if _, err := mgr.ScheduleRun(context.Background(), handle); err != nil {
 		t.Fatalf("unexpected error scheduling run: %v", err)
 	}
 	// Read the 1 chunk we're expecting so Waiting completes.
 	<-mgr.Chunks()
 	// Wait for all resources to complete.
-	if err := mgr.Wait(); err != nil {
-		t.Fatalf("unexpected error waiting: %v", err)
-	}
+	mgr.Wait()
 	// Enroll and run should return an error now.
 	if _, err := enrollDummy(mgr, &counterChunker{count: 1}); err == nil {
 		t.Fatalf("expected enroll to fail")
 	}
-	if err := mgr.ScheduleRun(context.Background(), handle); err == nil {
+	if _, err := mgr.ScheduleRun(context.Background(), handle); err == nil {
 		t.Fatalf("expected scheduling run to fail")
 	}
 }
@@ -152,15 +150,17 @@ func TestSourceManagerError(t *testing.T) {
 		t.Fatalf("unexpected error enrolling source: %v", err)
 	}
 	// A synchronous run should fail.
-	if err := mgr.Run(context.Background(), handle); err == nil {
+	if _, err := mgr.Run(context.Background(), handle); err == nil {
 		t.Fatalf("expected run to fail")
 	}
 	// Scheduling a run should not fail, but the error should surface in
 	// Wait().
-	if err := mgr.ScheduleRun(context.Background(), handle); err != nil {
+	ref, err := mgr.ScheduleRun(context.Background(), handle)
+	if err != nil {
 		t.Fatalf("unexpected error scheduling run: %v", err)
 	}
-	if err := mgr.Wait(); err == nil {
+	<-ref.Done()
+	if err := ref.Snapshot().FatalError(); err == nil {
 		t.Fatalf("expected wait to fail")
 	}
 }
@@ -177,18 +177,15 @@ func TestSourceManagerReport(t *testing.T) {
 			t.Fatalf("unexpected error enrolling source: %v", err)
 		}
 		// Synchronously run the source.
-		if err := mgr.Run(context.Background(), handle); err != nil {
+		ref, err := mgr.Run(context.Background(), handle)
+		if err != nil {
 			t.Fatalf("unexpected error running source: %v", err)
 		}
-		report := mgr.Report(handle)
-		if report == nil {
-			t.Fatalf("expected a report")
+		if errs := ref.Snapshot().Errors; len(errs) > 0 {
+			t.Fatalf("unexpected errors in report: %v", errs)
 		}
-		if err := report.Errors(); err != nil {
-			t.Fatalf("unexpected error in report: %v", err)
-		}
-		if report.TotalChunks != 4 {
-			t.Fatalf("expected report to have 4 chunks, got: %d", report.TotalChunks)
+		if count := ref.Snapshot().TotalChunks; count != 4 {
+			t.Fatalf("expected report to have 4 chunks, got: %d", count)
 		}
 	}
 }

proto/sources.proto

@@ -185,6 +185,7 @@ message GitHub {
   repeated string includeRepos = 12;
   bool includePullRequestComments = 14;
   bool includeIssueComments = 15;
+  bool includeGistComments = 16;
 }
 
 message GoogleDrive {