Add panic test case to fix to gitparse

* Initial implementation of JobReport with SourceManager usage

* Limit concurrent units

* Only save the last JobReport per handle

pkg/gitparse/gitparse.go

@@ -56,6 +56,7 @@ type ParseState int
 const (
 	Initial ParseState = iota
 	CommitLine
+	MergeLine
 	AuthorLine
 	DateLine
 	MessageStartLine
@@ -76,6 +77,7 @@ func (state ParseState) String() string {
 	return [...]string{
 		"Initial",
 		"CommitLine",
+		"MergeLine",
 		"AuthorLine",
 		"DateLine",
 		"MessageStartLine",
@@ -276,6 +278,8 @@ func (c *Parser) FromReader(ctx context.Context, stdOut io.Reader, commitChan ch
 			if len(line) >= 47 {
 				currentCommit.Hash = string(line[7:47])
 			}
+		case isMergeLine(isStaged, latestState, line):
+			latestState = MergeLine
 		case isAuthorLine(isStaged, latestState, line):
 			latestState = AuthorLine
 
@@ -428,6 +432,16 @@ func (c *Parser) FromReader(ctx context.Context, stdOut io.Reader, commitChan ch
 	ctx.Logger().V(2).Info("finished parsing git log.", "total_log_size", totalLogSize)
 }
 
+func isMergeLine(isStaged bool, latestState ParseState, line []byte) bool {
+	if isStaged || latestState != CommitLine {
+		return false
+	}
+	if len(line) > 6 && bytes.Equal(line[:6], []byte("Merge:")) {
+		return true
+	}
+	return false
+}
+
 // commit 7a95bbf0199e280a0e42dbb1d1a3f56cdd0f6e05
 func isCommitLine(isStaged bool, latestState ParseState, line []byte) bool {
 	if isStaged || !(latestState == Initial ||
@@ -450,7 +464,7 @@ func isCommitLine(isStaged bool, latestState ParseState, line []byte) bool {
 
 // Author: Bill Rich <bill.rich@trufflesec.com>
 func isAuthorLine(isStaged bool, latestState ParseState, line []byte) bool {
-	if isStaged || !(latestState == CommitLine) {
+	if isStaged || !(latestState == CommitLine || latestState == MergeLine) {
 		return false
 	}
 	if len(line) > 8 && bytes.Equal(line[:7], []byte("Author:")) {

pkg/gitparse/gitparse_test.go

@@ -2,11 +2,20 @@ package gitparse
 
 import (
 	"bytes"
+	_ "embed"
+	"io"
+	"os"
 	"strings"
 	"testing"
 	"time"
 
 	"github.com/trufflesecurity/trufflehog/v3/pkg/context"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/log"
+)
+
+var (
+	//go:embed testdata/panic_diff_1.txt
+	panicDiff1 []byte
 )
 
 type testCaseLine struct {
@@ -20,6 +29,20 @@ type testCase struct {
 	function func(bool, ParseState, []byte) bool
 }
 
+func TestParsePanicDiff(t *testing.T) {
+	// setup logger
+	logger, sync := log.New("forager/scanner",
+		log.WithConsoleSink(os.Stderr),
+	)
+	defer func() { _ = sync() }()
+	context.SetDefaultLogger(logger)
+
+	ctx := context.Background()
+	parser := NewParser()
+	commitCh := make(chan Commit, 64)
+	parser.FromReader(ctx, io.NopCloser(bytes.NewReader(panicDiff1)), commitCh, false)
+}
+
 func TestLineChecks(t *testing.T) {
 	tests := map[string]testCase{
 		"commitLine": {
@@ -67,6 +90,25 @@ func TestLineChecks(t *testing.T) {
 			},
 			function: isCommitLine,
 		},
+		"mergeLine": {
+			passes: []testCaseLine{
+				{
+					CommitLine,
+					[]byte("Merge: f21a95535a2 ed08d10bcf5"),
+				},
+			},
+			fails: []testCaseLine{
+				{
+					DateLine,
+					[]byte("    Merge pull request #34511 from cescoffier/duplicated-context-doc"),
+				},
+				{
+					CommitLine,
+					[]byte("notcorrect"),
+				},
+			},
+			function: isMergeLine,
+		},
 		"authorLine": {
 			passes: []testCaseLine{
 				{

pkg/gitparse/testdata/panic_diff_1.txt

@@ -0,0 +1,413 @@
+diff --git a/server/src/main/java/org/opensearch/common/lucene/search/MultiPhrasePrefixQuery.java b/server/src/main/java/org/opensearch/common/lucene/search/MultiPhrasePrefixQuery.java
+index 98ebb34fc040..cc0468efb243 100644
+--- a/server/src/main/java/org/opensearch/common/lucene/search/MultiPhrasePrefixQuery.java
++++ b/server/src/main/java/org/opensearch/common/lucene/search/MultiPhrasePrefixQuery.java
+@@ -39,6 +39,7 @@
+ import org.apache.lucene.index.TermsEnum;
+ import org.apache.lucene.search.BooleanClause;
+ import org.apache.lucene.search.BooleanQuery;
++import org.apache.lucene.search.IndexSearcher;
+ import org.apache.lucene.search.MatchNoDocsQuery;
+ import org.apache.lucene.search.MultiPhraseQuery;
+ import org.apache.lucene.search.Query;
+@@ -159,8 +160,8 @@ public int[] getPositions() {
+     }
+ 
+     @Override
+-    public Query rewrite(IndexReader reader) throws IOException {
+-        Query rewritten = super.rewrite(reader);
++    public Query rewrite(IndexSearcher searcher) throws IOException {
++        Query rewritten = super.rewrite(searcher);
+         if (rewritten != this) {
+             return rewritten;
+         }
+@@ -177,7 +178,7 @@ public Query rewrite(IndexReader reader) throws IOException {
+         int position = positions.get(sizeMinus1);
+         Set<Term> terms = new HashSet<>();
+         for (Term term : suffixTerms) {
+-            getPrefixTerms(terms, term, reader);
++            getPrefixTerms(terms, term, searcher.getIndexReader());
+             if (terms.size() > maxExpansions) {
+                 break;
+             }
+diff --git a/server/src/main/java/org/opensearch/common/lucene/search/function/FunctionScoreQuery.java b/server/src/main/java/org/opensearch/common/lucene/search/function/FunctionScoreQuery.java
+index ebd155b8a567..db92d4f1cba9 100644
+--- a/server/src/main/java/org/opensearch/common/lucene/search/function/FunctionScoreQuery.java
++++ b/server/src/main/java/org/opensearch/common/lucene/search/function/FunctionScoreQuery.java
+@@ -128,7 +128,7 @@ protected int doHashCode() {
+ 
+         @Override
+         protected ScoreFunction rewrite(IndexReader reader) throws IOException {
+-            Query newFilter = filter.rewrite(reader);
++            Query newFilter = filter.rewrite(new IndexSearcher(reader));
+             if (newFilter == filter) {
+                 return this;
+             }
+@@ -322,16 +322,16 @@ public void visit(QueryVisitor visitor) {
+     }
+ 
+     @Override
+-    public Query rewrite(IndexReader reader) throws IOException {
+-        Query rewritten = super.rewrite(reader);
++    public Query rewrite(IndexSearcher searcher) throws IOException {
++        Query rewritten = super.rewrite(searcher);
+         if (rewritten != this) {
+             return rewritten;
+         }
+-        Query newQ = subQ2023-07-28T08:38:41-07:00     info-0  forager/scanner loaded decoders {"count": 3}
+uery.rewrite(reader);
++        Query newQ = subQuery.rewrite(searcher);
+         ScoreFunction[] newFunctions = new ScoreFunction[functions.length];
+         boolean needsRewrite = (newQ != subQuery);
+         for (int i = 0; i < functions.length; i++) {
+-            newFunctions[i] = functions[i].rewrite(reader);
++            newFunctions[i] = functions[i].rewrite(searcher.getIndexReader());
+             needsRewrite |= (newFunctions[i] != functions[i]);
+         }
+         if (needsRewrite) {
+diff --git a/server/src/main/java/org/opensearch/common/lucene/search/function/ScriptScoreQuery.java b/server/src/main/java/org/opensearch/common/lucene/search/function/ScriptScoreQuery.java
+index 8bf5fc0f89d3..07084a6f690e 100644
+--- a/server/src/main/java/org/opensearch/common/lucene/search/function/ScriptScoreQuery.java
++++ b/server/src/main/java/org/opensearch/common/lucene/search/function/ScriptScoreQuery.java
+@@ -32,7 +32,6 @@
+ 
+ package org.opensearch.common.lucene.search.function;
+ 
+-import org.apache.lucene.index.IndexReader;
+ import org.apache.lucene.index.LeafReaderContext;
+ import org.apache.lucene.search.BooleanClause;
+ import org.apache.lucene.search.DocIdSetIterator;
+@@ -105,12 +104,12 @@ public ScriptScoreQuery(
+     }
+ 
+     @Override
+-    public Query rewrite(IndexReader reader) throws IOException {
+-        Query newQ = subQuery.rewrite(reader);
++    public Query rewrite(IndexSearcher searcher) throws IOException {
++        Query newQ = subQuery.rewrite(searcher);
+         if (newQ != subQuery) {
+             return new ScriptScoreQuery(newQ, queryName, script, scriptBuilder, minScore, indexName, shardId, indexVersion);
+         }
+-        return super.rewrite(reader);
++        return super.rewrite(searcher);
+     }
+ 
+     @Override
+diff --git a/server/src/main/java/org/opensearch/index/document/SortedUnsignedLongDocValuesRangeQuery.java b/server/src/main/java/org/opensearch/index/document/SortedUnsignedLongDocValuesRangeQuery.java
+index 04dd669d6088..234c67cc637f 100644
+--- a/server/src/main/java/org/opensearch/index/document/SortedUnsignedLongDocValuesRangeQuery.java
++++ b/server/src/main/java/org/opensearch/index/document/SortedUnsignedLongDocValuesRangeQuery.java
+@@ -10,7 +10,6 @@
+ 
+ import org.apache.lucene.index.DocValues;
+ import org.apache.lucene.index.FieldInfo;
+-import org.apache.lucene.index.IndexReader;
+ import org.apache.lucene.index.LeafReader;
+ import org.apache.lucene.index.LeafReaderContext;
+ import org.apache.lucene.index.NumericDocValues;
+@@ -87,12 +86,12 @@ public String toString(String field) {
+     }
+ 
+     @Override
+-    public Query rewrite(IndexReader reader) throws IOException {
++    public Query rewrite(IndexSearcher searcher) throws IOException {
+         if (Long.compareUnsigned(lowerValue, Numbers.MIN_UNSIGNED_LONG_VALUE_AS_LONG) == 0
+             && Long.compareUnsigned(upperValue, Numbers.MAX_UNSIGNED_LONG_VALUE_AS_LONG) == 0) {
+             return new FieldExistsQuery(field);
+         }
+-        return super.rewrite(reader);
++        return super.rewrite(searcher);
+     }
+ 
+     abstract SortedNumericDocValues getValues(LeafReader reader, String field) throws IOException;
+diff --git a/server/src/main/java/org/opensearch/index/query/DateRangeIncludingNowQuery.java b/server/src/main/java/org/opensearch/index/query/DateRangeIncludingNowQuery.java
+index 2a7dbf314103..001d77facb82 100644
+--- a/server/src/main/java/org/opensearch/index/query/DateRangeIncludingNowQuery.java
++++ b/server/src/main/java/org/opensearch/index/query/DateRangeIncludingNowQuery.java
+@@ -32,8 +32,8 @@
+ 
+ package org.opensearch.index.query;
+ 
+-import org.apache.lucene.index.IndexReader;
+ import org.apache.lucene.search.BooleanClause;
++import org.apache.lucene.search.IndexSearcher;
+ import org.apache.lucene.search.Query;
+ import org.apache.lucene.search.QueryVisitor;
+ 
+@@ -60,7 +60,7 @@ public Query getQuery() {
+     }
+ 
+     @Override
+-    public Query r2023-07-28T08:38:41-07:00    info-0  forager/scanner loaded detectors        {"total": 748, "verification_enabled": 748, "verification_disabled": 0}
+ewrite(IndexReader reader) throws IOException {
++    public Query rewrite(IndexSearcher searcher) throws IOException {
+         return in;
+     }
+ 
+diff --git a/server/src/main/java/org/opensearch/index/search/OpenSearchToParentBlockJoinQuery.java b/server/src/main/java/org/opensearch/index/search/OpenSearchToParentBlockJoinQuery.java
+index 770301629d96..cd28d8b6ef04 100644
+--- a/server/src/main/java/org/opensearch/index/search/OpenSearchToParentBlockJoinQuery.java
++++ b/server/src/main/java/org/opensearch/index/search/OpenSearchToParentBlockJoinQuery.java
+@@ -32,7 +32,6 @@
+ 
+ package org.opensearch.index.search;
+ 
+-import org.apache.lucene.index.IndexReader;
+ import org.apache.lucene.search.BooleanClause;
+ import org.apache.lucene.search.IndexSearcher;
+ import org.apache.lucene.search.Query;
+@@ -82,8 +81,8 @@ public ScoreMode getScoreMode() {
+     }
+ 
+     @Override
+-    public Query rewrite(IndexReader reader) throws IOException {
+-        Query innerRewrite = query.rewrite(reader);
++    public Query rewrite(IndexSearcher searcher) throws IOException {
++        Query innerRewrite = query.rewrite(searcher);
+         if (innerRewrite != query) {
+             // Right now ToParentBlockJoinQuery always rewrites to a ToParentBlockJoinQuery
+             // so the else block will never be used. It is useful in the case that
+@@ -97,7 +96,7 @@ public Query rewrite(IndexReader reader) throws IOException {
+                 return innerRewrite;
+             }
+         }
+-        return super.rewrite(reader);
++        return super.rewrite(searcher);
+     }
+ 
+     @Override
+diff --git a/server/src/main/java/org/opensearch/lucene/queries/MinDocQuery.java b/server/src/main/java/org/opensearch/lucene/queries/MinDocQuery.java
+index 2af0237e800c..5c904d8a7770 100644
+--- a/server/src/main/java/org/opensearch/lucene/queries/MinDocQuery.java
++++ b/server/src/main/java/org/opensearch/lucene/queries/MinDocQuery.java
+@@ -86,7 +86,8 @@ public boolean equals(Object obj) {
+     }
+ 
+     @Override
+-    public Query rewrite(IndexReader reader) throws IOException {
++    public Query rewrite(IndexSearcher searcher) throws IOException {
++        final IndexReader reader = searcher.getIndexReader();
+         if (Objects.equals(reader.getContext().id(), readerId) == false) {
+             return new MinDocQuery(minDoc, reader.getContext().id());
+         }
+diff --git a/server/src/main/java/org/opensearch/search/internal/ContextIndexSearcher.java b/server/src/main/java/org/opensearch/search/internal/ContextIndexSearcher.java
+index 5384b47cc69e..575133fcffad 100644
+--- a/server/src/main/java/org/opensearch/search/internal/ContextIndexSearcher.java
++++ b/server/src/main/java/org/opensearch/search/internal/ContextIndexSearcher.java
+@@ -339,14 +339,9 @@ private void searchLeaf(LeafReaderContext ctx, Weight weight, Collector collecto
+                     // continue with the following leaf
+                 } catch (QueryPhase.TimeExceededException e) {
+                     searchContext.setSearchTimedOut(true);
+-                    return;
+                 }
+             }
+         }
+-
+-        // Note: this is called if collection ran successfully, including the above special cases of
+-        // CollectionTerminatedException and TimeExceededException, but no other exception.
+-        leafCollector.finish();
+     }
+ 
+     private Weight wrapWeight(Weight weight) {
+diff --git a/server/src/main/java/org/opensearch/search/suggest/completion/CompletionSuggester.java b/server/src/main/java/org/opensearch/search/suggest/completion/CompletionSuggester.java
+index df18e874be9c..af7b1a798a77 100644
+--- a/server/src/main/java/org/opensearch/search/suggest/completion/CompletionSuggester.java
++++ b/server/src/main/java/org/opensearch/search/suggest/completion/CompletionSuggester.java
+@@ -35,7 +35,6 @@
+ import org.apache.lucene.search.BulkScorer;
+ import org.apache.lucene.search.CollectionTerminatedException;
+ import org.apache.lucene.search.IndexSearcher;
+-import org.apache.lucene.search.LeafCollector;
+ import org.apache.lucene.search.Weight;
+ import org.apache.lucene.search.suggest.document.CompletionQuery;
+ import org.apache.lucene.search.suggest.document.TopSuggestDocs;
+@@ -104,24 +103,17 @@ protected Suggest.Suggestion<? extends Suggest.Suggestion.Entry<? extends Sugges
+     }
+ 
+     private static void suggest(IndexSearcher searcher, CompletionQuery query, TopSuggestDocsCollector collector) throws IOException {
+-        query = (CompletionQuery) query.rewrite(searcher.getIndexReader());
++        query = (CompletionQuery) query.rewrite(searcher);
+         Weight weight = query.createWeight(searcher, collector.scoreMode(), 1f);
+         for (LeafReaderContext context : searcher.getIndexReader().leaves()) {
+             BulkScorer scorer = weight.bulkScorer(context);
+             if (scorer != null) {
+-                LeafCollector leafCollector = null;
+                 try {
+-                    leafCollector = collector.getLeafCollector(context);
+-                    scorer.score(leafCollector, context.reader().getLiveDocs());
++                    scorer.score(collector.getLeafCollector(context), context.reader().getLiveDocs());
+                 } catch (CollectionTerminatedException e) {
+                     // collection was terminated prematurely
+                     // continue with the following leaf
+                 }
+-                // Note: this is called if collection ran successfully, including the above special cases of
+-                // CollectionTerminatedException and TimeExceededException, but no other exception.
+-                if (leafCollector != null) {
+-                    leafCollector.finish();
+-                }
+             }
+         }
+     }
+diff --git a/server/src/test/java/org/opensearch/deps/lucene/VectorHighlighterTests.java b/server/src/test/java/org/opensearch/deps/lucene/VectorHighlighterTests.java
+index 0f831ff28af7..880eaeca99bb 100644
+--- a/server/src/test/java/org/opensearch/deps/lucene/VectorHighlighterTests.java
++++ b/server/src/test/java/org/opensearch/deps/lucene/VectorHighlighterTests.java
+@@ -125,7 +125,7 @@ public void testVectorHighlighterPrefixQuery() throws Exception {
+         assertThat(fragment, nullValue());
+ 
+         prefixQuery = new PrefixQuery(new Term("content", "ba"), PrefixQuery.SCORING_BOOLEAN_REWRITE);
+-        Query rewriteQuery = prefixQuery.rewrite(reader);
++        Query rewriteQuery = prefixQuery.rewrite(searcher);
+         fragment = highlighter.getBestFragment(highlighter.getFieldQuery(rewriteQuery), reader, topDocs.scoreDocs[0].doc, "content", 30);
+         assertThat(fragment, notNullValue());
+ 
+diff --git a/server/src/test/java/org/opensearch/index/mapper/DateFieldTypeTests.java b/server/src/test/java/org/opensearch/index/mapper/DateFieldTypeTests.java
+index d360a2a767e8..510dd3456924 100644
+--- a/server/src/test/java/org/opensearch/index/mapper/DateFieldTypeTests.java
++++ b/server/src/test/java/org/opensearch/index/mapper/DateFieldTypeTests.java
+@@ -40,6 +40,7 @@
+ import org.apache.lucene.index.IndexWriterConfig;
+ import org.apache.lucene.index.MultiReader;
+ import org.apache.lucene.index.SortedNumericDocValues;
++import org.apache.lucene.search.IndexSearcher;
+ import org.apache.lucene.search.IndexSortSortedNumericDocValuesRangeQuery;
+ import org.apache.lucene.search.DocIdSetIterator;
+ import org.apache.lucene.search.IndexOrDocValuesQuery;
+@@ -259,7 +260,10 @@ public void testRangeQuery() throws IOException {
+             LongPoint.newRangeQuery("field", instant1, instant2),
+             SortedNumericDocValuesField.newSlowRangeQuery("field", instant1, instant2)
+         );
+-        assertEquals(expected, ft.rangeQuery(date1, date2, true, true, null, null, null, context).rewrite(new MultiReader()));
++        assertEquals(
++            expected,
++            ft.rangeQuery(date1, date2, true, true, null, null, null, context).rewrite(new IndexSearcher(new MultiReader()))
++        );
+ 
+         instant1 = nowInMillis;
+         instant2 = instant1 + 100;
+diff --git a/server/src/test/java/org/opensearch/index/query/SpanMultiTermQueryBuilderTests.java b/server/src/test/java/org/opensearch/index/query/SpanMultiTermQueryBuilderTests.java
+index bc514208c803..be87f7ca1590 100644
+--- a/server/src/test/java/org/opensearch/index/query/SpanMultiTermQueryBuilderTests.java
++++ b/server/src/test/java/org/opensearch/index/query/SpanMultiTermQueryBuilderTests.java
+@@ -284,8 +284,9 @@ public void testTermExpansionExceptionOnSpanFailure() throws Exception {
+                     BooleanQuery.setMaxClauseCount(1);
+                     try {
+                         QueryBuilder queryBuilder = new SpanMultiTermQueryBuilder(QueryBuilders.prefixQuery("body", "bar"));
+-                        Query query = queryBuilder.toQuery(createShardContext(new IndexSearcher(reader)));
+-                        RuntimeException exc = expectThrows(RuntimeException.class, () -> query.rewrite(reader));
++                        IndexSearcher searcher = new IndexSearcher(reader);
++                        Query query = queryBuilder.toQuery(createShardContext(searcher));
++                        RuntimeException exc = expectThrows(RuntimeException.class, () -> query.rewrite(searcher));
+                         assertThat(exc.getMessage(), containsString("maxClauseCount"));
+                     } finally {
+                         BooleanQuery.setMaxClauseCount(origBoolMaxClauseCount);
+diff --git a/server/src/test/java/org/opensearch/index/search/OpenSearchToParentBlockJoinQueryTests.java b/server/src/test/java/org/opensearch/index/search/OpenSearchToParentBlockJoinQueryTests.java
+index 39160b1ac49b..45172c946ab3 100644
+--- a/server/src/test/java/org/opensearch/index/search/OpenSearchToParentBlockJoinQueryTests.java
++++ b/server/src/test/java/org/opensearch/index/search/OpenSearchToParentBlockJoinQueryTests.java
+@@ -34,6 +34,7 @@
+ 
+ import org.apache.lucene.index.MultiReader;
+ import org.apache.lucene.index.Term;
++import org.apache.lucene.search.IndexSearcher;
+ import org.apache.lucene.search.PhraseQuery;
+ import org.apache.lucene.search.Query;
+ import org.apache.lucene.search.TermQuery;
+@@ -112,7 +113,7 @@ public void testRewrite() throws IOException {
+             ScoreMode.Avg,
+             "nested"
+         );
+-        Query rewritten = q.rewrite(new MultiReader());
++        Query rewritten = q.rewrite(new IndexSearcher(new MultiReader()));
+         assertEquals(expected, rewritten);
+     }
+ }
+diff --git a/server/src/test/java/org/opensearch/lucene/queries/MinDocQueryTests.java b/server/src/test/java/org/opensearch/lucene/queries/MinDocQueryTests.java
+index a75a1d5b2fdb..701ba5a756b4 100644
+--- a/server/src/test/java/org/opensearch/lucene/queries/MinDocQueryTests.java
++++ b/server/src/test/java/org/opensearch/lucene/queries/MinDocQueryTests.java
+@@ -61,10 +61,11 @@ public void testBasics() {
+ 
+     public void testRewrite() throws Exception {
+         IndexReader reader = new MultiReader();
++        IndexSearcher searcher = new IndexSearcher(reader);
+         MinDocQuery query = new MinDocQuery(42);
+-        Query rewritten = query.rewrite(reader);
++        Query rewritten = query.rewrite(searcher);
+         QueryUtils.checkUnequal(query, rewritten);
+-        Query rewritten2 = rewritten.rewrite(reader);
++        Query rewritten2 = rewritten.rewrite(searcher);
+         assertSame(rewritten, rewritten2);
+     }
+ 
+diff --git a/server/src/test/java/org/opensearch/lucene/queries/SpanMatchNoDocsQueryTests.java b/server/src/test/java/org/opensearch/lucene/queries/SpanMatchNoDocsQueryTests.java
+index 727377c9a5d5..81f4290b6588 100644
+--- a/server/src/test/java/org/opensearch/lucene/queries/SpanMatchNoDocsQueryTests.java
++++ b/server/src/test/java/org/opensearch/lucene/queries/SpanMatchNoDocsQueryTests.java
+@@ -56,19 +56,11 @@
+ 
+ public class SpanMatchNoDocsQueryTests extends OpenSearchTestCase {
+     public void testSimple() throws Exception {
+-        Directory dir = newDirectory();
+-        IndexWriter iw = new IndexWriter(dir, newIndexWriterConfig());
+-        IndexReader ir = DirectoryReader.open(iw);
+-
+         SpanMatchNoDocsQuery query = new SpanMatchNoDocsQuery("field", "a good reason");
+         assertEquals(query.toString(), "SpanMatchNoDocsQuery(\"a good reason\")");
+-        Query rewrite = query.rewrite(ir);
++        Query rewrite = query.rewrite((IndexSearcher) null);
+         assertTrue(rewrite instanceof SpanMatchNoDocsQuery);
+         assertEquals(rewrite.toString(), "SpanMatchNoDocsQuery(\"a good reason\")");
+-
+-        iw.close();
+-        ir.close();
+-        dir.close();
+     }
+ 
+     public void testQuery() throws Exception {
+@@ -101,7 +93,7 @@ public void testQuery() throws Exception {
+         assertEquals(searcher.count(orQuery), 1);
+         hits = searcher.search(orQuery, 1000).scoreDocs;
+         assertEquals(1, hits.length);
+-        Query rewrite = orQuery.rewrite(ir);
++        Query rewrite = orQuery.rewrite(searcher);
+         assertEquals(rewrite, orQuery);
+ 
+         SpanNearQuery nearQuery = new SpanNearQuery(
+@@ -112,7 +104,7 @@ public void testQuery() throws Exception {
+         assertEquals(searcher.count(nearQuery), 0);
+         hits = searcher.search(nearQuery, 1000).scoreDocs;
+         assertEquals(0, hits.length);
+-        rewrite = nearQuery.rewrite(ir);
++        rewrite = nearQuery.rewrite(searcher);
+         assertEquals(rewrite, nearQuery);
+ 
+         iw.close();
+diff --git a/server/src/test/java/org/opensearch/search/internal/ContextIndexSearcherTests.java b/server/src/test/java/org/opensearch/search/internal/ContextIndexSearcherTests.java
+index f3907355ac6e..823fc6b46390 100644
+--- a/server/src/test/java/org/opensearch/search/internal/ContextIndexSearcherTests.java
++++ b/server/src/test/java/org/opensearch/search/internal/ContextIndexSearcherTests.java
+@@ -40,7 +40,6 @@
+ import org.apache.lucene.document.StringField;
+ import org.apache.lucene.index.DirectoryReader;
+ import org.apache.lucene.index.FilterDirectoryReader;
+-import org.apache.lucene.index.IndexReader;
+ import org.apache.lucene.index.IndexWriter;
+ import org.apache.lucene.index.IndexWriterConfig;
+ import org.apache.lucene.index.LeafReader;
+@@ -466,12 +465,12 @@ public String toString(String field) {
+         }
+ 
+         @Override
+-        public Query rewrite(IndexReader reader) throws IOException {
+-            Query queryRewritten = query.rewrite(reader);
++        public Query rewrite(IndexSearcher searcher) throws IOException {
++            Query queryRewritten = query.rewrite(searcher);
+             if (query != queryRewritten) {
+                 return new CreateScorerOnceQuery(queryRewritten);
+             }
+-            return super.rewrite(reader);
++            return super.rewrite(searcher);
+         }
+ 
+         @Override

pkg/pb/source_metadatapb/source_metadata.pb.go

@@ -2294,6 +2294,7 @@ type GoogleDrive struct {
 	Timestamp      string `protobuf:"bytes,4,opt,name=timestamp,proto3" json:"timestamp,omitempty"`
 	Shared         bool   `protobuf:"varint,5,opt,name=shared,proto3" json:"shared,omitempty"`
 	LastModifiedBy string `protobuf:"bytes,6,opt,name=last_modified_by,json=lastModifiedBy,proto3" json:"last_modified_by,omitempty"`
+	Path           string `protobuf:"bytes,7,opt,name=path,proto3" json:"path,omitempty"`
 }
 
 func (x *GoogleDrive) Reset() {
@@ -2370,6 +2371,13 @@ func (x *GoogleDrive) GetLastModifiedBy() string {
 	return ""
 }
 
+func (x *GoogleDrive) GetPath() string {
+	if x != nil {
+		return x.Path
+	}
+	return ""
+}
+
 type AzureRepos struct {
 	state         protoimpl.MessageState
 	sizeCache     protoimpl.SizeCache
@@ -3190,7 +3198,7 @@ var file_source_metadata_proto_rawDesc = []byte{
 	0x73, 0x18, 0x05, 0x20, 0x01, 0x28, 0x03, 0x52, 0x05, 0x76, 0x69, 0x65, 0x77, 0x73, 0x12, 0x14,
 	0x0a, 0x05, 0x64, 0x6f, 0x63, 0x69, 0x64, 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x64,
 	0x6f, 0x63, 0x69, 0x64, 0x12, 0x14, 0x0a, 0x05, 0x65, 0x6d, 0x61, 0x69, 0x6c, 0x18, 0x07, 0x20,
-	0x01, 0x28, 0x09, 0x52, 0x05, 0x65, 0x6d, 0x61, 0x69, 0x6c, 0x22, 0xab, 0x01, 0x0a, 0x0b, 0x47,
+	0x01, 0x28, 0x09, 0x52, 0x05, 0x65, 0x6d, 0x61, 0x69, 0x6c, 0x22, 0xbf, 0x01, 0x0a, 0x0b, 0x47,
 	0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x44, 0x72, 0x69, 0x76, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x66, 0x69,
 	0x6c, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x66, 0x69, 0x6c, 0x65, 0x12, 0x12,
 	0x0a, 0x04, 0x6c, 0x69, 0x6e, 0x6b, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6c, 0x69,
@@ -3201,129 +3209,130 @@ var file_source_metadata_proto_rawDesc = []byte{
 	0x18, 0x05, 0x20, 0x01, 0x28, 0x08, 0x52, 0x06, 0x73, 0x68, 0x61, 0x72, 0x65, 0x64, 0x12, 0x28,
 	0x0a, 0x10, 0x6c, 0x61, 0x73, 0x74, 0x5f, 0x6d, 0x6f, 0x64, 0x69, 0x66, 0x69, 0x65, 0x64, 0x5f,
 	0x62, 0x79, 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0e, 0x6c, 0x61, 0x73, 0x74, 0x4d, 0x6f,
-	0x64, 0x69, 0x66, 0x69, 0x65, 0x64, 0x42, 0x79, 0x22, 0xcb, 0x02, 0x0a, 0x0a, 0x41, 0x7a, 0x75,
+	0x64, 0x69, 0x66, 0x69, 0x65, 0x64, 0x42, 0x79, 0x12, 0x12, 0x0a, 0x04, 0x70, 0x61, 0x74, 0x68,
-	0x72, 0x65, 0x52, 0x65, 0x70, 0x6f, 0x73, 0x12, 0x12, 0x0a, 0x04, 0x6c, 0x69, 0x6e, 0x6b, 0x18,
+	0x18, 0x07, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x70, 0x61, 0x74, 0x68, 0x22, 0xcb, 0x02, 0x0a,
-	0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6c, 0x69, 0x6e, 0x6b, 0x12, 0x1a, 0x0a, 0x08, 0x75,
+	0x0a, 0x41, 0x7a, 0x75, 0x72, 0x65, 0x52, 0x65, 0x70, 0x6f, 0x73, 0x12, 0x12, 0x0a, 0x04, 0x6c,
-	0x73, 0x65, 0x72, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x75,
+	0x69, 0x6e, 0x6b, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6c, 0x69, 0x6e, 0x6b, 0x12,
-	0x73, 0x65, 0x72, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x1e, 0x0a, 0x0a, 0x72, 0x65, 0x70, 0x6f, 0x73,
+	0x1a, 0x0a, 0x08, 0x75, 0x73, 0x65, 0x72, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28,
-	0x69, 0x74, 0x6f, 0x72, 0x79, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x72, 0x65, 0x70,
+	0x09, 0x52, 0x08, 0x75, 0x73, 0x65, 0x72, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x1e, 0x0a, 0x0a, 0x72,
-	0x6f, 0x73, 0x69, 0x74, 0x6f, 0x72, 0x79, 0x12, 0x16, 0x0a, 0x06, 0x63, 0x6f, 0x6d, 0x6d, 0x69,
+	0x65, 0x70, 0x6f, 0x73, 0x69, 0x74, 0x6f, 0x72, 0x79, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52,
-	0x74, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x63, 0x6f, 0x6d, 0x6d, 0x69, 0x74, 0x12,
+	0x0a, 0x72, 0x65, 0x70, 0x6f, 0x73, 0x69, 0x74, 0x6f, 0x72, 0x79, 0x12, 0x16, 0x0a, 0x06, 0x63,
-	0x14, 0x0a, 0x05, 0x65, 0x6d, 0x61, 0x69, 0x6c, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05,
+	0x6f, 0x6d, 0x6d, 0x69, 0x74, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x63, 0x6f, 0x6d,
-	0x65, 0x6d, 0x61, 0x69, 0x6c, 0x12, 0x12, 0x0a, 0x04, 0x66, 0x69, 0x6c, 0x65, 0x18, 0x06, 0x20,
+	0x6d, 0x69, 0x74, 0x12, 0x14, 0x0a, 0x05, 0x65, 0x6d, 0x61, 0x69, 0x6c, 0x18, 0x05, 0x20, 0x01,
-	0x01, 0x28, 0x09, 0x52, 0x04, 0x66, 0x69, 0x6c, 0x65, 0x12, 0x1c, 0x0a, 0x09, 0x74, 0x69, 0x6d,
+	0x28, 0x09, 0x52, 0x05, 0x65, 0x6d, 0x61, 0x69, 0x6c, 0x12, 0x12, 0x0a, 0x04, 0x66, 0x69, 0x6c,
-	0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x18, 0x07, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x74, 0x69,
+	0x65, 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x66, 0x69, 0x6c, 0x65, 0x12, 0x1c, 0x0a,
-	0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x12, 0x12, 0x0a, 0x04, 0x6c, 0x69, 0x6e, 0x65, 0x18,
+	0x09, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x18, 0x07, 0x20, 0x01, 0x28, 0x09,
-	0x08, 0x20, 0x01, 0x28, 0x03, 0x52, 0x04, 0x6c, 0x69, 0x6e, 0x65, 0x12, 0x3b, 0x0a, 0x0a, 0x76,
+	0x52, 0x09, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x12, 0x12, 0x0a, 0x04, 0x6c,
-	0x69, 0x73, 0x69, 0x62, 0x69, 0x6c, 0x69, 0x74, 0x79, 0x18, 0x09, 0x20, 0x01, 0x28, 0x0e, 0x32,
+	0x69, 0x6e, 0x65, 0x18, 0x08, 0x20, 0x01, 0x28, 0x03, 0x52, 0x04, 0x6c, 0x69, 0x6e, 0x65, 0x12,
-	0x1b, 0x2e, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74,
+	0x3b, 0x0a, 0x0a, 0x76, 0x69, 0x73, 0x69, 0x62, 0x69, 0x6c, 0x69, 0x74, 0x79, 0x18, 0x09, 0x20,
-	0x61, 0x2e, 0x56, 0x69, 0x73, 0x69, 0x62, 0x69, 0x6c, 0x69, 0x74, 0x79, 0x52, 0x0a, 0x76, 0x69,
+	0x01, 0x28, 0x0e, 0x32, 0x1b, 0x2e, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f, 0x6d, 0x65, 0x74,
-	0x73, 0x69, 0x62, 0x69, 0x6c, 0x69, 0x74, 0x79, 0x12, 0x18, 0x0a, 0x07, 0x70, 0x72, 0x6f, 0x6a,
+	0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x56, 0x69, 0x73, 0x69, 0x62, 0x69, 0x6c, 0x69, 0x74, 0x79,
-	0x65, 0x63, 0x74, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x70, 0x72, 0x6f, 0x6a, 0x65,
+	0x52, 0x0a, 0x76, 0x69, 0x73, 0x69, 0x62, 0x69, 0x6c, 0x69, 0x74, 0x79, 0x12, 0x18, 0x0a, 0x07,
-	0x63, 0x74, 0x12, 0x22, 0x0a, 0x0c, 0x6f, 0x72, 0x67, 0x61, 0x6e, 0x69, 0x7a, 0x61, 0x74, 0x69,
+	0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x70,
-	0x6f, 0x6e, 0x18, 0x0b, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0c, 0x6f, 0x72, 0x67, 0x61, 0x6e, 0x69,
+	0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x12, 0x22, 0x0a, 0x0c, 0x6f, 0x72, 0x67, 0x61, 0x6e, 0x69,
-	0x7a, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x22, 0xbe, 0x0b, 0x0a, 0x08, 0x4d, 0x65, 0x74, 0x61, 0x44,
+	0x7a, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x0b, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0c, 0x6f, 0x72,
-	0x61, 0x74, 0x61, 0x12, 0x2e, 0x0a, 0x05, 0x61, 0x7a, 0x75, 0x72, 0x65, 0x18, 0x01, 0x20, 0x01,
+	0x67, 0x61, 0x6e, 0x69, 0x7a, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x22, 0xbe, 0x0b, 0x0a, 0x08, 0x4d,
+	0x65, 0x74, 0x61, 0x44, 0x61, 0x74, 0x61, 0x12, 0x2e, 0x0a, 0x05, 0x61, 0x7a, 0x75, 0x72, 0x65,
+	0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x16, 0x2e, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f,
+	0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x41, 0x7a, 0x75, 0x72, 0x65, 0x48, 0x00,
+	0x52, 0x05, 0x61, 0x7a, 0x75, 0x72, 0x65, 0x12, 0x3a, 0x0a, 0x09, 0x62, 0x69, 0x74, 0x62, 0x75,
+	0x63, 0x6b, 0x65, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x73, 0x6f, 0x75,
+	0x72, 0x63, 0x65, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x42, 0x69, 0x74,
+	0x62, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x48, 0x00, 0x52, 0x09, 0x62, 0x69, 0x74, 0x62, 0x75, 0x63,
+	0x6b, 0x65, 0x74, 0x12, 0x37, 0x0a, 0x08, 0x63, 0x69, 0x72, 0x63, 0x6c, 0x65, 0x63, 0x69, 0x18,
+	0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x19, 0x2e, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f, 0x6d,
+	0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x43, 0x69, 0x72, 0x63, 0x6c, 0x65, 0x43, 0x49,
+	0x48, 0x00, 0x52, 0x08, 0x63, 0x69, 0x72, 0x63, 0x6c, 0x65, 0x63, 0x69, 0x12, 0x3d, 0x0a, 0x0a,
+	0x63, 0x6f, 0x6e, 0x66, 0x6c, 0x75, 0x65, 0x6e, 0x63, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b,
+	0x32, 0x1b, 0x2e, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61,
+	0x74, 0x61, 0x2e, 0x43, 0x6f, 0x6e, 0x66, 0x6c, 0x75, 0x65, 0x6e, 0x63, 0x65, 0x48, 0x00, 0x52,
+	0x0a, 0x63, 0x6f, 0x6e, 0x66, 0x6c, 0x75, 0x65, 0x6e, 0x63, 0x65, 0x12, 0x31, 0x0a, 0x06, 0x64,
+	0x6f, 0x63, 0x6b, 0x65, 0x72, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x17, 0x2e, 0x73, 0x6f,
+	0x75, 0x72, 0x63, 0x65, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x44, 0x6f,
+	0x63, 0x6b, 0x65, 0x72, 0x48, 0x00, 0x52, 0x06, 0x64, 0x6f, 0x63, 0x6b, 0x65, 0x72, 0x12, 0x28,
+	0x0a, 0x03, 0x65, 0x63, 0x72, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x73, 0x6f,
+	0x75, 0x72, 0x63, 0x65, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x45, 0x43,
+	0x52, 0x48, 0x00, 0x52, 0x03, 0x65, 0x63, 0x72, 0x12, 0x28, 0x0a, 0x03, 0x67, 0x63, 0x73, 0x18,
+	0x07, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f, 0x6d,
+	0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x47, 0x43, 0x53, 0x48, 0x00, 0x52, 0x03, 0x67,
+	0x63, 0x73, 0x12, 0x31, 0x0a, 0x06, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x18, 0x08, 0x20, 0x01,
+	0x28, 0x0b, 0x32, 0x17, 0x2e, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f, 0x6d, 0x65, 0x74, 0x61,
+	0x64, 0x61, 0x74, 0x61, 0x2e, 0x47, 0x69, 0x74, 0x68, 0x75, 0x62, 0x48, 0x00, 0x52, 0x06, 0x67,
+	0x69, 0x74, 0x68, 0x75, 0x62, 0x12, 0x31, 0x0a, 0x06, 0x67, 0x69, 0x74, 0x6c, 0x61, 0x62, 0x18,
+	0x09, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x17, 0x2e, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f, 0x6d,
+	0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x47, 0x69, 0x74, 0x6c, 0x61, 0x62, 0x48, 0x00,
+	0x52, 0x06, 0x67, 0x69, 0x74, 0x6c, 0x61, 0x62, 0x12, 0x2b, 0x0a, 0x04, 0x6a, 0x69, 0x72, 0x61,
+	0x18, 0x0a, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x15, 0x2e, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f,
+	0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x4a, 0x69, 0x72, 0x61, 0x48, 0x00, 0x52,
+	0x04, 0x6a, 0x69, 0x72, 0x61, 0x12, 0x28, 0x0a, 0x03, 0x6e, 0x70, 0x6d, 0x18, 0x0b, 0x20, 0x01,
+	0x28, 0x0b, 0x32, 0x14, 0x2e, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f, 0x6d, 0x65, 0x74, 0x61,
+	0x64, 0x61, 0x74, 0x61, 0x2e, 0x4e, 0x50, 0x4d, 0x48, 0x00, 0x52, 0x03, 0x6e, 0x70, 0x6d, 0x12,
+	0x2b, 0x0a, 0x04, 0x70, 0x79, 0x70, 0x69, 0x18, 0x0c, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x15, 0x2e,
+	0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e,
+	0x50, 0x79, 0x50, 0x69, 0x48, 0x00, 0x52, 0x04, 0x70, 0x79, 0x70, 0x69, 0x12, 0x25, 0x0a, 0x02,
+	0x73, 0x33, 0x18, 0x0d, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x13, 0x2e, 0x73, 0x6f, 0x75, 0x72, 0x63,
+	0x65, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x53, 0x33, 0x48, 0x00, 0x52,
+	0x02, 0x73, 0x33, 0x12, 0x2e, 0x0a, 0x05, 0x73, 0x6c, 0x61, 0x63, 0x6b, 0x18, 0x0e, 0x20, 0x01,
 	0x28, 0x0b, 0x32, 0x16, 0x2e, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f, 0x6d, 0x65, 0x74, 0x61,
-	0x64, 0x61, 0x74, 0x61, 0x2e, 0x41, 0x7a, 0x75, 0x72, 0x65, 0x48, 0x00, 0x52, 0x05, 0x61, 0x7a,
+	0x64, 0x61, 0x74, 0x61, 0x2e, 0x53, 0x6c, 0x61, 0x63, 0x6b, 0x48, 0x00, 0x52, 0x05, 0x73, 0x6c,
-	0x75, 0x72, 0x65, 0x12, 0x3a, 0x0a, 0x09, 0x62, 0x69, 0x74, 0x62, 0x75, 0x63, 0x6b, 0x65, 0x74,
+	0x61, 0x63, 0x6b, 0x12, 0x3d, 0x0a, 0x0a, 0x66, 0x69, 0x6c, 0x65, 0x73, 0x79, 0x73, 0x74, 0x65,
-	0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f,
+	0x6d, 0x18, 0x0f, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1b, 0x2e, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65,
-	0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x42, 0x69, 0x74, 0x62, 0x75, 0x63, 0x6b,
+	0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x46, 0x69, 0x6c, 0x65, 0x73, 0x79,
-	0x65, 0x74, 0x48, 0x00, 0x52, 0x09, 0x62, 0x69, 0x74, 0x62, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x12,
+	0x73, 0x74, 0x65, 0x6d, 0x48, 0x00, 0x52, 0x0a, 0x66, 0x69, 0x6c, 0x65, 0x73, 0x79, 0x73, 0x74,
-	0x37, 0x0a, 0x08, 0x63, 0x69, 0x72, 0x63, 0x6c, 0x65, 0x63, 0x69, 0x18, 0x03, 0x20, 0x01, 0x28,
+	0x65, 0x6d, 0x12, 0x28, 0x0a, 0x03, 0x67, 0x69, 0x74, 0x18, 0x10, 0x20, 0x01, 0x28, 0x0b, 0x32,
-	0x0b, 0x32, 0x19, 0x2e, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64,
+	0x14, 0x2e, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74,
-	0x61, 0x74, 0x61, 0x2e, 0x43, 0x69, 0x72, 0x63, 0x6c, 0x65, 0x43, 0x49, 0x48, 0x00, 0x52, 0x08,
+	0x61, 0x2e, 0x47, 0x69, 0x74, 0x48, 0x00, 0x52, 0x03, 0x67, 0x69, 0x74, 0x12, 0x2b, 0x0a, 0x04,
-	0x63, 0x69, 0x72, 0x63, 0x6c, 0x65, 0x63, 0x69, 0x12, 0x3d, 0x0a, 0x0a, 0x63, 0x6f, 0x6e, 0x66,
+	0x74, 0x65, 0x73, 0x74, 0x18, 0x11, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x15, 0x2e, 0x73, 0x6f, 0x75,
-	0x6c, 0x75, 0x65, 0x6e, 0x63, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1b, 0x2e, 0x73,
+	0x72, 0x63, 0x65, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x54, 0x65, 0x73,
-	0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x43,
+	0x74, 0x48, 0x00, 0x52, 0x04, 0x74, 0x65, 0x73, 0x74, 0x12, 0x3a, 0x0a, 0x09, 0x62, 0x75, 0x69,
-	0x6f, 0x6e, 0x66, 0x6c, 0x75, 0x65, 0x6e, 0x63, 0x65, 0x48, 0x00, 0x52, 0x0a, 0x63, 0x6f, 0x6e,
+	0x6c, 0x64, 0x6b, 0x69, 0x74, 0x65, 0x18, 0x12, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x73,
-	0x66, 0x6c, 0x75, 0x65, 0x6e, 0x63, 0x65, 0x12, 0x31, 0x0a, 0x06, 0x64, 0x6f, 0x63, 0x6b, 0x65,
+	0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x42,
-	0x72, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x17, 0x2e, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65,
+	0x75, 0x69, 0x6c, 0x64, 0x6b, 0x69, 0x74, 0x65, 0x48, 0x00, 0x52, 0x09, 0x62, 0x75, 0x69, 0x6c,
-	0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x44, 0x6f, 0x63, 0x6b, 0x65, 0x72,
+	0x64, 0x6b, 0x69, 0x74, 0x65, 0x12, 0x31, 0x0a, 0x06, 0x67, 0x65, 0x72, 0x72, 0x69, 0x74, 0x18,
-	0x48, 0x00, 0x52, 0x06, 0x64, 0x6f, 0x63, 0x6b, 0x65, 0x72, 0x12, 0x28, 0x0a, 0x03, 0x65, 0x63,
+	0x13, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x17, 0x2e, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f, 0x6d,
-	0x72, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65,
+	0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x47, 0x65, 0x72, 0x72, 0x69, 0x74, 0x48, 0x00,
-	0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x45, 0x43, 0x52, 0x48, 0x00, 0x52,
+	0x52, 0x06, 0x67, 0x65, 0x72, 0x72, 0x69, 0x74, 0x12, 0x34, 0x0a, 0x07, 0x6a, 0x65, 0x6e, 0x6b,
-	0x03, 0x65, 0x63, 0x72, 0x12, 0x28, 0x0a, 0x03, 0x67, 0x63, 0x73, 0x18, 0x07, 0x20, 0x01, 0x28,
+	0x69, 0x6e, 0x73, 0x18, 0x14, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x18, 0x2e, 0x73, 0x6f, 0x75, 0x72,
-	0x0b, 0x32, 0x14, 0x2e, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64,
+	0x63, 0x65, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x4a, 0x65, 0x6e, 0x6b,
-	0x61, 0x74, 0x61, 0x2e, 0x47, 0x43, 0x53, 0x48, 0x00, 0x52, 0x03, 0x67, 0x63, 0x73, 0x12, 0x31,
+	0x69, 0x6e, 0x73, 0x48, 0x00, 0x52, 0x07, 0x6a, 0x65, 0x6e, 0x6b, 0x69, 0x6e, 0x73, 0x12, 0x2e,
-	0x0a, 0x06, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x18, 0x08, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x17,
+	0x0a, 0x05, 0x74, 0x65, 0x61, 0x6d, 0x73, 0x18, 0x15, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x16, 0x2e,
-	0x2e, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61,
+	0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e,
-	0x2e, 0x47, 0x69, 0x74, 0x68, 0x75, 0x62, 0x48, 0x00, 0x52, 0x06, 0x67, 0x69, 0x74, 0x68, 0x75,
+	0x54, 0x65, 0x61, 0x6d, 0x73, 0x48, 0x00, 0x52, 0x05, 0x74, 0x65, 0x61, 0x6d, 0x73, 0x12, 0x40,
-	0x62, 0x12, 0x31, 0x0a, 0x06, 0x67, 0x69, 0x74, 0x6c, 0x61, 0x62, 0x18, 0x09, 0x20, 0x01, 0x28,
+	0x0a, 0x0b, 0x61, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x6f, 0x72, 0x79, 0x18, 0x16, 0x20,
-	0x0b, 0x32, 0x17, 0x2e, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64,
-	0x61, 0x74, 0x61, 0x2e, 0x47, 0x69, 0x74, 0x6c, 0x61, 0x62, 0x48, 0x00, 0x52, 0x06, 0x67, 0x69,
-	0x74, 0x6c, 0x61, 0x62, 0x12, 0x2b, 0x0a, 0x04, 0x6a, 0x69, 0x72, 0x61, 0x18, 0x0a, 0x20, 0x01,
-	0x28, 0x0b, 0x32, 0x15, 0x2e, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f, 0x6d, 0x65, 0x74, 0x61,
-	0x64, 0x61, 0x74, 0x61, 0x2e, 0x4a, 0x69, 0x72, 0x61, 0x48, 0x00, 0x52, 0x04, 0x6a, 0x69, 0x72,
-	0x61, 0x12, 0x28, 0x0a, 0x03, 0x6e, 0x70, 0x6d, 0x18, 0x0b, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x14,
-	0x2e, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61,
-	0x2e, 0x4e, 0x50, 0x4d, 0x48, 0x00, 0x52, 0x03, 0x6e, 0x70, 0x6d, 0x12, 0x2b, 0x0a, 0x04, 0x70,
-	0x79, 0x70, 0x69, 0x18, 0x0c, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x15, 0x2e, 0x73, 0x6f, 0x75, 0x72,
-	0x63, 0x65, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x50, 0x79, 0x50, 0x69,
-	0x48, 0x00, 0x52, 0x04, 0x70, 0x79, 0x70, 0x69, 0x12, 0x25, 0x0a, 0x02, 0x73, 0x33, 0x18, 0x0d,
-	0x20, 0x01, 0x28, 0x0b, 0x32, 0x13, 0x2e, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f, 0x6d, 0x65,
-	0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x53, 0x33, 0x48, 0x00, 0x52, 0x02, 0x73, 0x33, 0x12,
-	0x2e, 0x0a, 0x05, 0x73, 0x6c, 0x61, 0x63, 0x6b, 0x18, 0x0e, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x16,
-	0x2e, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61,
-	0x2e, 0x53, 0x6c, 0x61, 0x63, 0x6b, 0x48, 0x00, 0x52, 0x05, 0x73, 0x6c, 0x61, 0x63, 0x6b, 0x12,
-	0x3d, 0x0a, 0x0a, 0x66, 0x69, 0x6c, 0x65, 0x73, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x18, 0x0f, 0x20,
-	0x01, 0x28, 0x0b, 0x32, 0x1b, 0x2e, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f, 0x6d, 0x65, 0x74,
-	0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x46, 0x69, 0x6c, 0x65, 0x73, 0x79, 0x73, 0x74, 0x65, 0x6d,
-	0x48, 0x00, 0x52, 0x0a, 0x66, 0x69, 0x6c, 0x65, 0x73, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x12, 0x28,
-	0x0a, 0x03, 0x67, 0x69, 0x74, 0x18, 0x10, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x73, 0x6f,
-	0x75, 0x72, 0x63, 0x65, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x47, 0x69,
-	0x74, 0x48, 0x00, 0x52, 0x03, 0x67, 0x69, 0x74, 0x12, 0x2b, 0x0a, 0x04, 0x74, 0x65, 0x73, 0x74,
-	0x18, 0x11, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x15, 0x2e, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f,
-	0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x54, 0x65, 0x73, 0x74, 0x48, 0x00, 0x52,
-	0x04, 0x74, 0x65, 0x73, 0x74, 0x12, 0x3a, 0x0a, 0x09, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x6b, 0x69,
-	0x74, 0x65, 0x18, 0x12, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x73, 0x6f, 0x75, 0x72, 0x63,
-	0x65, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x42, 0x75, 0x69, 0x6c, 0x64,
-	0x6b, 0x69, 0x74, 0x65, 0x48, 0x00, 0x52, 0x09, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x6b, 0x69, 0x74,
-	0x65, 0x12, 0x31, 0x0a, 0x06, 0x67, 0x65, 0x72, 0x72, 0x69, 0x74, 0x18, 0x13, 0x20, 0x01, 0x28,
-	0x0b, 0x32, 0x17, 0x2e, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64,
-	0x61, 0x74, 0x61, 0x2e, 0x47, 0x65, 0x72, 0x72, 0x69, 0x74, 0x48, 0x00, 0x52, 0x06, 0x67, 0x65,
-	0x72, 0x72, 0x69, 0x74, 0x12, 0x34, 0x0a, 0x07, 0x6a, 0x65, 0x6e, 0x6b, 0x69, 0x6e, 0x73, 0x18,
-	0x14, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x18, 0x2e, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f, 0x6d,
-	0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x4a, 0x65, 0x6e, 0x6b, 0x69, 0x6e, 0x73, 0x48,
-	0x00, 0x52, 0x07, 0x6a, 0x65, 0x6e, 0x6b, 0x69, 0x6e, 0x73, 0x12, 0x2e, 0x0a, 0x05, 0x74, 0x65,
-	0x61, 0x6d, 0x73, 0x18, 0x15, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x16, 0x2e, 0x73, 0x6f, 0x75, 0x72,
-	0x63, 0x65, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x54, 0x65, 0x61, 0x6d,
-	0x73, 0x48, 0x00, 0x52, 0x05, 0x74, 0x65, 0x61, 0x6d, 0x73, 0x12, 0x40, 0x0a, 0x0b, 0x61, 0x72,
-	0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x6f, 0x72, 0x79, 0x18, 0x16, 0x20, 0x01, 0x28, 0x0b, 0x32,
-	0x1c, 0x2e, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74,
-	0x61, 0x2e, 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x6f, 0x72, 0x79, 0x48, 0x00, 0x52,
-	0x0b, 0x61, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x6f, 0x72, 0x79, 0x12, 0x31, 0x0a, 0x06,
-	0x73, 0x79, 0x73, 0x6c, 0x6f, 0x67, 0x18, 0x17, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x17, 0x2e, 0x73,
-	0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x53,
-	0x79, 0x73, 0x6c, 0x6f, 0x67, 0x48, 0x00, 0x52, 0x06, 0x73, 0x79, 0x73, 0x6c, 0x6f, 0x67, 0x12,
-	0x5e, 0x0a, 0x15, 0x70, 0x75, 0x62, 0x6c, 0x69, 0x63, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x4d, 0x6f,
-	0x6e, 0x69, 0x74, 0x6f, 0x72, 0x69, 0x6e, 0x67, 0x18, 0x18, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x26,
-	0x2e, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61,
-	0x2e, 0x50, 0x75, 0x62, 0x6c, 0x69, 0x63, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x4d, 0x6f, 0x6e, 0x69,
-	0x74, 0x6f, 0x72, 0x69, 0x6e, 0x67, 0x48, 0x00, 0x52, 0x15, 0x70, 0x75, 0x62, 0x6c, 0x69, 0x63,
-	0x45, 0x76, 0x65, 0x6e, 0x74, 0x4d, 0x6f, 0x6e, 0x69, 0x74, 0x6f, 0x72, 0x69, 0x6e, 0x67, 0x12,
-	0x3d, 0x0a, 0x0a, 0x73, 0x68, 0x61, 0x72, 0x65, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x18, 0x19, 0x20,
-	0x01, 0x28, 0x0b, 0x32, 0x1b, 0x2e, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f, 0x6d, 0x65, 0x74,
-	0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x53, 0x68, 0x61, 0x72, 0x65, 0x50, 0x6f, 0x69, 0x6e, 0x74,
-	0x48, 0x00, 0x52, 0x0a, 0x73, 0x68, 0x61, 0x72, 0x65, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x12, 0x40,
-	0x0a, 0x0b, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x44, 0x72, 0x69, 0x76, 0x65, 0x18, 0x1a, 0x20,
 	0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f, 0x6d, 0x65, 0x74,
-	0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x47, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x44, 0x72, 0x69, 0x76,
+	0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x6f, 0x72,
-	0x65, 0x48, 0x00, 0x52, 0x0b, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x44, 0x72, 0x69, 0x76, 0x65,
+	0x79, 0x48, 0x00, 0x52, 0x0b, 0x61, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x6f, 0x72, 0x79,
-	0x12, 0x3d, 0x0a, 0x0a, 0x61, 0x7a, 0x75, 0x72, 0x65, 0x52, 0x65, 0x70, 0x6f, 0x73, 0x18, 0x1b,
+	0x12, 0x31, 0x0a, 0x06, 0x73, 0x79, 0x73, 0x6c, 0x6f, 0x67, 0x18, 0x17, 0x20, 0x01, 0x28, 0x0b,
-	0x20, 0x01, 0x28, 0x0b, 0x32, 0x1b, 0x2e, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f, 0x6d, 0x65,
+	0x32, 0x17, 0x2e, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61,
-	0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x41, 0x7a, 0x75, 0x72, 0x65, 0x52, 0x65, 0x70, 0x6f,
+	0x74, 0x61, 0x2e, 0x53, 0x79, 0x73, 0x6c, 0x6f, 0x67, 0x48, 0x00, 0x52, 0x06, 0x73, 0x79, 0x73,
-	0x73, 0x48, 0x00, 0x52, 0x0a, 0x61, 0x7a, 0x75, 0x72, 0x65, 0x52, 0x65, 0x70, 0x6f, 0x73, 0x42,
+	0x6c, 0x6f, 0x67, 0x12, 0x5e, 0x0a, 0x15, 0x70, 0x75, 0x62, 0x6c, 0x69, 0x63, 0x45, 0x76, 0x65,
-	0x06, 0x0a, 0x04, 0x64, 0x61, 0x74, 0x61, 0x2a, 0x3e, 0x0a, 0x0a, 0x56, 0x69, 0x73, 0x69, 0x62,
+	0x6e, 0x74, 0x4d, 0x6f, 0x6e, 0x69, 0x74, 0x6f, 0x72, 0x69, 0x6e, 0x67, 0x18, 0x18, 0x20, 0x01,
-	0x69, 0x6c, 0x69, 0x74, 0x79, 0x12, 0x0a, 0x0a, 0x06, 0x70, 0x75, 0x62, 0x6c, 0x69, 0x63, 0x10,
+	0x28, 0x0b, 0x32, 0x26, 0x2e, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f, 0x6d, 0x65, 0x74, 0x61,
-	0x00, 0x12, 0x0b, 0x0a, 0x07, 0x70, 0x72, 0x69, 0x76, 0x61, 0x74, 0x65, 0x10, 0x01, 0x12, 0x0a,
+	0x64, 0x61, 0x74, 0x61, 0x2e, 0x50, 0x75, 0x62, 0x6c, 0x69, 0x63, 0x45, 0x76, 0x65, 0x6e, 0x74,
-	0x0a, 0x06, 0x73, 0x68, 0x61, 0x72, 0x65, 0x64, 0x10, 0x02, 0x12, 0x0b, 0x0a, 0x07, 0x75, 0x6e,
+	0x4d, 0x6f, 0x6e, 0x69, 0x74, 0x6f, 0x72, 0x69, 0x6e, 0x67, 0x48, 0x00, 0x52, 0x15, 0x70, 0x75,
-	0x6b, 0x6e, 0x6f, 0x77, 0x6e, 0x10, 0x03, 0x42, 0x43, 0x5a, 0x41, 0x67, 0x69, 0x74, 0x68, 0x75,
+	0x62, 0x6c, 0x69, 0x63, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x4d, 0x6f, 0x6e, 0x69, 0x74, 0x6f, 0x72,
-	0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x74, 0x72, 0x75, 0x66, 0x66, 0x6c, 0x65, 0x73, 0x65, 0x63,
+	0x69, 0x6e, 0x67, 0x12, 0x3d, 0x0a, 0x0a, 0x73, 0x68, 0x61, 0x72, 0x65, 0x70, 0x6f, 0x69, 0x6e,
-	0x75, 0x72, 0x69, 0x74, 0x79, 0x2f, 0x74, 0x72, 0x75, 0x66, 0x66, 0x6c, 0x65, 0x68, 0x6f, 0x67,
+	0x74, 0x18, 0x19, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1b, 0x2e, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65,
-	0x2f, 0x76, 0x33, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x70, 0x62, 0x2f, 0x73, 0x6f, 0x75, 0x72, 0x63,
+	0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x53, 0x68, 0x61, 0x72, 0x65, 0x50,
-	0x65, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x70, 0x62, 0x62, 0x06, 0x70, 0x72,
+	0x6f, 0x69, 0x6e, 0x74, 0x48, 0x00, 0x52, 0x0a, 0x73, 0x68, 0x61, 0x72, 0x65, 0x70, 0x6f, 0x69,
-	0x6f, 0x74, 0x6f, 0x33,
+	0x6e, 0x74, 0x12, 0x40, 0x0a, 0x0b, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x44, 0x72, 0x69, 0x76,
+	0x65, 0x18, 0x1a, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65,
+	0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x47, 0x6f, 0x6f, 0x67, 0x6c, 0x65,
+	0x44, 0x72, 0x69, 0x76, 0x65, 0x48, 0x00, 0x52, 0x0b, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x44,
+	0x72, 0x69, 0x76, 0x65, 0x12, 0x3d, 0x0a, 0x0a, 0x61, 0x7a, 0x75, 0x72, 0x65, 0x52, 0x65, 0x70,
+	0x6f, 0x73, 0x18, 0x1b, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1b, 0x2e, 0x73, 0x6f, 0x75, 0x72, 0x63,
+	0x65, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x41, 0x7a, 0x75, 0x72, 0x65,
+	0x52, 0x65, 0x70, 0x6f, 0x73, 0x48, 0x00, 0x52, 0x0a, 0x61, 0x7a, 0x75, 0x72, 0x65, 0x52, 0x65,
+	0x70, 0x6f, 0x73, 0x42, 0x06, 0x0a, 0x04, 0x64, 0x61, 0x74, 0x61, 0x2a, 0x3e, 0x0a, 0x0a, 0x56,
+	0x69, 0x73, 0x69, 0x62, 0x69, 0x6c, 0x69, 0x74, 0x79, 0x12, 0x0a, 0x0a, 0x06, 0x70, 0x75, 0x62,
+	0x6c, 0x69, 0x63, 0x10, 0x00, 0x12, 0x0b, 0x0a, 0x07, 0x70, 0x72, 0x69, 0x76, 0x61, 0x74, 0x65,
+	0x10, 0x01, 0x12, 0x0a, 0x0a, 0x06, 0x73, 0x68, 0x61, 0x72, 0x65, 0x64, 0x10, 0x02, 0x12, 0x0b,
+	0x0a, 0x07, 0x75, 0x6e, 0x6b, 0x6e, 0x6f, 0x77, 0x6e, 0x10, 0x03, 0x42, 0x43, 0x5a, 0x41, 0x67,
+	0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x74, 0x72, 0x75, 0x66, 0x66, 0x6c,
+	0x65, 0x73, 0x65, 0x63, 0x75, 0x72, 0x69, 0x74, 0x79, 0x2f, 0x74, 0x72, 0x75, 0x66, 0x66, 0x6c,
+	0x65, 0x68, 0x6f, 0x67, 0x2f, 0x76, 0x33, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x70, 0x62, 0x2f, 0x73,
+	0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x70, 0x62,
+	0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
 }
 
 var (

pkg/pb/source_metadatapb/source_metadata.pb.validate.go

@@ -2865,6 +2865,8 @@ func (m *GoogleDrive) validate(all bool) error {
 
 	// no validation rules for LastModifiedBy
 
+	// no validation rules for Path
+
 	if len(errors) > 0 {
 		return GoogleDriveMultiError(errors)
 	}

pkg/sources/job_report.go

@@ -0,0 +1,37 @@
+package sources
+
+import (
+	"errors"
+	"sync"
+	"time"
+)
+
+// JobReport aggregates information about a run of a Source.
+type JobReport struct {
+	SourceID    int64
+	JobID       int64
+	StartTime   time.Time
+	EndTime     time.Time
+	TotalChunks uint64
+	errors      []error
+	errorsLock  sync.Mutex
+}
+
+// AddError adds a non-nil error to the aggregate of errors encountered during
+// scanning.
+func (jr *JobReport) AddError(err error) {
+	if err == nil {
+		return
+	}
+	jr.errorsLock.Lock()
+	defer jr.errorsLock.Unlock()
+	jr.errors = append(jr.errors, err)
+}
+
+// Errors joins all aggregated errors into one. If there were no errors, nil is
+// returned. errors.Is can be used to check for specific errors.
+func (jr *JobReport) Errors() error {
+	jr.errorsLock.Lock()
+	defer jr.errorsLock.Unlock()
+	return errors.Join(jr.errors...)
+}

pkg/sources/source_manager.go

@@ -1,9 +1,11 @@
 package sources
 
 import (
+	"errors"
 	"fmt"
 	"sync"
 	"sync/atomic"
+	"time"
 
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/context"
@@ -25,12 +27,20 @@ type SourceManager struct {
 	// Map of handle to source initializer.
 	handles     map[handle]SourceInitFunc
 	handlesLock sync.Mutex
+	// Map of handle to job reports.
+	// TODO: Manage culling and flushing to the API.
+	report     map[handle]*JobReport
+	reportLock sync.Mutex
 	// Pool limiting the amount of concurrent sources running.
 	pool            errgroup.Group
+	concurrentUnits int
+	// Run the sources using source unit enumeration / chunking if available.
+	useSourceUnits bool
 	// Downstream chunks channel to be scanned.
 	outputChunks chan *Chunk
-	// Set to true when Wait() returns.
+	// Set when Wait() returns.
 	done    bool
+	doneErr error
 }
 
 // apiClient is an interface for optionally communicating with an external API.
@@ -56,12 +66,25 @@ func WithBufferedOutput(size int) func(*SourceManager) {
 	return func(mgr *SourceManager) { mgr.outputChunks = make(chan *Chunk, size) }
 }
 
+// WithSourceUnits enables using source unit enumeration and chunking if the
+// source supports it.
+func WithSourceUnits() func(*SourceManager) {
+	return func(mgr *SourceManager) { mgr.useSourceUnits = true }
+}
+
+// WithConcurrentUnits limits the number of units to be scanned concurrently.
+// The default is unlimited.
+func WithConcurrentUnits(n int) func(*SourceManager) {
+	return func(mgr *SourceManager) { mgr.concurrentUnits = n }
+}
+
 // NewManager creates a new manager with the provided options.
 func NewManager(opts ...func(*SourceManager)) *SourceManager {
 	mgr := SourceManager{
 		// Default to the headless API. Can be overwritten by the WithAPI option.
 		api:          &headlessAPI{},
 		handles:      make(map[handle]SourceInitFunc),
+		report:       make(map[handle]*JobReport),
 		outputChunks: make(chan *Chunk),
 	}
 	for _, opt := range opts {
@@ -100,15 +123,24 @@ func (s *SourceManager) Run(ctx context.Context, handle handle) error {
 	ch := make(chan error)
 	s.pool.Go(func() error {
 		defer common.Recover(ctx)
-		// TODO: The manager should record these errors.
+		report, err := s.run(ctx, handle)
-		ch <- s.run(ctx, handle)
+		if report != nil {
+			s.reportLock.Lock()
+			s.report[handle] = report
+			s.reportLock.Unlock()
+		}
+		if err != nil {
+			ch <- err
+			return nil
+		}
+		ch <- report.Errors()
 		return nil
 	})
 	return <-ch
 }
 
 // ScheduleRun blocks until a resource is available to run the source, then
-// asynchronously runs it. Error information is lost in this case.
+// asynchronously runs it. Error information is stored and returned by Wait().
 func (s *SourceManager) ScheduleRun(ctx context.Context, handle handle) error {
 	// Do preflight checks before waiting on the pool.
 	if err := s.preflightChecks(ctx, handle); err != nil {
@@ -116,8 +148,14 @@ func (s *SourceManager) ScheduleRun(ctx context.Context, handle handle) error {
 	}
 	s.pool.Go(func() error {
 		defer common.Recover(ctx)
-		// TODO: The manager should record these errors.
+		// The error is already saved in the report, so we can ignore
-		_ = s.run(ctx, handle)
+		// it here.
+		report, _ := s.run(ctx, handle)
+		if report != nil {
+			s.reportLock.Lock()
+			s.report[handle] = report
+			s.reportLock.Unlock()
+		}
 		return nil
 	})
 	// TODO: Maybe wait for a signal here that initialization was successful?
@@ -136,12 +174,34 @@ func (s *SourceManager) Chunks() <-chan *Chunk {
 func (s *SourceManager) Wait() error {
 	// Check if the manager has been Waited.
 	if s.done {
-		return nil
+		return s.doneErr
 	}
-	// TODO: Aggregate all errors from all sources.
 	defer close(s.outputChunks)
 	defer func() { s.done = true }()
-	return s.pool.Wait()
+
+	// We are only using the errgroup for limiting concurrency.
+	// TODO: Maybe switch to using a semaphore.Weighted.
+	_ = s.pool.Wait()
+
+	// Aggregate all errors from all job reports.
+	// TODO: This should probably only be the fatal errors. We'll also need
+	//       to rewrite this for when the reports start getting culled.
+	s.reportLock.Lock()
+	defer s.reportLock.Unlock()
+	errs := make([]error, 0, len(s.report))
+	for _, report := range s.report {
+		errs = append(errs, report.Errors())
+	}
+	s.doneErr = errors.Join(errs...)
+	return s.doneErr
+}
+
+// Report retrieves a scan report for a given handle. If no report exists or
+// the Source has not finished, nil will be returned.
+func (s *SourceManager) Report(handle handle) *JobReport {
+	s.reportLock.Lock()
+	defer s.reportLock.Unlock()
+	return s.report[handle]
 }
 
 // preflightChecks is a helper method to check the Manager or the context isn't
@@ -159,23 +219,127 @@ func (s *SourceManager) preflightChecks(ctx context.Context, handle handle) erro
 }
 
 // run is a helper method to sychronously run the source. It does not check for
-// acquired resources.
+// acquired resources. Possible return values are:
-func (s *SourceManager) run(ctx context.Context, handle handle) error {
+//
+//   - *JobReport, nil
+//     Successfully ran the source, but the report could have errors.
+//
+//   - *JobReport, error
+//     There was an error calling Init or Chunks. This sort of error indicates
+//     a fatal error and is also recorded in the report.
+//
+//   - nil, error:
+//     There was an error from the API or the handle is invalid. The latter of
+//     which should never happen due to the preflightChecks.
+func (s *SourceManager) run(ctx context.Context, handle handle) (*JobReport, error) {
 	jobID, err := s.api.GetJobID(ctx, int64(handle))
 	if err != nil {
-		return err
+		return nil, err
 	}
 	initFunc, ok := s.getInitFunc(handle)
 	if !ok {
-		return fmt.Errorf("unrecognized handle")
+		return nil, fmt.Errorf("unrecognized handle")
 	}
+	// Create a report for this run.
+	report := &JobReport{
+		SourceID:  int64(handle),
+		JobID:     jobID,
+		StartTime: time.Now(),
+	}
+	defer func() { report.EndTime = time.Now() }()
+
+	// Initialize the source.
 	source, err := initFunc(ctx, jobID, int64(handle))
 	if err != nil {
-		return err
+		report.AddError(err)
+		return report, err
 	}
-	// TODO: Support UnitChunker and SourceUnitEnumerator.
+	// Check for the preferred method of tracking source units.
-	// TODO: This is where we can introspect on the chunks collected.
+	if enumChunker, ok := source.(SourceUnitEnumChunker); ok && s.useSourceUnits {
-	return source.Chunks(ctx, s.outputChunks)
+		return s.runWithUnits(ctx, handle, enumChunker, report)
+	}
+	return s.runWithoutUnits(ctx, handle, source, report)
+}
+
+// runWithoutUnits is a helper method to run a Source. It has coarse-grained
+// job reporting.
+func (s *SourceManager) runWithoutUnits(ctx context.Context, handle handle, source Source, report *JobReport) (*JobReport, error) {
+	// Introspect on the chunks we get from the Chunks method.
+	ch := make(chan *Chunk)
+	var wg sync.WaitGroup
+	// Consume chunks and export chunks.
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		for chunk := range ch {
+			atomic.AddUint64(&report.TotalChunks, 1)
+			_ = common.CancellableWrite(ctx, s.outputChunks, chunk)
+		}
+	}()
+	// Don't return from this function until the goroutine has finished
+	// outputting chunks to the downstream channel. Closing the channel
+	// will stop the goroutine, so that needs to happen first in the defer
+	// stack.
+	defer wg.Wait()
+	defer close(ch)
+	if err := source.Chunks(ctx, ch); err != nil {
+		report.AddError(err)
+		return report, err
+	}
+	return report, nil
+}
+
+// runWithUnits is a helper method to run a Source that is also a
+// SourceUnitEnumChunker. This allows better introspection of what is getting
+// scanned and any errors encountered.
+func (s *SourceManager) runWithUnits(ctx context.Context, handle handle, source SourceUnitEnumChunker, report *JobReport) (*JobReport, error) {
+	reporter := &mgrUnitReporter{
+		unitCh: make(chan SourceUnit),
+	}
+	// Produce units.
+	go func() {
+		// TODO: Catch panics and add to report.
+		defer close(reporter.unitCh)
+		if err := source.Enumerate(ctx, reporter); err != nil {
+			report.AddError(err)
+		}
+	}()
+	var wg sync.WaitGroup
+	// TODO: Maybe switch to using a semaphore.Weighted.
+	var unitPool errgroup.Group
+	if s.concurrentUnits != 0 {
+		// Negative values indicated no limit.
+		unitPool.SetLimit(s.concurrentUnits)
+	}
+	for unit := range reporter.unitCh {
+		reporter := &mgrChunkReporter{
+			unitID:  unit.SourceUnitID(),
+			chunkCh: make(chan *Chunk),
+		}
+		unit := unit
+		// Consume units and produce chunks.
+		unitPool.Go(func() error {
+			// TODO: Catch panics and add to report.
+			defer close(reporter.chunkCh)
+			if err := source.ChunkUnit(ctx, unit, reporter); err != nil {
+				report.AddError(err)
+			}
+			return nil
+		})
+		// Consume chunks and export chunks.
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			for chunk := range reporter.chunkCh {
+				// TODO: Introspect on the chunks we got from this unit.
+				atomic.AddUint64(&report.TotalChunks, 1)
+				_ = common.CancellableWrite(ctx, s.outputChunks, chunk)
+			}
+		}()
+	}
+	wg.Wait()
+	// TODO: Return fatal errors.
+	return report, nil
 }
 
 // getInitFunc is a helper method for safe concurrent access to the
@@ -201,3 +365,40 @@ func (api *headlessAPI) RegisterSource(ctx context.Context, name string, kind so
 func (api *headlessAPI) GetJobID(ctx context.Context, id int64) (int64, error) {
 	return atomic.AddInt64(&api.jobIDCounter, 1), nil
 }
+
+// mgrUnitReporter implements the UnitReporter interface.
+type mgrUnitReporter struct {
+	unitCh       chan SourceUnit
+	unitErrs     []error
+	unitErrsLock sync.Mutex
+}
+
+func (s *mgrUnitReporter) UnitOk(ctx context.Context, unit SourceUnit) error {
+	return common.CancellableWrite(ctx, s.unitCh, unit)
+}
+
+func (s *mgrUnitReporter) UnitErr(ctx context.Context, err error) error {
+	s.unitErrsLock.Lock()
+	defer s.unitErrsLock.Unlock()
+	s.unitErrs = append(s.unitErrs, err)
+	return nil
+}
+
+// mgrChunkReporter implements the ChunkReporter interface.
+type mgrChunkReporter struct {
+	unitID        string
+	chunkCh       chan *Chunk
+	chunkErrs     []error
+	chunkErrsLock sync.Mutex
+}
+
+func (s *mgrChunkReporter) ChunkOk(ctx context.Context, chunk Chunk) error {
+	return common.CancellableWrite(ctx, s.chunkCh, &chunk)
+}
+
+func (s *mgrChunkReporter) ChunkErr(ctx context.Context, err error) error {
+	s.chunkErrsLock.Lock()
+	defer s.chunkErrsLock.Unlock()
+	s.chunkErrs = append(s.chunkErrs, err)
+	return nil
+}

pkg/sources/source_manager_test.go

@@ -29,6 +29,8 @@ func (d *DummySource) GetProgress() *Progress { return nil }
 // Interface to easily test different chunking methods.
 type chunker interface {
 	Chunks(context.Context, chan *Chunk) error
+	ChunkUnit(ctx context.Context, unit SourceUnit, reporter ChunkReporter) error
+	Enumerate(ctx context.Context, reporter UnitReporter) error
 }
 
 // Chunk method that writes count bytes to the channel before returning.
@@ -45,6 +47,31 @@ func (c *counterChunker) Chunks(_ context.Context, ch chan *Chunk) error {
 	return nil
 }
 
+// countChunk implements SourceUnit.
+type countChunk byte
+
+func (c countChunk) SourceUnitID() string { return fmt.Sprintf("countChunk(%d)", c) }
+
+func (c *counterChunker) Enumerate(ctx context.Context, reporter UnitReporter) error {
+	for i := 0; i < c.count; i++ {
+		if err := reporter.UnitOk(ctx, countChunk(byte(i))); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func (c *counterChunker) ChunkUnit(ctx context.Context, unit SourceUnit, reporter ChunkReporter) error {
+	return reporter.ChunkOk(ctx, Chunk{Data: []byte{byte(unit.(countChunk))}})
+}
+
+// Chunk method that always returns an error.
+type errorChunker struct{ error }
+
+func (c errorChunker) Chunks(context.Context, chan *Chunk) error                  { return c }
+func (c errorChunker) Enumerate(context.Context, UnitReporter) error              { return c }
+func (c errorChunker) ChunkUnit(context.Context, SourceUnit, ChunkReporter) error { return c }
+
 // enrollDummy is a helper function to enroll a DummySource with a SourceManager.
 func enrollDummy(mgr *SourceManager, chunkMethod chunker) (handle, error) {
 	return mgr.Enroll(context.Background(), "dummy", 1337,
@@ -117,3 +144,51 @@ func TestSourceManagerWait(t *testing.T) {
 		t.Fatalf("expected scheduling run to fail")
 	}
 }
+
+func TestSourceManagerError(t *testing.T) {
+	mgr := NewManager()
+	handle, err := enrollDummy(mgr, errorChunker{fmt.Errorf("oops")})
+	if err != nil {
+		t.Fatalf("unexpected error enrolling source: %v", err)
+	}
+	// A synchronous run should fail.
+	if err := mgr.Run(context.Background(), handle); err == nil {
+		t.Fatalf("expected run to fail")
+	}
+	// Scheduling a run should not fail, but the error should surface in
+	// Wait().
+	if err := mgr.ScheduleRun(context.Background(), handle); err != nil {
+		t.Fatalf("unexpected error scheduling run: %v", err)
+	}
+	if err := mgr.Wait(); err == nil {
+		t.Fatalf("expected wait to fail")
+	}
+}
+
+func TestSourceManagerReport(t *testing.T) {
+	for _, opts := range [][]func(*SourceManager){
+		{WithBufferedOutput(8)},
+		{WithBufferedOutput(8), WithSourceUnits()},
+		{WithBufferedOutput(8), WithSourceUnits(), WithConcurrentUnits(1)},
+	} {
+		mgr := NewManager(opts...)
+		handle, err := enrollDummy(mgr, &counterChunker{count: 4})
+		if err != nil {
+			t.Fatalf("unexpected error enrolling source: %v", err)
+		}
+		// Synchronously run the source.
+		if err := mgr.Run(context.Background(), handle); err != nil {
+			t.Fatalf("unexpected error running source: %v", err)
+		}
+		report := mgr.Report(handle)
+		if report == nil {
+			t.Fatalf("expected a report")
+		}
+		if err := report.Errors(); err != nil {
+			t.Fatalf("unexpected error in report: %v", err)
+		}
+		if report.TotalChunks != 4 {
+			t.Fatalf("expected report to have 4 chunks, got: %d", report.TotalChunks)
+		}
+	}
+}

pkg/sources/sources.go

@@ -44,6 +44,13 @@ type Source interface {
 	GetProgress() *Progress
 }
 
+// SourceUnitEnumChunker are the two required interfaces to support enumerating
+// and chunking of units.
+type SourceUnitEnumChunker interface {
+	SourceUnitEnumerator
+	SourceUnitChunker
+}
+
 // SourceUnitUnmarshaller defines an optional interface a Source can implement
 // to support units coming from an external source.
 type SourceUnitUnmarshaller interface {

proto/source_metadata.proto

@@ -248,6 +248,7 @@ message GoogleDrive {
   string timestamp = 4;
   bool shared = 5;
   string last_modified_by = 6;
+  string path = 7;
 }
 
 message AzureRepos {