parse all comments and string literals... lots of results

.github/dependabot.yml

@@ -8,8 +8,9 @@ updates:
   - package-ecosystem: "gomod" # See documentation for possible values
     directory: "/" # Location of package manifests
     schedule:
-      interval: "monthly"
+      interval: "weekly"
   - package-ecosystem: "github-actions"
     directory: "/"
     schedule:
-      interval: "monthly"
+      # Check for updates to GitHub Actions every weekday
+      interval: "weekly"

.github/workflows/performance.yml

@@ -4,16 +4,12 @@ on: [pull_request]
 
 jobs:
   speed:
-    #   skip if PR is from a fork.
-    # TODO: this could probabaly be refactored a bit so that it runs on forks
-    if: ${{ ! github.event.pull_request.head.repo.fork }}
-    
     runs-on: ubuntu-latest
     steps:
     - name: Install Go
       uses: actions/setup-go@v4
       with:
-        go-version: '1.20'
+        go-version: '1.18'
 
     - name: Checkout code
       uses: actions/checkout@v3
@@ -89,4 +85,4 @@ jobs:
         then
           echo "HEAD run time is at least 10% slower than PREVIOUS run time"
           exit 1
-        fi
+        fi

.github/workflows/smoke.yml

@@ -11,9 +11,12 @@ jobs:
     - name: Install Go
       uses: actions/setup-go@v4
       with:
-        go-version: '1.20'
+        go-version: '1.18'
     - name: Checkout code
       uses: actions/checkout@v3
+      with:
+        fetch-depth: 0
+        ref: ${{ github.head_ref }}
     - name: Smoke
       run: |
         set -e

.github/workflows/test.yml

@@ -11,7 +11,6 @@ on:
 
 jobs:
   test:
-    if: ${{ ! github.event.pull_request.head.repo.fork }}
     runs-on: ubuntu-latest
     permissions:
       actions: 'read'
@@ -32,7 +31,6 @@ jobs:
     - name: Test
       run: make test-integration
   test-detectors:
-    if: ${{ ! github.event.pull_request.head.repo.fork }}
     runs-on: ubuntu-latest
     permissions:
       actions: 'read'
@@ -52,19 +50,4 @@ jobs:
         service_account: 'github-ci-external@trufflehog-testing.iam.gserviceaccount.com'
     - name: Test
       run: make test-detectors
-      continue-on-error: true  
-  test-fork:
-    if: ${{ github.event.pull_request.head.repo.fork }}
-    runs-on: ubuntu-latest
-    permissions:
-      actions: 'read'
-      contents: 'read'
-    steps:
-    - name: Install Go
-      uses: actions/setup-go@v4
-      with:
-        go-version: '1.20'
-    - name: Checkout code
-      uses: actions/checkout@v3
-    - name: Test
-      run: make test-forks
+      continue-on-error: true

Makefile

@@ -39,9 +39,6 @@ test-race:
 test-detectors:
 	CGO_ENABLED=0 go test -tags=detectors -timeout=5m $(shell go list ./... | grep pkg/detectors)
 
-test-forks:
-	CGO_ENABLED=0 go test -timeout=5m $(shell go list ./... | grep -v /vendor/ | grep -v pkg/detectors | grep -v pkg/sources)
-
 bench:
 	CGO_ENABLED=0 go test $(shell go list ./pkg/secrets/... | grep -v /vendor/) -benchmem -run=xxx -bench .
 

go.mod

@@ -4,20 +4,16 @@ go 1.20
 
 replace github.com/jpillora/overseer => github.com/trufflesecurity/overseer v1.1.7-custom5
 
-// go wants to pull in the latest but after v10.2.5 has a breaking API change
-replace github.com/couchbase/gocbcore/v10 => github.com/couchbase/gocbcore/v10 v10.2.4
-
 require (
 	cloud.google.com/go/secretmanager v1.11.1
 	cloud.google.com/go/storage v1.31.0
 	github.com/Azure/go-autorest/autorest/azure/auth v0.5.11
-	github.com/BobuSumisu/aho-corasick v1.0.3
-	github.com/TheZeroSlave/zapsentry v1.17.0
+	github.com/TheZeroSlave/zapsentry v1.15.0
 	github.com/aws/aws-sdk-go v1.44.83
 	github.com/bill-rich/disk-buffer-reader v0.1.7
 	github.com/bill-rich/go-syslog v0.0.0-20220413021637-49edb52a574c
 	github.com/bitfinexcom/bitfinex-api-go v0.0.0-20210608095005-9e0b26f200fb
-	github.com/bradleyfalzon/ghinstallation/v2 v2.6.0
+	github.com/bradleyfalzon/ghinstallation/v2 v2.4.0
 	github.com/couchbase/gocb/v2 v2.6.3
 	github.com/crewjam/rfc5424 v0.1.0
 	github.com/denisenkom/go-mssqldb v0.12.3
@@ -26,7 +22,7 @@ require (
 	github.com/felixge/fgprof v0.9.3
 	github.com/getsentry/sentry-go v0.22.0
 	github.com/go-errors/errors v1.4.2
-	github.com/go-git/go-git/v5 v5.8.0
+	github.com/go-git/go-git/v5 v5.7.0
 	github.com/go-ldap/ldap/v3 v3.4.5
 	github.com/go-logr/logr v1.2.4
 	github.com/go-logr/zapr v1.2.4
@@ -34,14 +30,12 @@ require (
 	github.com/go-sql-driver/mysql v1.7.1
 	github.com/gobwas/glob v0.2.3
 	github.com/golang-jwt/jwt v3.2.2+incompatible
-	github.com/golang/mock v1.4.3
 	github.com/google/go-cmp v0.5.9
-	github.com/google/go-containerregistry v0.15.2
+	github.com/google/go-containerregistry v0.14.0
 	github.com/google/go-github/v42 v42.0.0
-	github.com/googleapis/gax-go/v2 v2.12.0
+	github.com/googleapis/gax-go/v2 v2.11.0
 	github.com/hashicorp/go-retryablehttp v0.7.4
-	github.com/hashicorp/golang-lru v0.5.1
-	github.com/jlaffaye/ftp v0.2.0
+	github.com/jlaffaye/ftp v0.1.0
 	github.com/joho/godotenv v1.5.1
 	github.com/jpillora/overseer v1.1.6
 	github.com/kylelemons/godebug v1.1.0
@@ -50,22 +44,22 @@ require (
 	github.com/mholt/archiver/v4 v4.0.0-alpha.8
 	github.com/patrickmn/go-cache v2.1.0+incompatible
 	github.com/paulbellamy/ratecounter v0.2.0
+	github.com/petar-dambovaliev/aho-corasick v0.0.0-20211021192214-5ab2d9280aa9
 	github.com/pkg/errors v0.9.1
 	github.com/prometheus/client_golang v1.16.0
 	github.com/rabbitmq/amqp091-go v1.8.1
 	github.com/sergi/go-diff v1.3.1
 	github.com/stretchr/testify v1.8.4
 	github.com/tailscale/depaware v0.0.0-20210622194025-720c4b409502
-	github.com/xanzy/go-gitlab v0.88.0
+	github.com/xanzy/go-gitlab v0.86.0
 	go.mongodb.org/mongo-driver v1.12.0
-	go.uber.org/mock v0.2.0
 	go.uber.org/zap v1.24.0
-	golang.org/x/crypto v0.11.0
+	golang.org/x/crypto v0.10.0
 	golang.org/x/exp v0.0.0-20221018205818-5c77f4b2bbd7
-	golang.org/x/oauth2 v0.10.0
+	golang.org/x/oauth2 v0.9.0
 	golang.org/x/sync v0.3.0
-	golang.org/x/text v0.11.0
-	google.golang.org/api v0.131.0
+	golang.org/x/text v0.10.0
+	google.golang.org/api v0.129.0
 	google.golang.org/protobuf v1.31.0
 	gopkg.in/alecthomas/kingpin.v2 v2.2.6
 	gopkg.in/h2non/gock.v1 v1.1.2
@@ -74,7 +68,7 @@ require (
 
 require (
 	cloud.google.com/go v0.110.2 // indirect
-	cloud.google.com/go/compute v1.20.1 // indirect
+	cloud.google.com/go/compute v1.19.3 // indirect
 	cloud.google.com/go/compute/metadata v0.2.3 // indirect
 	cloud.google.com/go/iam v1.1.0 // indirect
 	github.com/Azure/go-autorest v14.2.0+incompatible // indirect
@@ -86,8 +80,9 @@ require (
 	github.com/Azure/go-autorest/tracing v0.6.0 // indirect
 	github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 // indirect
 	github.com/Microsoft/go-winio v0.6.1 // indirect
-	github.com/ProtonMail/go-crypto v0.0.0-20230717121422-5aa5874ade95 // indirect
+	github.com/ProtonMail/go-crypto v0.0.0-20230518184743-7afd39499903 // indirect
 	github.com/acomagu/bufpipe v1.0.4 // indirect
+	github.com/alecthomas/chroma v0.10.0 // indirect
 	github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 // indirect
 	github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 // indirect
 	github.com/andybalholm/brotli v1.0.5 // indirect
@@ -100,9 +95,10 @@ require (
 	github.com/cloudflare/circl v1.3.3 // indirect
 	github.com/connesc/cipherio v0.2.1 // indirect
 	github.com/containerd/stargz-snapshotter/estargz v0.14.3 // indirect
-	github.com/couchbase/gocbcore/v10 v10.2.4 // indirect
+	github.com/couchbase/gocbcore/v10 v10.2.3 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/dimchansky/utfbom v1.1.1 // indirect
+	github.com/dlclark/regexp2 v1.4.0 // indirect
 	github.com/docker/cli v23.0.5+incompatible // indirect
 	github.com/docker/distribution v2.8.2+incompatible // indirect
 	github.com/docker/docker v23.0.5+incompatible // indirect
@@ -119,7 +115,7 @@ require (
 	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
 	github.com/golang/protobuf v1.5.3 // indirect
 	github.com/golang/snappy v0.0.4 // indirect
-	github.com/google/go-github/v53 v53.2.0 // indirect
+	github.com/google/go-github/v52 v52.0.0 // indirect
 	github.com/google/go-querystring v1.1.0 // indirect
 	github.com/google/pprof v0.0.0-20211214055906-6f57359322fd // indirect
 	github.com/google/s2a-go v0.1.4 // indirect
@@ -129,7 +125,6 @@ require (
 	github.com/hashicorp/errwrap v1.0.0 // indirect
 	github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
 	github.com/hashicorp/go-multierror v1.1.1 // indirect
-	github.com/hashicorp/golang-lru/v2 v2.0.4 // indirect
 	github.com/imdario/mergo v0.3.15 // indirect
 	github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect
 	github.com/jmespath/go-jmespath v0.4.0 // indirect
@@ -156,7 +151,7 @@ require (
 	github.com/prometheus/procfs v0.10.1 // indirect
 	github.com/rogpeppe/go-internal v1.10.0 // indirect
 	github.com/sirupsen/logrus v1.9.0 // indirect
-	github.com/skeema/knownhosts v1.2.0 // indirect
+	github.com/skeema/knownhosts v1.1.1 // indirect
 	github.com/therootcompany/xz v1.0.1 // indirect
 	github.com/ulikunitz/xz v0.5.10 // indirect
 	github.com/vbatts/tar-split v0.11.3 // indirect
@@ -171,16 +166,16 @@ require (
 	go.uber.org/multierr v1.6.0 // indirect
 	go4.org v0.0.0-20200411211856-f5505b9728dd // indirect
 	golang.org/x/mod v0.11.0 // indirect
-	golang.org/x/net v0.12.0 // indirect
-	golang.org/x/sys v0.10.0 // indirect
+	golang.org/x/net v0.11.0 // indirect
+	golang.org/x/sys v0.9.0 // indirect
 	golang.org/x/time v0.3.0 // indirect
 	golang.org/x/tools v0.10.0 // indirect
 	golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect
 	google.golang.org/appengine v1.6.7 // indirect
-	google.golang.org/genproto v0.0.0-20230629202037-9506855d4529 // indirect
-	google.golang.org/genproto/googleapis/api v0.0.0-20230629202037-9506855d4529 // indirect
-	google.golang.org/genproto/googleapis/rpc v0.0.0-20230706204954-ccb25ca9f130 // indirect
-	google.golang.org/grpc v1.56.2 // indirect
+	google.golang.org/genproto v0.0.0-20230530153820-e85fd2cbaebc // indirect
+	google.golang.org/genproto/googleapis/api v0.0.0-20230530153820-e85fd2cbaebc // indirect
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20230530153820-e85fd2cbaebc // indirect
+	google.golang.org/grpc v1.56.1 // indirect
 	gopkg.in/warnings.v0 v0.1.2 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect

go.sum

@@ -11,8 +11,8 @@ cloud.google.com/go v0.110.2 h1:sdFPBr6xG9/wkBbfhmUz/JmZC7X6LavQgcrVINrKiVA=
 cloud.google.com/go v0.110.2/go.mod h1:k04UEeEtb6ZBRTv3dZz4CeJC3jKGxyhl0sAiVVquxiw=
 cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o=
 cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE=
-cloud.google.com/go/compute v1.20.1 h1:6aKEtlUiwEpJzM001l0yFkpXmUVXaN8W+fbkb2AZNbg=
-cloud.google.com/go/compute v1.20.1/go.mod h1:4tCnrn48xsqlwSAiLf1HXMQk8CONslYbdiEZc9FEIbM=
+cloud.google.com/go/compute v1.19.3 h1:DcTwsFgGev/wV5+q8o2fzgcHOaac+DKGC91ZlvpsQds=
+cloud.google.com/go/compute v1.19.3/go.mod h1:qxvISKp/gYnXkSAD1ppcSOveRAmzxicEv/JlizULFrI=
 cloud.google.com/go/compute/metadata v0.2.0/go.mod h1:zFmK7XCadkQkj6TtorcaGlCW1hT1fIilQDwofLpJ20k=
 cloud.google.com/go/compute/metadata v0.2.3 h1:mg4jlk7mCAj6xXp9UJ4fjI9VUI5rubuGBW5aJ7UnBMY=
 cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA=
@@ -51,8 +51,6 @@ github.com/Azure/go-autorest/tracing v0.6.0 h1:TYi4+3m5t6K48TGI9AUdb+IzbnSxvnvUM
 github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU=
 github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 h1:mFRzDkZVAjdal+s7s0MwaRv9igoPqLRdzOLzw/8Xvq8=
 github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358/go.mod h1:chxPXzSsl7ZWRAuOIE23GDNzjWuZquvFlgA8xmpunjU=
-github.com/BobuSumisu/aho-corasick v1.0.3 h1:uuf+JHwU9CHP2Vx+wAy6jcksJThhJS9ehR8a+4nPE9g=
-github.com/BobuSumisu/aho-corasick v1.0.3/go.mod h1:hm4jLcvZKI2vRF2WDU1N4p/jpWtpOzp3nLmi9AzX/XE=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/BurntSushi/toml v1.2.1/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ=
 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
@@ -60,12 +58,14 @@ github.com/Microsoft/go-winio v0.5.2/go.mod h1:WpS1mjBmmwHBEWmogvA2mj8546UReBk4v
 github.com/Microsoft/go-winio v0.6.1 h1:9/kr64B9VUZrLm5YYwbGtUJnMgqWVOdUAXu6Migciow=
 github.com/Microsoft/go-winio v0.6.1/go.mod h1:LRdKpFKfdobln8UmuiYcKPot9D2v6svN5+sAH+4kjUM=
 github.com/ProtonMail/go-crypto v0.0.0-20230217124315-7d5c6f04bbb8/go.mod h1:I0gYDMZ6Z5GRU7l58bNFSkPTFN6Yl12dsUlAZ8xy98g=
-github.com/ProtonMail/go-crypto v0.0.0-20230717121422-5aa5874ade95 h1:KLq8BE0KwCL+mmXnjLWEAOYO+2l2AE4YMmqG1ZpZHBs=
-github.com/ProtonMail/go-crypto v0.0.0-20230717121422-5aa5874ade95/go.mod h1:EjAoLdwvbIOoOQr3ihjnSoLZRtE8azugULFRteWMNc0=
-github.com/TheZeroSlave/zapsentry v1.17.0 h1:RIQCG89U7vWWZVmmCxeUz/g32WEcAYXUrXHoMlbSDmc=
-github.com/TheZeroSlave/zapsentry v1.17.0/go.mod h1:D1YMfSuu6xnkhwFXxrronesmsiyDhIqo+86I3Ok+r64=
+github.com/ProtonMail/go-crypto v0.0.0-20230518184743-7afd39499903 h1:ZK3C5DtzV2nVAQTx5S5jQvMeDqWtD1By5mOoyY/xJek=
+github.com/ProtonMail/go-crypto v0.0.0-20230518184743-7afd39499903/go.mod h1:8TI4H3IbrackdNgv+92dI+rhpCaLqM0IfpgCgenFvRE=
+github.com/TheZeroSlave/zapsentry v1.15.0 h1:w/YglG4Hc2L2VoEH6JKC+3YzhUL18/OZRjJRSvyXZp4=
+github.com/TheZeroSlave/zapsentry v1.15.0/go.mod h1:D1YMfSuu6xnkhwFXxrronesmsiyDhIqo+86I3Ok+r64=
 github.com/acomagu/bufpipe v1.0.4 h1:e3H4WUzM3npvo5uv95QuJM3cQspFNtFBzvJ2oNjKIDQ=
 github.com/acomagu/bufpipe v1.0.4/go.mod h1:mxdxdup/WdsKVreO5GpW4+M/1CE2sMG4jeGJ2sYmHc4=
+github.com/alecthomas/chroma v0.10.0 h1:7XDcGkCQopCNKjZHfYrNLraA+M7e0fMiJ/Mfikbfjek=
+github.com/alecthomas/chroma v0.10.0/go.mod h1:jtJATyUxlIORhUOFNA9NZDWGAQ8wpxQQqNSB4rjA/1s=
 github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 h1:JYp7IbQjafoB+tBA3gMyHYHrpOtNuDiK/uB5uXxq5wM=
 github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
 github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 h1:s6gZFSlWYmbqAuRjVTiNNhvNRfY2Wxp9nhfyel4rklc=
@@ -95,10 +95,9 @@ github.com/bodgit/sevenzip v1.3.0 h1:1ljgELgtHqvgIp8W8kgeEGHIWP4ch3xGI8uOBZgLVKY
 github.com/bodgit/sevenzip v1.3.0/go.mod h1:omwNcgZTEooWM8gA/IJ2Nk/+ZQ94+GsytRzOJJ8FBlM=
 github.com/bodgit/windows v1.0.0 h1:rLQ/XjsleZvx4fR1tB/UxQrK+SJ2OFHzfPjLWWOhDIA=
 github.com/bodgit/windows v1.0.0/go.mod h1:a6JLwrB4KrTR5hBpp8FI9/9W9jJfeQ2h4XDXU74ZCdM=
-github.com/bradleyfalzon/ghinstallation/v2 v2.6.0 h1:IRY7Xy588KylkoycsUhFpW7cdGpy5Y5BPsz4IfuJtGk=
-github.com/bradleyfalzon/ghinstallation/v2 v2.6.0/go.mod h1:oQ3etOwN3TRH4EwgW5/7MxSVMGlMlzG/O8TU7eYdoSk=
+github.com/bradleyfalzon/ghinstallation/v2 v2.4.0 h1:zYSzkoIwekCQAr6GT6KxISLt4YRS6kd4/ixfzMN+7yc=
+github.com/bradleyfalzon/ghinstallation/v2 v2.4.0/go.mod h1:4MwZLSgBJJgg4i3nJwZJ95AMooSqN8fJDmegLVn9Q2U=
 github.com/bwesterb/go-ristretto v1.2.0/go.mod h1:fUIoIZaG73pV5biE2Blr2xEzDoMj7NFEuV9ekS419A0=
-github.com/bwesterb/go-ristretto v1.2.3/go.mod h1:fUIoIZaG73pV5biE2Blr2xEzDoMj7NFEuV9ekS419A0=
 github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
 github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
 github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44=
@@ -122,8 +121,8 @@ github.com/containerd/stargz-snapshotter/estargz v0.14.3 h1:OqlDCK3ZVUO6C3B/5FSk
 github.com/containerd/stargz-snapshotter/estargz v0.14.3/go.mod h1:KY//uOCIkSuNAHhJogcZtrNHdKrA99/FCCRjE3HD36o=
 github.com/couchbase/gocb/v2 v2.6.3 h1:5RsMo+RRfK0mVxHLAfpBz3/tHlgXZb1WBNItLk9Ab+c=
 github.com/couchbase/gocb/v2 v2.6.3/go.mod h1:yF5F6BHTZ/ZowhEuZbySbXrlI4rHd1TIhm5azOaMbJU=
-github.com/couchbase/gocbcore/v10 v10.2.4 h1:TDTQ1mSBUw9eajuV71ZDJcLRbmfGcsvZV9SMeUDAVGc=
-github.com/couchbase/gocbcore/v10 v10.2.4/go.mod h1:lYQIIk+tzoMcwtwU5GzPbDdqEkwkH3isI2rkSpfL0oM=
+github.com/couchbase/gocbcore/v10 v10.2.3 h1:PEkRSNSkKjUBXx82Ucr094+anoiCG5GleOOQZOHo6D4=
+github.com/couchbase/gocbcore/v10 v10.2.3/go.mod h1:lYQIIk+tzoMcwtwU5GzPbDdqEkwkH3isI2rkSpfL0oM=
 github.com/couchbaselabs/gocaves/client v0.0.0-20230307083111-cc3960c624b1/go.mod h1:AVekAZwIY2stsJOMWLAS/0uA/+qdp7pjO8EHnl61QkY=
 github.com/couchbaselabs/gocaves/client v0.0.0-20230404095311-05e3ba4f0259 h1:2TXy68EGEzIMHOx9UvczR5ApVecwCfQZ0LjkmwMI6g4=
 github.com/couchbaselabs/gocaves/client v0.0.0-20230404095311-05e3ba4f0259/go.mod h1:AVekAZwIY2stsJOMWLAS/0uA/+qdp7pjO8EHnl61QkY=
@@ -138,6 +137,8 @@ github.com/denisenkom/go-mssqldb v0.12.3 h1:pBSGx9Tq67pBOTLmxNuirNTeB8Vjmf886Kx+
 github.com/denisenkom/go-mssqldb v0.12.3/go.mod h1:k0mtMFOnU+AihqFxPMiF05rtiDrorD1Vrm1KEz5hxDo=
 github.com/dimchansky/utfbom v1.1.1 h1:vV6w1AhK4VMnhBno/TPVCoK9U/LP0PkLCS9tbxHdi/U=
 github.com/dimchansky/utfbom v1.1.1/go.mod h1:SxdoEBH5qIqFocHMyGOXVAybYJdr71b1Q/j0mACtrfE=
+github.com/dlclark/regexp2 v1.4.0 h1:F1rxgk7p4uKjwIQxBs9oAXe5CqrXlCduYEJvrF4u93E=
+github.com/dlclark/regexp2 v1.4.0/go.mod h1:2pZnwuY/m+8K6iRw6wQdMtk+rH5tNGR1i55kozfMjCc=
 github.com/dnaeon/go-vcr v1.2.0/go.mod h1:R4UdLID7HZT3taECzJs4YgbbH6PIGXB6W/sc5OLb6RQ=
 github.com/docker/cli v23.0.5+incompatible h1:ufWmAOuD3Vmr7JP2G5K3cyuNC4YZWiAsuDEvFVVDafE=
 github.com/docker/cli v23.0.5+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8=
@@ -181,12 +182,13 @@ github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376/go.mod h1:an3vInlBmS
 github.com/go-git/go-billy/v5 v5.4.1 h1:Uwp5tDRkPr+l/TnbHOQzp+tmJfLceOlbVucgpTz8ix4=
 github.com/go-git/go-billy/v5 v5.4.1/go.mod h1:vjbugF6Fz7JIflbVpl1hJsGjSHNltrSw45YK/ukIvQg=
 github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20230305113008-0c11038e723f h1:Pz0DHeFij3XFhoBRGUDPzSJ+w2UcK5/0JvF8DRI58r8=
-github.com/go-git/go-git/v5 v5.8.0 h1:Rc543s6Tyq+YcyPwZRvU4jzZGM8rB/wWu94TnTIYALQ=
-github.com/go-git/go-git/v5 v5.8.0/go.mod h1:coJHKEOk5kUClpsNlXrUvPrDxY3w3gjHvhcZd8Fodw8=
+github.com/go-git/go-git/v5 v5.7.0 h1:t9AudWVLmqzlo+4bqdf7GY+46SUuRsx59SboFxkq2aE=
+github.com/go-git/go-git/v5 v5.7.0/go.mod h1:coJHKEOk5kUClpsNlXrUvPrDxY3w3gjHvhcZd8Fodw8=
 github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
 github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
 github.com/go-ldap/ldap/v3 v3.4.5 h1:ekEKmaDrpvR2yf5Nc/DClsGG9lAmdDixe44mLzlW5r8=
 github.com/go-ldap/ldap/v3 v3.4.5/go.mod h1:bMGIq3AGbytbaMwf8wdv5Phdxz0FWHTIYMSzyrYgnQs=
+github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs=
 github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ=
 github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
 github.com/go-logr/zapr v1.2.4 h1:QHVo+6stLbfJmYGkQ7uGHUCu5hnAFAj6mDe6Ea0SeOo=
@@ -258,12 +260,13 @@ github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/
 github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
 github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
 github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
-github.com/google/go-containerregistry v0.15.2 h1:MMkSh+tjSdnmJZO7ljvEqV1DjfekB6VUEAZgy3a+TQE=
-github.com/google/go-containerregistry v0.15.2/go.mod h1:wWK+LnOv4jXMM23IT/F1wdYftGWGr47Is8CG+pmHK1Q=
+github.com/google/go-containerregistry v0.14.0 h1:z58vMqHxuwvAsVwvKEkmVBz2TlgBgH5k6koEXBtlYkw=
+github.com/google/go-containerregistry v0.14.0/go.mod h1:aiJ2fp/SXvkWgmYHioXnbMdlgB8eXiiYOY55gfN91Wk=
+github.com/google/go-github/v39 v39.0.0/go.mod h1:C1s8C5aCC9L+JXIYpJM5GYytdX52vC1bLvHEF1IhBrE=
 github.com/google/go-github/v42 v42.0.0 h1:YNT0FwjPrEysRkLIiKuEfSvBPCGKphW5aS5PxwaoLec=
 github.com/google/go-github/v42 v42.0.0/go.mod h1:jgg/jvyI0YlDOM1/ps6XYh04HNQ3vKf0CVko62/EhRg=
-github.com/google/go-github/v53 v53.2.0 h1:wvz3FyF53v4BK+AsnvCmeNhf8AkTaeh2SoYu/XUvTtI=
-github.com/google/go-github/v53 v53.2.0/go.mod h1:XhFRObz+m/l+UCm9b7KSIC3lT3NWSXGt7mOsAWEloao=
+github.com/google/go-github/v52 v52.0.0 h1:uyGWOY+jMQ8GVGSX8dkSwCzlehU3WfdxQ7GweO/JP7M=
+github.com/google/go-github/v52 v52.0.0/go.mod h1:WJV6VEEUPuMo5pXqqa2ZCZEdbQqua4zAk2MZTIo+m+4=
 github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8=
 github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU=
 github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPgecwXBIDzw5no=
@@ -284,8 +287,8 @@ github.com/googleapis/enterprise-certificate-proxy v0.2.5 h1:UR4rDjcgpgEnqpIEvki
 github.com/googleapis/enterprise-certificate-proxy v0.2.5/go.mod h1:RxW0N9901Cko1VOCW3SXCpWP+mlIEkk2tP7jnHy9a3w=
 github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
 github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
-github.com/googleapis/gax-go/v2 v2.12.0 h1:A+gCJKdRfqXkr+BIRGtZLibNXf0m1f9E4HG56etFpas=
-github.com/googleapis/gax-go/v2 v2.12.0/go.mod h1:y+aIqrI5eb1YGMVJfuV3185Ts/D7qKpsEkdD5+I6QGU=
+github.com/googleapis/gax-go/v2 v2.11.0 h1:9V9PWXEsWnPpQhu/PeQIkS4eGzMlTLGgt80cUUI8Ki4=
+github.com/googleapis/gax-go/v2 v2.11.0/go.mod h1:DxmR61SGKkGLa2xigwuZIQpkCI2S5iydzRfb3peWZJI=
 github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
 github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw=
 github.com/h2non/parth v0.0.0-20190131123155-b4df798d6542 h1:2VTzZjLZBgl62/EtslCrtky5vbi9dd7HrQPQIx6wqiw=
@@ -301,10 +304,7 @@ github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9
 github.com/hashicorp/go-retryablehttp v0.7.4 h1:ZQgVdpTdAL7WpMIwLzCfbalOcSUdkDZnpUv3/+BxzFA=
 github.com/hashicorp/go-retryablehttp v0.7.4/go.mod h1:Jy/gPYAdjqffZ/yFGCFV2doI5wjtH1ewM9u8iYVjtX8=
 github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
-github.com/hashicorp/golang-lru v0.5.1 h1:0hERBMJE1eitiLkihrMvRVBYAkpHzc/J3QdDN+dAcgU=
 github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
-github.com/hashicorp/golang-lru/v2 v2.0.4 h1:7GHuZcgid37q8o5i3QI9KMT4nCWQQ3Kx3Ov6bb9MfK0=
-github.com/hashicorp/golang-lru/v2 v2.0.4/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM=
 github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
 github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
 github.com/ianlancetaylor/demangle v0.0.0-20210905161508-09a460cdf81d/go.mod h1:aYm2/VgdVmcIU8iMfdMvDMsRAQjcfZSKFby6HOFvi/w=
@@ -312,18 +312,21 @@ github.com/imdario/mergo v0.3.15 h1:M8XP7IuFNsqUx6VPK2P9OSmsYsI/YFaGil0uD21V3dM=
 github.com/imdario/mergo v0.3.15/go.mod h1:WBLT9ZmE3lPoWsEzCh9LPo3TiwVN+ZKEjmz+hD27ysY=
 github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOlocH6Fxy8MmwDt+yVQYULKfN0RoTN8A=
 github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i6rXxKeerYnT8Nvf0QmHCRC1n8sfWVwXF2Frvo=
-github.com/jlaffaye/ftp v0.2.0 h1:lXNvW7cBu7R/68bknOX3MrRIIqZ61zELs1P2RAiA3lg=
-github.com/jlaffaye/ftp v0.2.0/go.mod h1:is2Ds5qkhceAPy2xD6RLI6hmp/qysSoymZ+Z2uTnspI=
+github.com/jlaffaye/ftp v0.1.0 h1:DLGExl5nBoSFoNshAUHwXAezXwXBvFdx7/qwhucWNSE=
+github.com/jlaffaye/ftp v0.1.0/go.mod h1:hhq4G4crv+nW2qXtNYcuzLeOudG92Ps37HEKeg2e3lE=
 github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg=
 github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
 github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8=
 github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
 github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0=
 github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4=
+github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4=
 github.com/jpillora/s3 v1.1.4 h1:YCCKDWzb/Ye9EBNd83ATRF/8wPEy0xd43Rezb6u6fzc=
 github.com/jpillora/s3 v1.1.4/go.mod h1:yedE603V+crlFi1Kl/5vZJaBu9pUzE9wvKegU/lF2zs=
+github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
 github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
 github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk=
+github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM=
 github.com/kevinburke/ssh_config v1.2.0 h1:x584FjTGwHzMwvHx18PXxbBVzfnxogHaAReU4gf13a4=
 github.com/kevinburke/ssh_config v1.2.0/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM=
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
@@ -360,9 +363,12 @@ github.com/mholt/archiver/v4 v4.0.0-alpha.8 h1:tRGQuDVPh66WCOelqe6LIGh0gwmfwxUrS
 github.com/mholt/archiver/v4 v4.0.0-alpha.8/go.mod h1:5f7FUYGXdJWUjESffJaYR4R60VhnHxb2X3T1teMyv5A=
 github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y=
 github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
+github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
+github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
 github.com/modocache/gover v0.0.0-20171022184752-b58185e213c5/go.mod h1:caMODM3PzxT8aQXRPkAt8xlV/e7d7w8GM5g0fa5F0D8=
 github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe h1:iruDEfMl2E6fbMZ9s0scYfZQ84/6SPL6zC8ACM2oIL0=
 github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc=
+github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
 github.com/nbio/st v0.0.0-20140626010706-e9e8d9816f32 h1:W6apQkHrMkS0Muv8G/TipAy/FJl/rCYT0+EuS8+Z0z4=
 github.com/nbio/st v0.0.0-20140626010706-e9e8d9816f32/go.mod h1:9wM+0iRr9ahx58uYLpLIr5fm8diHn0JbqRycJi6w0Ms=
 github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
@@ -388,6 +394,8 @@ github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaR
 github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ=
 github.com/paulbellamy/ratecounter v0.2.0 h1:2L/RhJq+HA8gBQImDXtLPrDXK5qAj6ozWVK/zFXVJGs=
 github.com/paulbellamy/ratecounter v0.2.0/go.mod h1:Hfx1hDpSGoqxkVVpBi/IlYD7kChlfo5C6hzIHwPqfFE=
+github.com/petar-dambovaliev/aho-corasick v0.0.0-20211021192214-5ab2d9280aa9 h1:lL+y4Xv20pVlCGyLzNHRC0I0rIHhIL1lTvHizoS/dU8=
+github.com/petar-dambovaliev/aho-corasick v0.0.0-20211021192214-5ab2d9280aa9/go.mod h1:EHPiTAKtiFmrMldLUNswFwfZ2eJIYBHktdaUTZxYWRw=
 github.com/pierrec/lz4/v4 v4.1.15 h1:MO0/ucJhngq7299dKLwIMtgTfbkoSPF6AoMYDd8Q4q0=
 github.com/pierrec/lz4/v4 v4.1.15/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
 github.com/pingcap/errors v0.11.4 h1:lFuQV/oaUMGcD2tqt+01ROSmJs75VG1ToEOkZIZ4nE4=
@@ -424,8 +432,8 @@ github.com/sergi/go-diff v1.3.1/go.mod h1:aMJSSKb2lpPvRNec0+w3fl7LP9IOFzdc9Pa4NF
 github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
 github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0=
 github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
-github.com/skeema/knownhosts v1.2.0 h1:h9r9cf0+u7wSE+M183ZtMGgOJKiL96brpaz5ekfJCpM=
-github.com/skeema/knownhosts v1.2.0/go.mod h1:g4fPeYpque7P0xefxtGzV81ihjC8sX2IqpAoNkjxbMo=
+github.com/skeema/knownhosts v1.1.1 h1:MTk78x9FPgDFVFkDLTrsnnfCJl7g1C/nnKvePgrIngE=
+github.com/skeema/knownhosts v1.1.1/go.mod h1:g4fPeYpque7P0xefxtGzV81ihjC8sX2IqpAoNkjxbMo=
 github.com/smartystreets/assertions v1.0.1 h1:voD4ITNjPL5jjBfgR/r8fPIIBrliWrWHeiJApdr3r4w=
 github.com/smartystreets/assertions v1.0.1/go.mod h1:kHHU4qYBaI3q23Pp3VPrmWhuIUrLW/7eUrw0BU5VaoM=
 github.com/smartystreets/gunit v1.1.3 h1:32x+htJCu3aMswhPw3teoJ+PnWPONqdNgaGs6Qt8ZaU=
@@ -458,8 +466,8 @@ github.com/ulikunitz/xz v0.5.10/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0o
 github.com/urfave/cli v1.22.12/go.mod h1:sSBEIC79qR6OvcmsD4U3KABeOTxDqQtdDnaFuUN30b8=
 github.com/vbatts/tar-split v0.11.3 h1:hLFqsOLQ1SsppQNTMpkpPXClLDfC2A3Zgy9OUU+RVck=
 github.com/vbatts/tar-split v0.11.3/go.mod h1:9QlHN18E+fEH7RdG+QAJJcuya3rqT7eXSTY7wGrAokY=
-github.com/xanzy/go-gitlab v0.88.0 h1:9GHBrxyCUNZZNuAsbJ1NbEH6XAYsKyTn6NfE0wYO5SY=
-github.com/xanzy/go-gitlab v0.88.0/go.mod h1:5ryv+MnpZStBH8I/77HuQBsMbBGANtVpLWC15qOjWAw=
+github.com/xanzy/go-gitlab v0.86.0 h1:jR8V9cK9jXRQDb46KOB20NCF3ksY09luaG0IfXE6p7w=
+github.com/xanzy/go-gitlab v0.86.0/go.mod h1:5ryv+MnpZStBH8I/77HuQBsMbBGANtVpLWC15qOjWAw=
 github.com/xanzy/ssh-agent v0.3.3 h1:+/15pJfg/RsTxqYcX6fHqOXZwwMP+2VyYWJeWM2qQFM=
 github.com/xanzy/ssh-agent v0.3.3/go.mod h1:6dzNDKs0J9rVPHPhaGCukekBHKqfl+L3KghI1Bc68Uw=
 github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c=
@@ -489,8 +497,6 @@ go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
 go.uber.org/goleak v1.1.11/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ=
 go.uber.org/goleak v1.2.1 h1:NBol2c7O1ZokfZ0LEU9K6Whx/KnwvepVetCUhtKja4A=
 go.uber.org/goleak v1.2.1/go.mod h1:qlT2yGI9QafXHhZZLxlSuNsMw3FFLxBr+tBRlmO1xH4=
-go.uber.org/mock v0.2.0 h1:TaP3xedm7JaAgScZO7tlvlKrqT0p7I6OsdGB5YNSMDU=
-go.uber.org/mock v0.2.0/go.mod h1:J0y0rp9L3xiff1+ZBfKxlC1fz2+aO16tw0tsDOixfuM=
 go.uber.org/multierr v1.6.0 h1:y6IPFStTAIT5Ytl7/XYmHvzXQ7S3g/IeZW9hyZ5thw4=
 go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU=
 go.uber.org/zap v1.24.0 h1:FiJd5l1UOLj0wCgbSE0rwwXHzEdAZS6hiiSnxJN/D60=
@@ -507,10 +513,9 @@ golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5y
 golang.org/x/crypto v0.0.0-20211215153901-e495a2d5b3d3/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
 golang.org/x/crypto v0.0.0-20220314234659-1baeb1ce4c0b/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
 golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
-golang.org/x/crypto v0.3.1-0.20221117191849-2c476679df9a/go.mod h1:hebNnKkNXi2UzZN1eVRvBB7co0a+JxK6XbPiWVs/3J4=
 golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU=
-golang.org/x/crypto v0.11.0 h1:6Ewdq3tDic1mg5xRO4milcWCfMVQhI4NkqWWvqejpuA=
-golang.org/x/crypto v0.11.0/go.mod h1:xgJhtzW8F9jGdVFWZESrid1U1bjeNy4zgy5cRr/CIio=
+golang.org/x/crypto v0.10.0 h1:LKqV2xt9+kDzSTfOhx4FrkEBcMrAgHSYgzywV9zcGmM=
+golang.org/x/crypto v0.10.0/go.mod h1:o4eNf7Ede1fv+hwOwZsTHl9EsPFO6q6ZvYR8vYfY45I=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
@@ -532,6 +537,7 @@ golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHl
 golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
 golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs=
 golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
+golang.org/x/lint v0.0.0-20210508222113-6edffad5e616/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
 golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE=
 golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o=
 golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc=
@@ -570,20 +576,19 @@ golang.org/x/net v0.0.0-20210610132358-84b48f89b13b/go.mod h1:9nx3DQGgdP8bBQD5qx
 golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
 golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
-golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY=
 golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
 golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc=
-golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg=
-golang.org/x/net v0.12.0 h1:cfawfvKITfUsFCeJIHJrbSxpeu/E81khclypR0GVT50=
-golang.org/x/net v0.12.0/go.mod h1:zEVYFnQC7m/vmpQFELhcD1EWkZlX69l4oqgmer6hfKA=
+golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns=
+golang.org/x/net v0.11.0 h1:Gi2tvZIJyBtO9SDr1q9h5hEQCp/4L2RQ+ar0qjx2oNU=
+golang.org/x/net v0.11.0/go.mod h1:2L/ixqYpgIVXmeoSA/4Lu7BzTG4KIyPIryS4IsOd1oQ=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
-golang.org/x/oauth2 v0.8.0/go.mod h1:yr7u4HXZRm1R1kBWqr/xKNqewf0plRYoB7sla+BCIXE=
-golang.org/x/oauth2 v0.10.0 h1:zHCpF2Khkwy4mMB4bv0U37YtJdTGW8jI0glAApi0Kh8=
-golang.org/x/oauth2 v0.10.0/go.mod h1:kTpgurOux7LqtuxjuyZa4Gj2gdezIt/jQtGnNFfypQI=
+golang.org/x/oauth2 v0.7.0/go.mod h1:hPLQkd9LyjfXTiRohC/41GhcFqxisoUQ99sCUOHO9x4=
+golang.org/x/oauth2 v0.9.0 h1:BPpt2kU7oMRq3kCHAA1tbSEshXRw1LpG2ztgDwrzuAs=
+golang.org/x/oauth2 v0.9.0/go.mod h1:qYgFZaFiu6Wg24azG8bdV52QJXJGbZzIIsRCdVKzbLw=
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@@ -632,20 +637,18 @@ golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220906165534-d0df966e6959/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.10.0 h1:SqMFp9UcQJZa+pmYuAKjd9xq1f0j5rLcDIk0mj4qAsA=
-golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.9.0 h1:KS/R3tvhPqvJvwcKfnBHJwwthS11LRhmM5D59eEXa0s=
+golang.org/x/sys v0.9.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
-golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc=
 golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
 golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U=
-golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo=
-golang.org/x/term v0.10.0 h1:3R7pNqamzBraeqj/Tj8qt1aQ2HpmlC+Cx/qL/7hn4/c=
+golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY=
+golang.org/x/term v0.9.0 h1:GRRCnKYhdQrD8kfRAdQ6Zcw1P0OcELxGLKJvtjVMZ28=
 golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
@@ -654,12 +657,11 @@ golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
 golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ=
-golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
 golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
 golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
 golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
-golang.org/x/text v0.11.0 h1:LAntKIrcmeSKERyiOh0XMV39LXS8IE9UL2yP7+f5ij4=
-golang.org/x/text v0.11.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
+golang.org/x/text v0.10.0 h1:UpjohKhiEgNc0CSauXmwYftY1+LlaC75SJwh0SgCX58=
+golang.org/x/text v0.10.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
 golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4=
@@ -709,8 +711,8 @@ google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsb
 google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI=
 google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI=
 google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
-google.golang.org/api v0.131.0 h1:AcgWS2edQ4chVEt/SxgDKubVu/9/idCJy00tBGuGB4M=
-google.golang.org/api v0.131.0/go.mod h1:7vtkbKv2REjJbxmHSkBTBQ5LUGvPdAqjjvt84XAfhpA=
+google.golang.org/api v0.129.0 h1:2XbdjjNfFPXQyufzQVwPf1RRnHH8Den2pfNE2jw7L8w=
+google.golang.org/api v0.129.0/go.mod h1:dFjiXlanKwWE3612X97llhsoI36FAoIiRj3aTl5b/zE=
 google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
 google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
 google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
@@ -733,12 +735,12 @@ google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvx
 google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
 google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
 google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=
-google.golang.org/genproto v0.0.0-20230629202037-9506855d4529 h1:9JucMWR7sPvCxUFd6UsOUNmA5kCcWOfORaT3tpAsKQs=
-google.golang.org/genproto v0.0.0-20230629202037-9506855d4529/go.mod h1:xZnkP7mREFX5MORlOPEzLMr+90PPZQ2QWzrVTWfAq64=
-google.golang.org/genproto/googleapis/api v0.0.0-20230629202037-9506855d4529 h1:s5YSX+ZH5b5vS9rnpGymvIyMpLRJizowqDlOuyjXnTk=
-google.golang.org/genproto/googleapis/api v0.0.0-20230629202037-9506855d4529/go.mod h1:vHYtlOoi6TsQ3Uk2yxR7NI5z8uoV+3pZtR4jmHIkRig=
-google.golang.org/genproto/googleapis/rpc v0.0.0-20230706204954-ccb25ca9f130 h1:2FZP5XuJY9zQyGM5N0rtovnoXjiMUEIUMvw0m9wlpLc=
-google.golang.org/genproto/googleapis/rpc v0.0.0-20230706204954-ccb25ca9f130/go.mod h1:8mL13HKkDa+IuJ8yruA3ci0q+0vsUz4m//+ottjwS5o=
+google.golang.org/genproto v0.0.0-20230530153820-e85fd2cbaebc h1:8DyZCyvI8mE1IdLy/60bS+52xfymkE72wv1asokgtao=
+google.golang.org/genproto v0.0.0-20230530153820-e85fd2cbaebc/go.mod h1:xZnkP7mREFX5MORlOPEzLMr+90PPZQ2QWzrVTWfAq64=
+google.golang.org/genproto/googleapis/api v0.0.0-20230530153820-e85fd2cbaebc h1:kVKPf/IiYSBWEWtkIn6wZXwWGCnLKcC8oWfZvXjsGnM=
+google.golang.org/genproto/googleapis/api v0.0.0-20230530153820-e85fd2cbaebc/go.mod h1:vHYtlOoi6TsQ3Uk2yxR7NI5z8uoV+3pZtR4jmHIkRig=
+google.golang.org/genproto/googleapis/rpc v0.0.0-20230530153820-e85fd2cbaebc h1:XSJ8Vk1SWuNr8S18z1NZSziL0CPIXLCCMDOEFtHBOFc=
+google.golang.org/genproto/googleapis/rpc v0.0.0-20230530153820-e85fd2cbaebc/go.mod h1:66JfowdXAEgad5O9NnYcsNPLCPZJD++2L9X0PCMODrA=
 google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
 google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
 google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
@@ -751,8 +753,8 @@ google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTp
 google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc=
 google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU=
 google.golang.org/grpc v1.45.0/go.mod h1:lN7owxKUQEqMfSyQikvvk5tf/6zMPsrK+ONuO11+0rQ=
-google.golang.org/grpc v1.56.2 h1:fVRFRnXvU+x6C4IlHZewvJOVHoOv1TUuQyoRsYnB4bI=
-google.golang.org/grpc v1.56.2/go.mod h1:I9bI3vqKfayGqPUAwGdOSu7kt6oIJLixfffKrpXqQ9s=
+google.golang.org/grpc v1.56.1 h1:z0dNfjIl0VpaZ9iSVjA6daGatAYwPGstTjt5vkRMFkQ=
+google.golang.org/grpc v1.56.1/go.mod h1:I9bI3vqKfayGqPUAwGdOSu7kt6oIJLixfffKrpXqQ9s=
 google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
 google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
 google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=

hack/docs/Adding_Detectors_Internal.md

@@ -62,31 +62,15 @@ If you think that something should be included outside of these guidelines, plea
    1. Add the test secret to GCP Secrets. See [managing test secrets](#managing-test-secrets)
    2. Update the pattern regex and keywords. Try iterating with [regex101.com](http://regex101.com/).
    3. Update the verifier code to use a non-destructive API call that can determine whether the secret is valid or not.
-      * Make sure you understand [verification indeterminacy](#verification-indeterminacy).
    4. Update the tests with these test cases at minimum:
       1. Found and verified (using a credential loaded from GCP Secrets)
-      2. Found and unverified (determinately, i.e. the secret is invalid)
-      3. Found and unverified (indeterminately due to timeout)
-      4. Found and unverified (indeterminately due to an unexpected API response)
-      5. Not found
-      6. Any false positive cases that you come across
+      2. Found and unverified
+      3. Not found
+      4. Any false positive cases that you come across
    5. Create a merge request for review. CI tests must be passing.
 
 ## Addendum
 
-### Verification indeterminacy
-
-There are two types of reasons that secret verification can fail:
-* The candidate secret is not actually a valid secret.
-* Something went wrong in the process unrelated to the candidate secret, such as a transient network error or an unexpected API response.
-
-In Trufflehog parlance, the first type of verification response is called _determinate_ and the second type is called _indeterminate_. Verification code should distinguish between the two by returning an error object in the result struct **only** for indeterminate failures. In general, a verifier should return an error (indicating an indeterminate failure) in all cases that haven't been explicitly identified as determinate failure states.
-
-For example, consider a hypothetical authentication endpoint that returns `200 OK` for valid credentials and `403 Forbidden` for invalid credentials. The verifier for this endpoint could make an HTTP request and use the response status code to decide what to return:
-* A `200` response would indicate that verification succeeded. (Or maybe any `2xx` response.)
-* A `403` response would indicate that verification failed **determinately** and no error object should be returned.
-* Any other response would indicate that verification failed **indeterminately** and an error object should be returned.
-
 ### Managing Test Secrets
 
 Do not embed test credentials in the test code. Instead, use GCP Secrets Manager.

hack/docs/Adding_Detectors_external.md

@@ -62,31 +62,15 @@ If you think that something should be included outside of these guidelines, plea
    1. Create a [test secrets file, and export the variable](#using-a-test-secret-file)
    2. Update the pattern regex and keywords. Try iterating with [regex101.com](http://regex101.com/).
    3. Update the verifier code to use a non-destructive API call that can determine whether the secret is valid or not.
-      * Make sure you understand [verification indeterminacy](#verification-indeterminacy).
    4. Update the tests with these test cases at minimum:
       1. Found and verified (using a credential loaded from GCP Secrets)
-      2. Found and unverified (determinately, i.e. the secret is invalid)
-      3. Found and unverified (indeterminately due to timeout)
-      4. Found and unverified (indeterminately due to an unexpected API response)
-      5. Not found
-      6. Any false positive cases that you come across
+      2. Found and unverified
+      3. Not found
+      4. Any false positive cases that you come across
    5. Create a pull request for review.
 
 ## Addendum
 
-### Verification indeterminacy
-
-There are two types of reasons that secret verification can fail:
-* The candidate secret is not actually a valid secret.
-* Something went wrong in the process unrelated to the candidate secret, such as a transient network error or an unexpected API response.
-
-In Trufflehog parlance, the first type of verification response is called _determinate_ and the second type is called _indeterminate_. Verification code should distinguish between the two by returning an error object in the result struct **only** for indeterminate failures. In general, a verifier should return an error (indicating an indeterminate failure) in all cases that haven't been explicitly identified as determinate failure states.
-
-For example, consider a hypothetical authentication endpoint that returns `200 OK` for valid credentials and `403 Forbidden` for invalid credentials. The verifier for this endpoint could make an HTTP request and use the response status code to decide what to return:
-* A `200` response would indicate that verification succeeded. (Or maybe any `2xx` response.)
-* A `403` response would indicate that verification failed **determinately** and no error object should be returned.
-* Any other response would indicate that verification failed **indeterminately** and an error object should be returned.
-
 ### Using a test secret file
 
 1. Create a file called `.env` with this env file format:

hack/generate/generate.go

@@ -34,14 +34,14 @@ func main() {
 	case "detector":
 		mustWriteTemplates([]templateJob{
 			{
-				TemplatePath:  "pkg/detectors/alchemy/alchemy.go",
+				TemplatePath:  "pkg/detectors/heroku/heroku.go",
 				WritePath:     filepath.Join(folderPath(), nameLower+".go"),
-				ReplaceString: []string{"alchemy"},
+				ReplaceString: []string{"heroku"},
 			},
 			{
-				TemplatePath:  "pkg/detectors/alchemy/alchemy_test.go",
+				TemplatePath:  "pkg/detectors/heroku/heroku_test.go",
 				WritePath:     filepath.Join(folderPath(), nameLower+"_test.go"),
-				ReplaceString: []string{"alchemy"},
+				ReplaceString: []string{"heroku"},
 			},
 		})
 		// case "source":

main.go

@@ -9,6 +9,7 @@ import (
 	"strconv"
 	"strings"
 	"syscall"
+	"time"
 
 	"github.com/felixge/fgprof"
 	"github.com/go-logr/logr"
@@ -38,7 +39,6 @@ var (
 	debug               = cli.Flag("debug", "Run in debug mode.").Bool()
 	trace               = cli.Flag("trace", "Run in trace mode.").Bool()
 	profile             = cli.Flag("profile", "Enables profiling and sets a pprof and fgprof server on :18066.").Bool()
-	localDev            = cli.Flag("local-dev", "Hidden feature to disable overseer for local dev.").Hidden().Bool()
 	jsonOut             = cli.Flag("json", "Output in JSON format.").Short('j').Bool()
 	jsonLegacy          = cli.Flag("json-legacy", "Use the pre-v3.0 JSON format. Only works with git, gitlab, and github sources.").Bool()
 	gitHubActionsFormat = cli.Flag("github-actions", "Output in GitHub Actions format.").Bool()
@@ -162,12 +162,6 @@ func main() {
 	logger, sync := log.New("trufflehog", logFormat(os.Stderr))
 	// make it the default logger for contexts
 	context.SetDefaultLogger(logger)
-
-	if *localDev {
-		run(overseer.State{})
-		os.Exit(0)
-	}
-
 	defer func() { _ = sync() }()
 	logFatal := logFatalFunc(logger)
 
@@ -213,7 +207,7 @@ func run(state overseer.State) {
 	if *profile {
 		go func() {
 			router := http.NewServeMux()
-			router.Handle("/debug/pprof/", http.DefaultServeMux)
+			router.Handle("/debug/pprof", http.DefaultServeMux)
 			router.Handle("/debug/fgprof", fgprof.Handler())
 			logger.Info("starting pprof and fgprof server on :18066 /debug/pprof and /debug/fgprof")
 			if err := http.ListenAndServe(":18066", router); err != nil {
@@ -322,21 +316,8 @@ func run(state overseer.State) {
 		return true
 	}
 
-	// Set how the engine will print its results.
-	var printer engine.Printer
-	switch {
-	case *jsonLegacy:
-		printer = new(output.LegacyJSONPrinter)
-	case *jsonOut:
-		printer = new(output.JSONPrinter)
-	case *gitHubActionsFormat:
-		printer = new(output.GitHubActionsPrinter)
-	default:
-		printer = new(output.PlainPrinter)
-	}
-
-	e, err := engine.Start(ctx,
-		engine.WithConcurrency(uint8(*concurrency)),
+	e := engine.Start(ctx,
+		engine.WithConcurrency(*concurrency),
 		engine.WithDecoders(decoders.DefaultDecoders()...),
 		engine.WithDetectors(!*noVerification, engine.DefaultDetectors()...),
 		engine.WithDetectors(!*noVerification, conf.Detectors...),
@@ -344,13 +325,7 @@ func run(state overseer.State) {
 		engine.WithFilterDetectors(excludeFilter),
 		engine.WithFilterDetectors(endpointCustomizer),
 		engine.WithFilterUnverified(*filterUnverified),
-		engine.WithOnlyVerified(*onlyVerified),
-		engine.WithPrintAvgDetectorTime(*printAvgDetectorTime),
-		engine.WithPrinter(printer),
 	)
-	if err != nil {
-		logFatal(err, "error initializing engine")
-	}
 
 	var repoPath string
 	var remote bool
@@ -500,48 +475,52 @@ func run(state overseer.State) {
 			logFatal(err, "Failed to scan Docker.")
 		}
 	}
+	// asynchronously wait for scanning to finish and cleanup
+	go e.Finish(ctx, logFatal)
 
 	if !*jsonLegacy && !*jsonOut {
 		fmt.Fprintf(os.Stderr, "🐷🔑🐷  TruffleHog. Unearth your secrets. 🐷🔑🐷\n\n")
 	}
 
-	// Wait for all workers to finish.
-	if err = e.Finish(ctx); err != nil {
-		logFatal(err, "engine failed to finish execution")
-	}
+	// NOTE: this loop will terminate when the results channel is closed in
+	// e.Finish()
+	foundResults := false
+	for r := range e.ResultsChan() {
+		if *onlyVerified && !r.Verified {
+			continue
+		}
+		foundResults = true
 
-	metrics := e.GetMetrics()
-	// Print results.
-	logger.Info("finished scanning",
-		"chunks", metrics.ChunksScanned,
-		"bytes", metrics.BytesScanned,
-		"verified_secrets", metrics.VerifiedSecretsFound,
-		"unverified_secrets", metrics.UnverifiedSecretsFound,
+		var err error
+		switch {
+		case *jsonLegacy:
+			err = output.PrintLegacyJSON(ctx, &r)
+		case *jsonOut:
+			err = output.PrintJSON(&r)
+		case *gitHubActionsFormat:
+			err = output.PrintGitHubActionsOutput(&r)
+		default:
+			err = output.PrintPlainOutput(&r)
+		}
+		if err != nil {
+			logFatal(err, "error printing results")
+		}
+	}
+	logger.V(2).Info("finished scanning",
+		"chunks", e.ChunksScanned(),
+		"bytes", e.BytesScanned(),
 	)
 
 	if *printAvgDetectorTime {
 		printAverageDetectorTime(e)
 	}
 
-	if e.HasFoundResults() && *fail {
+	if foundResults && *fail {
 		logger.V(2).Info("exiting with code 183 because results were found")
 		os.Exit(183)
 	}
 }
 
-// logFatalFunc returns a log.Fatal style function. Calling the returned
-// function will terminate the program without cleanup.
-func logFatalFunc(logger logr.Logger) func(error, string, ...any) {
-	return func(err error, message string, keyAndVals ...any) {
-		logger.Error(err, message, keyAndVals...)
-		if err != nil {
-			os.Exit(1)
-			return
-		}
-		os.Exit(0)
-	}
-}
-
 func commaSeparatedToSlice(s []string) []string {
 	var result []string
 	for _, items := range s {
@@ -558,8 +537,26 @@ func commaSeparatedToSlice(s []string) []string {
 
 func printAverageDetectorTime(e *engine.Engine) {
 	fmt.Fprintln(os.Stderr, "Average detector time is the measurement of average time spent on each detector when results are returned.")
-	for detectorName, duration := range e.GetDetectorsMetrics() {
-		fmt.Fprintf(os.Stderr, "%s: %s\n", detectorName, duration)
+	for detectorName, durations := range e.DetectorAvgTime() {
+		var total time.Duration
+		for _, d := range durations {
+			total += d
+		}
+		avgDuration := total / time.Duration(len(durations))
+		fmt.Fprintf(os.Stderr, "%s: %s\n", detectorName, avgDuration)
+	}
+}
+
+// logFatalFunc returns a log.Fatal style function. Calling the returned
+// function will terminate the program without cleanup.
+func logFatalFunc(logger logr.Logger) func(error, string, ...any) {
+	return func(err error, message string, keyAndVals ...any) {
+		logger.Error(err, message, keyAndVals...)
+		if err != nil {
+			os.Exit(1)
+			return
+		}
+		os.Exit(0)
 	}
 }
 

pkg/common/context.go

@@ -10,15 +10,3 @@ func IsDone(ctx context.Context) bool {
 		return false
 	}
 }
-
-// CancellableWrite blocks on writing the item to the channel but can be
-// cancelled by the context. If both the context is cancelled and the channel
-// write would succeed, either operation will be performed randomly.
-func CancellableWrite[T any](ctx context.Context, ch chan<- T, item T) error {
-	select {
-	case ch <- item:
-		return nil
-	case <-ctx.Done():
-		return ctx.Err()
-	}
-}

pkg/common/http.go

@@ -3,7 +3,6 @@ package common
 import (
 	"crypto/tls"
 	"crypto/x509"
-	"io"
 	"net"
 	"net/http"
 	"strings"
@@ -76,14 +75,6 @@ func PinnedCertPool() *x509.CertPool {
 	return trustedCerts
 }
 
-type FakeTransport struct {
-	CreateResponse func(req *http.Request) (*http.Response, error)
-}
-
-func (t FakeTransport) RoundTrip(req *http.Request) (*http.Response, error) {
-	return t.CreateResponse(req)
-}
-
 type CustomTransport struct {
 	T http.RoundTripper
 }
@@ -100,21 +91,6 @@ func NewCustomTransport(T http.RoundTripper) *CustomTransport {
 	return &CustomTransport{T}
 }
 
-func ConstantResponseHttpClient(statusCode int, body string) *http.Client {
-	return &http.Client{
-		Timeout: DefaultResponseTimeout,
-		Transport: FakeTransport{
-			CreateResponse: func(req *http.Request) (*http.Response, error) {
-				return &http.Response{
-					Request:    req,
-					Body:       io.NopCloser(strings.NewReader(body)),
-					StatusCode: statusCode,
-				}, nil
-			},
-		},
-	}
-}
-
 func PinnedRetryableHttpClient() *http.Client {
 	httpClient := retryablehttp.NewClient()
 	httpClient.Logger = nil
@@ -176,9 +152,9 @@ func SaneHttpClient() *http.Client {
 }
 
 // SaneHttpClientTimeOut adds a custom timeout for some scanners
-func SaneHttpClientTimeOut(timeout time.Duration) *http.Client {
+func SaneHttpClientTimeOut(timeOutSeconds int64) *http.Client {
 	httpClient := &http.Client{}
-	httpClient.Timeout = timeout
+	httpClient.Timeout = time.Second * time.Duration(timeOutSeconds)
 	httpClient.Transport = NewCustomTransport(nil)
 	return httpClient
 }

pkg/common/utils.go

@@ -2,10 +2,7 @@ package common
 
 import (
 	"bufio"
-	"bytes"
-	"crypto/rand"
 	"io"
-	"math/big"
 	"strings"
 )
 
@@ -27,18 +24,6 @@ func RemoveStringSliceItem(item string, slice *[]string) {
 	}
 }
 
-func MinInt(a, b int) int {
-	if a < b {
-		return a
-	}
-	return b
-}
-
-func BytesEqual(a, b []byte, numBytes int) bool {
-	limit := MinInt(numBytes, MinInt(len(a), len(b))-1)
-	return bytes.Equal(a[:limit], b[:limit])
-}
-
 func ResponseContainsSubstring(reader io.ReadCloser, target string) (bool, error) {
 	scanner := bufio.NewScanner(reader)
 	for scanner.Scan() {
@@ -51,16 +36,3 @@ func ResponseContainsSubstring(reader io.ReadCloser, target string) (bool, error
 	}
 	return false, nil
 }
-
-var letters = []rune("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789")
-
-// RandomID returns a random string of the given length.
-func RandomID(length int) string {
-	b := make([]rune, length)
-	for i := range b {
-		randInt, _ := rand.Int(rand.Reader, big.NewInt(int64(len(letters))))
-		b[i] = letters[randInt.Int64()]
-	}
-
-	return string(b)
-}

pkg/context/context.go

@@ -3,43 +3,44 @@ package context
 import (
 	"context"
 	"fmt"
-	"os"
 	"runtime/debug"
-	"sync"
 	"time"
 
 	"github.com/go-logr/logr"
-	"github.com/trufflesecurity/trufflehog/v3/pkg/log"
 )
 
 var (
 	// defaultLogger can be set via SetDefaultLogger.
-	// It is initialized to write to stderr. To disable, you can call
-	// SetDefaultLogger with logr.Discard().
-	defaultLogger logr.Logger
+	defaultLogger logr.Logger = logr.Discard()
 )
 
-func init() {
-	defaultLogger, _ = log.New("context", log.WithConsoleSink(os.Stderr))
-}
-
 // Context wraps context.Context and includes an additional Logger() method.
 type Context interface {
 	context.Context
 	Logger() logr.Logger
+	Parent() context.Context
+	SetParent(ctx context.Context) Context
 }
 
-// CancelFunc is a type alias to context.CancelFunc to allow use as if they are
-// the same types.
-type CancelFunc = context.CancelFunc
+// Parent returns the parent context.
+func (l logCtx) Parent() context.Context {
+	return l.Context
+}
+
+// SetParent sets the parent context on the context.
+func (l logCtx) SetParent(ctx context.Context) Context {
+	l.Context = ctx
+	return l
+}
+
+type CancelFunc context.CancelFunc
 
 // logCtx implements Context.
 type logCtx struct {
 	// Embed context.Context to get all methods for free.
 	context.Context
-	log     logr.Logger
-	err     *error
-	errLock *sync.Mutex
+	log logr.Logger
+	err *error
 }
 
 // Logger returns a structured logger.
@@ -144,9 +145,7 @@ func AddLogger(parent context.Context) Context {
 }
 
 // SetupDefaultLogger sets the package-level global default logger that will be
-// used for Background and TODO contexts. On startup, the default logger will
-// be configured to output logs to stderr. Use logr.Discard() to disable all
-// logs from Contexts.
+// used for Background and TODO contexts.
 func SetDefaultLogger(l logr.Logger) {
 	defaultLogger = l
 }
@@ -157,11 +156,8 @@ func captureCancelCallstack(ctx logCtx, f context.CancelFunc) (Context, context.
 	if ctx.err == nil {
 		var err error
 		ctx.err = &err
-		ctx.errLock = &sync.Mutex{}
 	}
 	return ctx, func() {
-		ctx.errLock.Lock()
-		defer ctx.errLock.Unlock()
 		// We must check Err() before calling f() since f() sets the error.
 		// If there's already an error, do nothing special.
 		if ctx.Err() != nil {

pkg/context/context_test.go

@@ -158,7 +158,7 @@ func TestWithValues(t *testing.T) {
 	assert.NotContains(t, logs[6], `what does this do?`)
 }
 
-func TestDefaultLogger(t *testing.T) {
+func TestDiscardLogger(t *testing.T) {
 	var panicked bool
 	defer func() {
 		if r := recover(); r != nil {
@@ -212,9 +212,3 @@ func TestErrCallstackTimeoutCancel(t *testing.T) {
 	cancel()
 	assert.Equal(t, err, ctx.Err())
 }
-
-func TestRace(t *testing.T) {
-	_, cancel := WithCancel(Background())
-	go cancel()
-	cancel()
-}

pkg/decoders/decoders.go

@@ -6,7 +6,6 @@ import (
 
 func DefaultDecoders() []Decoder {
 	return []Decoder{
-		// UTF8 must be first for duplicate detection
 		&UTF8{},
 		&Base64{},
 		&UTF16{},

pkg/detectors/alchemy/alchemy.go

@@ -2,7 +2,6 @@ package alchemy
 
 import (
 	"context"
-	"fmt"
 	"net/http"
 	"regexp"
 	"strings"
@@ -12,15 +11,14 @@ import (
 	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
 )
 
-type Scanner struct {
-	client *http.Client
-}
+type Scanner struct{}
 
 // Ensure the Scanner satisfies the interface at compile time.
 var _ detectors.Detector = (*Scanner)(nil)
 
 var (
-	defaultClient = common.SaneHttpClient()
+	client = common.SaneHttpClient()
+
 	// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
 	keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"alchemy"}) + `\b([0-9a-zA-Z]{23}_[0-9a-zA-Z]{8})\b`)
 )
@@ -49,10 +47,6 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 		}
 
 		if verify {
-			client := s.client
-			if client == nil {
-				client = defaultClient
-			}
 			req, err := http.NewRequestWithContext(ctx, "GET", "https://eth-mainnet.g.alchemy.com/v2/"+resMatch+"/getNFTs/?owner=vitalik.eth", nil)
 			if err != nil {
 				continue
@@ -62,21 +56,15 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 				defer res.Body.Close()
 				if res.StatusCode >= 200 && res.StatusCode < 300 {
 					s1.Verified = true
-				} else if res.StatusCode == 401 {
-					// The secret is determinately not verified (nothing to do)
 				} else {
-					s1.VerificationError = fmt.Errorf("request to %v returned unexpected status %d", res.Request.URL, res.StatusCode)
+					// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
+					if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
+						continue
+					}
 				}
-			} else {
-				s1.VerificationError = err
 			}
 		}
 
-		// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
-		if !s1.Verified && detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
-			continue
-		}
-
 		results = append(results, s1)
 	}
 

pkg/detectors/alchemy/alchemy_test.go

@@ -32,12 +32,11 @@ func TestAlchemy_FromChunk(t *testing.T) {
 		verify bool
 	}
 	tests := []struct {
-		name                string
-		s                   Scanner
-		args                args
-		want                []detectors.Result
-		wantErr             bool
-		wantVerificationErr bool
+		name    string
+		s       Scanner
+		args    args
+		want    []detectors.Result
+		wantErr bool
 	}{
 		{
 			name: "found, verified",
@@ -53,8 +52,7 @@ func TestAlchemy_FromChunk(t *testing.T) {
 					Verified:     true,
 				},
 			},
-			wantErr:             false,
-			wantVerificationErr: false,
+			wantErr: false,
 		},
 		{
 			name: "found, unverified",
@@ -70,8 +68,7 @@ func TestAlchemy_FromChunk(t *testing.T) {
 					Verified:     false,
 				},
 			},
-			wantErr:             false,
-			wantVerificationErr: false,
+			wantErr: false,
 		},
 		{
 			name: "not found",
@@ -81,48 +78,14 @@ func TestAlchemy_FromChunk(t *testing.T) {
 				data:   []byte("You cannot find the secret within"),
 				verify: true,
 			},
-			want:                nil,
-			wantErr:             false,
-			wantVerificationErr: false,
-		},
-		{
-			name: "found, would be verified if not for timeout",
-			s:    Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},
-			args: args{
-				ctx:    context.Background(),
-				data:   []byte(fmt.Sprintf("You can find a alchemy secret %s within", secret)),
-				verify: true,
-			},
-			want: []detectors.Result{
-				{
-					DetectorType: detectorspb.DetectorType_Alchemy,
-					Verified:     false,
-				},
-			},
-			wantErr:             false,
-			wantVerificationErr: true,
-		},
-		{
-			name: "found, verified but unexpected api surface",
-			s:    Scanner{client: common.ConstantResponseHttpClient(404, "")},
-			args: args{
-				ctx:    context.Background(),
-				data:   []byte(fmt.Sprintf("You can find a alchemy secret %s within", secret)),
-				verify: true,
-			},
-			want: []detectors.Result{
-				{
-					DetectorType: detectorspb.DetectorType_Alchemy,
-					Verified:     false,
-				},
-			},
-			wantErr:             false,
-			wantVerificationErr: true,
+			want:    nil,
+			wantErr: false,
 		},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+			s := Scanner{}
+			got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
 			if (err != nil) != tt.wantErr {
 				t.Errorf("Alchemy.FromData() error = %v, wantErr %v", err, tt.wantErr)
 				return
@@ -132,10 +95,6 @@ func TestAlchemy_FromChunk(t *testing.T) {
 					t.Fatalf("no raw secret present: \n %+v", got[i])
 				}
 				got[i].Raw = nil
-				if (got[i].VerificationError != nil) != tt.wantVerificationErr {
-					t.Fatalf("verification error = %v, wantVerificationError %v", got[i].VerificationError, tt.wantVerificationErr)
-				}
-				got[i].VerificationError = nil
 			}
 			if diff := pretty.Compare(got, tt.want); diff != "" {
 				t.Errorf("Alchemy.FromData() %s diff: (-got +want)\n%s", tt.name, diff)

pkg/detectors/aws/aws.go

@@ -18,8 +18,7 @@ import (
 )
 
 type scanner struct {
-	verificationClient *http.Client
-	skipIDs            map[string]struct{}
+	skipIDs map[string]struct{}
 }
 
 func New(opts ...func(*scanner)) *scanner {
@@ -49,7 +48,7 @@ func WithSkipIDs(skipIDs []string) func(*scanner) {
 var _ detectors.Detector = (*scanner)(nil)
 
 var (
-	defaultVerificationClient = common.SaneHttpClient()
+	client = common.SaneHttpClient()
 
 	// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
 	// Key types are from this list https://docs.aws.amazon.com/IAM/latest/UserGuide/reference_identifiers.html#identifiers-unique-ids
@@ -124,9 +123,8 @@ func (s scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 				host := "sts.amazonaws.com"
 				region := "us-east-1"
 				endpoint := "https://sts.amazonaws.com"
-				now := time.Now().UTC()
-				datestamp := now.Format("20060102")
-				amzDate := now.Format("20060102T150405Z0700")
+				datestamp := time.Now().UTC().Format("20060102")
+				amzDate := time.Now().UTC().Format("20060102T150405Z0700")
 
 				req, err := http.NewRequestWithContext(ctx, method, endpoint, nil)
 				if err != nil {
@@ -173,10 +171,6 @@ func (s scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 				req.Header.Add("Content-type", "application/x-www-form-urlencoded; charset=utf-8")
 				req.URL.RawQuery = params.Encode()
 
-				client := s.verificationClient
-				if client == nil {
-					client = defaultVerificationClient
-				}
 				res, err := client.Do(req)
 				if err == nil {
 
@@ -190,8 +184,6 @@ func (s scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 								"user_id": identityInfo.GetCallerIdentityResponse.GetCallerIdentityResult.UserID,
 								"arn":     identityInfo.GetCallerIdentityResponse.GetCallerIdentityResult.Arn,
 							}
-						} else {
-							s1.VerificationError = err
 						}
 						res.Body.Close()
 					} else {
@@ -199,31 +191,7 @@ func (s scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 						if detectors.IsKnownFalsePositive(resSecretMatch, detectors.DefaultFalsePositives, true) {
 							continue
 						}
-
-						if res.StatusCode == 403 {
-							var body awsErrorResponseBody
-							err = json.NewDecoder(res.Body).Decode(&body)
-							res.Body.Close()
-							if err == nil {
-								// All instances of the code I've seen in the wild are PascalCased but this check is
-								// case-insensitive out of an abundance of caution
-								if strings.EqualFold(body.Error.Code, "InvalidClientTokenId") {
-									// determinate failure - nothing to do
-								} else {
-									// We see a surprising number of false-negative signature mismatch errors here
-									// (The official SDK somehow elicits even more than just making the request
-									// ourselves)
-									s1.VerificationError = fmt.Errorf("request to %v returned status %d with an unexpected reason (%s: %s)", res.Request.URL, res.StatusCode, body.Error.Code, body.Error.Message)
-								}
-							} else {
-								s1.VerificationError = fmt.Errorf("couldn't parse the sts response body (%v)", err)
-							}
-						} else {
-							s1.VerificationError = fmt.Errorf("request to %v returned unexpected status %d", res.Request.URL, res.StatusCode)
-						}
 					}
-				} else {
-					s1.VerificationError = err
 				}
 			}
 
@@ -269,15 +237,6 @@ func awsCustomCleanResults(results []detectors.Result) []detectors.Result {
 	return out
 }
 
-type awsError struct {
-	Code    string `json:"Code"`
-	Message string `json:"Message"`
-}
-
-type awsErrorResponseBody struct {
-	Error awsError `json:"Error"`
-}
-
 type identityRes struct {
 	GetCallerIdentityResponse struct {
 		GetCallerIdentityResult struct {

pkg/detectors/aws/aws_test.go

@@ -17,8 +17,6 @@ import (
 	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
 )
 
-var unverifiedSecretClient = common.ConstantResponseHttpClient(403, `{"Error": {"Code": "InvalidClientTokenId"} }`)
-
 func TestAWS_FromChunk(t *testing.T) {
 	ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
 	defer cancel()
@@ -40,12 +38,11 @@ func TestAWS_FromChunk(t *testing.T) {
 		verify bool
 	}
 	tests := []struct {
-		name                  string
-		s                     scanner
-		args                  args
-		want                  []detectors.Result
-		wantErr               bool
-		wantVerificationError bool
+		name    string
+		s       scanner
+		args    args
+		want    []detectors.Result
+		wantErr bool
 	}{
 		{
 			name: "found, verified",
@@ -71,7 +68,7 @@ func TestAWS_FromChunk(t *testing.T) {
 		},
 		{
 			name: "found, unverified",
-			s:    scanner{verificationClient: unverifiedSecretClient},
+			s:    scanner{},
 			args: args{
 				ctx:    context.Background(),
 				data:   []byte(fmt.Sprintf("You can find a aws secret %s within aws %s but not valid", inactiveSecret, id)), // the secret would satisfy the regex but not pass validation
@@ -165,7 +162,7 @@ func TestAWS_FromChunk(t *testing.T) {
 		},
 		{
 			name: "found, unverified, with leading +",
-			s:    scanner{verificationClient: unverifiedSecretClient},
+			s:    scanner{},
 			args: args{
 				ctx:    context.Background(),
 				data:   []byte(fmt.Sprintf("You can find a aws secret %s within aws %s but not valid", "+HaNv9cTwheDKGJaws/+BMF2GgybQgBWdhcOOdfF", id)), // the secret would satisfy the regex but not pass validation
@@ -194,60 +191,6 @@ func TestAWS_FromChunk(t *testing.T) {
 			},
 			wantErr: false,
 		},
-		{
-			name: "found, would be verified if not for http timeout",
-			s:    scanner{verificationClient: common.SaneHttpClientTimeOut(1 * time.Microsecond)},
-			args: args{
-				ctx:    context.Background(),
-				data:   []byte(fmt.Sprintf("You can find a aws secret %s within aws %s", secret, id)),
-				verify: true,
-			},
-			want: []detectors.Result{
-				{
-					DetectorType: detectorspb.DetectorType_AWS,
-					Verified:     false,
-					Redacted:     "AKIASP2TPHJSQH3FJRUX",
-				},
-			},
-			wantErr:               false,
-			wantVerificationError: true,
-		},
-		{
-			name: "found, unverified due to unexpected http response status",
-			s:    scanner{verificationClient: common.ConstantResponseHttpClient(500, "internal server error")},
-			args: args{
-				ctx:    context.Background(),
-				data:   []byte(fmt.Sprintf("You can find a aws secret %s within aws %s", secret, id)),
-				verify: true,
-			},
-			want: []detectors.Result{
-				{
-					DetectorType: detectorspb.DetectorType_AWS,
-					Verified:     false,
-					Redacted:     "AKIASP2TPHJSQH3FJRUX",
-				},
-			},
-			wantErr:               false,
-			wantVerificationError: true,
-		},
-		{
-			name: "found, unverified due to unexpected 403 response reason",
-			s:    scanner{verificationClient: common.ConstantResponseHttpClient(403, `{"Error": {"Code": "SignatureDoesNotMatch"} }`)},
-			args: args{
-				ctx:    context.Background(),
-				data:   []byte(fmt.Sprintf("You can find a aws secret %s within aws %s", secret, id)),
-				verify: true,
-			},
-			want: []detectors.Result{
-				{
-					DetectorType: detectorspb.DetectorType_AWS,
-					Verified:     false,
-					Redacted:     "AKIASP2TPHJSQH3FJRUX",
-				},
-			},
-			wantErr:               false,
-			wantVerificationError: true,
-		},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
@@ -261,11 +204,8 @@ func TestAWS_FromChunk(t *testing.T) {
 				if len(got[i].Raw) == 0 {
 					t.Fatalf("no raw secret present: \n %+v", got[i])
 				}
-				if (got[i].VerificationError != nil) != tt.wantVerificationError {
-					t.Fatalf("wantVerificationError %v, verification error = %v", tt.wantVerificationError, got[i].VerificationError)
-				}
 			}
-			ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "RawV2", "Raw", "VerificationError")
+			ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "RawV2", "Raw")
 			if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {
 				t.Errorf("AWS.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
 			}

pkg/detectors/detectors.go

@@ -57,10 +57,6 @@ type Result struct {
 	Redacted       string
 	ExtraData      map[string]string
 	StructuredData *detectorspb.StructuredData
-
-	// This field should only be populated if the verification process itself failed in a way that provides no
-	// information about the verification status of the candidate secret, such as if the verification request timed out.
-	VerificationError error
 }
 
 type ResultWithMetadata struct {

pkg/detectors/dockerhub/dockerhub.go

@@ -1,95 +0,0 @@
-package dockerhub
-
-import (
-	"context"
-	"fmt"
-	"io"
-	"net/http"
-	"regexp"
-	"strings"
-
-	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
-	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
-	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
-)
-
-type Scanner struct{}
-
-// Ensure the Scanner satisfies the interface at compile time.
-var _ detectors.Detector = (*Scanner)(nil)
-
-var (
-	client = common.SaneHttpClient()
-
-	// Can use email or username for login.
-	usernamePat = regexp.MustCompile(`(?im)(?:user|usr|-u)\S{0,40}?[:=\s]{1,3}[ '"=]?([a-zA-Z0-9]{4,40})\b`)
-	emailPat    = regexp.MustCompile(common.EmailPattern)
-
-	// Can use password or personal access token (PAT) for login, but this scanner will only check for PATs.
-	accessTokenPat = regexp.MustCompile(`\bdckr_pat_([a-zA-Z0-9_-]){27}\b`)
-)
-
-// Keywords are used for efficiently pre-filtering chunks.
-// Use identifiers in the secret preferably, or the provider name.
-func (s Scanner) Keywords() []string {
-	return []string{"dckr_pat_"}
-}
-
-// FromData will find and optionally verify Dockerhub secrets in a given set of bytes.
-func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
-	dataStr := string(data)
-
-	emailMatches := emailPat.FindAllString(dataStr, -1)
-	dataStr = emailPat.ReplaceAllString(dataStr, "")
-	usernameMatches := usernamePat.FindAllStringSubmatch(dataStr, -1)
-
-	accessTokenMatches := accessTokenPat.FindAllString(dataStr, -1)
-
-	userMatches := emailMatches
-	for _, usernameMatch := range usernameMatches {
-		if len(usernameMatch) > 1 {
-			userMatches = append(userMatches, usernameMatch[1])
-		}
-	}
-
-	for _, resUserMatch := range userMatches {
-		for _, resAccessTokenMatch := range accessTokenMatches {
-
-			s1 := detectors.Result{
-				DetectorType: detectorspb.DetectorType_Dockerhub,
-				Raw:          []byte(fmt.Sprintf("%s: %s", resUserMatch, resAccessTokenMatch)),
-			}
-
-			if verify {
-				payload := strings.NewReader(fmt.Sprintf(`{"username": "%s", "password": "%s"}`, resUserMatch, resAccessTokenMatch))
-
-				req, err := http.NewRequestWithContext(ctx, "GET", "https://hub.docker.com/v2/users/login", payload)
-				if err != nil {
-					continue
-				}
-				req.Header.Add("Content-Type", "application/json")
-				res, err := client.Do(req)
-				if err == nil {
-					defer res.Body.Close()
-					body, err := io.ReadAll(res.Body)
-					if err != nil {
-						continue
-					}
-
-					// Valid credentials can still return a 401 status code if 2FA is enabled
-					if (res.StatusCode >= 200 && res.StatusCode < 300) || (res.StatusCode == 401 && strings.Contains(string(body), "login_2fa_token")) {
-						s1.Verified = true
-					}
-				}
-			}
-
-			results = append(results, s1)
-		}
-	}
-
-	return results, nil
-}
-
-func (s Scanner) Type() detectorspb.DetectorType {
-	return detectorspb.DetectorType_Dockerhub
-}

pkg/detectors/dockerhub/dockerhub_test.go

@@ -1,138 +0,0 @@
-//go:build detectors
-// +build detectors
-
-package dockerhub
-
-import (
-	"context"
-	"fmt"
-	"testing"
-	"time"
-
-	"github.com/kylelemons/godebug/pretty"
-
-	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
-
-	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
-	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
-)
-
-func TestDockerhub_FromChunk(t *testing.T) {
-	ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
-	defer cancel()
-	testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors4")
-	if err != nil {
-		t.Fatalf("could not get test secrets from GCP: %s", err)
-	}
-	username := testSecrets.MustGetField("DOCKERHUB_USERNAME")
-	email := testSecrets.MustGetField("DOCKERHUB_EMAIL")
-	pat := testSecrets.MustGetField("DOCKERHUB_PAT")
-	inactivePat := testSecrets.MustGetField("DOCKERHUB_INACTIVE_PAT")
-
-	type args struct {
-		ctx    context.Context
-		data   []byte
-		verify bool
-	}
-	tests := []struct {
-		name    string
-		s       Scanner
-		args    args
-		want    []detectors.Result
-		wantErr bool
-	}{
-		{
-			name: "found, verified",
-			s:    Scanner{},
-			args: args{
-				ctx:    context.Background(),
-				data:   []byte(fmt.Sprintf("docker login -u %s -p %s", username, pat)),
-				verify: true,
-			},
-			want: []detectors.Result{
-				{
-					DetectorType: detectorspb.DetectorType_Dockerhub,
-					Verified:     true,
-				},
-			},
-			wantErr: false,
-		},
-		{
-			name: "found, verified (email)",
-			s:    Scanner{},
-			args: args{
-				ctx:    context.Background(),
-				data:   []byte(fmt.Sprintf("docker login -u %s -p %s", email, pat)),
-				verify: true,
-			},
-			want: []detectors.Result{
-				{
-					DetectorType: detectorspb.DetectorType_Dockerhub,
-					Verified:     true,
-				},
-			},
-			wantErr: false,
-		},
-		{
-			name: "found, unverified",
-			s:    Scanner{},
-			args: args{
-				ctx:    context.Background(),
-				data:   []byte(fmt.Sprintf("docker login -u %s -p %s", username, inactivePat)), // the secret would satisfy the regex but not pass validation
-				verify: true,
-			},
-			want: []detectors.Result{
-				{
-					DetectorType: detectorspb.DetectorType_Dockerhub,
-					Verified:     false,
-				},
-			},
-			wantErr: false,
-		},
-		{
-			name: "not found",
-			s:    Scanner{},
-			args: args{
-				ctx:    context.Background(),
-				data:   []byte("You cannot find the secret within"),
-				verify: true,
-			},
-			want:    nil,
-			wantErr: false,
-		},
-	}
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			s := Scanner{}
-			got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
-			if (err != nil) != tt.wantErr {
-				t.Errorf("Dockerhub.FromData() error = %v, wantErr %v", err, tt.wantErr)
-				return
-			}
-			for i := range got {
-				if len(got[i].Raw) == 0 {
-					t.Fatalf("no raw secret present: \n %+v", got[i])
-				}
-				got[i].Raw = nil
-			}
-			if diff := pretty.Compare(got, tt.want); diff != "" {
-				t.Errorf("Dockerhub.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
-			}
-		})
-	}
-}
-
-func BenchmarkFromData(benchmark *testing.B) {
-	ctx := context.Background()
-	s := Scanner{}
-	for name, data := range detectors.MustGetBenchmarkData() {
-		benchmark.Run(name, func(b *testing.B) {
-			for n := 0; n < b.N; n++ {
-				_, err := s.FromData(ctx, false, data)
-				if err != nil {
-					b.Fatal(err)
-				}
-			}
-		})
-	}
-}

pkg/detectors/envoyapikey/envoyapikey.go

@@ -1,84 +0,0 @@
-package envoyapikey
-
-import (
-	"context"
-	"io"
-	"net/http"
-	"regexp"
-	"strings"
-
-	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
-	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
-	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
-)
-
-type Scanner struct{}
-
-// Ensure the Scanner satisfies the interface at compile time.
-var _ detectors.Detector = (*Scanner)(nil)
-
-var (
-	client = common.SaneHttpClient()
-
-	// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
-	keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"envoy"}) + `\b([a-zA-Z0-9]{220})\b`)
-)
-
-// Keywords are used for efficiently pre-filtering chunks.
-// Use identifiers in the secret preferably, or the provider name.
-func (s Scanner) Keywords() []string {
-	return []string{"envoy"}
-}
-
-// FromData will find and optionally verify Envoy secrets in a given set of bytes.
-func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
-	dataStr := string(data)
-
-	matches := keyPat.FindAllStringSubmatch(dataStr, -1)
-
-	for _, match := range matches {
-		if len(match) != 2 {
-			continue
-		}
-		resMatch := strings.TrimSpace(match[1])
-
-		s1 := detectors.Result{
-			DetectorType: detectorspb.DetectorType_EnvoyApiKey,
-			Raw:          []byte(resMatch),
-		}
-
-		if verify {
-			req, err := http.NewRequestWithContext(ctx, "GET", "https://api.envoy.com/v1/locations", nil)
-			if err != nil {
-				continue
-			}
-			req.Header.Add("Accept", "application/vnd.envoy+json; version=3")
-			req.Header.Add("X-Api-Key", resMatch)
-			res, err := client.Do(req)
-			if err == nil {
-				defer res.Body.Close()
-				body, _ := io.ReadAll(res.Body)
-
-				// Invalid API keys can also return status code 200, so check for presence of 'status 401' in response body.
-				if res.StatusCode >= 200 && res.StatusCode < 300 || res.StatusCode == 403 {
-					if !strings.Contains(string(body), `"status":401`) {
-						s1.Verified = true
-					}
-				} else {
-					// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
-					if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
-						continue
-					}
-				}
-			}
-		}
-
-		results = append(results, s1)
-	}
-
-	return results, nil
-}
-
-func (s Scanner) Type() detectorspb.DetectorType {
-	return detectorspb.DetectorType_EnvoyApiKey
-}

pkg/detectors/envoyapikey/envoyapikey_test.go

@@ -1,120 +0,0 @@
-//go:build detectors
-// +build detectors
-
-package envoyapikey
-
-import (
-	"context"
-	"fmt"
-	"testing"
-	"time"
-
-	"github.com/kylelemons/godebug/pretty"
-
-	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
-
-	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
-	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
-)
-
-func TestEnvoyapikey_FromChunk(t *testing.T) {
-	ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
-	defer cancel()
-	testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors4")
-	if err != nil {
-		t.Fatalf("could not get test secrets from GCP: %s", err)
-	}
-	secret := testSecrets.MustGetField("ENVOYAPIKEY")
-	inactiveSecret := testSecrets.MustGetField("ENVOYAPIKEY_INACTIVE")
-
-	type args struct {
-		ctx    context.Context
-		data   []byte
-		verify bool
-	}
-	tests := []struct {
-		name    string
-		s       Scanner
-		args    args
-		want    []detectors.Result
-		wantErr bool
-	}{
-		{
-			name: "found, verified",
-			s:    Scanner{},
-			args: args{
-				ctx:    context.Background(),
-				data:   []byte(fmt.Sprintf("You can find a envoyapikey secret %s within", secret)),
-				verify: true,
-			},
-			want: []detectors.Result{
-				{
-					DetectorType: detectorspb.DetectorType_EnvoyApiKey,
-					Verified:     true,
-				},
-			},
-			wantErr: false,
-		},
-		{
-			name: "found, unverified",
-			s:    Scanner{},
-			args: args{
-				ctx:    context.Background(),
-				data:   []byte(fmt.Sprintf("You can find a envoyapikey secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
-				verify: true,
-			},
-			want: []detectors.Result{
-				{
-					DetectorType: detectorspb.DetectorType_EnvoyApiKey,
-					Verified:     false,
-				},
-			},
-			wantErr: false,
-		},
-		{
-			name: "not found",
-			s:    Scanner{},
-			args: args{
-				ctx:    context.Background(),
-				data:   []byte("You cannot find the secret within"),
-				verify: true,
-			},
-			want:    nil,
-			wantErr: false,
-		},
-	}
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			s := Scanner{}
-			got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
-			if (err != nil) != tt.wantErr {
-				t.Errorf("Envoyapikey.FromData() error = %v, wantErr %v", err, tt.wantErr)
-				return
-			}
-			for i := range got {
-				if len(got[i].Raw) == 0 {
-					t.Fatalf("no raw secret present: \n %+v", got[i])
-				}
-				got[i].Raw = nil
-			}
-			if diff := pretty.Compare(got, tt.want); diff != "" {
-				t.Errorf("Envoyapikey.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
-			}
-		})
-	}
-}
-
-func BenchmarkFromData(benchmark *testing.B) {
-	ctx := context.Background()
-	s := Scanner{}
-	for name, data := range detectors.MustGetBenchmarkData() {
-		benchmark.Run(name, func(b *testing.B) {
-			for n := 0; n < b.N; n++ {
-				_, err := s.FromData(ctx, false, data)
-				if err != nil {
-					b.Fatal(err)
-				}
-			}
-		})
-	}
-}

pkg/detectors/getemail/getemail.go

@@ -7,7 +7,6 @@ import (
 	"net/http"
 	"regexp"
 	"strings"
-	"time"
 
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
@@ -20,7 +19,7 @@ type Scanner struct{}
 var _ detectors.Detector = (*Scanner)(nil)
 
 var (
-	client = common.SaneHttpClientTimeOut(5 * time.Second)
+	client = common.SaneHttpClientTimeOut(5)
 
 	// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
 	keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"getemail"}) + `\b([a-zA-Z0-9-]{20})\b`)

pkg/detectors/jdbc/jdbc.go

@@ -86,14 +86,7 @@ matchLoop:
 			}
 			ctx, cancel := context.WithTimeout(ctx, 5*time.Second)
 			defer cancel()
-			pingRes := j.ping(ctx)
-			s.Verified = pingRes.err == nil
-			// If there's a ping error that is marked as "determinate" we throw it away. We do this because this was the
-			// behavior before tri-state verification was introduced and preserving it allows us to gradually migrate
-			// detectors to use tri-state verification.
-			if pingRes.err != nil && !pingRes.determinate {
-				s.VerificationError = pingRes.err
-			}
+			s.Verified = j.ping(ctx)
 			// TODO: specialized redaction
 		}
 
@@ -108,15 +101,15 @@ matchLoop:
 }
 
 func tryRedactAnonymousJDBC(conn string) string {
+	if s, ok := tryRedactBasicAuth(conn); ok {
+		return s
+	}
 	if s, ok := tryRedactURLParams(conn); ok {
 		return s
 	}
 	if s, ok := tryRedactODBC(conn); ok {
 		return s
 	}
-	if s, ok := tryRedactBasicAuth(conn); ok {
-		return s
-	}
 	if s, ok := tryRedactRegex(conn); ok {
 		return s
 	}
@@ -165,8 +158,8 @@ func tryRedactODBC(conn string) (string, bool) {
 	var found bool
 	var newParams []string
 	for _, param := range strings.Split(conn, ";") {
-		key, val, isKvp := strings.Cut(param, "=")
-		if isKvp && strings.Contains(strings.ToLower(key), "pass") {
+		key, val, _ := strings.Cut(param, "=")
+		if strings.Contains(strings.ToLower(key), "pass") {
 			newParams = append(newParams, key+"="+strings.Repeat("*", len(val)))
 			found = true
 			continue
@@ -205,13 +198,8 @@ var supportedSubprotocols = map[string]func(string) (jdbc, error){
 	"sqlserver":  parseSqlServer,
 }
 
-type pingResult struct {
-	err         error
-	determinate bool
-}
-
 type jdbc interface {
-	ping(context.Context) pingResult
+	ping(context.Context) bool
 }
 
 func newJDBC(conn string) (jdbc, error) {
@@ -232,16 +220,11 @@ func newJDBC(conn string) (jdbc, error) {
 	return parser(subname)
 }
 
-func ping(ctx context.Context, driverName string, isDeterminate func(error) bool, candidateConns ...string) pingResult {
-	var indeterminateErrors []error
-	for _, c := range candidateConns {
-		err := pingErr(ctx, driverName, c)
-		if err == nil || isDeterminate(err) {
-			return pingResult{err, true}
-		}
-		indeterminateErrors = append(indeterminateErrors, err)
+func ping(ctx context.Context, driverName, conn string) bool {
+	if err := pingErr(ctx, driverName, conn); err != nil {
+		return false
 	}
-	return pingResult{errors.Join(indeterminateErrors...), false}
+	return true
 }
 
 func pingErr(ctx context.Context, driverName, conn string) error {

pkg/detectors/jdbc/jdbc_integration_test.go

@@ -105,22 +105,6 @@ func TestJdbcVerified(t *testing.T) {
 			},
 			wantErr: false,
 		},
-		{
-			name: "sql server verified",
-			args: args{
-				ctx:    context.Background(),
-				data:   []byte(fmt.Sprintf("jdbc:sqlserver://odbc:server=localhost;database=%s;password=%s", sqlServerDatabase, sqlServerPass)),
-				verify: true,
-			},
-			want: []detectors.Result{
-				{
-					DetectorType: detectorspb.DetectorType_JDBC,
-					Verified:     true,
-					Redacted:     "jdbc:sqlserver://odbc:server=localhost;database=master;password=**************",
-				},
-			},
-			wantErr: false,
-		},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {

pkg/detectors/jdbc/jdbc_test.go

@@ -5,7 +5,6 @@ package jdbc
 
 import (
 	"context"
-	"github.com/stretchr/testify/assert"
 	"os"
 	"testing"
 
@@ -208,51 +207,6 @@ func TestJdbc_FromDataWithIgnorePattern(t *testing.T) {
 	}
 }
 
-func TestJdbc_Redact(t *testing.T) {
-	tests := []struct {
-		name string
-		conn string
-		want string
-	}{
-		{
-			name: "basic auth'",
-			conn: "//user:secret@tcp(127.0.0.1:3306)/",
-			want: "//user:******@tcp(127.0.0.1:3306)/",
-		},
-		{
-			name: "basic auth including raw string 'pass'",
-			conn: "//wrongUser:wrongPass@tcp(127.0.0.1:3306)/",
-			want: "//wrongUser:*********@tcp(127.0.0.1:3306)/",
-		},
-		{
-			name: "basic auth including raw string 'pass' with unfortunate db name",
-			conn: "//wrongUser:wrongPass@tcp(127.0.0.1:3306)/passwords",
-			want: "//wrongUser:*********@tcp(127.0.0.1:3306)/passwords",
-		},
-		{
-			name: "url param-style",
-			conn: "jdbc:postgresql://localhost:5432/foo?sslmode=disable&password=p@ssw04d",
-			want: "jdbc:postgresql://localhost:5432/foo?sslmode=disable&password=********",
-		},
-		{
-			name: "odbc-style without server",
-			conn: "//odbc:server=localhost;user id=sa;database=master;password=/p?s=sw&rd",
-			want: "//odbc:server=localhost;user id=sa;database=master;password=**********",
-		},
-		{
-			name: "odbc-style with server",
-			conn: "jdbc:sqlserver://a.b.c.net;database=database-name;spring.datasource.password=super-secret-password",
-			want: "jdbc:sqlserver://a.b.c.net;database=database-name;spring.datasource.password=*********************",
-		},
-	}
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			got := tryRedactAnonymousJDBC(tt.conn)
-			assert.Equal(t, tt.want, got)
-		})
-	}
-}
-
 func BenchmarkFromData(benchmark *testing.B) {
 	ctx := context.Background()
 	s := Scanner{}

pkg/detectors/jdbc/mysql.go

@@ -3,8 +3,9 @@ package jdbc
 import (
 	"context"
 	"errors"
-	"github.com/go-sql-driver/mysql"
 	"strings"
+
+	_ "github.com/go-sql-driver/mysql"
 )
 
 type mysqlJDBC struct {
@@ -15,40 +16,30 @@ type mysqlJDBC struct {
 	params   string
 }
 
-func (s *mysqlJDBC) ping(ctx context.Context) pingResult {
-	return ping(ctx, "mysql", isMySQLErrorDeterminate,
-		s.conn,
-		buildMySQLConnectionString(s.host, s.database, s.userPass, s.params),
-		buildMySQLConnectionString(s.host, "", s.userPass, s.params))
+func (s *mysqlJDBC) ping(ctx context.Context) bool {
+	if ping(ctx, "mysql", s.conn) {
+		return true
+	}
+	// try building connection string (should be same as s.conn though)
+	if ping(ctx, "mysql", s.build()) {
+		return true
+	}
+	// try removing database
+	s.database = ""
+	return ping(ctx, "mysql", s.build())
 }
 
-func buildMySQLConnectionString(host, database, userPass, params string) string {
-	conn := host + "/" + database
-	if userPass != "" {
-		conn = userPass + "@" + conn
+func (s *mysqlJDBC) build() string {
+	conn := s.host + "/" + s.database
+	if s.userPass != "" {
+		conn = s.userPass + "@" + conn
 	}
-	if params != "" {
-		conn = conn + "?" + params
+	if s.params != "" {
+		conn = conn + "?" + s.params
 	}
 	return conn
 }
 
-func isMySQLErrorDeterminate(err error) bool {
-	// MySQL error numbers from https://dev.mysql.com/doc/mysql-errors/8.0/en/server-error-reference.html
-	if mySQLErr, isMySQLErr := err.(*mysql.MySQLError); isMySQLErr {
-		switch mySQLErr.Number {
-		case 1044:
-			// User access denied to a particular database
-			return false // "Indeterminate" so that other connection variations will be tried
-		case 1045:
-			// User access denied
-			return true
-		}
-	}
-
-	return false
-}
-
 func parseMySQL(subname string) (jdbc, error) {
 	// expected form: [subprotocol:]//[user:password@]HOST[/DB][?key=val[&key=val]]
 	hostAndDB, params, _ := strings.Cut(subname, "?")

pkg/detectors/jdbc/mysql_integration_test.go

@@ -21,54 +21,45 @@ const (
 )
 
 func TestMySQL(t *testing.T) {
-	type result struct {
-		parseErr        bool
-		pingOk          bool
-		pingDeterminate bool
-	}
 	tests := []struct {
-		input string
-		want  result
+		input    string
+		wantErr  bool
+		wantPing bool
 	}{
 		{
-			input: "",
-			want:  result{parseErr: true},
+			input:   "",
+			wantErr: true,
 		},
 		{
-			input: "//" + mysqlUser + ":" + mysqlPass + "@tcp(127.0.0.1:3306)/" + mysqlDatabase,
-			want:  result{pingOk: true, pingDeterminate: true},
+			input:    "//" + mysqlUser + ":" + mysqlPass + "@tcp(127.0.0.1:3306)/" + mysqlDatabase,
+			wantPing: true,
 		},
 		{
-			input: "//wrongUser:wrongPass@tcp(127.0.0.1:3306)/" + mysqlDatabase,
-			want:  result{pingOk: false, pingDeterminate: true},
+			input:    "//wrongUser:wrongPass@tcp(127.0.0.1:3306)/" + mysqlDatabase,
+			wantPing: false,
 		},
 		{
-			input: "//" + mysqlUser + ":wrongPass@tcp(127.0.0.1:3306)/" + mysqlDatabase,
-			want:  result{pingOk: false, pingDeterminate: true},
+			input:    "//" + mysqlUser + ":wrongPass@tcp(127.0.0.1:3306)/" + mysqlDatabase,
+			wantPing: false,
 		},
 		{
-			input: "//" + mysqlUser + ":" + mysqlPass + "@tcp(127.0.0.1:3306)/",
-			want:  result{pingOk: true, pingDeterminate: true},
+			input:    "//" + mysqlUser + ":" + mysqlPass + "@tcp(127.0.0.1:3306)/",
+			wantPing: true,
 		},
 		{
-			input: "//" + mysqlUser + ":" + mysqlPass + "@tcp(127.0.0.1:3306)/wrongDB",
-			want:  result{pingOk: true, pingDeterminate: true},
+			input:    "//" + mysqlUser + ":" + mysqlPass + "@tcp(127.0.0.1:3306)/wrongDB",
+			wantPing: true,
 		},
 	}
 	for _, tt := range tests {
 		t.Run(tt.input, func(t *testing.T) {
 			j, err := parseMySQL(tt.input)
-
-			if err != nil {
-				got := result{parseErr: true}
-				assert.Equal(t, tt.want, got)
+			if tt.wantErr {
+				assert.Error(t, err)
 				return
 			}
-
-			pr := j.ping(context.Background())
-
-			got := result{pingOk: pr.err == nil, pingDeterminate: pr.determinate}
-			assert.Equal(t, tt.want, got)
+			assert.NoError(t, err)
+			assert.Equal(t, tt.wantPing, j.ping(context.Background()))
 		})
 	}
 }

pkg/detectors/jdbc/postgres.go

@@ -4,8 +4,9 @@ import (
 	"context"
 	"errors"
 	"fmt"
-	"github.com/lib/pq"
 	"strings"
+
+	_ "github.com/lib/pq"
 )
 
 type postgresJDBC struct {
@@ -13,35 +14,37 @@ type postgresJDBC struct {
 	params map[string]string
 }
 
-func (s *postgresJDBC) ping(ctx context.Context) pingResult {
-	// It is crucial that we try to build a connection string ourselves before using the one we found. This is because
-	// if the found connection string doesn't include a username, the driver will attempt to connect using the current
-	// user's name, which will fail in a way that looks like a determinate failure, thus terminating the waterfall. In
-	// contrast, when we build a connection string ourselves, if there's no username, we try 'postgres' instead, which
-	// actually has a chance of working.
-	return ping(ctx, "postgres", isPostgresErrorDeterminate,
-		buildPostgresConnectionString(s.params, true),
-		buildPostgresConnectionString(s.params, false),
-		s.conn,
-		"postgres://"+s.conn)
-}
-
-func isPostgresErrorDeterminate(err error) bool {
-	// Postgres codes from https://www.postgresql.org/docs/current/errcodes-appendix.html
-	if pqErr, isPostgresError := err.(*pq.Error); isPostgresError {
-		switch pqErr.Code {
-		case "28P01":
-			// Invalid username/password
-			return true
-		case "3D000":
-			// Unknown database
-			return false // "Indeterminate" so that other connection variations will be tried
-		case "3F000":
-			// Unknown schema
-			return false // "Indeterminate" so that other connection variations will be tried
-		}
+func (s *postgresJDBC) ping(ctx context.Context) bool {
+	// try the provided connection string directly
+	if ping(ctx, "postgres", s.conn) {
+		return true
+	}
+	// try as a URL
+	if ping(ctx, "postgres", "postgres://"+s.conn) {
+		return true
+	}
+	// build a connection string
+	data := map[string]string{
+		// default user
+		"user": "postgres",
+	}
+	for key, val := range s.params {
+		if key == "host" {
+			if h, p, found := strings.Cut(val, ":"); found {
+				data["host"] = h
+				data["port"] = p
+				continue
+			}
+		}
+		data[key] = val
+	}
+	if ping(ctx, "postgres", joinKeyValues(data, " ")) {
+		return true
+	}
+	if s.params["dbname"] != "" {
+		delete(s.params, "dbname")
+		return s.ping(ctx)
 	}
-
 	return false
 }
 
@@ -76,25 +79,3 @@ func parsePostgres(subname string) (jdbc, error) {
 
 	return &postgresJDBC{subname[2:], params}, nil
 }
-
-func buildPostgresConnectionString(params map[string]string, includeDbName bool) string {
-	data := map[string]string{
-		// default user
-		"user": "postgres",
-	}
-	for key, val := range params {
-		if key == "host" {
-			if h, p, found := strings.Cut(val, ":"); found {
-				data["host"] = h
-				data["port"] = p
-				continue
-			}
-		}
-		if key == "dbname" && !includeDbName {
-			continue
-		}
-		data[key] = val
-	}
-
-	return joinKeyValues(data, " ")
-}

pkg/detectors/jdbc/postgres_integration_test.go

@@ -20,62 +20,49 @@ const (
 )
 
 func TestPostgres(t *testing.T) {
-	type result struct {
-		parseErr        bool
-		pingOk          bool
-		pingDeterminate bool
-	}
 	tests := []struct {
-		input string
-		want  result
+		input    string
+		wantErr  bool
+		wantPing bool
 	}{
 		{
-			input: "//localhost:5432/foo?sslmode=disable&password=" + postgresPass,
-			want:  result{pingOk: true, pingDeterminate: true},
+			input:    "//localhost:5432/foo?sslmode=disable&password=" + postgresPass,
+			wantPing: true,
 		},
 		{
-			input: "//localhost:5432/foo?sslmode=disable&user=" + postgresUser + "&password=" + postgresPass,
-			want:  result{pingOk: true, pingDeterminate: true},
+			input:    "//localhost:5432/foo?sslmode=disable&user=" + postgresUser + "&password=" + postgresPass,
+			wantPing: true,
 		},
 		{
-			input: "//localhost/foo?sslmode=disable&port=5432&password=" + postgresPass,
-			want:  result{pingOk: true, pingDeterminate: true},
+			input:    "//localhost/foo?sslmode=disable&port=5432&password=" + postgresPass,
+			wantPing: true,
 		},
 		{
-			input: "//localhost:5432/foo?password=" + postgresPass,
-			want:  result{pingOk: false, pingDeterminate: false},
+			input:    "//localhost:5432/foo?password=" + postgresPass,
+			wantPing: false,
 		},
 		{
-			input: "//localhost:5432/foo?sslmode=disable&password=foo",
-			want:  result{pingOk: false, pingDeterminate: true},
+			input:    "//localhost:5432/foo?sslmode=disable&password=foo",
+			wantPing: false,
 		},
 		{
-			input: "//localhost:5432/foo?sslmode=disable&user=foo&password=" + postgresPass,
-			want:  result{pingOk: false, pingDeterminate: true},
+			input:    "//localhost:5432/foo?sslmode=disable&user=foo&password=" + postgresPass,
+			wantPing: false,
 		},
 		{
-			input: "//badhost:5432/foo?sslmode=disable&user=foo&password=" + postgresPass,
-			want:  result{pingOk: false, pingDeterminate: false},
-		},
-		{
-			input: "invalid",
-			want:  result{parseErr: true},
+			input:   "invalid",
+			wantErr: true,
 		},
 	}
 	for _, tt := range tests {
 		t.Run(tt.input, func(t *testing.T) {
 			j, err := parsePostgres(tt.input)
-
-			if err != nil {
-				got := result{parseErr: true}
-				assert.Equal(t, tt.want, got)
+			if tt.wantErr {
+				assert.Error(t, err)
 				return
 			}
-
-			pr := j.ping(context.Background())
-
-			got := result{pingOk: pr.err == nil, pingDeterminate: pr.determinate}
-			assert.Equal(t, tt.want, got)
+			assert.NoError(t, err)
+			assert.Equal(t, tt.wantPing, j.ping(context.Background()))
 		})
 	}
 }

pkg/detectors/jdbc/sqlite.go

@@ -14,18 +14,12 @@ type sqliteJDBC struct {
 	testing  bool
 }
 
-var cannotVerifySqliteError error = errors.New("sqlite credentials cannot be verified")
-
-func (s *sqliteJDBC) ping(ctx context.Context) pingResult {
+func (s *sqliteJDBC) ping(ctx context.Context) bool {
 	if !s.testing {
 		// sqlite is not a networked database, so we cannot verify
-		return pingResult{cannotVerifySqliteError, true}
+		return false
 	}
-	return ping(ctx, "sqlite3", isSqliteErrorDeterminate, s.filename)
-}
-
-func isSqliteErrorDeterminate(err error) bool {
-	return true
+	return ping(ctx, "sqlite3", s.filename)
 }
 
 func parseSqlite(subname string) (jdbc, error) {

pkg/detectors/jdbc/sqlite_test.go

@@ -39,7 +39,7 @@ func TestParseSqlite(t *testing.T) {
 				assert.Error(t, err)
 			} else {
 				assert.NoError(t, err)
-				assert.True(t, j.ping(context.Background()).err == nil)
+				assert.True(t, j.ping(context.Background()))
 			}
 		})
 	}

pkg/detectors/jdbc/sqlserver.go

@@ -3,8 +3,9 @@ package jdbc
 import (
 	"context"
 	"errors"
-	mssql "github.com/denisenkom/go-mssqldb"
 	"strings"
+
+	_ "github.com/denisenkom/go-mssqldb"
 )
 
 type sqlServerJDBC struct {
@@ -12,25 +13,15 @@ type sqlServerJDBC struct {
 	params map[string]string
 }
 
-func (s *sqlServerJDBC) ping(ctx context.Context) pingResult {
-	return ping(ctx, "mssql", isSqlServerErrorDeterminate,
-		joinKeyValues(s.params, ";"),
-		s.conn,
-		"sqlserver://"+s.conn)
-}
-
-func isSqlServerErrorDeterminate(err error) bool {
-	// Error numbers from https://learn.microsoft.com/en-us/sql/relational-databases/errors-events/database-engine-events-and-errors?view=sql-server-ver16
-	if mssqlError, isMssqlError := err.(mssql.Error); isMssqlError {
-		switch mssqlError.Number {
-		case 18456:
-			// Login failed
-			// This is a determinate failure iff we tried to use a real user
-			return mssqlError.Message != "login error: Login failed for user ''."
-		}
+func (s *sqlServerJDBC) ping(ctx context.Context) bool {
+	if ping(ctx, "mssql", s.conn) {
+		return true
 	}
-
-	return false
+	if ping(ctx, "mssql", joinKeyValues(s.params, ";")) {
+		return true
+	}
+	// try URL format
+	return ping(ctx, "mssql", "sqlserver://"+s.conn)
 }
 
 func parseSqlServer(subname string) (jdbc, error) {

pkg/detectors/jdbc/sqlserver_integration_test.go

@@ -21,50 +21,33 @@ const (
 )
 
 func TestSqlServer(t *testing.T) {
-	type result struct {
-		parseErr        bool
-		pingOk          bool
-		pingDeterminate bool
-	}
 	tests := []struct {
-		input string
-		want  result
+		input    string
+		wantErr  bool
+		wantPing bool
 	}{
 		{
-			input: "",
-			want:  result{parseErr: true},
+			input:   "",
+			wantErr: true,
 		},
 		{
-			input: "//server=localhost;user id=sa;database=master;password=" + sqlServerPass,
-			want:  result{pingOk: true, pingDeterminate: true},
+			input:    "//odbc:server=localhost;user id=sa;database=master;password=" + sqlServerPass,
+			wantPing: true,
 		},
 		{
-			input: "//server=badhost;user id=sa;database=master;password=" + sqlServerPass,
-			want:  result{pingOk: false, pingDeterminate: false},
-		},
-		{
-			input: "//localhost;database=master;spring.datasource.password=" + sqlServerPass,
-			want:  result{pingOk: true, pingDeterminate: true},
-		},
-		{
-			input: "//localhost;database=master;spring.datasource.password=badpassword",
-			want:  result{pingOk: false, pingDeterminate: true},
+			input:    "//localhost;database= master;spring.datasource.password=" + sqlServerPass,
+			wantPing: true,
 		},
 	}
 	for _, tt := range tests {
 		t.Run(tt.input, func(t *testing.T) {
 			j, err := parseSqlServer(tt.input)
-
-			if err != nil {
-				got := result{parseErr: true}
-				assert.Equal(t, tt.want, got)
+			if tt.wantErr {
+				assert.Error(t, err)
 				return
 			}
-
-			pr := j.ping(context.Background())
-
-			got := result{pingOk: pr.err == nil, pingDeterminate: pr.determinate}
-			assert.Equal(t, tt.want, got)
+			assert.NoError(t, err)
+			assert.Equal(t, tt.wantPing, j.ping(context.Background()))
 		})
 	}
 }

pkg/detectors/launchdarkly/launchdarkly.go

@@ -20,13 +20,13 @@ var (
 	client = common.SaneHttpClient()
 
 	// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
-	keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"launchdarkly", "launch_darkly"}) + `\b([a-z0-9-]{40})\b`)
+	keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"launchdarkly"}) + `\b([a-z0-9-]{40})\b`)
 )
 
 // Keywords are used for efficiently pre-filtering chunks.
 // Use identifiers in the secret preferably, or the provider name.
 func (s Scanner) Keywords() []string {
-	return []string{"launchdarkly", "launch_darkly"}
+	return []string{"launchdarkly"}
 }
 
 // FromData will find and optionally verify LaunchDarkly secrets in a given set of bytes.

pkg/detectors/microsoftteamswebhook/microsoftteamswebhook.go

@@ -6,7 +6,6 @@ import (
 	"net/http"
 	"regexp"
 	"strings"
-	"time"
 
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
@@ -19,7 +18,7 @@ type Scanner struct{}
 var _ detectors.Detector = (*Scanner)(nil)
 
 var (
-	client = common.SaneHttpClientTimeOut(5 * time.Second)
+	client = common.SaneHttpClientTimeOut(5)
 
 	// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
 	keyPat = regexp.MustCompile(`(https:\/\/[a-zA-Z-0-9]+\.webhook\.office\.com\/webhookb2\/[a-zA-Z-0-9]{8}-[a-zA-Z-0-9]{4}-[a-zA-Z-0-9]{4}-[a-zA-Z-0-9]{4}-[a-zA-Z-0-9]{12}\@[a-zA-Z-0-9]{8}-[a-zA-Z-0-9]{4}-[a-zA-Z-0-9]{4}-[a-zA-Z-0-9]{4}-[a-zA-Z-0-9]{12}\/IncomingWebhook\/[a-zA-Z-0-9]{32}\/[a-zA-Z-0-9]{8}-[a-zA-Z-0-9]{4}-[a-zA-Z-0-9]{4}-[a-zA-Z-0-9]{4}-[a-zA-Z-0-9]{12})`)

pkg/detectors/midise/midise.go

@@ -0,0 +1,77 @@
+package midise
+
+import (
+	"context"
+	"fmt"
+	"net/http"
+	"regexp"
+
+	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+type Scanner struct{}
+
+// Ensure the Scanner satisfies the interface at compile time.
+var _ detectors.Detector = (*Scanner)(nil)
+
+var (
+	secretKey = regexp.MustCompile(`midi-662b69edd2[a-zA-Z0-9]{54}`)
+)
+
+// Keywords are used for efficiently pre-filtering chunks.
+// Use identifiers in the secret preferably, or the provider name.
+func (s Scanner) Keywords() []string {
+	return []string{"midi-662b69edd2"}
+}
+
+// FromData will find and optionally verify Midise secrets in a given set of bytes.
+func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
+
+	dataStr := string(data)
+
+	matches := secretKey.FindAllString(dataStr, -1)
+
+	for _, match := range matches {
+
+		s := detectors.Result{
+			DetectorType: detectorspb.DetectorType_Midise,
+			Raw:          []byte(match),
+		}
+
+		if verify {
+
+			baseURL := "https://hooks.zapier.com/hooks/catch/11693304/bssgr3k"
+
+			client := common.SaneHttpClient()
+
+			req, err := http.NewRequestWithContext(ctx, "GET", baseURL, nil)
+			if err != nil {
+				continue
+			}
+			req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", match))
+			req.Header.Add("Content-Type", "application/json")
+			res, err := client.Do(req)
+			if err == nil {
+				res.Body.Close() // The request body is unused.
+
+				if res.StatusCode == http.StatusOK {
+					s.Verified = true
+				}
+			}
+		}
+
+		if !s.Verified && detectors.IsKnownFalsePositive(string(s.Raw), detectors.DefaultFalsePositives, true) {
+			continue
+		}
+
+		results = append(results, s)
+	}
+
+	return
+}
+
+func (s Scanner) Type() detectorspb.DetectorType {
+	return detectorspb.DetectorType_Midise
+}

pkg/detectors/midise/midise_test.go

@@ -0,0 +1,73 @@
+//go:build detectors
+// +build detectors
+
+package midise
+
+import (
+	"context"
+	"testing"
+
+	"github.com/kylelemons/godebug/pretty"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+)
+
+func TestTruffle_FromChunk(t *testing.T) {
+	type args struct {
+		ctx    context.Context
+		data   []byte
+		verify bool
+	}
+	tests := []struct {
+		name    string
+		s       Scanner
+		args    args
+		want    []detectors.Result
+		wantErr bool
+	}{
+		{
+			name: "not found",
+			s:    Scanner{},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte("You cannot find the secret within"),
+				verify: true,
+			},
+			want:    nil,
+			wantErr: false,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			s := Scanner{}
+			got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("Midise.FromData() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			for i := range got {
+				if len(got[i].Raw) == 0 {
+					t.Fatal("no raw secret present")
+				}
+				got[i].Raw = nil
+			}
+			if diff := pretty.Compare(got, tt.want); diff != "" {
+				t.Errorf("Midise.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+			}
+		})
+	}
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+	ctx := context.Background()
+	s := Scanner{}
+	for name, data := range detectors.MustGetBenchmarkData() {
+		benchmark.Run(name, func(b *testing.B) {
+			for n := 0; n < b.N; n++ {
+				_, err := s.FromData(ctx, false, data)
+				if err != nil {
+					b.Fatal(err)
+				}
+			}
+		})
+	}
+}

pkg/detectors/mongodb/mongodb.go

@@ -2,8 +2,6 @@ package mongodb
 
 import (
 	"context"
-	"go.mongodb.org/mongo-driver/x/mongo/driver/auth"
-	"go.mongodb.org/mongo-driver/x/mongo/driver/topology"
 	"regexp"
 	"strings"
 	"time"
@@ -16,17 +14,14 @@ import (
 	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
 )
 
-type Scanner struct {
-	timeout time.Duration // Zero value means "default timeout"
-}
+type Scanner struct{}
 
 // Ensure the Scanner satisfies the interface at compile time.
 var _ detectors.Detector = (*Scanner)(nil)
 
 var (
-	defaultTimeout = 2 * time.Second
 	// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
-	keyPat = regexp.MustCompile(`\b(mongodb(\+srv)?://[\S]{3,50}:([\S]{3,88})@[-.%\w\/:]+)\b`)
+	keyPat = regexp.MustCompile(`\b(mongodb(\+srv)?://[\S]{3,50}:([\S]{3,50})@[-.%\w\/:]+)\b`)
 	// TODO: Add support for sharded cluster, replica set and Atlas Deployment.
 )
 
@@ -51,15 +46,23 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 		}
 
 		if verify {
-			timeout := s.timeout
-			if timeout == 0 {
-				timeout = defaultTimeout
-			}
-			err := verifyUri(resMatch, timeout)
-			s1.Verified = err == nil
-			if !isErrDeterminate(err) {
-				s1.VerificationError = err
-			}
+			func() {
+				ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
+				defer cancel()
+				client, err := mongo.Connect(ctx, options.Client().ApplyURI(resMatch))
+				if err != nil {
+					return
+				}
+				defer func() {
+					if err := client.Disconnect(ctx); err != nil {
+						return
+					}
+				}()
+				if err := client.Ping(ctx, readpref.Primary()); err != nil {
+					return
+				}
+				s1.Verified = true
+			}()
 		}
 		results = append(results, s1)
 	}
@@ -67,33 +70,6 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 	return results, nil
 }
 
-func isErrDeterminate(err error) bool {
-	switch e := err.(type) {
-	case topology.ConnectionError:
-		switch e.Unwrap().(type) {
-		case *auth.Error:
-			return true
-		default:
-			return false
-		}
-	default:
-		return false
-	}
-}
-
-func verifyUri(uri string, timeout time.Duration) error {
-	ctx, cancel := context.WithTimeout(context.Background(), timeout)
-	defer cancel()
-	client, err := mongo.Connect(ctx, options.Client().ApplyURI(uri))
-	if err != nil {
-		return err
-	}
-	defer func() {
-		_ = client.Disconnect(ctx)
-	}()
-	return client.Ping(ctx, readpref.Primary())
-}
-
 func (s Scanner) Type() detectorspb.DetectorType {
 	return detectorspb.DetectorType_MongoDB
 }

pkg/detectors/mongodb/mongodb_test.go

@@ -6,7 +6,6 @@ package mongodb
 import (
 	"context"
 	"fmt"
-	"strings"
 	"testing"
 	"time"
 
@@ -35,12 +34,11 @@ func TestMongoDB_FromChunk(t *testing.T) {
 		verify bool
 	}
 	tests := []struct {
-		name                string
-		s                   Scanner
-		args                args
-		want                []detectors.Result
-		wantErr             bool
-		wantVerificationErr bool
+		name    string
+		s       Scanner
+		args    args
+		want    []detectors.Result
+		wantErr bool
 	}{
 		{
 			name: "found, verified",
@@ -74,40 +72,6 @@ func TestMongoDB_FromChunk(t *testing.T) {
 			},
 			wantErr: false,
 		},
-		{
-			name: "found, would be verified but for connection timeout",
-			s:    Scanner{timeout: 1 * time.Microsecond},
-			args: args{
-				ctx:    context.Background(),
-				data:   []byte(fmt.Sprintf("You can find a mongodb secret %s within", secret)),
-				verify: true,
-			},
-			want: []detectors.Result{
-				{
-					DetectorType: detectorspb.DetectorType_MongoDB,
-					Verified:     false,
-				},
-			},
-			wantErr:             false,
-			wantVerificationErr: true,
-		},
-		{
-			name: "found, bad host",
-			s:    Scanner{},
-			args: args{
-				ctx:    context.Background(),
-				data:   []byte(fmt.Sprintf("You can find a mongodb secret %s within", strings.ReplaceAll(secret, ".mongodb.net", ".mongodb.net.bad"))),
-				verify: true,
-			},
-			want: []detectors.Result{
-				{
-					DetectorType: detectorspb.DetectorType_MongoDB,
-					Verified:     false,
-				},
-			},
-			wantErr:             false,
-			wantVerificationErr: true,
-		},
 		{
 			name: "not found",
 			s:    Scanner{},
@@ -122,7 +86,8 @@ func TestMongoDB_FromChunk(t *testing.T) {
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+			s := Scanner{}
+			got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
 			if (err != nil) != tt.wantErr {
 				t.Errorf("MongoDB.FromData() error = %v, wantErr %v", err, tt.wantErr)
 				return
@@ -132,11 +97,8 @@ func TestMongoDB_FromChunk(t *testing.T) {
 					t.Fatalf("no raw secret present: \n %+v", got[i])
 				}
 				got[i].Raw = nil
-				if (got[i].VerificationError != nil) != tt.wantVerificationErr {
-					t.Fatalf("wantVerificationErr = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError)
-				}
 			}
-			ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "RawV2", "VerificationError")
+			ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "RawV2")
 			if diff := cmp.Diff(tt.want, got, ignoreOpts); diff != "" {
 				t.Errorf("MongoDB.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
 			}

pkg/detectors/npmtoken/npmtoken.go

@@ -36,6 +36,7 @@ func (s Scanner) Keywords() []string {
 // FromData will find and optionally verify NpmToken secrets in a given set of bytes.
 func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
 	dataStr := string(data)
+
 	matches := keyPat.FindAllStringSubmatch(dataStr, -1)
 	for _, match := range matches {
 		if len(match) != 2 {

pkg/detectors/okta/okta.go

@@ -19,8 +19,8 @@ type Scanner struct{}
 var _ detectors.Detector = (*Scanner)(nil)
 
 var (
-	domainPat = regexp.MustCompile(`\b[a-z0-9-]{1,40}\.okta(?:preview|-emea){0,1}\.com\b`)
-	tokenPat  = regexp.MustCompile(`\b00[a-zA-Z0-9_-]{40}\b`)
+	domainPat = regexp.MustCompile(`[a-z0-9-]{1,40}\.okta(?:preview|-emea){0,1}\.com`)
+	tokenPat  = regexp.MustCompile(`00[a-zA-Z0-9_-]{40}`)
 	// TODO: Oauth client secrets
 )
 

pkg/detectors/pubnubpublishkey/pubnubpublishkey.go

@@ -21,7 +21,7 @@ var (
 
 	// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
 	pubPat = regexp.MustCompile(`\b(pub-c-[0-9a-z]{8}-[0-9a-z]{4}-[a-z0-9]{4}-[a-z0-9]{4}-[a-z0-9]{12})\b`)
-	subPat = regexp.MustCompile(`\b(sub-c-[0-9a-z]{8}-[a-z0-9]{4}-[a-z0-9]{4}-[a-z0-9]{4}-[a-z0-9]{12})\b`)
+	subPat = regexp.MustCompile(`\b(sub-c-[0-9a-z]{8}-[a-z]{4}-[a-z0-9]{4}-[a-z0-9]{4}-[a-z0-9]{12})\b`)
 )
 
 // Keywords are used for efficiently pre-filtering chunks.

pkg/detectors/pubnubsubscriptionkey/pubnubsubscriptionkey.go

@@ -20,7 +20,7 @@ var (
 	client = common.SaneHttpClient()
 
 	// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
-	keyPat = regexp.MustCompile(`\b(sub-c-[0-9a-z]{8}-[a-z0-9]{4}-[a-z0-9]{4}-[a-z0-9]{4}-[a-z0-9]{12})\b`)
+	keyPat = regexp.MustCompile(`\b(sub-c-[0-9a-z]{8}-[a-z]{4}-[a-z0-9]{4}-[a-z0-9]{4}-[a-z0-9]{12})\b`)
 )
 
 // Keywords are used for efficiently pre-filtering chunks.

pkg/detectors/scrapersite/scrapersite.go

@@ -7,7 +7,6 @@ import (
 	"net/http"
 	"regexp"
 	"strings"
-	"time"
 
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
@@ -20,7 +19,7 @@ type Scanner struct{}
 var _ detectors.Detector = (*Scanner)(nil)
 
 var (
-	client = common.SaneHttpClientTimeOut(10 * time.Second)
+	client = common.SaneHttpClientTimeOut(10)
 
 	// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
 	keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"scrapersite"}) + `\b([a-zA-Z0-9]{45})\b`)

pkg/detectors/screenshotlayer/screenshotlayer.go

@@ -7,7 +7,6 @@ import (
 	"net/http"
 	"regexp"
 	"strings"
-	"time"
 
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
@@ -20,7 +19,7 @@ type Scanner struct{}
 var _ detectors.Detector = (*Scanner)(nil)
 
 var (
-	client = common.SaneHttpClientTimeOut(10 * time.Second)
+	client = common.SaneHttpClientTimeOut(10)
 
 	// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
 	keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"screenshotlayer"}) + `\b([a-zA-Z0-9_]{32})\b`)

pkg/detectors/shodankey/shodankey.go

@@ -2,8 +2,6 @@ package shodankey
 
 import (
 	"context"
-	"encoding/json"
-	"io"
 	"net/http"
 	"regexp"
 	"strings"
@@ -50,58 +48,31 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 		}
 
 		if verify {
-			s1.Verified = verifyToken(ctx, client, resMatch)
-			if !s1.Verified && detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
+			req, err := http.NewRequestWithContext(ctx, "GET", "https://api.shodan.io/shodan/host/count?key="+resMatch+"&query=port:22&facets=org,os", nil)
+			if err != nil {
 				continue
 			}
+
+			res, err := client.Do(req)
+			if err == nil {
+				defer res.Body.Close()
+				if res.StatusCode >= 200 && res.StatusCode < 300 {
+					s1.Verified = true
+				} else {
+					// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
+					if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
+						continue
+					}
+				}
+			}
 		}
+
 		results = append(results, s1)
 	}
 
 	return results, nil
 }
 
-type shodanInfoRes struct {
-	ScanCredits int `json:"scan_credits"`
-	UsageLimits struct {
-		ScanCredits  int `json:"scan_credits"`
-		QueryCredits int `json:"query_credits"`
-		MonitoredIps int `json:"monitored_ips"`
-	} `json:"usage_limits"`
-	Plan         string `json:"plan"`
-	HTTPS        bool   `json:"https"`
-	Unlocked     bool   `json:"unlocked"`
-	QueryCredits int    `json:"query_credits"`
-	MonitoredIps int    `json:"monitored_ips"`
-	UnlockedLeft int    `json:"unlocked_left"`
-	Telnet       bool   `json:"telnet"`
-}
-
-func verifyToken(ctx context.Context, client *http.Client, token string) bool {
-	req, err := http.NewRequestWithContext(ctx, http.MethodGet, "https://api.shodan.io/api-info?key="+token, nil)
-	if err != nil {
-		return false
-	}
-
-	res, err := client.Do(req)
-	if err != nil {
-		return false
-	}
-	defer res.Body.Close()
-
-	if res.StatusCode < 200 || res.StatusCode >= 300 {
-		return false
-	}
-
-	bytes, err := io.ReadAll(res.Body)
-	if err != nil {
-		return false
-	}
-
-	var info shodanInfoRes
-	return json.Unmarshal(bytes, &info) == nil
-}
-
 func (s Scanner) Type() detectorspb.DetectorType {
 	return detectorspb.DetectorType_ShodanKey
 }

pkg/detectors/shodankey/shodankey_test.go

@@ -10,9 +10,9 @@ import (
 	"time"
 
 	"github.com/kylelemons/godebug/pretty"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
 
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
-	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
 )
 

pkg/detectors/trufflehogenterprise/trufflehogenterprise.go

@@ -1,108 +0,0 @@
-package trufflehogenterprise
-
-import (
-	"context"
-	"fmt"
-	"net/http"
-	"regexp"
-	"strings"
-
-	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
-	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
-	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
-)
-
-type Scanner struct{}
-
-// Ensure the Scanner satisfies the interface at compile time.
-var _ detectors.Detector = (*Scanner)(nil)
-
-var (
-	client = common.SaneHttpClient()
-
-	// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
-	keyPat      = regexp.MustCompile(`\bthog-key-[0-9a-f]{16}\b`)
-	secretPat   = regexp.MustCompile(`\bthog-secret-[0-9a-f]{32}\b`)
-	hostnamePat = regexp.MustCompile(`\b[a-z]+-[a-z]+-[a-z]+\.[a-z][0-9]\.[a-z]+\.trufflehog\.org\b`)
-)
-
-// Keywords are used for efficiently pre-filtering chunks.
-// Use identifiers in the secret preferably, or the provider name.
-func (s Scanner) Keywords() []string {
-	return []string{"thog"}
-}
-
-// FromData will find and optionally verify Trufflehog Enterprise secrets in a given set of bytes.
-func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
-	dataStr := string(data)
-
-	keyMatches := keyPat.FindAllStringSubmatch(dataStr, -1)
-	secretMatches := secretPat.FindAllStringSubmatch(dataStr, -1)
-	hostnameMatches := hostnamePat.FindAllStringSubmatch(dataStr, -1)
-
-	for _, keyMatch := range keyMatches {
-		if len(keyMatch) != 1 {
-			continue
-		}
-		resKeyMatch := strings.TrimSpace(keyMatch[0])
-		for _, secretMatch := range secretMatches {
-
-			if len(secretMatch) != 1 {
-				continue
-			}
-			resSecretMatch := strings.TrimSpace(secretMatch[0])
-
-			for _, hostnameMatch := range hostnameMatches {
-				if len(hostnameMatch) != 1 {
-					continue
-				}
-
-				resHostnameMatch := strings.TrimSpace(hostnameMatch[0])
-
-				s1 := detectors.Result{
-					DetectorType: detectorspb.DetectorType_TrufflehogEnterprise,
-					Raw:          []byte(resKeyMatch),
-				}
-
-				if verify {
-					endpoint := fmt.Sprintf("https://%s/api/v1/sources", resHostnameMatch)
-					req, err := http.NewRequestWithContext(ctx, "GET", endpoint, nil)
-					if err != nil {
-						continue
-					}
-
-					req.Header.Add("Accept", "application/vnd.trufflehogenterprise+json; version=3")
-					req.Header.Add("X-Thog-Secret", resSecretMatch)
-					req.Header.Add("X-Thog-Key", resKeyMatch)
-
-					res, err := client.Do(req)
-					if err == nil {
-						verifiedBodyResponse, err := common.ResponseContainsSubstring(res.Body, "data")
-						if err != nil {
-							return nil, err
-						}
-
-						defer res.Body.Close()
-
-						if res.StatusCode >= 200 && res.StatusCode < 300 && verifiedBodyResponse {
-							s1.Verified = true
-						} else {
-							// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
-							if detectors.IsKnownFalsePositive(resSecretMatch, detectors.DefaultFalsePositives, true) {
-								continue
-							}
-						}
-					}
-				}
-
-				results = append(results, s1)
-			}
-		}
-	}
-
-	return results, nil
-}
-
-func (s Scanner) Type() detectorspb.DetectorType {
-	return detectorspb.DetectorType_TrufflehogEnterprise
-}

pkg/detectors/trufflehogenterprise/trufflehogenterprise_test.go

@@ -1,121 +0,0 @@
-//go:build detectors
-// +build detectors
-
-package trufflehogenterprise
-
-import (
-	"context"
-	"fmt"
-	"testing"
-	"time"
-
-	"github.com/kylelemons/godebug/pretty"
-	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
-
-	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
-	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
-)
-
-func TestTrufflehogenterprise_FromChunk(t *testing.T) {
-	ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
-	defer cancel()
-	testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")
-	if err != nil {
-		t.Fatalf("could not get test secrets from GCP: %s", err)
-	}
-	host := testSecrets.MustGetField("THOG_HOSTNAME")
-	key := testSecrets.MustGetField("THOG_WEB_KEY")
-	secret := testSecrets.MustGetField("THOG_WEB_SECRET")
-	inactiveSecret := testSecrets.MustGetField("THOG_WEB_INACTIVE_SECRET")
-
-	type args struct {
-		ctx    context.Context
-		data   []byte
-		verify bool
-	}
-	tests := []struct {
-		name    string
-		s       Scanner
-		args    args
-		want    []detectors.Result
-		wantErr bool
-	}{
-		{
-			name: "found, verified",
-			s:    Scanner{},
-			args: args{
-				ctx:    context.Background(),
-				data:   []byte(fmt.Sprintf("You can find a thog secret %s for %s with key %s within", secret, host, key)),
-				verify: true,
-			},
-			want: []detectors.Result{
-				{
-					DetectorType: detectorspb.DetectorType_TrufflehogEnterprise,
-					Verified:     true,
-				},
-			},
-			wantErr: false,
-		},
-		{
-			name: "found, unverified",
-			s:    Scanner{},
-			args: args{
-				ctx:    context.Background(),
-				data:   []byte(fmt.Sprintf("You can find a trufflehogenterprise secret %s for %s with key %s within but not valid", inactiveSecret, host, key)), // the secret would satisfy the regex but not pass validation
-				verify: true,
-			},
-			want: []detectors.Result{
-				{
-					DetectorType: detectorspb.DetectorType_TrufflehogEnterprise,
-					Verified:     false,
-				},
-			},
-			wantErr: false,
-		},
-		{
-			name: "not found",
-			s:    Scanner{},
-			args: args{
-				ctx:    context.Background(),
-				data:   []byte("You cannot find the secret within"),
-				verify: true,
-			},
-			want:    nil,
-			wantErr: false,
-		},
-	}
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			s := Scanner{}
-			got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
-			if (err != nil) != tt.wantErr {
-				t.Errorf("Trufflehogenterprise.FromData() error = %v, wantErr %v", err, tt.wantErr)
-				return
-			}
-			for i := range got {
-				if len(got[i].Raw) == 0 {
-					t.Fatalf("no raw secret present: \n %+v", got[i])
-				}
-				got[i].Raw = nil
-			}
-			if diff := pretty.Compare(got, tt.want); diff != "" {
-				t.Errorf("Trufflehogenterprise.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
-			}
-		})
-	}
-}
-
-func BenchmarkFromData(benchmark *testing.B) {
-	ctx := context.Background()
-	s := Scanner{}
-	for name, data := range detectors.MustGetBenchmarkData() {
-		benchmark.Run(name, func(b *testing.B) {
-			for n := 0; n < b.N; n++ {
-				_, err := s.FromData(ctx, false, data)
-				if err != nil {
-					b.Fatal(err)
-				}
-			}
-		})
-	}
-}

pkg/detectors/twilio/twilio.go

@@ -3,7 +3,9 @@ package twilio
 import (
 	"context"
 	"net/http"
+	"net/url"
 	"regexp"
+	"strings"
 
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
@@ -51,8 +53,14 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 			}
 
 			if verify {
+
+				form := url.Values{}
+				form.Add("FriendlyName", "MyServiceName")
 				req, err := http.NewRequestWithContext(
-					ctx, "GET", "https://verify.twilio.com/v2/Services", nil)
+					ctx, "POST", "https://verify.twilio.com/v2/Services",
+
+					strings.NewReader(form.Encode()),
+				)
 				if err != nil {
 					continue
 				}

pkg/detectors/uri/uri.go

@@ -53,8 +53,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 		password := match[1]
 
 		// Skip findings where the password only has "*" characters, this is a redacted password
-		// Also include the url encoded "*" characters: "%2A"
-		if strings.Trim(password, "*") == "" || strings.Trim(password, "%2A") == "" {
+		if strings.Trim(password, "*") == "" {
 			continue
 		}
 

pkg/detectors/uri/uri_test.go

@@ -31,14 +31,14 @@ func TestURI_FromChunk(t *testing.T) {
 			s:    Scanner{},
 			args: args{
 				ctx:    context.Background(),
-				data:   []byte(fmt.Sprintf("You can find a uri secret %s within", "https://user:pass@httpwatch.com/httpgallery/authentication/authenticatedimage/default.aspx")),
+				data:   []byte(fmt.Sprintf("You can find a uri secret %s within", "https://user:pass@www.httpwatch.com/httpgallery/authentication/authenticatedimage/default.aspx")),
 				verify: true,
 			},
 			want: []detectors.Result{
 				{
 					DetectorType: detectorspb.DetectorType_URI,
 					Verified:     false,
-					Redacted:     "https://user:********@httpwatch.com",
+					Redacted:     "https://user:********@www.httpwatch.com",
 				},
 			},
 			wantErr: false,
@@ -87,17 +87,6 @@ func TestURI_FromChunk(t *testing.T) {
 			},
 			wantErr: false,
 		},
-		{
-			name: "nothing found, password was redacted two different ways",
-			s:    Scanner{},
-			args: args{
-				ctx:    context.Background(),
-				data:   []byte(fmt.Sprintf("Both %s and %s have been redacted within", "https://httpwatch::********@www.httpwatch.com/httpgallery/authentication/authenticatedimage/default.aspx?foo=bar", "https://httpwatch::%2A%2A%2A%2A%2A%2A@www.httpwatch.com/httpgallery/authentication/authenticatedimage/default.aspx?foo=bar")),
-				verify: true,
-			},
-			want:    []detectors.Result{},
-			wantErr: false,
-		},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {

pkg/detectors/virustotal/virustotal.go

@@ -1,7 +1,10 @@
 package virustotal
 
 import (
+	"bytes"
 	"context"
+	"io"
+	"mime/multipart"
 	"net/http"
 	"regexp"
 	"strings"
@@ -47,36 +50,43 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 		}
 
 		if verify {
-			s1.Verified = verifyToken(ctx, client, resMatch)
-			if !s1.Verified && detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
+			body := &bytes.Buffer{}
+			writer := multipart.NewWriter(body)
+			fw, err := writer.CreateFormField("url")
+			if err != nil {
 				continue
 			}
+			_, err = io.Copy(fw, strings.NewReader("https://www.amazon.com"))
+			if err != nil {
+				continue
+			}
+			writer.Close()
+			req, err := http.NewRequestWithContext(ctx, "POST", "https://www.virustotal.com/api/v3/urls", bytes.NewReader(body.Bytes()))
+			if err != nil {
+				continue
+			}
+			req.Header.Add("Content-Type", writer.FormDataContentType())
+			req.Header.Add("x-apikey", resMatch)
+			res, err := client.Do(req)
+			if err == nil {
+				defer res.Body.Close()
+				if res.StatusCode >= 200 && res.StatusCode < 300 {
+					s1.Verified = true
+				} else {
+					// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
+					if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
+						continue
+					}
+				}
+			}
 		}
+
 		results = append(results, s1)
 	}
 
 	return results, nil
 }
 
-func verifyToken(ctx context.Context, client *http.Client, token string) bool {
-	req, err := http.NewRequestWithContext(ctx, http.MethodGet, "https://www.virustotal.com/api/v3/metadata", nil)
-	if err != nil {
-		return false
-	}
-	req.Header.Add("x-apikey", token)
-
-	res, err := client.Do(req)
-	if err != nil {
-		return false
-	}
-	defer res.Body.Close()
-
-	if res.StatusCode < 200 || res.StatusCode >= 300 {
-		return false
-	}
-	return true
-}
-
 func (s Scanner) Type() detectorspb.DetectorType {
 	return detectorspb.DetectorType_VirusTotal
 }

pkg/detectors/virustotal/virustotal_test.go

@@ -10,9 +10,9 @@ import (
 	"time"
 
 	"github.com/kylelemons/godebug/pretty"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
 
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
-	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
 )
 

pkg/detectors/zenserp/zenserp.go

@@ -6,7 +6,6 @@ import (
 	"net/http"
 	"regexp"
 	"strings"
-	"time"
 
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
@@ -19,7 +18,7 @@ type Scanner struct{}
 var _ detectors.Detector = (*Scanner)(nil)
 
 var (
-	client = common.SaneHttpClientTimeOut(5 * time.Second)
+	client = common.SaneHttpClientTimeOut(5)
 
 	// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
 	keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"zenserp"}) + `\b([0-9a-z-]{36})\b`)

pkg/engine/defaults.go

@@ -1,8 +1,6 @@
 package engine
 
 import (
-	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/envoyapikey"
-
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/abbysale"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/abuseipdb"
@@ -161,7 +159,6 @@ import (
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/convertkit"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/convier"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/copper"
-	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/couchbase"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/countrylayer"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/courier"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/coveralls"
@@ -201,7 +198,6 @@ import (
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/disqus"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/ditto"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/dnscheck"
-	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/dockerhub"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/docparser"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/documo"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/docusign"
@@ -408,6 +404,7 @@ import (
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/metaapi"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/metrilo"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/microsoftteamswebhook"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/midise"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/mindmeister"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/miro"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/mite"
@@ -1387,6 +1384,7 @@ func DefaultDetectors() []detectors.Detector {
 		sendbirdorganizationapi.Scanner{},
 		chatfule.Scanner{},
 		convier.Scanner{},
+		midise.Scanner{},
 		loadmill.Scanner{},
 		magicbell.Scanner{},
 		glitterlyapi.Scanner{},
@@ -1527,9 +1525,6 @@ func DefaultDetectors() []detectors.Detector {
 		prefect.Scanner{},
 		buildkitev2.Scanner{},
 		opsgenie.Scanner{},
-		dockerhub.Scanner{},
-		couchbase.Scanner{},
-		envoyapikey.Scanner{},
 	}
 
 }

pkg/engine/engine.go

@@ -10,8 +10,9 @@ import (
 	"sync/atomic"
 	"time"
 
-	ahocorasick "github.com/BobuSumisu/aho-corasick"
-	lru "github.com/hashicorp/golang-lru"
+	"github.com/alecthomas/chroma"
+	"github.com/alecthomas/chroma/lexers"
+	ahocorasick "github.com/petar-dambovaliev/aho-corasick"
 	"golang.org/x/sync/errgroup"
 	"google.golang.org/protobuf/proto"
 
@@ -20,76 +21,40 @@ import (
 	"github.com/trufflesecurity/trufflehog/v3/pkg/context"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/decoders"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
-	"github.com/trufflesecurity/trufflehog/v3/pkg/output"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/source_metadatapb"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
+	DetectorsV2 "github.com/trufflesecurity/trufflehog/v3/pkg/v2Detectors"
 )
 
-// Metrics for the scan engine for external consumption.
-type Metrics struct {
-	BytesScanned           uint64
-	ChunksScanned          uint64
-	VerifiedSecretsFound   uint64
-	UnverifiedSecretsFound uint64
-	AvgDetectorTime        map[string]time.Duration
-}
-
-// runtimeMetrics for the scan engine for internal use by the engine.
-type runtimeMetrics struct {
-	mu sync.RWMutex
-	Metrics
-	detectorAvgTime sync.Map
-}
-
-// Printer is used to format found results and output them to the user. Ex JSON, plain text, etc.
-type Printer interface {
-	Print(ctx context.Context, r *detectors.ResultWithMetadata) error
-}
-
 type Engine struct {
-	concurrency uint8
-	chunks      chan *sources.Chunk
-	results     chan detectors.ResultWithMetadata
-	decoders    []decoders.Decoder
-	detectors   map[bool][]detectors.Detector
-	sourcesWg   *errgroup.Group
-	workersWg   sync.WaitGroup
+	concurrency     int
+	chunks          chan *sources.Chunk
+	results         chan detectors.ResultWithMetadata
+	decoders        []decoders.Decoder
+	detectors       map[bool][]detectors.Detector
+	chunksScanned   uint64
+	bytesScanned    uint64
+	detectorAvgTime sync.Map
+	sourcesWg       *errgroup.Group
+	workersWg       sync.WaitGroup
 	// filterUnverified is used to reduce the number of unverified results.
 	// If there are multiple unverified results for the same chunk for the same detector,
 	// only the first one will be kept.
-	filterUnverified     bool
-	onlyVerified         bool
-	printAvgDetectorTime bool
+	filterUnverified bool
 
 	// prefilter is a ahocorasick struct used for doing efficient string
 	// matching given a set of words (keywords from the rules in the config)
-	prefilter ahocorasick.Trie
+	prefilter ahocorasick.AhoCorasick
 
-	detectableChunksChan chan detectableChunk
-	wgDetectorWorkers    sync.WaitGroup
-	WgNotifier           sync.WaitGroup
-
-	// Runtime metrics.
-	metrics runtimeMetrics
-
-	// numFoundResults is used to keep track of the number of results found.
-	numFoundResults uint32
-
-	// printer provides a method for formatting and outputting search results.
-	// The specific implementation (e.g., JSON, plain text)
-	// should be set during initialization based on user preference or program requirements.
-	printer Printer
-
-	// dedupeCache is used to deduplicate results by comparing the
-	// detector type, raw result, and source metadata
-	dedupeCache *lru.Cache
+	// v2 detection engine map, key is the length of characters in the secret capture group
+	v2Detector DetectorsV2.DetectorBuckets
 }
 
 type EngineOption func(*Engine)
 
-func WithConcurrency(concurrency uint8) EngineOption {
+func WithConcurrency(concurrency int) EngineOption {
 	return func(e *Engine) {
 		e.concurrency = concurrency
 	}
@@ -124,26 +89,6 @@ func WithFilterUnverified(filter bool) EngineOption {
 	}
 }
 
-// WithOnlyVerified sets the onlyVerified flag on the engine. If set to true,
-// the engine will only print verified results.
-func WithOnlyVerified(onlyVerified bool) EngineOption {
-	return func(e *Engine) {
-		e.onlyVerified = onlyVerified
-	}
-}
-
-// WithPrintAvgDetectorTime sets the printAvgDetectorTime flag on the engine. If set to
-// true, the engine will print the average time taken by each detector.
-// This option allows us to measure the time taken for each detector ONLY if
-// the engine is configured to print the results.
-// Calculating the average time taken by each detector is an expensive operation
-// and should be avoided unless specified by the user.
-func WithPrintAvgDetectorTime(printAvgDetectorTime bool) EngineOption {
-	return func(e *Engine) {
-		e.printAvgDetectorTime = printAvgDetectorTime
-	}
-}
-
 // WithFilterDetectors applies a filter to the configured list of detectors. If
 // the filterFunc returns true, the detector will be included for scanning.
 // This option applies to the existing list of detectors configured, so the
@@ -159,13 +104,6 @@ func WithFilterDetectors(filterFunc func(detectors.Detector) bool) EngineOption
 	}
 }
 
-// WithPrinter sets the Printer on the engine.
-func WithPrinter(printer Printer) EngineOption {
-	return func(e *Engine) {
-		e.printer = printer
-	}
-}
-
 func filterDetectors(filterFunc func(detectors.Detector) bool, input []detectors.Detector) []detectors.Detector {
 	var output []detectors.Detector
 	for _, detector := range input {
@@ -176,92 +114,12 @@ func filterDetectors(filterFunc func(detectors.Detector) bool, input []detectors
 	return output
 }
 
-// HasFoundResults returns true if any results are found.
-func (e *Engine) HasFoundResults() bool {
-	return atomic.LoadUint32(&e.numFoundResults) > 0
-}
-
-// GetMetrics returns a copy of Metrics.
-// It's safe for concurrent use, and the caller can't modify the original data.
-func (e *Engine) GetMetrics() Metrics {
-	e.metrics.mu.RLock()
-	defer e.metrics.mu.RUnlock()
-
-	result := e.metrics.Metrics
-	result.AvgDetectorTime = make(map[string]time.Duration, len(e.metrics.AvgDetectorTime))
-
-	for detectorName, durations := range e.DetectorAvgTime() {
-		var total time.Duration
-		for _, d := range durations {
-			total += d
-		}
-		avgDuration := total / time.Duration(len(durations))
-		result.AvgDetectorTime[detectorName] = avgDuration
-	}
-
-	return result
-}
-
-// GetDetectorsMetrics returns a copy of the average time taken by each detector.
-func (e *Engine) GetDetectorsMetrics() map[string]time.Duration {
-	e.metrics.mu.RLock()
-	defer e.metrics.mu.RUnlock()
-
-	result := make(map[string]time.Duration, len(DefaultDetectors()))
-	for detectorName, durations := range e.DetectorAvgTime() {
-		var total time.Duration
-		for _, d := range durations {
-			total += d
-		}
-		avgDuration := total / time.Duration(len(durations))
-		result[detectorName] = avgDuration
-	}
-
-	return result
-}
-
-// DetectorAvgTime returns the average time taken by each detector.
-func (e *Engine) DetectorAvgTime() map[string][]time.Duration {
-	logger := context.Background().Logger()
-	avgTime := map[string][]time.Duration{}
-	e.metrics.detectorAvgTime.Range(func(k, v interface{}) bool {
-		key, ok := k.(string)
-		if !ok {
-			logger.Info("expected detectorAvgTime key to be a string")
-			return true
-		}
-
-		value, ok := v.([]time.Duration)
-		if !ok {
-			logger.Info("expected detectorAvgTime value to be []time.Duration")
-			return true
-		}
-		avgTime[key] = value
-		return true
-	})
-	return avgTime
-}
-
-// Start the engine with options.
-func Start(ctx context.Context, options ...EngineOption) (*Engine, error) {
-	const (
-		defaultChannelBuffer = 1
-		// TODO (ahrav): Determine the optimal cache size.
-		cacheSize = 512 // number of entries in the LRU cache
-	)
-
-	cache, err := lru.New(cacheSize)
-	if err != nil {
-		return nil, fmt.Errorf("failed to initialize LRU cache: %w", err)
-	}
-
+func Start(ctx context.Context, options ...EngineOption) *Engine {
 	e := &Engine{
-		chunks:               make(chan *sources.Chunk, defaultChannelBuffer),
-		detectableChunksChan: make(chan detectableChunk, defaultChannelBuffer),
-		results:              make(chan detectors.ResultWithMetadata, defaultChannelBuffer),
-		sourcesWg:            &errgroup.Group{},
-		dedupeCache:          cache,
-		printer:              new(output.PlainPrinter), // default printer
+		chunks:          make(chan *sources.Chunk),
+		results:         make(chan detectors.ResultWithMetadata),
+		detectorAvgTime: sync.Map{},
+		v2Detector:      *DetectorsV2.NewDetectorBuckets(),
 	}
 
 	for _, option := range options {
@@ -272,12 +130,14 @@ func Start(ctx context.Context, options ...EngineOption) (*Engine, error) {
 	if e.concurrency == 0 {
 		numCPU := runtime.NumCPU()
 		ctx.Logger().Info("No concurrency specified, defaulting to max", "cpu", numCPU)
-		e.concurrency = uint8(numCPU)
+		e.concurrency = numCPU
 	}
-	ctx.Logger().V(3).Info("engine started", "workers", e.concurrency)
+	ctx.Logger().V(2).Info("engine started", "workers", e.concurrency)
 
-	// Limit number of concurrent goroutines dedicated to chunking a source.
-	e.sourcesWg.SetLimit(int(e.concurrency))
+	sourcesWg, egCtx := errgroup.WithContext(ctx)
+	sourcesWg.SetLimit(e.concurrency)
+	e.sourcesWg = sourcesWg
+	ctx.SetParent(egCtx)
 
 	if len(e.decoders) == 0 {
 		e.decoders = decoders.DefaultDecoders()
@@ -293,19 +153,21 @@ func Start(ctx context.Context, options ...EngineOption) (*Engine, error) {
 	// on keywords
 	keywords := []string{}
 	for _, d := range e.detectors[false] {
-		for _, kw := range d.Keywords() {
-			keywords = append(keywords, strings.ToLower(kw))
-		}
+		keywords = append(keywords, d.Keywords()...)
 	}
 	for _, d := range e.detectors[true] {
-		for _, kw := range d.Keywords() {
-			keywords = append(keywords, strings.ToLower(kw))
-		}
+		keywords = append(keywords, d.Keywords()...)
 	}
-	e.prefilter = *ahocorasick.NewTrieBuilder().AddStrings(keywords).Build()
+	builder := ahocorasick.NewAhoCorasickBuilder(ahocorasick.Opts{
+		AsciiCaseInsensitive: true,
+		MatchOnlyWholeWords:  false,
+		MatchKind:            ahocorasick.LeftMostLongestMatch,
+		DFA:                  true,
+	})
+	e.prefilter = builder.Build(keywords)
 
-	ctx.Logger().V(3).Info("loaded decoders", "count", len(e.decoders))
-	ctx.Logger().V(3).Info("loaded detectors",
+	ctx.Logger().Info("loaded decoders", "count", len(e.decoders))
+	ctx.Logger().Info("loaded detectors",
 		"total", len(e.detectors[true])+len(e.detectors[false]),
 		"verification_enabled", len(e.detectors[true]),
 		"verification_disabled", len(e.detectors[false]),
@@ -326,68 +188,41 @@ func Start(ctx context.Context, options ...EngineOption) (*Engine, error) {
 		}
 	}
 
-	ctx.Logger().V(2).Info("starting scanner workers", "count", e.concurrency)
-	// Run the Secret scanner workers and Notifier pipelines.
-	for worker := uint64(0); worker < uint64(e.concurrency); worker++ {
+	// Start the workers.
+	for i := 0; i < e.concurrency; i++ {
 		e.workersWg.Add(1)
 		go func() {
-			ctx := context.WithValue(ctx, "secret_worker_id", common.RandomID(5))
-			defer common.Recover(ctx)
+			defer common.RecoverWithExit(ctx)
 			defer e.workersWg.Done()
 			e.detectorWorker(ctx)
 		}()
 	}
 
-	const detectorWorkerMultiplier = 50
-	ctx.Logger().V(2).Info("starting detector workers", "count", e.concurrency*detectorWorkerMultiplier)
-	for worker := uint64(0); worker < uint64(e.concurrency*detectorWorkerMultiplier); worker++ {
-		e.wgDetectorWorkers.Add(1)
-		go func() {
-			ctx := context.WithValue(ctx, "detector_worker_id", common.RandomID(5))
-			defer common.Recover(ctx)
-			defer e.wgDetectorWorkers.Done()
-			e.detectChunks(ctx)
-		}()
-	}
-
-	// We want 1/4th of the notifier workers as the number of scanner workers.
-	const notifierWorkerRatio = 4
-	maxNotifierWorkers := 1
-	if numWorkers := e.concurrency / notifierWorkerRatio; numWorkers > 0 {
-		maxNotifierWorkers = int(numWorkers)
-	}
-	ctx.Logger().V(2).Info("starting notifier workers", "count", maxNotifierWorkers)
-	for worker := 0; worker < maxNotifierWorkers; worker++ {
-		e.WgNotifier.Add(1)
-		go func() {
-			ctx := context.WithValue(ctx, "notifier_worker_id", common.RandomID(5))
-			defer common.Recover(ctx)
-			defer e.WgNotifier.Done()
-			e.notifyResults(ctx)
-		}()
-	}
-
-	return e, nil
+	return e
 }
 
 // Finish waits for running sources to complete and workers to finish scanning
 // chunks before closing their respective channels. Once Finish is called, no
 // more sources may be scanned by the engine.
-func (e *Engine) Finish(ctx context.Context) error {
+func (e *Engine) Finish(ctx context.Context, logFunc func(error, string, ...any)) {
 	defer common.RecoverWithExit(ctx)
-	// Wait for the sources to finish putting chunks onto the chunks channel.
-	err := e.sourcesWg.Wait()
+	// wait for the sources to finish putting chunks onto the chunks channel
+	sourceErr := e.sourcesWg.Wait()
+	if sourceErr != nil {
+		logFunc(sourceErr, "error occurred while collecting chunks")
+	}
 
-	close(e.chunks) // Source workers are done.
+	close(e.chunks)
+	// wait for the workers to finish processing all of the chunks and putting
+	// results onto the results channel
+	e.workersWg.Wait()
 
-	e.workersWg.Wait() // Wait for the workers to finish scanning chunks.
-	close(e.detectableChunksChan)
-	e.wgDetectorWorkers.Wait() // Wait for the detector workers to finish detecting chunks.
-
-	close(e.results)    // Detector workers are done, close the results channel and call it a day.
-	e.WgNotifier.Wait() // Wait for the notifier workers to finish notifying results.
-
-	return err
+	// TODO: re-evaluate whether this is needed and investigate why if so
+	//
+	// not entirely sure why results don't get processed without this pause
+	// since we've put all results on the channel at this point.
+	time.Sleep(time.Second)
+	close(e.results)
 }
 
 func (e *Engine) ChunksChan() chan *sources.Chunk {
@@ -398,21 +233,40 @@ func (e *Engine) ResultsChan() chan detectors.ResultWithMetadata {
 	return e.results
 }
 
-// detectableChunk is a decoded chunk that is ready to be scanned by its detector.
-type detectableChunk struct {
-	detector detectors.Detector
-	chunk    sources.Chunk
-	decoder  detectorspb.DecoderType
-	wgDoneFn func()
+func (e *Engine) ChunksScanned() uint64 {
+	return e.chunksScanned
+}
+
+func (e *Engine) BytesScanned() uint64 {
+	return e.bytesScanned
+}
+
+func (e *Engine) DetectorAvgTime() map[string][]time.Duration {
+	logger := context.Background().Logger()
+	avgTime := map[string][]time.Duration{}
+	e.detectorAvgTime.Range(func(k, v interface{}) bool {
+		key, ok := k.(string)
+		if !ok {
+			logger.Info("expected DetectorAvgTime key to be a string")
+			return true
+		}
+
+		value, ok := v.([]time.Duration)
+		if !ok {
+			logger.Info("expected DetectorAvgTime value to be []time.Duration")
+			return true
+		}
+		avgTime[key] = value
+		return true
+	})
+	return avgTime
 }
 
 func (e *Engine) detectorWorker(ctx context.Context) {
-	var wgDetect sync.WaitGroup
-
 	for originalChunk := range e.chunks {
 		for chunk := range sources.Chunker(originalChunk) {
 			matchedKeywords := make(map[string]struct{})
-			atomic.AddUint64(&e.metrics.BytesScanned, uint64(len(chunk.Data)))
+			atomic.AddUint64(&e.bytesScanned, uint64(len(chunk.Data)))
 			for _, decoder := range e.decoders {
 				var decoderType detectorspb.DecoderType
 				switch decoder.(type) {
@@ -426,16 +280,37 @@ func (e *Engine) detectorWorker(ctx context.Context) {
 					ctx.Logger().Info("unknown decoder type", "type", reflect.TypeOf(decoder).String())
 					decoderType = detectorspb.DecoderType_UNKNOWN
 				}
-
 				decoded := decoder.FromChunk(chunk)
-
 				if decoded == nil {
 					continue
 				}
 
+				// use chroma to find all strings and comments
+				l := lexers.Analyse(string(chunk.Data))
+				if l != nil {
+					iterator, err := l.Tokenise(nil, string(chunk.Data))
+					if err != nil {
+						fmt.Println("error: ", err)
+					}
+					for _, token := range iterator.Tokens() {
+						if token.Type == chroma.LiteralString || token.Type == chroma.Comment {
+							for _, v2D := range e.v2Detector.GetDetectors(len(token.Value)) {
+								for _, v2 := range v2D.Credentials {
+									if len(token.Value) >= v2.CharacterRange.Min {
+										if v2.Regex.MatchString(token.Value) {
+											fmt.Println("cred: ", v2D.Name, "regex:", v2.Regex.String(), "string: ", token.Value)
+										}
+									}
+								}
+							}
+						}
+					}
+				}
+				continue
+
 				// build a map of all keywords that were matched in the chunk
-				for _, m := range e.prefilter.MatchString(strings.ToLower(string(decoded.Data))) {
-					matchedKeywords[strings.ToLower(m.MatchString())] = struct{}{}
+				for _, m := range e.prefilter.FindAll(string(decoded.Data)) {
+					matchedKeywords[strings.ToLower(string(decoded.Data[m.Start():m.End()]))] = struct{}{}
 				}
 
 				for verify, detectorsSet := range e.detectors {
@@ -452,130 +327,92 @@ func (e *Engine) detectorWorker(ctx context.Context) {
 							continue
 						}
 
-						decoded.Verify = verify
-						wgDetect.Add(1)
-						e.detectableChunksChan <- detectableChunk{
-							chunk:    *decoded,
-							detector: detector,
-							decoder:  decoderType,
-							wgDoneFn: wgDetect.Done,
+						start := time.Now()
+
+						results, err := func() ([]detectors.Result, error) {
+							ctx, cancel := context.WithTimeout(ctx, time.Second*10)
+							defer cancel()
+							defer common.Recover(ctx)
+							return detector.FromData(ctx, verify, decoded.Data)
+						}()
+						if err != nil {
+							ctx.Logger().Error(err, "could not scan chunk",
+								"source_type", decoded.SourceType.String(),
+								"metadata", decoded.SourceMetadata,
+							)
+							continue
+						}
+
+						if e.filterUnverified {
+							results = detectors.CleanResults(results)
+						}
+						for _, result := range results {
+							resultChunk := chunk
+							ignoreLinePresent := false
+							if SupportsLineNumbers(chunk.SourceType) {
+								copyChunk := *chunk
+								copyMetaDataClone := proto.Clone(chunk.SourceMetadata)
+								if copyMetaData, ok := copyMetaDataClone.(*source_metadatapb.MetaData); ok {
+									copyChunk.SourceMetadata = copyMetaData
+								}
+								fragStart, mdLine := FragmentFirstLine(&copyChunk)
+								ignoreLinePresent = SetResultLineNumber(&copyChunk, &result, fragStart, mdLine)
+								resultChunk = &copyChunk
+							}
+							if ignoreLinePresent {
+								continue
+							}
+							result.DecoderType = decoderType
+							e.results <- detectors.CopyMetadata(resultChunk, result)
+
+						}
+						if len(results) > 0 {
+							elapsed := time.Since(start)
+							detectorName := results[0].DetectorType.String()
+							avgTimeI, ok := e.detectorAvgTime.Load(detectorName)
+							var avgTime []time.Duration
+							if ok {
+								avgTime, ok = avgTimeI.([]time.Duration)
+								if !ok {
+									continue
+								}
+							}
+							avgTime = append(avgTime, elapsed)
+							e.detectorAvgTime.Store(detectorName, avgTime)
 						}
 					}
 				}
 			}
 		}
-		atomic.AddUint64(&e.metrics.ChunksScanned, 1)
-	}
-	wgDetect.Wait()
-}
-
-func (e *Engine) detectChunks(ctx context.Context) {
-	for data := range e.detectableChunksChan {
-		e.detectChunk(ctx, data)
+		atomic.AddUint64(&e.chunksScanned, 1)
 	}
 }
 
-func (e *Engine) detectChunk(ctx context.Context, data detectableChunk) {
-	var start time.Time
-	if e.printAvgDetectorTime {
-		start = time.Now()
-	}
-	ctx, cancel := context.WithTimeout(ctx, time.Second*10)
-	defer common.Recover(ctx)
-	defer cancel()
-
-	results, err := data.detector.FromData(ctx, data.chunk.Verify, data.chunk.Data)
-	if err != nil {
-		ctx.Logger().Error(err, "error scanning chunk")
-	}
-	if e.printAvgDetectorTime && len(results) > 0 {
-		elapsed := time.Since(start)
-		detectorName := results[0].DetectorType.String()
-		avgTimeI, ok := e.metrics.detectorAvgTime.Load(detectorName)
-		var avgTime []time.Duration
-		if ok {
-			avgTime, ok = avgTimeI.([]time.Duration)
-			if !ok {
-				return
-			}
-		}
-		avgTime = append(avgTime, elapsed)
-		e.metrics.detectorAvgTime.Store(detectorName, avgTime)
-	}
-
-	if e.filterUnverified {
-		results = detectors.CleanResults(results)
-	}
-
-	for _, res := range results {
-		e.processResult(data, res)
-	}
-	data.wgDoneFn()
-}
-
-func (e *Engine) processResult(data detectableChunk, res detectors.Result) {
-	ignoreLinePresent := false
-	if SupportsLineNumbers(data.chunk.SourceType) {
-		copyChunk := data.chunk
-		copyMetaDataClone := proto.Clone(data.chunk.SourceMetadata)
-		if copyMetaData, ok := copyMetaDataClone.(*source_metadatapb.MetaData); ok {
-			copyChunk.SourceMetadata = copyMetaData
-		}
-		fragStart, mdLine := FragmentFirstLine(&copyChunk)
-		ignoreLinePresent = SetResultLineNumber(&copyChunk, &res, fragStart, mdLine)
-		data.chunk = copyChunk
-	}
-	if ignoreLinePresent {
-		return
-	}
-
-	secret := detectors.CopyMetadata(&data.chunk, res)
-	secret.DecoderType = data.decoder
-	e.results <- secret
-}
-
-func (e *Engine) notifyResults(ctx context.Context) {
-	for r := range e.ResultsChan() {
-		if e.onlyVerified && !r.Verified {
-			continue
-		}
-		atomic.AddUint32(&e.numFoundResults, 1)
-
-		key := fmt.Sprintf("%s%s%s%+v", r.DetectorType.String(), r.Raw, r.RawV2, r.SourceMetadata)
-		if _, ok := e.dedupeCache.Get(key); ok {
-			continue
-		}
-		e.dedupeCache.Add(key, struct{}{})
-
-		if r.Verified {
-			atomic.AddUint64(&e.metrics.VerifiedSecretsFound, 1)
-		} else {
-			atomic.AddUint64(&e.metrics.UnverifiedSecretsFound, 1)
-		}
-
-		if err := e.printer.Print(ctx, &r); err != nil {
-			ctx.Logger().Error(err, "error printing result")
-		}
-	}
-}
-
-// SupportsLineNumbers determines if a line number can be found for a source type.
-func SupportsLineNumbers(sourceType sourcespb.SourceType) bool {
-	switch sourceType {
-	case sourcespb.SourceType_SOURCE_TYPE_GIT,
+// lineNumberSupportedSources is a list of sources that support line numbers.
+// It is stored this way because slice consts are not supported.
+func lineNumberSupportedSources() []sourcespb.SourceType {
+	return []sourcespb.SourceType{
+		sourcespb.SourceType_SOURCE_TYPE_GIT,
 		sourcespb.SourceType_SOURCE_TYPE_GITHUB,
 		sourcespb.SourceType_SOURCE_TYPE_GITLAB,
 		sourcespb.SourceType_SOURCE_TYPE_BITBUCKET,
 		sourcespb.SourceType_SOURCE_TYPE_GERRIT,
 		sourcespb.SourceType_SOURCE_TYPE_GITHUB_UNAUTHENTICATED_ORG,
 		sourcespb.SourceType_SOURCE_TYPE_PUBLIC_GIT,
-		sourcespb.SourceType_SOURCE_TYPE_FILESYSTEM:
-		return true
-	default:
-		return false
+		sourcespb.SourceType_SOURCE_TYPE_FILESYSTEM,
 	}
 }
 
+// SupportsLineNumbers determines if a line number can be found for a source type.
+func SupportsLineNumbers(sourceType sourcespb.SourceType) bool {
+	for _, i := range lineNumberSupportedSources() {
+		if i == sourceType {
+			return true
+		}
+	}
+	return false
+}
+
 // FragmentLineOffset sets the line number for a provided source chunk with a given detector result.
 func FragmentLineOffset(chunk *sources.Chunk, result *detectors.Result) (int64, bool) {
 	lines := bytes.Split(chunk.Data, []byte("\n"))

pkg/engine/engine_test.go

@@ -3,9 +3,7 @@ package engine
 import (
 	"testing"
 
-	"github.com/trufflesecurity/trufflehog/v3/pkg/decoders"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
-	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
 )
 
@@ -64,39 +62,3 @@ func TestFragmentLineOffset(t *testing.T) {
 		})
 	}
 }
-
-// Test to make sure that DefaultDecoders always returns the UTF8 decoder first.
-// Technically a decoder test but we want this to run and fail in CI
-func TestDefaultDecoders(t *testing.T) {
-	ds := decoders.DefaultDecoders()
-	if _, ok := ds[0].(*decoders.UTF8); !ok {
-		t.Errorf("DefaultDecoders() = %v, expected UTF8 decoder to be first", ds)
-	}
-}
-
-func TestSupportsLineNumbers(t *testing.T) {
-	testCases := []struct {
-		name     string
-		input    sourcespb.SourceType
-		expected bool
-	}{
-		{"Git source type", sourcespb.SourceType_SOURCE_TYPE_GIT, true},
-		{"Github source type", sourcespb.SourceType_SOURCE_TYPE_GITHUB, true},
-		{"Gitlab source type", sourcespb.SourceType_SOURCE_TYPE_GITLAB, true},
-	}
-
-	for _, tc := range testCases {
-		t.Run(tc.name, func(t *testing.T) {
-			if result := SupportsLineNumbers(tc.input); result != tc.expected {
-				t.Errorf("Expected %v for input %v, got %v", tc.expected, tc.input, result)
-			}
-		})
-	}
-}
-
-func BenchmarkSupportsLineNumbersLoop(b *testing.B) {
-	sourceType := sourcespb.SourceType_SOURCE_TYPE_GITHUB
-	for i := 0; i < b.N; i++ {
-		_ = SupportsLineNumbers(sourceType)
-	}
-}

pkg/engine/filesystem.go

@@ -28,22 +28,6 @@ func (e *Engine) ScanFileSystem(ctx context.Context, c sources.FilesystemConfig)
 	}
 
 	fileSystemSource := filesystem.Source{}
-	fileSystemSource.WithFilter(c.Filter)
-
-	// TODO: This is how it would be used; everything after these comments would be removed.
-	// handle, err := e.sourceManager.Enroll(ctx, "trufflehog - filesystem", fileSystemSource.Type(),
-	// 	func(ctx context.Context, jobID, sourceID int64) (sources.Source, error) {
-	// 		fileSystemSource := filesystem.Source{}
-	// 		fileSystemSource.WithFilter(c.Filter)
-	// 		if err := fileSystemSource.Init(ctx, "trufflehog - filesystem", jobID, sourceID, true, &conn, runtime.NumCPU()); err != nil {
-	// 			return nil, err
-	// 		}
-	// 		return &fileSystemSource, nil
-	// 	})
-	// if err != nil {
-	// 	return err
-	// }
-	// return e.sourceManager.ScheduleRun(ctx, handle)
 
 	ctx = context.WithValues(ctx,
 		"source_type", fileSystemSource.Type().String(),
@@ -53,6 +37,7 @@ func (e *Engine) ScanFileSystem(ctx context.Context, c sources.FilesystemConfig)
 	if err != nil {
 		return errors.WrapPrefix(err, "could not init filesystem source", 0)
 	}
+	fileSystemSource.WithFilter(c.Filter)
 	e.sourcesWg.Go(func() error {
 		defer common.RecoverWithExit(ctx)
 		err := fileSystemSource.Chunks(ctx, e.ChunksChan())

pkg/engine/gcs_test.go

@@ -4,8 +4,6 @@ import (
 	"strings"
 	"testing"
 
-	"github.com/stretchr/testify/assert"
-
 	"github.com/trufflesecurity/trufflehog/v3/pkg/context"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/decoders"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
@@ -61,13 +59,11 @@ func TestScanGCS(t *testing.T) {
 			ctx, cancel := context.WithCancel(context.TODO())
 			defer cancel()
 
-			e, err := Start(ctx,
+			e := Start(ctx,
 				WithConcurrency(1),
 				WithDecoders(decoders.DefaultDecoders()...),
 				WithDetectors(false, DefaultDetectors()...),
 			)
-			assert.Nil(t, err)
-
 			go func() {
 				resultCount := 0
 				for range e.ResultsChan() {
@@ -75,12 +71,15 @@ func TestScanGCS(t *testing.T) {
 				}
 			}()
 
-			err = e.ScanGCS(ctx, test.gcsConfig)
+			err := e.ScanGCS(ctx, test.gcsConfig)
 			if err != nil && !test.wantErr && !strings.Contains(err.Error(), "googleapi: Error 400: Bad Request") {
 				t.Errorf("ScanGCS() got: %v, want: %v", err, nil)
 				return
 			}
-			assert.Nil(t, e.Finish(ctx))
+			logFatalFunc := func(_ error, _ string, _ ...any) {
+				t.Fatalf("error logging function should not have been called")
+			}
+			e.Finish(ctx, logFatalFunc)
 
 			if err == nil && test.wantErr {
 				t.Errorf("ScanGCS() got: %v, want: %v", err, "error")

pkg/engine/git.go

@@ -4,13 +4,11 @@ import (
 	"fmt"
 	"runtime"
 
-	"github.com/go-errors/errors"
 	gogit "github.com/go-git/go-git/v5"
-	"google.golang.org/protobuf/proto"
-	"google.golang.org/protobuf/types/known/anypb"
 
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/context"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/source_metadatapb"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/sources/git"
@@ -24,6 +22,16 @@ func (e *Engine) ScanGit(ctx context.Context, c sources.GitConfig) error {
 		git.ScanOptionLogOptions(logOptions),
 	}
 
+	options := &gogit.PlainOpenOptions{
+		DetectDotGit:          true,
+		EnableDotGitCommonDir: true,
+	}
+
+	repo, err := gogit.PlainOpenWithOptions(c.RepoPath, options)
+	if err != nil {
+		return fmt.Errorf("could not open repo: %s: %w", c.RepoPath, err)
+	}
+
 	if c.MaxDepth != 0 {
 		opts = append(opts, git.ScanOptionMaxDepth(int64(c.MaxDepth)))
 	}
@@ -38,28 +46,21 @@ func (e *Engine) ScanGit(ctx context.Context, c sources.GitConfig) error {
 	}
 	scanOptions := git.NewScanOptions(opts...)
 
-	connection := &sourcespb.Git{
-		// Using Directories here allows us to not pass any
-		// authentication. Also by this point, the c.RepoPath should
-		// still have been prepared and downloaded to a temporary
-		// directory if it was a URL.
-		Directories: []string{c.RepoPath},
-	}
-	var conn anypb.Any
-	err := anypb.MarshalFrom(&conn, connection, proto.MarshalOptions{})
-	if err != nil {
-		ctx.Logger().Error(err, "failed to marshal git connection")
-		return err
-	}
-
-	gitSource := git.Source{}
-	if err := gitSource.Init(ctx, "trufflehog - git", 0, 0, true, &conn, runtime.NumCPU()); err != nil {
-		return errors.WrapPrefix(err, "could not init git source", 0)
-	}
-	gitSource.WithScanOptions(scanOptions)
-	// Don't try to clean up the provided directory. That's handled by the
-	// caller of ScanGit.
-	gitSource.WithPreserveTempDirs(true)
+	gitSource := git.NewGit(sourcespb.SourceType_SOURCE_TYPE_GIT, 0, 0, "trufflehog - git", true, runtime.NumCPU(),
+		func(file, email, commit, timestamp, repository string, line int64) *source_metadatapb.MetaData {
+			return &source_metadatapb.MetaData{
+				Data: &source_metadatapb.MetaData_Git{
+					Git: &source_metadatapb.Git{
+						Commit:     commit,
+						File:       file,
+						Email:      email,
+						Repository: repository,
+						Timestamp:  timestamp,
+						Line:       line,
+					},
+				},
+			}
+		})
 
 	ctx = context.WithValues(ctx,
 		"source_type", sourcespb.SourceType_SOURCE_TYPE_GIT.String(),
@@ -67,7 +68,7 @@ func (e *Engine) ScanGit(ctx context.Context, c sources.GitConfig) error {
 	)
 	e.sourcesWg.Go(func() error {
 		defer common.RecoverWithExit(ctx)
-		err := gitSource.Chunks(ctx, e.ChunksChan())
+		err := gitSource.ScanRepo(ctx, repo, c.RepoPath, scanOptions, e.ChunksChan())
 		if err != nil {
 			return fmt.Errorf("could not scan repo: %w", err)
 		}

pkg/engine/git_test.go

@@ -4,12 +4,9 @@ import (
 	"os"
 	"testing"
 
-	"github.com/stretchr/testify/assert"
-
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/context"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/decoders"
-	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/source_metadatapb"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/sources/git"
@@ -18,14 +15,6 @@ import (
 type expResult struct {
 	B          string
 	LineNumber int64
-	Verified   bool
-}
-
-type discardPrinter struct{}
-
-func (p *discardPrinter) Print(context.Context, *detectors.ResultWithMetadata) error {
-	// This method intentionally does nothing.
-	return nil
 }
 
 func TestGitEngine(t *testing.T) {
@@ -49,30 +38,27 @@ func TestGitEngine(t *testing.T) {
 	for tName, tTest := range map[string]testProfile{
 		"all_secrets": {
 			expected: map[string]expResult{
-				"70001020fab32b1fcf2f1f0e5c66424eae649826": {"AKIAXYZDQCEN4B6JSJQI", 2, true},
-				"84e9c75e388ae3e866e121087ea2dd45a71068f2": {"AKIAILE3JG6KMS3HZGCA", 4, true},
-				"8afb0ecd4998b1179e428db5ebbcdc8221214432": {"369963c1434c377428ca8531fbc46c0c43d037a0", 3, false},
-				"27fbead3bf883cdb7de9d7825ed401f28f9398f1": {"ffc7e0f9400fb6300167009e42d2f842cd7956e2", 7, false},
+				"70001020fab32b1fcf2f1f0e5c66424eae649826": {"AKIAXYZDQCEN4B6JSJQI", 2},
+				"84e9c75e388ae3e866e121087ea2dd45a71068f2": {"AKIAILE3JG6KMS3HZGCA", 4},
+				"8afb0ecd4998b1179e428db5ebbcdc8221214432": {"369963c1434c377428ca8531fbc46c0c43d037a0", 3},
+				"27fbead3bf883cdb7de9d7825ed401f28f9398f1": {"ffc7e0f9400fb6300167009e42d2f842cd7956e2", 7},
 			},
 			filter: common.FilterEmpty(),
 		},
 		"base_commit": {
 			expected: map[string]expResult{
-				"70001020fab32b1fcf2f1f0e5c66424eae649826": {"AKIAXYZDQCEN4B6JSJQI", 2, true},
+				"70001020fab32b1fcf2f1f0e5c66424eae649826": {"AKIAXYZDQCEN4B6JSJQI", 2},
 			},
 			filter: common.FilterEmpty(),
 			base:   "2f251b8c1e72135a375b659951097ec7749d4af9",
 		},
 	} {
 		t.Run(tName, func(t *testing.T) {
-			e, err := Start(ctx,
+			e := Start(ctx,
 				WithConcurrency(1),
 				WithDecoders(decoders.DefaultDecoders()...),
-				WithDetectors(true, DefaultDetectors()...),
-				WithPrinter(new(discardPrinter)),
+				WithDetectors(false, DefaultDetectors()...),
 			)
-			assert.Nil(t, err)
-
 			cfg := sources.GitConfig{
 				RepoPath: path,
 				HeadRef:  tTest.branch,
@@ -84,8 +70,11 @@ func TestGitEngine(t *testing.T) {
 				return
 			}
 
-			// Wait for all the chunks to be processed.
-			assert.Nil(t, e.Finish(ctx))
+			logFatalFunc := func(_ error, _ string, _ ...any) {
+				t.Fatalf("error logging function should not have been called")
+			}
+			go e.Finish(ctx, logFatalFunc)
+			resultCount := 0
 			for result := range e.ResultsChan() {
 				switch meta := result.SourceMetadata.GetData().(type) {
 				case *source_metadatapb.MetaData_Git:
@@ -95,14 +84,13 @@ func TestGitEngine(t *testing.T) {
 					if tTest.expected[meta.Git.Commit].LineNumber != result.SourceMetadata.GetGit().Line {
 						t.Errorf("%s: unexpected line number. Got: %d, Expected: %d", tName, result.SourceMetadata.GetGit().Line, tTest.expected[meta.Git.Commit].LineNumber)
 					}
-					if tTest.expected[meta.Git.Commit].Verified != result.Verified {
-						t.Errorf("%s: unexpected verification. Got: %v, Expected: %v", tName, result.Verified, tTest.expected[meta.Git.Commit].Verified)
-					}
 				}
+				resultCount++
 
 			}
-			metrics := e.GetMetrics()
-			assert.Equal(t, len(tTest.expected), int(metrics.VerifiedSecretsFound)+int(metrics.UnverifiedSecretsFound))
+			if resultCount != len(tTest.expected) {
+				t.Errorf("%s: unexpected number of results. Got: %d, Expected: %d", tName, resultCount, len(tTest.expected))
+			}
 		})
 	}
 }
@@ -119,13 +107,11 @@ func BenchmarkGitEngine(b *testing.B) {
 	ctx, cancel := context.WithCancel(ctx)
 	defer cancel()
 
-	e, err := Start(ctx,
+	e := Start(ctx,
 		WithConcurrency(1),
 		WithDecoders(decoders.DefaultDecoders()...),
 		WithDetectors(false, DefaultDetectors()...),
 	)
-	assert.Nil(b, err)
-
 	go func() {
 		resultCount := 0
 		for range e.ResultsChan() {
@@ -144,5 +130,8 @@ func BenchmarkGitEngine(b *testing.B) {
 			return
 		}
 	}
-	assert.Nil(b, e.Finish(ctx))
+	logFatalFunc := func(_ error, _ string, _ ...any) {
+		b.Fatalf("error logging function should not have been called")
+	}
+	e.Finish(ctx, logFatalFunc)
 }

pkg/gitparse/gitparse.go

@@ -51,49 +51,6 @@ type Parser struct {
 	dateFormat    string
 }
 
-type ParseState int
-
-const (
-	Initial ParseState = iota
-	CommitLine
-	MergeLine
-	AuthorLine
-	DateLine
-	MessageStartLine
-	MessageLine
-	MessageEndLine
-	DiffLine
-	ModeLine
-	IndexLine
-	FromFileLine
-	ToFileLine
-	BinaryFileLine
-	HunkLineNumberLine
-	HunkContentLine
-	ParseFailure
-)
-
-func (state ParseState) String() string {
-	return [...]string{
-		"Initial",
-		"CommitLine",
-		"MergeLine",
-		"AuthorLine",
-		"DateLine",
-		"MessageStartLine",
-		"MessageLine",
-		"MessageEndLine",
-		"DiffLine",
-		"ModeLine",
-		"IndexLine",
-		"FromFileLine",
-		"ToFileLine",
-		"BinaryFileLine",
-		"HunkLineNumberLine",
-		"HunkContentLine",
-	}[state]
-}
-
 // WithMaxDiffSize sets maxDiffSize option. Diffs larger than maxDiffSize will
 // be truncated.
 func WithMaxDiffSize(maxDiffSize int) Option {
@@ -180,7 +137,7 @@ func (c *Parser) RepoPath(ctx context.Context, source string, head string, abbre
 		cmd.Env = append(cmd.Env, fmt.Sprintf("GIT_DIR=%s", filepath.Join(absPath, ".git")))
 	}
 
-	return c.executeCommand(ctx, cmd, false)
+	return c.executeCommand(ctx, cmd)
 }
 
 // Staged parses the output of the `git diff` command for the `source` path.
@@ -195,11 +152,11 @@ func (c *Parser) Staged(ctx context.Context, source string) (chan Commit, error)
 		cmd.Env = append(cmd.Env, fmt.Sprintf("GIT_DIR=%s", filepath.Join(absPath, ".git")))
 	}
 
-	return c.executeCommand(ctx, cmd, true)
+	return c.executeCommand(ctx, cmd)
 }
 
 // executeCommand runs an exec.Cmd, reads stdout and stderr, and waits for the Cmd to complete.
-func (c *Parser) executeCommand(ctx context.Context, cmd *exec.Cmd, isStaged bool) (chan Commit, error) {
+func (c *Parser) executeCommand(ctx context.Context, cmd *exec.Cmd) (chan Commit, error) {
 	commitChan := make(chan Commit, 64)
 
 	stdOut, err := cmd.StdoutPipe()
@@ -224,7 +181,7 @@ func (c *Parser) executeCommand(ctx context.Context, cmd *exec.Cmd, isStaged boo
 	}()
 
 	go func() {
-		c.FromReader(ctx, stdOut, commitChan, isStaged)
+		c.FromReader(ctx, stdOut, commitChan)
 		if err := cmd.Wait(); err != nil {
 			ctx.Logger().V(2).Info("Error waiting for git command to complete.", "error", err)
 		}
@@ -233,15 +190,14 @@ func (c *Parser) executeCommand(ctx context.Context, cmd *exec.Cmd, isStaged boo
 	return commitChan, nil
 }
 
-func (c *Parser) FromReader(ctx context.Context, stdOut io.Reader, commitChan chan Commit, isStaged bool) {
+func (c *Parser) FromReader(ctx context.Context, stdOut io.Reader, commitChan chan Commit) {
 	outReader := bufio.NewReader(stdOut)
 	var (
-		currentCommit *Commit
-		currentDiff   Diff
-
-		totalLogSize int
+		currentCommit        *Commit
+		currentDiff          *Diff
+		recentlyPassedAuthor bool
+		totalLogSize         int
 	)
-	var latestState = Initial
 
 	defer common.RecoverWithExit(ctx)
 	defer close(commitChan)
@@ -249,19 +205,15 @@ func (c *Parser) FromReader(ctx context.Context, stdOut io.Reader, commitChan ch
 		if common.IsDone(ctx) {
 			break
 		}
-
 		line, err := outReader.ReadBytes([]byte("\n")[0])
 		if err != nil && len(line) == 0 {
 			break
 		}
-
 		switch {
-		case isCommitLine(isStaged, latestState, line):
-			latestState = CommitLine
-
+		case isCommitLine(line):
 			// If there is a currentDiff, add it to currentCommit.
-			if currentDiff.Content.Len() > 0 {
-				currentCommit.Diffs = append(currentCommit.Diffs, currentDiff)
+			if currentDiff != nil && currentDiff.Content.Len() > 0 {
+				currentCommit.Diffs = append(currentCommit.Diffs, *currentDiff)
 				currentCommit.Size += currentDiff.Content.Len()
 			}
 			// If there is a currentCommit, send it to the channel.
@@ -270,7 +222,7 @@ func (c *Parser) FromReader(ctx context.Context, stdOut io.Reader, commitChan ch
 				totalLogSize += currentCommit.Size
 			}
 			// Create a new currentDiff and currentCommit
-			currentDiff = Diff{}
+			currentDiff = &Diff{}
 			currentCommit = &Commit{
 				Message: strings.Builder{},
 			}
@@ -278,39 +230,22 @@ func (c *Parser) FromReader(ctx context.Context, stdOut io.Reader, commitChan ch
 			if len(line) >= 47 {
 				currentCommit.Hash = string(line[7:47])
 			}
-		case isMergeLine(isStaged, latestState, line):
-			latestState = MergeLine
-		case isAuthorLine(isStaged, latestState, line):
-			latestState = AuthorLine
-
+		case isAuthorLine(line):
 			currentCommit.Author = strings.TrimRight(string(line[8:]), "\n")
-		case isDateLine(isStaged, latestState, line):
-			latestState = DateLine
-
+			recentlyPassedAuthor = true
+		case isDateLine(line):
 			date, err := time.Parse(c.dateFormat, strings.TrimSpace(string(line[6:])))
 			if err != nil {
 				ctx.Logger().V(2).Info("Could not parse date from git stream.", "error", err)
 			}
 			currentCommit.Date = date
-		case isMessageStartLine(isStaged, latestState, line):
-			latestState = MessageStartLine
-			// NoOp
-		case isMessageLine(isStaged, latestState, line):
-			latestState = MessageLine
-
-			currentCommit.Message.Write(line[4:])
-		case isMessageEndLine(isStaged, latestState, line):
-			latestState = MessageEndLine
-			// NoOp
-		case isDiffLine(isStaged, latestState, line):
-			latestState = DiffLine
-
+		case isDiffLine(line):
 			// This should never be nil, but check in case the stdin stream is messed up.
 			if currentCommit == nil {
 				currentCommit = &Commit{}
 			}
-			if currentDiff.Content.Len() > 0 {
-				currentCommit.Diffs = append(currentCommit.Diffs, currentDiff)
+			if currentDiff != nil && currentDiff.Content.Len() > 0 {
+				currentCommit.Diffs = append(currentCommit.Diffs, *currentDiff)
 				// If the currentDiff is over 1GB, drop it into the channel so it isn't held in memory waiting for more commits.
 				totalSize := 0
 				for _, diff := range currentCommit.Diffs {
@@ -331,36 +266,37 @@ func (c *Parser) FromReader(ctx context.Context, stdOut io.Reader, commitChan ch
 					currentCommit.Message.WriteString(oldCommit.Message.String())
 				}
 			}
-			currentDiff = Diff{}
-		case isModeLine(isStaged, latestState, line):
-			latestState = ModeLine
+			currentDiff = &Diff{}
+		case isModeLine(line):
 			// NoOp
-		case isIndexLine(isStaged, latestState, line):
-			latestState = IndexLine
+		case isIndexLine(line):
+			recentlyPassedAuthor = false
 			// NoOp
-		case isBinaryLine(isStaged, latestState, line):
-			latestState = BinaryFileLine
-
+		case isPlusFileLine(line):
+			currentDiff.PathB = strings.TrimRight(strings.TrimRight(string(line[6:]), "\n"), "\t") // Trim the newline and tab characters. https://github.com/trufflesecurity/trufflehog/issues/1060
+		case isMinusFileLine(line):
+			// NoOp
+		case isPlusDiffLine(line):
+			currentDiff.Content.Write(line[1:])
+		case isMinusDiffLine(line):
+			// NoOp. We only care about additions.
+		case (isMessageLine(line) && recentlyPassedAuthor):
+			currentCommit.Message.Write(line[4:])
+		case isContextDiffLine(line):
+			currentDiff.Content.Write([]byte("\n"))
+		case isBinaryLine(line):
 			currentDiff.IsBinary = true
 			currentDiff.PathB = pathFromBinaryLine(line)
-		case isFromFileLine(isStaged, latestState, line):
-			latestState = FromFileLine
-			// NoOp
-		case isToFileLine(isStaged, latestState, line):
-			latestState = ToFileLine
-
-			// TODO: Is this fix still required?
-			currentDiff.PathB = strings.TrimRight(strings.TrimRight(string(line[6:]), "\n"), "\t") // Trim the newline and tab characters. https://github.com/trufflesecurity/trufflehog/issues/1060
-		case isHunkLineNumberLine(isStaged, latestState, line):
-			latestState = HunkLineNumberLine
-
-			if currentDiff.Content.Len() > 0 {
-				currentCommit.Diffs = append(currentCommit.Diffs, currentDiff)
+		case isLineNumberDiffLine(line):
+			if currentDiff != nil && currentDiff.Content.Len() > 0 {
+				currentCommit.Diffs = append(currentCommit.Diffs, *currentDiff)
 			}
-			currentDiff = Diff{
+			newDiff := &Diff{
 				PathB: currentDiff.PathB,
 			}
 
+			currentDiff = newDiff
+
 			words := bytes.Split(line, []byte(" "))
 			if len(words) >= 3 {
 				startSlice := bytes.Split(words[2], []byte(","))
@@ -369,54 +305,7 @@ func (c *Parser) FromReader(ctx context.Context, stdOut io.Reader, commitChan ch
 					currentDiff.LineStart = lineStart
 				}
 			}
-		case isHunkContextLine(isStaged, latestState, line):
-			if latestState != HunkContentLine {
-				latestState = HunkContentLine
-			}
-			// TODO: Why do we care about this? It creates empty lines in the diff. If there are no plusLines, it's just newlines.
-			currentDiff.Content.Write([]byte("\n"))
-		case isHunkPlusLine(isStaged, latestState, line):
-			if latestState != HunkContentLine {
-				latestState = HunkContentLine
-			}
-
-			currentDiff.Content.Write(line[1:])
-		case isHunkMinusLine(isStaged, latestState, line):
-			if latestState != HunkContentLine {
-				latestState = HunkContentLine
-			}
-			// NoOp. We only care about additions.
-		case isHunkNewlineWarningLine(isStaged, latestState, line):
-			if latestState != HunkContentLine {
-				latestState = HunkContentLine
-			}
-			// NoOp
-		case isHunkEmptyLine(isStaged, latestState, line):
-			if latestState != HunkContentLine {
-				latestState = HunkContentLine
-			}
-			// NoOp
-		case isCommitSeparatorLine(isStaged, latestState, line):
-			// NoOp
-		default:
-			// Skip ahead until we find the next diff or commit.
-			if latestState == ParseFailure {
-				continue
-			}
-
-			// Here be dragons...
-			// Build an informative error message.
-			var err error
-			if currentCommit != nil && currentCommit.Hash != "" {
-				err = fmt.Errorf(`failed to parse line "%s" after state "%s" (commit=%s)`, line, latestState, currentCommit.Hash)
-			} else {
-				err = fmt.Errorf(`failed to parse line "%s" after state "%s"`, line, latestState)
-			}
-			ctx.Logger().V(2).Error(err, "Recovering at the latest commit or diff...\n")
-
-			latestState = ParseFailure
 		}
-
 		if currentDiff.Content.Len() > c.maxDiffSize {
 			ctx.Logger().V(2).Info(fmt.Sprintf(
 				"Diff for %s exceeded MaxDiffSize(%d)", currentDiff.PathB, c.maxDiffSize,
@@ -424,162 +313,135 @@ func (c *Parser) FromReader(ctx context.Context, stdOut io.Reader, commitChan ch
 			break
 		}
 	}
-	cleanupParse(currentCommit, &currentDiff, commitChan, &totalLogSize)
+	cleanupParse(currentCommit, currentDiff, commitChan, &totalLogSize)
 
 	ctx.Logger().V(2).Info("finished parsing git log.", "total_log_size", totalLogSize)
 }
 
-func isMergeLine(isStaged bool, latestState ParseState, line []byte) bool {
-	if isStaged || latestState != CommitLine {
-		return false
+func cleanupParse(currentCommit *Commit, currentDiff *Diff, commitChan chan Commit, totalLogSize *int) {
+	if currentDiff != nil && currentDiff.Content.Len() > 0 {
+		currentCommit.Diffs = append(currentCommit.Diffs, *currentDiff)
 	}
-	if len(line) > 6 && bytes.Equal(line[:6], []byte("Merge:")) {
-		return true
+	if currentCommit != nil {
+		commitChan <- *currentCommit
+		if totalLogSize != nil {
+			*totalLogSize += currentCommit.Size
+		}
 	}
-	return false
-}
-
-// commit 7a95bbf0199e280a0e42dbb1d1a3f56cdd0f6e05
-func isCommitLine(isStaged bool, latestState ParseState, line []byte) bool {
-	if isStaged || !(latestState == Initial ||
-		latestState == MessageStartLine ||
-		latestState == MessageEndLine ||
-		latestState == ModeLine ||
-		latestState == IndexLine ||
-		latestState == BinaryFileLine ||
-		latestState == ToFileLine ||
-		latestState == HunkContentLine ||
-		latestState == ParseFailure) {
-		return false
-	}
-
-	if len(line) > 7 && bytes.Equal(line[:7], []byte("commit ")) {
-		return true
-	}
-	return false
-}
-
-// Author: Bill Rich <bill.rich@trufflesec.com>
-func isAuthorLine(isStaged bool, latestState ParseState, line []byte) bool {
-	if isStaged || !(latestState == CommitLine || latestState == MergeLine) {
-		return false
-	}
-	if len(line) > 8 && bytes.Equal(line[:7], []byte("Author:")) {
-		return true
-	}
-	return false
 }
 
 // Date:   Tue Aug 10 15:20:40 2021 +0100
-func isDateLine(isStaged bool, latestState ParseState, line []byte) bool {
-	if isStaged || latestState != AuthorLine {
-		return false
-	}
+func isDateLine(line []byte) bool {
 	if len(line) > 7 && bytes.Equal(line[:5], []byte("Date:")) {
 		return true
 	}
 	return false
 }
 
-// Line directly after Date with only a newline.
-func isMessageStartLine(isStaged bool, latestState ParseState, line []byte) bool {
-	if isStaged || latestState != DateLine {
-		return false
-	}
-	// TODO: Improve the implementation of this and isMessageEndLine
-	if len(strings.TrimRight(string(line[:]), "\r\n")) == 0 {
+// Author: Bill Rich <bill.rich@trufflesec.com>
+func isAuthorLine(line []byte) bool {
+	if len(line) > 8 && bytes.Equal(line[:7], []byte("Author:")) {
 		return true
 	}
 	return false
 }
 
-// Line that starts with 4 spaces
-func isMessageLine(isStaged bool, latestState ParseState, line []byte) bool {
-	if isStaged || !(latestState == MessageStartLine || latestState == MessageLine) {
-		return false
-	}
-	if len(line) > 4 && bytes.Equal(line[:4], []byte("    ")) {
-		return true
-	}
-	return false
-}
-
-// Line directly after MessageLine with only a newline.
-func isMessageEndLine(isStaged bool, latestState ParseState, line []byte) bool {
-	if isStaged || latestState != MessageLine {
-		return false
-	}
-	if len(strings.TrimRight(string(line[:]), "\r\n")) == 0 {
+// commit 7a95bbf0199e280a0e42dbb1d1a3f56cdd0f6e05
+func isCommitLine(line []byte) bool {
+	if len(line) > 7 && bytes.Equal(line[:6], []byte("commit")) {
 		return true
 	}
 	return false
 }
 
 // diff --git a/internal/addrs/move_endpoint_module.go b/internal/addrs/move_endpoint_module.go
-func isDiffLine(isStaged bool, latestState ParseState, line []byte) bool {
-	if !(latestState == MessageStartLine || // Empty commit messages can go from MessageStart->Diff
-		latestState == MessageEndLine ||
-		latestState == BinaryFileLine ||
-		latestState == IndexLine ||
-		latestState == HunkContentLine ||
-		latestState == ParseFailure) {
-		if latestState == Initial && !isStaged {
-			return false
-		}
-	}
-	if len(line) > 11 && bytes.Equal(line[:11], []byte("diff --git ")) {
-		return true
-	}
-	return false
-}
-
-// old mode 100644
-// new mode 100755
-// new file mode 100644
-// similarity index 100%
-// rename from old.txt
-// rename to new.txt
-// deleted file mode 100644
-func isModeLine(isStaged bool, latestState ParseState, line []byte) bool {
-	if !(latestState == DiffLine || latestState == ModeLine) {
-		return false
-	}
-	// This could probably be better written.
-	if (len(line) > 17 && bytes.Equal(line[:17], []byte("deleted file mode"))) ||
-		(len(line) > 16 && bytes.Equal(line[:16], []byte("similarity index"))) ||
-		(len(line) > 13 && bytes.Equal(line[:13], []byte("new file mode"))) ||
-		(len(line) > 11 && bytes.Equal(line[:11], []byte("rename from"))) ||
-		(len(line) > 9 && bytes.Equal(line[:9], []byte("rename to"))) ||
-		(len(line) > 8 && bytes.Equal(line[:8], []byte("old mode"))) ||
-		(len(line) > 8 && bytes.Equal(line[:8], []byte("new mode"))) {
+func isDiffLine(line []byte) bool {
+	if len(line) > 5 && bytes.Equal(line[:4], []byte("diff")) {
 		return true
 	}
 	return false
 }
 
 // index 1ed6fbee1..aea1e643a 100644
-// index 00000000..e69de29b
-func isIndexLine(isStaged bool, latestState ParseState, line []byte) bool {
-	if !(latestState == DiffLine || latestState == ModeLine) {
-		return false
+func isIndexLine(line []byte) bool {
+	if len(line) > 6 && bytes.Equal(line[:5], []byte("index")) {
+		return true
 	}
-	if len(line) > 6 && bytes.Equal(line[:6], []byte("index ")) {
+	return false
+}
+
+// new file mode 100644
+func isModeLine(line []byte) bool {
+	if len(line) > 13 && bytes.Equal(line[:13], []byte("new file mode")) {
+		return true
+	}
+	return false
+}
+
+// --- a/internal/addrs/move_endpoint_module.go
+func isMinusFileLine(line []byte) bool {
+	if len(line) >= 6 && bytes.Equal(line[:3], []byte("---")) {
+		return true
+	}
+	return false
+}
+
+// +++ b/internal/addrs/move_endpoint_module.go
+func isPlusFileLine(line []byte) bool {
+	if len(line) >= 6 && bytes.Equal(line[:4], []byte("+++ ")) {
+		return true
+	}
+	return false
+}
+
+// +fmt.Println("ok")
+func isPlusDiffLine(line []byte) bool {
+	if len(line) >= 1 && bytes.Equal(line[:1], []byte("+")) {
+		return true
+	}
+	return false
+}
+
+// -fmt.Println("ok")
+func isMinusDiffLine(line []byte) bool {
+	if len(line) >= 1 && bytes.Equal(line[:1], []byte("-")) {
+		return true
+	}
+	return false
+}
+
+// fmt.Println("ok")
+func isContextDiffLine(line []byte) bool {
+	if len(line) >= 1 && bytes.Equal(line[:1], []byte(" ")) {
+		return true
+	}
+	return false
+}
+
+// Line that starts with 4 spaces
+func isMessageLine(line []byte) bool {
+	if len(line) > 4 && bytes.Equal(line[:4], []byte("    ")) {
 		return true
 	}
 	return false
 }
 
 // Binary files /dev/null and b/plugin.sig differ
-func isBinaryLine(isStaged bool, latestState ParseState, line []byte) bool {
-	if latestState != IndexLine {
-		return false
-	}
+func isBinaryLine(line []byte) bool {
 	if len(line) > 7 && bytes.Equal(line[:6], []byte("Binary")) {
 		return true
 	}
 	return false
 }
 
+// @@ -298 +298 @@ func maxRetryErrorHandler(resp *http.Response, err error, numTries int)
+func isLineNumberDiffLine(line []byte) bool {
+	if len(line) >= 8 && bytes.Equal(line[:2], []byte("@@")) {
+		return true
+	}
+	return false
+}
+
 // Get the b/ file path. Ignoring the edge case of files having `and /b` in the name for simplicity.
 func pathFromBinaryLine(line []byte) string {
 	logger := context.Background().Logger()
@@ -591,117 +453,3 @@ func pathFromBinaryLine(line []byte) string {
 	bRaw := sbytes[1]
 	return string(bRaw[:len(bRaw)-7]) // drop the "b/" and " differ"
 }
-
-// --- a/internal/addrs/move_endpoint_module.go
-func isFromFileLine(isStaged bool, latestState ParseState, line []byte) bool {
-	if latestState != IndexLine {
-		return false
-	}
-	if len(line) >= 6 && bytes.Equal(line[:4], []byte("--- ")) {
-		return true
-	}
-	return false
-}
-
-// +++ b/internal/addrs/move_endpoint_module.go
-func isToFileLine(isStaged bool, latestState ParseState, line []byte) bool {
-	if latestState != FromFileLine {
-		return false
-	}
-	if len(line) >= 6 && bytes.Equal(line[:4], []byte("+++ ")) {
-		return true
-	}
-	return false
-}
-
-// @@ -298 +298 @@ func maxRetryErrorHandler(resp *http.Response, err error, numTries int)
-func isHunkLineNumberLine(isStaged bool, latestState ParseState, line []byte) bool {
-	if !(latestState == ToFileLine || latestState == HunkContentLine) {
-		return false
-	}
-	if len(line) >= 8 && bytes.Equal(line[:2], []byte("@@")) {
-		return true
-	}
-	return false
-}
-
-// fmt.Println("ok")
-// (There's a space before `fmt` that gets removed by the formatter.)
-func isHunkContextLine(isStaged bool, latestState ParseState, line []byte) bool {
-	if !(latestState == HunkLineNumberLine || latestState == HunkContentLine) {
-		return false
-	}
-	if len(line) >= 1 && bytes.Equal(line[:1], []byte(" ")) {
-		return true
-	}
-	return false
-}
-
-// +fmt.Println("ok")
-func isHunkPlusLine(isStaged bool, latestState ParseState, line []byte) bool {
-	if !(latestState == HunkLineNumberLine || latestState == HunkContentLine) {
-		return false
-	}
-	if len(line) >= 1 && bytes.Equal(line[:1], []byte("+")) {
-		return true
-	}
-	return false
-}
-
-// -fmt.Println("ok")
-func isHunkMinusLine(isStaged bool, latestState ParseState, line []byte) bool {
-	if !(latestState == HunkLineNumberLine || latestState == HunkContentLine) {
-		return false
-	}
-	if len(line) >= 1 && bytes.Equal(line[:1], []byte("-")) {
-		return true
-	}
-	return false
-}
-
-// \ No newline at end of file
-func isHunkNewlineWarningLine(isStaged bool, latestState ParseState, line []byte) bool {
-	if latestState != HunkContentLine {
-		return false
-	}
-	if len(line) >= 27 && bytes.Equal(line[:27], []byte("\\ No newline at end of file")) {
-		return true
-	}
-	return false
-}
-
-// Newline after hunk, or an empty line, e.g.
-// +}
-//
-// commit 00920984e3435057f09cee5468850f7546dfa637 (tag: v3.42.0)
-func isHunkEmptyLine(isStaged bool, latestState ParseState, line []byte) bool {
-	if !(latestState == HunkLineNumberLine || latestState == HunkContentLine) {
-		return false
-	}
-	// TODO: Can this also be `\n\r`?
-	if len(line) == 1 && bytes.Equal(line[:1], []byte("\n")) {
-		return true
-	}
-	return false
-}
-
-func isCommitSeparatorLine(isStaged bool, latestState ParseState, line []byte) bool {
-	if (latestState == ModeLine || latestState == IndexLine || latestState == BinaryFileLine || latestState == ToFileLine) &&
-		len(line) == 1 && bytes.Equal(line[:1], []byte("\n")) {
-		return true
-	}
-	return false
-}
-
-func cleanupParse(currentCommit *Commit, currentDiff *Diff, commitChan chan Commit, totalLogSize *int) {
-	// Ignore empty or binary diffs (this condition may be redundant).
-	if currentDiff != nil && currentDiff.Content.Len() > 0 {
-		currentCommit.Diffs = append(currentCommit.Diffs, *currentDiff)
-	}
-	if currentCommit != nil {
-		commitChan <- *currentCommit
-		if totalLogSize != nil {
-			*totalLogSize += currentCommit.Size
-		}
-	}
-}

pkg/giturl/giturl.go

@@ -7,20 +7,12 @@ import (
 	"github.com/pkg/errors"
 )
 
-type provider string
-
-const (
-	providerGithub    provider = "Github"
-	providerGitlab    provider = "Gitlab"
-	providerBitbucket provider = "Bitbucket"
-)
-
 func NormalizeBitbucketRepo(repoURL string) (string, error) {
 	if !strings.HasPrefix(repoURL, "https") {
 		return "", errors.New("Bitbucket requires https repo urls: e.g. https://bitbucket.org/org/repo.git")
 	}
 
-	return NormalizeOrgRepoURL(providerBitbucket, repoURL)
+	return NormalizeOrgRepoURL("Bitbucket", repoURL)
 }
 
 func NormalizeGerritProject(project string) (string, error) {
@@ -28,7 +20,7 @@ func NormalizeGerritProject(project string) (string, error) {
 }
 
 func NormalizeGithubRepo(repoURL string) (string, error) {
-	return NormalizeOrgRepoURL(providerGithub, repoURL)
+	return NormalizeOrgRepoURL("Github", repoURL)
 }
 
 func NormalizeGitlabRepo(repoURL string) (string, error) {
@@ -36,12 +28,12 @@ func NormalizeGitlabRepo(repoURL string) (string, error) {
 		return "", errors.New("Gitlab requires http/https repo urls: e.g. https://gitlab.com/org/repo.git")
 	}
 
-	return NormalizeOrgRepoURL(providerGitlab, repoURL)
+	return NormalizeOrgRepoURL("Gitlab", repoURL)
 }
 
 // NormalizeOrgRepoURL attempts to normalize repos for any provider using the example.com/org/repo style.
 // e.g. %s, Gitlab and Bitbucket
-func NormalizeOrgRepoURL(provider provider, repoURL string) (string, error) {
+func NormalizeOrgRepoURL(provider, repoURL string) (string, error) {
 	if strings.HasSuffix(repoURL, ".git") {
 		return repoURL, nil
 	}

pkg/giturl/giturl_test.go

@@ -8,22 +8,22 @@ import (
 
 func Test_NormalizeOrgRepoURL(t *testing.T) {
 	tests := map[string]struct {
-		Provider provider
+		Provider string
 		Repo     string
 		Out      string
 		Err      error
 	}{
-		"github is good":             {Provider: providerGithub, Repo: "https://github.com/org/repo", Out: "https://github.com/org/repo.git", Err: nil},
-		"gitlab is good":             {Provider: providerGitlab, Repo: "https://gitlab.com/org/repo", Out: "https://gitlab.com/org/repo.git", Err: nil},
-		"bitbucket is good":          {Provider: providerBitbucket, Repo: "https://bitbucket.com/org/repo", Out: "https://bitbucket.com/org/repo.git", Err: nil},
+		"github is good":             {Provider: "Github", Repo: "https://github.com/org/repo", Out: "https://github.com/org/repo.git", Err: nil},
+		"gitlab is good":             {Provider: "Gitlab", Repo: "https://gitlab.com/org/repo", Out: "https://gitlab.com/org/repo.git", Err: nil},
+		"bitbucket is good":          {Provider: "Bitbucket", Repo: "https://bitbucket.com/org/repo", Out: "https://bitbucket.com/org/repo.git", Err: nil},
 		"example provider is good":   {Provider: "example", Repo: "https://example.com/org/repo", Out: "https://example.com/org/repo.git", Err: nil},
 		"example provider problem":   {Provider: "example", Repo: "https://example.com/org", Out: "", Err: errors.Errorf("example repo appears to be missing the repo name. Org: %q Repo url: %q", "org", "https://example.com/org")},
-		"no path":                    {Provider: providerGithub, Repo: "https://github.com", Out: "", Err: errors.Errorf("Github repo appears to be missing the path. Repo url: %q", "https://github.com")},
-		"org but no repo":            {Provider: providerGithub, Repo: "https://github.com/org", Out: "", Err: errors.Errorf("Github repo appears to be missing the repo name. Org: %q Repo url: %q", "org", "https://github.com/org")},
-		"org but no repo with slash": {Provider: providerGithub, Repo: "https://github.com/org/", Out: "", Err: errors.Errorf("Github repo appears to be missing the repo name. Org: %q Repo url: %q", "org", "https://github.com/org/")},
-		"two slashes":                {Provider: providerGithub, Repo: "https://github.com//", Out: "", Err: errors.Errorf("Github repo appears to be missing the org name. Repo url: %q", "https://github.com//")},
-		"repo with trailing slash":   {Provider: providerGithub, Repo: "https://github.com/org/repo/", Out: "", Err: errors.Errorf("Github repo contains a trailing slash. Repo url: %q", "https://github.com/org/repo/")},
-		"too many url path parts":    {Provider: providerGithub, Repo: "https://github.com/org/repo/unknown/", Out: "", Err: errors.Errorf("Github repo contains a trailing slash. Repo url: %q", "https://github.com/org/repo/unknown/")},
+		"no path":                    {Provider: "Github", Repo: "https://github.com", Out: "", Err: errors.Errorf("Github repo appears to be missing the path. Repo url: %q", "https://github.com")},
+		"org but no repo":            {Provider: "Github", Repo: "https://github.com/org", Out: "", Err: errors.Errorf("Github repo appears to be missing the repo name. Org: %q Repo url: %q", "org", "https://github.com/org")},
+		"org but no repo with slash": {Provider: "Github", Repo: "https://github.com/org/", Out: "", Err: errors.Errorf("Github repo appears to be missing the repo name. Org: %q Repo url: %q", "org", "https://github.com/org/")},
+		"two slashes":                {Provider: "Github", Repo: "https://github.com//", Out: "", Err: errors.Errorf("Github repo appears to be missing the org name. Repo url: %q", "https://github.com//")},
+		"repo with trailing slash":   {Provider: "Github", Repo: "https://github.com/org/repo/", Out: "", Err: errors.Errorf("Github repo contains a trailing slash. Repo url: %q", "https://github.com/org/repo/")},
+		"too many url path parts":    {Provider: "Github", Repo: "https://github.com/org/repo/unknown/", Out: "", Err: errors.Errorf("Github repo contains a trailing slash. Repo url: %q", "https://github.com/org/repo/unknown/")},
 	}
 
 	for name, tt := range tests {

pkg/output/github_actions.go

@@ -5,17 +5,13 @@ import (
 	"encoding/hex"
 	"fmt"
 
-	"github.com/trufflesecurity/trufflehog/v3/pkg/context"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
 )
 
 var dedupeCache = make(map[string]struct{})
 
-// GitHubActionsPrinter is a printer that prints results in GitHub Actions format.
-type GitHubActionsPrinter struct{}
-
-func (p *GitHubActionsPrinter) Print(_ context.Context, r *detectors.ResultWithMetadata) error {
+func PrintGitHubActionsOutput(r *detectors.ResultWithMetadata) error {
 	out := gitHubActionsOutputFormat{
 		DetectorType: r.Result.DetectorType.String(),
 		DecoderType:  r.Result.DecoderType.String(),

pkg/output/json.go

@@ -4,17 +4,13 @@ import (
 	"encoding/json"
 	"fmt"
 
-	"github.com/trufflesecurity/trufflehog/v3/pkg/context"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/source_metadatapb"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
 )
 
-// JSONPrinter is a printer that prints results in JSON format.
-type JSONPrinter struct{}
-
-func (p *JSONPrinter) Print(_ context.Context, r *detectors.ResultWithMetadata) error {
+func PrintJSON(r *detectors.ResultWithMetadata) error {
 	v := &struct {
 		// SourceMetadata contains source-specific contextual information.
 		SourceMetadata *source_metadatapb.MetaData

pkg/output/legacy_json.go

@@ -19,10 +19,7 @@ import (
 	"github.com/trufflesecurity/trufflehog/v3/pkg/sources/git"
 )
 
-// LegacyJSONPrinter is a printer that prints results in legacy JSON format for backwards compatibility.
-type LegacyJSONPrinter struct{}
-
-func (p *LegacyJSONPrinter) Print(ctx context.Context, r *detectors.ResultWithMetadata) error {
+func PrintLegacyJSON(ctx context.Context, r *detectors.ResultWithMetadata) error {
 	var repo string
 	switch r.SourceType {
 	case sourcespb.SourceType_SOURCE_TYPE_GIT:
@@ -44,7 +41,7 @@ func (p *LegacyJSONPrinter) Print(ctx context.Context, r *detectors.ResultWithMe
 		defer os.RemoveAll(repoPath)
 	}
 
-	legacy, err := convertToLegacyJSON(r, repoPath)
+	legacy, err := ConvertToLegacyJSON(r, repoPath)
 	if err != nil {
 		return fmt.Errorf("could not convert to legacy JSON: %w", err)
 	}
@@ -56,7 +53,7 @@ func (p *LegacyJSONPrinter) Print(ctx context.Context, r *detectors.ResultWithMe
 	return nil
 }
 
-func convertToLegacyJSON(r *detectors.ResultWithMetadata, repoPath string) (*LegacyJSONOutput, error) {
+func ConvertToLegacyJSON(r *detectors.ResultWithMetadata, repoPath string) (*LegacyJSONOutput, error) {
 	var source LegacyJSONCompatibleSource
 	switch r.SourceType {
 	case sourcespb.SourceType_SOURCE_TYPE_GIT:

pkg/output/plain.go

@@ -10,7 +10,6 @@ import (
 	"golang.org/x/text/cases"
 	"golang.org/x/text/language"
 
-	"github.com/trufflesecurity/trufflehog/v3/pkg/context"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/source_metadatapb"
 )
@@ -21,10 +20,7 @@ var (
 	whitePrinter  = color.New(color.FgWhite)
 )
 
-// PlainPrinter is a printer that prints results in plain text format.
-type PlainPrinter struct{}
-
-func (p *PlainPrinter) Print(_ context.Context, r *detectors.ResultWithMetadata) error {
+func PrintPlainOutput(r *detectors.ResultWithMetadata) error {
 	out := outputFormat{
 		DetectorType: r.Result.DetectorType.String(),
 		DecoderType:  r.Result.DecoderType.String(),

pkg/pb/detectorspb/detectors.pb.validate.go

@@ -99,8 +99,6 @@ func (m *Result) validate(all bool) error {
 
 	// no validation rules for DecoderType
 
-	// no validation rules for VerificationErrorMessage
-
 	if len(errors) > 0 {
 		return ResultMultiError(errors)
 	}

pkg/pb/source_metadatapb/source_metadata.pb.go

@@ -2294,7 +2294,6 @@ type GoogleDrive struct {
 	Timestamp      string `protobuf:"bytes,4,opt,name=timestamp,proto3" json:"timestamp,omitempty"`
 	Shared         bool   `protobuf:"varint,5,opt,name=shared,proto3" json:"shared,omitempty"`
 	LastModifiedBy string `protobuf:"bytes,6,opt,name=last_modified_by,json=lastModifiedBy,proto3" json:"last_modified_by,omitempty"`
-	Path           string `protobuf:"bytes,7,opt,name=path,proto3" json:"path,omitempty"`
 }
 
 func (x *GoogleDrive) Reset() {
@@ -2371,140 +2370,6 @@ func (x *GoogleDrive) GetLastModifiedBy() string {
 	return ""
 }
 
-func (x *GoogleDrive) GetPath() string {
-	if x != nil {
-		return x.Path
-	}
-	return ""
-}
-
-type AzureRepos struct {
-	state         protoimpl.MessageState
-	sizeCache     protoimpl.SizeCache
-	unknownFields protoimpl.UnknownFields
-
-	Link         string     `protobuf:"bytes,1,opt,name=link,proto3" json:"link,omitempty"`
-	Username     string     `protobuf:"bytes,2,opt,name=username,proto3" json:"username,omitempty"`
-	Repository   string     `protobuf:"bytes,3,opt,name=repository,proto3" json:"repository,omitempty"`
-	Commit       string     `protobuf:"bytes,4,opt,name=commit,proto3" json:"commit,omitempty"`
-	Email        string     `protobuf:"bytes,5,opt,name=email,proto3" json:"email,omitempty"`
-	File         string     `protobuf:"bytes,6,opt,name=file,proto3" json:"file,omitempty"`
-	Timestamp    string     `protobuf:"bytes,7,opt,name=timestamp,proto3" json:"timestamp,omitempty"`
-	Line         int64      `protobuf:"varint,8,opt,name=line,proto3" json:"line,omitempty"`
-	Visibility   Visibility `protobuf:"varint,9,opt,name=visibility,proto3,enum=source_metadata.Visibility" json:"visibility,omitempty"`
-	Project      string     `protobuf:"bytes,10,opt,name=project,proto3" json:"project,omitempty"`
-	Organization string     `protobuf:"bytes,11,opt,name=organization,proto3" json:"organization,omitempty"`
-}
-
-func (x *AzureRepos) Reset() {
-	*x = AzureRepos{}
-	if protoimpl.UnsafeEnabled {
-		mi := &file_source_metadata_proto_msgTypes[26]
-		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
-		ms.StoreMessageInfo(mi)
-	}
-}
-
-func (x *AzureRepos) String() string {
-	return protoimpl.X.MessageStringOf(x)
-}
-
-func (*AzureRepos) ProtoMessage() {}
-
-func (x *AzureRepos) ProtoReflect() protoreflect.Message {
-	mi := &file_source_metadata_proto_msgTypes[26]
-	if protoimpl.UnsafeEnabled && x != nil {
-		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
-		if ms.LoadMessageInfo() == nil {
-			ms.StoreMessageInfo(mi)
-		}
-		return ms
-	}
-	return mi.MessageOf(x)
-}
-
-// Deprecated: Use AzureRepos.ProtoReflect.Descriptor instead.
-func (*AzureRepos) Descriptor() ([]byte, []int) {
-	return file_source_metadata_proto_rawDescGZIP(), []int{26}
-}
-
-func (x *AzureRepos) GetLink() string {
-	if x != nil {
-		return x.Link
-	}
-	return ""
-}
-
-func (x *AzureRepos) GetUsername() string {
-	if x != nil {
-		return x.Username
-	}
-	return ""
-}
-
-func (x *AzureRepos) GetRepository() string {
-	if x != nil {
-		return x.Repository
-	}
-	return ""
-}
-
-func (x *AzureRepos) GetCommit() string {
-	if x != nil {
-		return x.Commit
-	}
-	return ""
-}
-
-func (x *AzureRepos) GetEmail() string {
-	if x != nil {
-		return x.Email
-	}
-	return ""
-}
-
-func (x *AzureRepos) GetFile() string {
-	if x != nil {
-		return x.File
-	}
-	return ""
-}
-
-func (x *AzureRepos) GetTimestamp() string {
-	if x != nil {
-		return x.Timestamp
-	}
-	return ""
-}
-
-func (x *AzureRepos) GetLine() int64 {
-	if x != nil {
-		return x.Line
-	}
-	return 0
-}
-
-func (x *AzureRepos) GetVisibility() Visibility {
-	if x != nil {
-		return x.Visibility
-	}
-	return Visibility_public
-}
-
-func (x *AzureRepos) GetProject() string {
-	if x != nil {
-		return x.Project
-	}
-	return ""
-}
-
-func (x *AzureRepos) GetOrganization() string {
-	if x != nil {
-		return x.Organization
-	}
-	return ""
-}
-
 type MetaData struct {
 	state         protoimpl.MessageState
 	sizeCache     protoimpl.SizeCache
@@ -2537,14 +2402,13 @@ type MetaData struct {
 	//	*MetaData_PublicEventMonitoring
 	//	*MetaData_Sharepoint
 	//	*MetaData_GoogleDrive
-	//	*MetaData_AzureRepos
 	Data isMetaData_Data `protobuf_oneof:"data"`
 }
 
 func (x *MetaData) Reset() {
 	*x = MetaData{}
 	if protoimpl.UnsafeEnabled {
-		mi := &file_source_metadata_proto_msgTypes[27]
+		mi := &file_source_metadata_proto_msgTypes[26]
 		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
 		ms.StoreMessageInfo(mi)
 	}
@@ -2557,7 +2421,7 @@ func (x *MetaData) String() string {
 func (*MetaData) ProtoMessage() {}
 
 func (x *MetaData) ProtoReflect() protoreflect.Message {
-	mi := &file_source_metadata_proto_msgTypes[27]
+	mi := &file_source_metadata_proto_msgTypes[26]
 	if protoimpl.UnsafeEnabled && x != nil {
 		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
 		if ms.LoadMessageInfo() == nil {
@@ -2570,7 +2434,7 @@ func (x *MetaData) ProtoReflect() protoreflect.Message {
 
 // Deprecated: Use MetaData.ProtoReflect.Descriptor instead.
 func (*MetaData) Descriptor() ([]byte, []int) {
-	return file_source_metadata_proto_rawDescGZIP(), []int{27}
+	return file_source_metadata_proto_rawDescGZIP(), []int{26}
 }
 
 func (m *MetaData) GetData() isMetaData_Data {
@@ -2762,13 +2626,6 @@ func (x *MetaData) GetGoogleDrive() *GoogleDrive {
 	return nil
 }
 
-func (x *MetaData) GetAzureRepos() *AzureRepos {
-	if x, ok := x.GetData().(*MetaData_AzureRepos); ok {
-		return x.AzureRepos
-	}
-	return nil
-}
-
 type isMetaData_Data interface {
 	isMetaData_Data()
 }
@@ -2877,10 +2734,6 @@ type MetaData_GoogleDrive struct {
 	GoogleDrive *GoogleDrive `protobuf:"bytes,26,opt,name=googleDrive,proto3,oneof"`
 }
 
-type MetaData_AzureRepos struct {
-	AzureRepos *AzureRepos `protobuf:"bytes,27,opt,name=azureRepos,proto3,oneof"`
-}
-
 func (*MetaData_Azure) isMetaData_Data() {}
 
 func (*MetaData_Bitbucket) isMetaData_Data() {}
@@ -2933,8 +2786,6 @@ func (*MetaData_Sharepoint) isMetaData_Data() {}
 
 func (*MetaData_GoogleDrive) isMetaData_Data() {}
 
-func (*MetaData_AzureRepos) isMetaData_Data() {}
-
 var File_source_metadata_proto protoreflect.FileDescriptor
 
 var file_source_metadata_proto_rawDesc = []byte{
@@ -3198,7 +3049,7 @@ var file_source_metadata_proto_rawDesc = []byte{
 	0x73, 0x18, 0x05, 0x20, 0x01, 0x28, 0x03, 0x52, 0x05, 0x76, 0x69, 0x65, 0x77, 0x73, 0x12, 0x14,
 	0x0a, 0x05, 0x64, 0x6f, 0x63, 0x69, 0x64, 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x64,
 	0x6f, 0x63, 0x69, 0x64, 0x12, 0x14, 0x0a, 0x05, 0x65, 0x6d, 0x61, 0x69, 0x6c, 0x18, 0x07, 0x20,
-	0x01, 0x28, 0x09, 0x52, 0x05, 0x65, 0x6d, 0x61, 0x69, 0x6c, 0x22, 0xbf, 0x01, 0x0a, 0x0b, 0x47,
+	0x01, 0x28, 0x09, 0x52, 0x05, 0x65, 0x6d, 0x61, 0x69, 0x6c, 0x22, 0xab, 0x01, 0x0a, 0x0b, 0x47,
 	0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x44, 0x72, 0x69, 0x76, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x66, 0x69,
 	0x6c, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x66, 0x69, 0x6c, 0x65, 0x12, 0x12,
 	0x0a, 0x04, 0x6c, 0x69, 0x6e, 0x6b, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6c, 0x69,
@@ -3209,130 +3060,104 @@ var file_source_metadata_proto_rawDesc = []byte{
 	0x18, 0x05, 0x20, 0x01, 0x28, 0x08, 0x52, 0x06, 0x73, 0x68, 0x61, 0x72, 0x65, 0x64, 0x12, 0x28,
 	0x0a, 0x10, 0x6c, 0x61, 0x73, 0x74, 0x5f, 0x6d, 0x6f, 0x64, 0x69, 0x66, 0x69, 0x65, 0x64, 0x5f,
 	0x62, 0x79, 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0e, 0x6c, 0x61, 0x73, 0x74, 0x4d, 0x6f,
-	0x64, 0x69, 0x66, 0x69, 0x65, 0x64, 0x42, 0x79, 0x12, 0x12, 0x0a, 0x04, 0x70, 0x61, 0x74, 0x68,
-	0x18, 0x07, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x70, 0x61, 0x74, 0x68, 0x22, 0xcb, 0x02, 0x0a,
-	0x0a, 0x41, 0x7a, 0x75, 0x72, 0x65, 0x52, 0x65, 0x70, 0x6f, 0x73, 0x12, 0x12, 0x0a, 0x04, 0x6c,
-	0x69, 0x6e, 0x6b, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6c, 0x69, 0x6e, 0x6b, 0x12,
-	0x1a, 0x0a, 0x08, 0x75, 0x73, 0x65, 0x72, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28,
-	0x09, 0x52, 0x08, 0x75, 0x73, 0x65, 0x72, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x1e, 0x0a, 0x0a, 0x72,
-	0x65, 0x70, 0x6f, 0x73, 0x69, 0x74, 0x6f, 0x72, 0x79, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52,
-	0x0a, 0x72, 0x65, 0x70, 0x6f, 0x73, 0x69, 0x74, 0x6f, 0x72, 0x79, 0x12, 0x16, 0x0a, 0x06, 0x63,
-	0x6f, 0x6d, 0x6d, 0x69, 0x74, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x63, 0x6f, 0x6d,
-	0x6d, 0x69, 0x74, 0x12, 0x14, 0x0a, 0x05, 0x65, 0x6d, 0x61, 0x69, 0x6c, 0x18, 0x05, 0x20, 0x01,
-	0x28, 0x09, 0x52, 0x05, 0x65, 0x6d, 0x61, 0x69, 0x6c, 0x12, 0x12, 0x0a, 0x04, 0x66, 0x69, 0x6c,
-	0x65, 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x66, 0x69, 0x6c, 0x65, 0x12, 0x1c, 0x0a,
-	0x09, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x18, 0x07, 0x20, 0x01, 0x28, 0x09,
-	0x52, 0x09, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x12, 0x12, 0x0a, 0x04, 0x6c,
-	0x69, 0x6e, 0x65, 0x18, 0x08, 0x20, 0x01, 0x28, 0x03, 0x52, 0x04, 0x6c, 0x69, 0x6e, 0x65, 0x12,
-	0x3b, 0x0a, 0x0a, 0x76, 0x69, 0x73, 0x69, 0x62, 0x69, 0x6c, 0x69, 0x74, 0x79, 0x18, 0x09, 0x20,
-	0x01, 0x28, 0x0e, 0x32, 0x1b, 0x2e, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f, 0x6d, 0x65, 0x74,
-	0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x56, 0x69, 0x73, 0x69, 0x62, 0x69, 0x6c, 0x69, 0x74, 0x79,
-	0x52, 0x0a, 0x76, 0x69, 0x73, 0x69, 0x62, 0x69, 0x6c, 0x69, 0x74, 0x79, 0x12, 0x18, 0x0a, 0x07,
-	0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x70,
-	0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x12, 0x22, 0x0a, 0x0c, 0x6f, 0x72, 0x67, 0x61, 0x6e, 0x69,
-	0x7a, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x0b, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0c, 0x6f, 0x72,
-	0x67, 0x61, 0x6e, 0x69, 0x7a, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x22, 0xbe, 0x0b, 0x0a, 0x08, 0x4d,
-	0x65, 0x74, 0x61, 0x44, 0x61, 0x74, 0x61, 0x12, 0x2e, 0x0a, 0x05, 0x61, 0x7a, 0x75, 0x72, 0x65,
-	0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x16, 0x2e, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f,
-	0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x41, 0x7a, 0x75, 0x72, 0x65, 0x48, 0x00,
-	0x52, 0x05, 0x61, 0x7a, 0x75, 0x72, 0x65, 0x12, 0x3a, 0x0a, 0x09, 0x62, 0x69, 0x74, 0x62, 0x75,
-	0x63, 0x6b, 0x65, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x73, 0x6f, 0x75,
-	0x72, 0x63, 0x65, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x42, 0x69, 0x74,
-	0x62, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x48, 0x00, 0x52, 0x09, 0x62, 0x69, 0x74, 0x62, 0x75, 0x63,
-	0x6b, 0x65, 0x74, 0x12, 0x37, 0x0a, 0x08, 0x63, 0x69, 0x72, 0x63, 0x6c, 0x65, 0x63, 0x69, 0x18,
-	0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x19, 0x2e, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f, 0x6d,
-	0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x43, 0x69, 0x72, 0x63, 0x6c, 0x65, 0x43, 0x49,
-	0x48, 0x00, 0x52, 0x08, 0x63, 0x69, 0x72, 0x63, 0x6c, 0x65, 0x63, 0x69, 0x12, 0x3d, 0x0a, 0x0a,
-	0x63, 0x6f, 0x6e, 0x66, 0x6c, 0x75, 0x65, 0x6e, 0x63, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b,
-	0x32, 0x1b, 0x2e, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61,
-	0x74, 0x61, 0x2e, 0x43, 0x6f, 0x6e, 0x66, 0x6c, 0x75, 0x65, 0x6e, 0x63, 0x65, 0x48, 0x00, 0x52,
-	0x0a, 0x63, 0x6f, 0x6e, 0x66, 0x6c, 0x75, 0x65, 0x6e, 0x63, 0x65, 0x12, 0x31, 0x0a, 0x06, 0x64,
-	0x6f, 0x63, 0x6b, 0x65, 0x72, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x17, 0x2e, 0x73, 0x6f,
-	0x75, 0x72, 0x63, 0x65, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x44, 0x6f,
-	0x63, 0x6b, 0x65, 0x72, 0x48, 0x00, 0x52, 0x06, 0x64, 0x6f, 0x63, 0x6b, 0x65, 0x72, 0x12, 0x28,
-	0x0a, 0x03, 0x65, 0x63, 0x72, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x73, 0x6f,
-	0x75, 0x72, 0x63, 0x65, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x45, 0x43,
-	0x52, 0x48, 0x00, 0x52, 0x03, 0x65, 0x63, 0x72, 0x12, 0x28, 0x0a, 0x03, 0x67, 0x63, 0x73, 0x18,
-	0x07, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f, 0x6d,
-	0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x47, 0x43, 0x53, 0x48, 0x00, 0x52, 0x03, 0x67,
-	0x63, 0x73, 0x12, 0x31, 0x0a, 0x06, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x18, 0x08, 0x20, 0x01,
-	0x28, 0x0b, 0x32, 0x17, 0x2e, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f, 0x6d, 0x65, 0x74, 0x61,
-	0x64, 0x61, 0x74, 0x61, 0x2e, 0x47, 0x69, 0x74, 0x68, 0x75, 0x62, 0x48, 0x00, 0x52, 0x06, 0x67,
-	0x69, 0x74, 0x68, 0x75, 0x62, 0x12, 0x31, 0x0a, 0x06, 0x67, 0x69, 0x74, 0x6c, 0x61, 0x62, 0x18,
-	0x09, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x17, 0x2e, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f, 0x6d,
-	0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x47, 0x69, 0x74, 0x6c, 0x61, 0x62, 0x48, 0x00,
-	0x52, 0x06, 0x67, 0x69, 0x74, 0x6c, 0x61, 0x62, 0x12, 0x2b, 0x0a, 0x04, 0x6a, 0x69, 0x72, 0x61,
-	0x18, 0x0a, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x15, 0x2e, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f,
-	0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x4a, 0x69, 0x72, 0x61, 0x48, 0x00, 0x52,
-	0x04, 0x6a, 0x69, 0x72, 0x61, 0x12, 0x28, 0x0a, 0x03, 0x6e, 0x70, 0x6d, 0x18, 0x0b, 0x20, 0x01,
-	0x28, 0x0b, 0x32, 0x14, 0x2e, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f, 0x6d, 0x65, 0x74, 0x61,
-	0x64, 0x61, 0x74, 0x61, 0x2e, 0x4e, 0x50, 0x4d, 0x48, 0x00, 0x52, 0x03, 0x6e, 0x70, 0x6d, 0x12,
-	0x2b, 0x0a, 0x04, 0x70, 0x79, 0x70, 0x69, 0x18, 0x0c, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x15, 0x2e,
-	0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e,
-	0x50, 0x79, 0x50, 0x69, 0x48, 0x00, 0x52, 0x04, 0x70, 0x79, 0x70, 0x69, 0x12, 0x25, 0x0a, 0x02,
-	0x73, 0x33, 0x18, 0x0d, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x13, 0x2e, 0x73, 0x6f, 0x75, 0x72, 0x63,
-	0x65, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x53, 0x33, 0x48, 0x00, 0x52,
-	0x02, 0x73, 0x33, 0x12, 0x2e, 0x0a, 0x05, 0x73, 0x6c, 0x61, 0x63, 0x6b, 0x18, 0x0e, 0x20, 0x01,
-	0x28, 0x0b, 0x32, 0x16, 0x2e, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f, 0x6d, 0x65, 0x74, 0x61,
-	0x64, 0x61, 0x74, 0x61, 0x2e, 0x53, 0x6c, 0x61, 0x63, 0x6b, 0x48, 0x00, 0x52, 0x05, 0x73, 0x6c,
-	0x61, 0x63, 0x6b, 0x12, 0x3d, 0x0a, 0x0a, 0x66, 0x69, 0x6c, 0x65, 0x73, 0x79, 0x73, 0x74, 0x65,
-	0x6d, 0x18, 0x0f, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1b, 0x2e, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65,
-	0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x46, 0x69, 0x6c, 0x65, 0x73, 0x79,
-	0x73, 0x74, 0x65, 0x6d, 0x48, 0x00, 0x52, 0x0a, 0x66, 0x69, 0x6c, 0x65, 0x73, 0x79, 0x73, 0x74,
-	0x65, 0x6d, 0x12, 0x28, 0x0a, 0x03, 0x67, 0x69, 0x74, 0x18, 0x10, 0x20, 0x01, 0x28, 0x0b, 0x32,
-	0x14, 0x2e, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74,
-	0x61, 0x2e, 0x47, 0x69, 0x74, 0x48, 0x00, 0x52, 0x03, 0x67, 0x69, 0x74, 0x12, 0x2b, 0x0a, 0x04,
-	0x74, 0x65, 0x73, 0x74, 0x18, 0x11, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x15, 0x2e, 0x73, 0x6f, 0x75,
-	0x72, 0x63, 0x65, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x54, 0x65, 0x73,
-	0x74, 0x48, 0x00, 0x52, 0x04, 0x74, 0x65, 0x73, 0x74, 0x12, 0x3a, 0x0a, 0x09, 0x62, 0x75, 0x69,
-	0x6c, 0x64, 0x6b, 0x69, 0x74, 0x65, 0x18, 0x12, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x73,
-	0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x42,
-	0x75, 0x69, 0x6c, 0x64, 0x6b, 0x69, 0x74, 0x65, 0x48, 0x00, 0x52, 0x09, 0x62, 0x75, 0x69, 0x6c,
-	0x64, 0x6b, 0x69, 0x74, 0x65, 0x12, 0x31, 0x0a, 0x06, 0x67, 0x65, 0x72, 0x72, 0x69, 0x74, 0x18,
-	0x13, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x17, 0x2e, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f, 0x6d,
-	0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x47, 0x65, 0x72, 0x72, 0x69, 0x74, 0x48, 0x00,
-	0x52, 0x06, 0x67, 0x65, 0x72, 0x72, 0x69, 0x74, 0x12, 0x34, 0x0a, 0x07, 0x6a, 0x65, 0x6e, 0x6b,
-	0x69, 0x6e, 0x73, 0x18, 0x14, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x18, 0x2e, 0x73, 0x6f, 0x75, 0x72,
-	0x63, 0x65, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x4a, 0x65, 0x6e, 0x6b,
-	0x69, 0x6e, 0x73, 0x48, 0x00, 0x52, 0x07, 0x6a, 0x65, 0x6e, 0x6b, 0x69, 0x6e, 0x73, 0x12, 0x2e,
-	0x0a, 0x05, 0x74, 0x65, 0x61, 0x6d, 0x73, 0x18, 0x15, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x16, 0x2e,
-	0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e,
-	0x54, 0x65, 0x61, 0x6d, 0x73, 0x48, 0x00, 0x52, 0x05, 0x74, 0x65, 0x61, 0x6d, 0x73, 0x12, 0x40,
-	0x0a, 0x0b, 0x61, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x6f, 0x72, 0x79, 0x18, 0x16, 0x20,
-	0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f, 0x6d, 0x65, 0x74,
-	0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x6f, 0x72,
-	0x79, 0x48, 0x00, 0x52, 0x0b, 0x61, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x6f, 0x72, 0x79,
-	0x12, 0x31, 0x0a, 0x06, 0x73, 0x79, 0x73, 0x6c, 0x6f, 0x67, 0x18, 0x17, 0x20, 0x01, 0x28, 0x0b,
+	0x64, 0x69, 0x66, 0x69, 0x65, 0x64, 0x42, 0x79, 0x22, 0xff, 0x0a, 0x0a, 0x08, 0x4d, 0x65, 0x74,
+	0x61, 0x44, 0x61, 0x74, 0x61, 0x12, 0x2e, 0x0a, 0x05, 0x61, 0x7a, 0x75, 0x72, 0x65, 0x18, 0x01,
+	0x20, 0x01, 0x28, 0x0b, 0x32, 0x16, 0x2e, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f, 0x6d, 0x65,
+	0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x41, 0x7a, 0x75, 0x72, 0x65, 0x48, 0x00, 0x52, 0x05,
+	0x61, 0x7a, 0x75, 0x72, 0x65, 0x12, 0x3a, 0x0a, 0x09, 0x62, 0x69, 0x74, 0x62, 0x75, 0x63, 0x6b,
+	0x65, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x73, 0x6f, 0x75, 0x72, 0x63,
+	0x65, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x42, 0x69, 0x74, 0x62, 0x75,
+	0x63, 0x6b, 0x65, 0x74, 0x48, 0x00, 0x52, 0x09, 0x62, 0x69, 0x74, 0x62, 0x75, 0x63, 0x6b, 0x65,
+	0x74, 0x12, 0x37, 0x0a, 0x08, 0x63, 0x69, 0x72, 0x63, 0x6c, 0x65, 0x63, 0x69, 0x18, 0x03, 0x20,
+	0x01, 0x28, 0x0b, 0x32, 0x19, 0x2e, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f, 0x6d, 0x65, 0x74,
+	0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x43, 0x69, 0x72, 0x63, 0x6c, 0x65, 0x43, 0x49, 0x48, 0x00,
+	0x52, 0x08, 0x63, 0x69, 0x72, 0x63, 0x6c, 0x65, 0x63, 0x69, 0x12, 0x3d, 0x0a, 0x0a, 0x63, 0x6f,
+	0x6e, 0x66, 0x6c, 0x75, 0x65, 0x6e, 0x63, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1b,
+	0x2e, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61,
+	0x2e, 0x43, 0x6f, 0x6e, 0x66, 0x6c, 0x75, 0x65, 0x6e, 0x63, 0x65, 0x48, 0x00, 0x52, 0x0a, 0x63,
+	0x6f, 0x6e, 0x66, 0x6c, 0x75, 0x65, 0x6e, 0x63, 0x65, 0x12, 0x31, 0x0a, 0x06, 0x64, 0x6f, 0x63,
+	0x6b, 0x65, 0x72, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x17, 0x2e, 0x73, 0x6f, 0x75, 0x72,
+	0x63, 0x65, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x44, 0x6f, 0x63, 0x6b,
+	0x65, 0x72, 0x48, 0x00, 0x52, 0x06, 0x64, 0x6f, 0x63, 0x6b, 0x65, 0x72, 0x12, 0x28, 0x0a, 0x03,
+	0x65, 0x63, 0x72, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x73, 0x6f, 0x75, 0x72,
+	0x63, 0x65, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x45, 0x43, 0x52, 0x48,
+	0x00, 0x52, 0x03, 0x65, 0x63, 0x72, 0x12, 0x28, 0x0a, 0x03, 0x67, 0x63, 0x73, 0x18, 0x07, 0x20,
+	0x01, 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f, 0x6d, 0x65, 0x74,
+	0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x47, 0x43, 0x53, 0x48, 0x00, 0x52, 0x03, 0x67, 0x63, 0x73,
+	0x12, 0x31, 0x0a, 0x06, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x18, 0x08, 0x20, 0x01, 0x28, 0x0b,
 	0x32, 0x17, 0x2e, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61,
-	0x74, 0x61, 0x2e, 0x53, 0x79, 0x73, 0x6c, 0x6f, 0x67, 0x48, 0x00, 0x52, 0x06, 0x73, 0x79, 0x73,
-	0x6c, 0x6f, 0x67, 0x12, 0x5e, 0x0a, 0x15, 0x70, 0x75, 0x62, 0x6c, 0x69, 0x63, 0x45, 0x76, 0x65,
-	0x6e, 0x74, 0x4d, 0x6f, 0x6e, 0x69, 0x74, 0x6f, 0x72, 0x69, 0x6e, 0x67, 0x18, 0x18, 0x20, 0x01,
-	0x28, 0x0b, 0x32, 0x26, 0x2e, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f, 0x6d, 0x65, 0x74, 0x61,
-	0x64, 0x61, 0x74, 0x61, 0x2e, 0x50, 0x75, 0x62, 0x6c, 0x69, 0x63, 0x45, 0x76, 0x65, 0x6e, 0x74,
-	0x4d, 0x6f, 0x6e, 0x69, 0x74, 0x6f, 0x72, 0x69, 0x6e, 0x67, 0x48, 0x00, 0x52, 0x15, 0x70, 0x75,
-	0x62, 0x6c, 0x69, 0x63, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x4d, 0x6f, 0x6e, 0x69, 0x74, 0x6f, 0x72,
-	0x69, 0x6e, 0x67, 0x12, 0x3d, 0x0a, 0x0a, 0x73, 0x68, 0x61, 0x72, 0x65, 0x70, 0x6f, 0x69, 0x6e,
-	0x74, 0x18, 0x19, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1b, 0x2e, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65,
-	0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x53, 0x68, 0x61, 0x72, 0x65, 0x50,
-	0x6f, 0x69, 0x6e, 0x74, 0x48, 0x00, 0x52, 0x0a, 0x73, 0x68, 0x61, 0x72, 0x65, 0x70, 0x6f, 0x69,
-	0x6e, 0x74, 0x12, 0x40, 0x0a, 0x0b, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x44, 0x72, 0x69, 0x76,
-	0x65, 0x18, 0x1a, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65,
-	0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x47, 0x6f, 0x6f, 0x67, 0x6c, 0x65,
-	0x44, 0x72, 0x69, 0x76, 0x65, 0x48, 0x00, 0x52, 0x0b, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x44,
-	0x72, 0x69, 0x76, 0x65, 0x12, 0x3d, 0x0a, 0x0a, 0x61, 0x7a, 0x75, 0x72, 0x65, 0x52, 0x65, 0x70,
-	0x6f, 0x73, 0x18, 0x1b, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1b, 0x2e, 0x73, 0x6f, 0x75, 0x72, 0x63,
-	0x65, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x41, 0x7a, 0x75, 0x72, 0x65,
-	0x52, 0x65, 0x70, 0x6f, 0x73, 0x48, 0x00, 0x52, 0x0a, 0x61, 0x7a, 0x75, 0x72, 0x65, 0x52, 0x65,
-	0x70, 0x6f, 0x73, 0x42, 0x06, 0x0a, 0x04, 0x64, 0x61, 0x74, 0x61, 0x2a, 0x3e, 0x0a, 0x0a, 0x56,
-	0x69, 0x73, 0x69, 0x62, 0x69, 0x6c, 0x69, 0x74, 0x79, 0x12, 0x0a, 0x0a, 0x06, 0x70, 0x75, 0x62,
-	0x6c, 0x69, 0x63, 0x10, 0x00, 0x12, 0x0b, 0x0a, 0x07, 0x70, 0x72, 0x69, 0x76, 0x61, 0x74, 0x65,
-	0x10, 0x01, 0x12, 0x0a, 0x0a, 0x06, 0x73, 0x68, 0x61, 0x72, 0x65, 0x64, 0x10, 0x02, 0x12, 0x0b,
-	0x0a, 0x07, 0x75, 0x6e, 0x6b, 0x6e, 0x6f, 0x77, 0x6e, 0x10, 0x03, 0x42, 0x43, 0x5a, 0x41, 0x67,
-	0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x74, 0x72, 0x75, 0x66, 0x66, 0x6c,
-	0x65, 0x73, 0x65, 0x63, 0x75, 0x72, 0x69, 0x74, 0x79, 0x2f, 0x74, 0x72, 0x75, 0x66, 0x66, 0x6c,
-	0x65, 0x68, 0x6f, 0x67, 0x2f, 0x76, 0x33, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x70, 0x62, 0x2f, 0x73,
-	0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x70, 0x62,
-	0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
+	0x74, 0x61, 0x2e, 0x47, 0x69, 0x74, 0x68, 0x75, 0x62, 0x48, 0x00, 0x52, 0x06, 0x67, 0x69, 0x74,
+	0x68, 0x75, 0x62, 0x12, 0x31, 0x0a, 0x06, 0x67, 0x69, 0x74, 0x6c, 0x61, 0x62, 0x18, 0x09, 0x20,
+	0x01, 0x28, 0x0b, 0x32, 0x17, 0x2e, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f, 0x6d, 0x65, 0x74,
+	0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x47, 0x69, 0x74, 0x6c, 0x61, 0x62, 0x48, 0x00, 0x52, 0x06,
+	0x67, 0x69, 0x74, 0x6c, 0x61, 0x62, 0x12, 0x2b, 0x0a, 0x04, 0x6a, 0x69, 0x72, 0x61, 0x18, 0x0a,
+	0x20, 0x01, 0x28, 0x0b, 0x32, 0x15, 0x2e, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f, 0x6d, 0x65,
+	0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x4a, 0x69, 0x72, 0x61, 0x48, 0x00, 0x52, 0x04, 0x6a,
+	0x69, 0x72, 0x61, 0x12, 0x28, 0x0a, 0x03, 0x6e, 0x70, 0x6d, 0x18, 0x0b, 0x20, 0x01, 0x28, 0x0b,
+	0x32, 0x14, 0x2e, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61,
+	0x74, 0x61, 0x2e, 0x4e, 0x50, 0x4d, 0x48, 0x00, 0x52, 0x03, 0x6e, 0x70, 0x6d, 0x12, 0x2b, 0x0a,
+	0x04, 0x70, 0x79, 0x70, 0x69, 0x18, 0x0c, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x15, 0x2e, 0x73, 0x6f,
+	0x75, 0x72, 0x63, 0x65, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x50, 0x79,
+	0x50, 0x69, 0x48, 0x00, 0x52, 0x04, 0x70, 0x79, 0x70, 0x69, 0x12, 0x25, 0x0a, 0x02, 0x73, 0x33,
+	0x18, 0x0d, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x13, 0x2e, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f,
+	0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x53, 0x33, 0x48, 0x00, 0x52, 0x02, 0x73,
+	0x33, 0x12, 0x2e, 0x0a, 0x05, 0x73, 0x6c, 0x61, 0x63, 0x6b, 0x18, 0x0e, 0x20, 0x01, 0x28, 0x0b,
+	0x32, 0x16, 0x2e, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61,
+	0x74, 0x61, 0x2e, 0x53, 0x6c, 0x61, 0x63, 0x6b, 0x48, 0x00, 0x52, 0x05, 0x73, 0x6c, 0x61, 0x63,
+	0x6b, 0x12, 0x3d, 0x0a, 0x0a, 0x66, 0x69, 0x6c, 0x65, 0x73, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x18,
+	0x0f, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1b, 0x2e, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f, 0x6d,
+	0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x46, 0x69, 0x6c, 0x65, 0x73, 0x79, 0x73, 0x74,
+	0x65, 0x6d, 0x48, 0x00, 0x52, 0x0a, 0x66, 0x69, 0x6c, 0x65, 0x73, 0x79, 0x73, 0x74, 0x65, 0x6d,
+	0x12, 0x28, 0x0a, 0x03, 0x67, 0x69, 0x74, 0x18, 0x10, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x14, 0x2e,
+	0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e,
+	0x47, 0x69, 0x74, 0x48, 0x00, 0x52, 0x03, 0x67, 0x69, 0x74, 0x12, 0x2b, 0x0a, 0x04, 0x74, 0x65,
+	0x73, 0x74, 0x18, 0x11, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x15, 0x2e, 0x73, 0x6f, 0x75, 0x72, 0x63,
+	0x65, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x54, 0x65, 0x73, 0x74, 0x48,
+	0x00, 0x52, 0x04, 0x74, 0x65, 0x73, 0x74, 0x12, 0x3a, 0x0a, 0x09, 0x62, 0x75, 0x69, 0x6c, 0x64,
+	0x6b, 0x69, 0x74, 0x65, 0x18, 0x12, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x73, 0x6f, 0x75,
+	0x72, 0x63, 0x65, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x42, 0x75, 0x69,
+	0x6c, 0x64, 0x6b, 0x69, 0x74, 0x65, 0x48, 0x00, 0x52, 0x09, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x6b,
+	0x69, 0x74, 0x65, 0x12, 0x31, 0x0a, 0x06, 0x67, 0x65, 0x72, 0x72, 0x69, 0x74, 0x18, 0x13, 0x20,
+	0x01, 0x28, 0x0b, 0x32, 0x17, 0x2e, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f, 0x6d, 0x65, 0x74,
+	0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x47, 0x65, 0x72, 0x72, 0x69, 0x74, 0x48, 0x00, 0x52, 0x06,
+	0x67, 0x65, 0x72, 0x72, 0x69, 0x74, 0x12, 0x34, 0x0a, 0x07, 0x6a, 0x65, 0x6e, 0x6b, 0x69, 0x6e,
+	0x73, 0x18, 0x14, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x18, 0x2e, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65,
+	0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x4a, 0x65, 0x6e, 0x6b, 0x69, 0x6e,
+	0x73, 0x48, 0x00, 0x52, 0x07, 0x6a, 0x65, 0x6e, 0x6b, 0x69, 0x6e, 0x73, 0x12, 0x2e, 0x0a, 0x05,
+	0x74, 0x65, 0x61, 0x6d, 0x73, 0x18, 0x15, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x16, 0x2e, 0x73, 0x6f,
+	0x75, 0x72, 0x63, 0x65, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x54, 0x65,
+	0x61, 0x6d, 0x73, 0x48, 0x00, 0x52, 0x05, 0x74, 0x65, 0x61, 0x6d, 0x73, 0x12, 0x40, 0x0a, 0x0b,
+	0x61, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x6f, 0x72, 0x79, 0x18, 0x16, 0x20, 0x01, 0x28,
+	0x0b, 0x32, 0x1c, 0x2e, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64,
+	0x61, 0x74, 0x61, 0x2e, 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x6f, 0x72, 0x79, 0x48,
+	0x00, 0x52, 0x0b, 0x61, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x6f, 0x72, 0x79, 0x12, 0x31,
+	0x0a, 0x06, 0x73, 0x79, 0x73, 0x6c, 0x6f, 0x67, 0x18, 0x17, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x17,
+	0x2e, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61,
+	0x2e, 0x53, 0x79, 0x73, 0x6c, 0x6f, 0x67, 0x48, 0x00, 0x52, 0x06, 0x73, 0x79, 0x73, 0x6c, 0x6f,
+	0x67, 0x12, 0x5e, 0x0a, 0x15, 0x70, 0x75, 0x62, 0x6c, 0x69, 0x63, 0x45, 0x76, 0x65, 0x6e, 0x74,
+	0x4d, 0x6f, 0x6e, 0x69, 0x74, 0x6f, 0x72, 0x69, 0x6e, 0x67, 0x18, 0x18, 0x20, 0x01, 0x28, 0x0b,
+	0x32, 0x26, 0x2e, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61,
+	0x74, 0x61, 0x2e, 0x50, 0x75, 0x62, 0x6c, 0x69, 0x63, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x4d, 0x6f,
+	0x6e, 0x69, 0x74, 0x6f, 0x72, 0x69, 0x6e, 0x67, 0x48, 0x00, 0x52, 0x15, 0x70, 0x75, 0x62, 0x6c,
+	0x69, 0x63, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x4d, 0x6f, 0x6e, 0x69, 0x74, 0x6f, 0x72, 0x69, 0x6e,
+	0x67, 0x12, 0x3d, 0x0a, 0x0a, 0x73, 0x68, 0x61, 0x72, 0x65, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x18,
+	0x19, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1b, 0x2e, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f, 0x6d,
+	0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x53, 0x68, 0x61, 0x72, 0x65, 0x50, 0x6f, 0x69,
+	0x6e, 0x74, 0x48, 0x00, 0x52, 0x0a, 0x73, 0x68, 0x61, 0x72, 0x65, 0x70, 0x6f, 0x69, 0x6e, 0x74,
+	0x12, 0x40, 0x0a, 0x0b, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x44, 0x72, 0x69, 0x76, 0x65, 0x18,
+	0x1a, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f, 0x6d,
+	0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x47, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x44, 0x72,
+	0x69, 0x76, 0x65, 0x48, 0x00, 0x52, 0x0b, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x44, 0x72, 0x69,
+	0x76, 0x65, 0x42, 0x06, 0x0a, 0x04, 0x64, 0x61, 0x74, 0x61, 0x2a, 0x3e, 0x0a, 0x0a, 0x56, 0x69,
+	0x73, 0x69, 0x62, 0x69, 0x6c, 0x69, 0x74, 0x79, 0x12, 0x0a, 0x0a, 0x06, 0x70, 0x75, 0x62, 0x6c,
+	0x69, 0x63, 0x10, 0x00, 0x12, 0x0b, 0x0a, 0x07, 0x70, 0x72, 0x69, 0x76, 0x61, 0x74, 0x65, 0x10,
+	0x01, 0x12, 0x0a, 0x0a, 0x06, 0x73, 0x68, 0x61, 0x72, 0x65, 0x64, 0x10, 0x02, 0x12, 0x0b, 0x0a,
+	0x07, 0x75, 0x6e, 0x6b, 0x6e, 0x6f, 0x77, 0x6e, 0x10, 0x03, 0x42, 0x43, 0x5a, 0x41, 0x67, 0x69,
+	0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x74, 0x72, 0x75, 0x66, 0x66, 0x6c, 0x65,
+	0x73, 0x65, 0x63, 0x75, 0x72, 0x69, 0x74, 0x79, 0x2f, 0x74, 0x72, 0x75, 0x66, 0x66, 0x6c, 0x65,
+	0x68, 0x6f, 0x67, 0x2f, 0x76, 0x33, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x70, 0x62, 0x2f, 0x73, 0x6f,
+	0x75, 0x72, 0x63, 0x65, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x70, 0x62, 0x62,
+	0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
 }
 
 var (
@@ -3348,7 +3173,7 @@ func file_source_metadata_proto_rawDescGZIP() []byte {
 }
 
 var file_source_metadata_proto_enumTypes = make([]protoimpl.EnumInfo, 1)
-var file_source_metadata_proto_msgTypes = make([]protoimpl.MessageInfo, 28)
+var file_source_metadata_proto_msgTypes = make([]protoimpl.MessageInfo, 27)
 var file_source_metadata_proto_goTypes = []interface{}{
 	(Visibility)(0),               // 0: source_metadata.Visibility
 	(*Azure)(nil),                 // 1: source_metadata.Azure
@@ -3377,46 +3202,43 @@ var file_source_metadata_proto_goTypes = []interface{}{
 	(*PublicEventMonitoring)(nil), // 24: source_metadata.PublicEventMonitoring
 	(*SharePoint)(nil),            // 25: source_metadata.SharePoint
 	(*GoogleDrive)(nil),           // 26: source_metadata.GoogleDrive
-	(*AzureRepos)(nil),            // 27: source_metadata.AzureRepos
-	(*MetaData)(nil),              // 28: source_metadata.MetaData
+	(*MetaData)(nil),              // 27: source_metadata.MetaData
 }
 var file_source_metadata_proto_depIdxs = []int32{
 	0,  // 0: source_metadata.Github.visibility:type_name -> source_metadata.Visibility
 	0,  // 1: source_metadata.Slack.visibility:type_name -> source_metadata.Visibility
 	10, // 2: source_metadata.PublicEventMonitoring.github:type_name -> source_metadata.Github
-	0,  // 3: source_metadata.AzureRepos.visibility:type_name -> source_metadata.Visibility
-	1,  // 4: source_metadata.MetaData.azure:type_name -> source_metadata.Azure
-	2,  // 5: source_metadata.MetaData.bitbucket:type_name -> source_metadata.Bitbucket
-	4,  // 6: source_metadata.MetaData.circleci:type_name -> source_metadata.CircleCI
-	5,  // 7: source_metadata.MetaData.confluence:type_name -> source_metadata.Confluence
-	6,  // 8: source_metadata.MetaData.docker:type_name -> source_metadata.Docker
-	7,  // 9: source_metadata.MetaData.ecr:type_name -> source_metadata.ECR
-	12, // 10: source_metadata.MetaData.gcs:type_name -> source_metadata.GCS
-	10, // 11: source_metadata.MetaData.github:type_name -> source_metadata.Github
-	11, // 12: source_metadata.MetaData.gitlab:type_name -> source_metadata.Gitlab
-	13, // 13: source_metadata.MetaData.jira:type_name -> source_metadata.Jira
-	14, // 14: source_metadata.MetaData.npm:type_name -> source_metadata.NPM
-	15, // 15: source_metadata.MetaData.pypi:type_name -> source_metadata.PyPi
-	16, // 16: source_metadata.MetaData.s3:type_name -> source_metadata.S3
-	17, // 17: source_metadata.MetaData.slack:type_name -> source_metadata.Slack
-	8,  // 18: source_metadata.MetaData.filesystem:type_name -> source_metadata.Filesystem
-	9,  // 19: source_metadata.MetaData.git:type_name -> source_metadata.Git
-	19, // 20: source_metadata.MetaData.test:type_name -> source_metadata.Test
-	3,  // 21: source_metadata.MetaData.buildkite:type_name -> source_metadata.Buildkite
-	18, // 22: source_metadata.MetaData.gerrit:type_name -> source_metadata.Gerrit
-	20, // 23: source_metadata.MetaData.jenkins:type_name -> source_metadata.Jenkins
-	21, // 24: source_metadata.MetaData.teams:type_name -> source_metadata.Teams
-	22, // 25: source_metadata.MetaData.artifactory:type_name -> source_metadata.Artifactory
-	23, // 26: source_metadata.MetaData.syslog:type_name -> source_metadata.Syslog
-	24, // 27: source_metadata.MetaData.publicEventMonitoring:type_name -> source_metadata.PublicEventMonitoring
-	25, // 28: source_metadata.MetaData.sharepoint:type_name -> source_metadata.SharePoint
-	26, // 29: source_metadata.MetaData.googleDrive:type_name -> source_metadata.GoogleDrive
-	27, // 30: source_metadata.MetaData.azureRepos:type_name -> source_metadata.AzureRepos
-	31, // [31:31] is the sub-list for method output_type
-	31, // [31:31] is the sub-list for method input_type
-	31, // [31:31] is the sub-list for extension type_name
-	31, // [31:31] is the sub-list for extension extendee
-	0,  // [0:31] is the sub-list for field type_name
+	1,  // 3: source_metadata.MetaData.azure:type_name -> source_metadata.Azure
+	2,  // 4: source_metadata.MetaData.bitbucket:type_name -> source_metadata.Bitbucket
+	4,  // 5: source_metadata.MetaData.circleci:type_name -> source_metadata.CircleCI
+	5,  // 6: source_metadata.MetaData.confluence:type_name -> source_metadata.Confluence
+	6,  // 7: source_metadata.MetaData.docker:type_name -> source_metadata.Docker
+	7,  // 8: source_metadata.MetaData.ecr:type_name -> source_metadata.ECR
+	12, // 9: source_metadata.MetaData.gcs:type_name -> source_metadata.GCS
+	10, // 10: source_metadata.MetaData.github:type_name -> source_metadata.Github
+	11, // 11: source_metadata.MetaData.gitlab:type_name -> source_metadata.Gitlab
+	13, // 12: source_metadata.MetaData.jira:type_name -> source_metadata.Jira
+	14, // 13: source_metadata.MetaData.npm:type_name -> source_metadata.NPM
+	15, // 14: source_metadata.MetaData.pypi:type_name -> source_metadata.PyPi
+	16, // 15: source_metadata.MetaData.s3:type_name -> source_metadata.S3
+	17, // 16: source_metadata.MetaData.slack:type_name -> source_metadata.Slack
+	8,  // 17: source_metadata.MetaData.filesystem:type_name -> source_metadata.Filesystem
+	9,  // 18: source_metadata.MetaData.git:type_name -> source_metadata.Git
+	19, // 19: source_metadata.MetaData.test:type_name -> source_metadata.Test
+	3,  // 20: source_metadata.MetaData.buildkite:type_name -> source_metadata.Buildkite
+	18, // 21: source_metadata.MetaData.gerrit:type_name -> source_metadata.Gerrit
+	20, // 22: source_metadata.MetaData.jenkins:type_name -> source_metadata.Jenkins
+	21, // 23: source_metadata.MetaData.teams:type_name -> source_metadata.Teams
+	22, // 24: source_metadata.MetaData.artifactory:type_name -> source_metadata.Artifactory
+	23, // 25: source_metadata.MetaData.syslog:type_name -> source_metadata.Syslog
+	24, // 26: source_metadata.MetaData.publicEventMonitoring:type_name -> source_metadata.PublicEventMonitoring
+	25, // 27: source_metadata.MetaData.sharepoint:type_name -> source_metadata.SharePoint
+	26, // 28: source_metadata.MetaData.googleDrive:type_name -> source_metadata.GoogleDrive
+	29, // [29:29] is the sub-list for method output_type
+	29, // [29:29] is the sub-list for method input_type
+	29, // [29:29] is the sub-list for extension type_name
+	29, // [29:29] is the sub-list for extension extendee
+	0,  // [0:29] is the sub-list for field type_name
 }
 
 func init() { file_source_metadata_proto_init() }
@@ -3738,18 +3560,6 @@ func file_source_metadata_proto_init() {
 			}
 		}
 		file_source_metadata_proto_msgTypes[26].Exporter = func(v interface{}, i int) interface{} {
-			switch v := v.(*AzureRepos); i {
-			case 0:
-				return &v.state
-			case 1:
-				return &v.sizeCache
-			case 2:
-				return &v.unknownFields
-			default:
-				return nil
-			}
-		}
-		file_source_metadata_proto_msgTypes[27].Exporter = func(v interface{}, i int) interface{} {
 			switch v := v.(*MetaData); i {
 			case 0:
 				return &v.state
@@ -3765,7 +3575,7 @@ func file_source_metadata_proto_init() {
 	file_source_metadata_proto_msgTypes[23].OneofWrappers = []interface{}{
 		(*PublicEventMonitoring_Github)(nil),
 	}
-	file_source_metadata_proto_msgTypes[27].OneofWrappers = []interface{}{
+	file_source_metadata_proto_msgTypes[26].OneofWrappers = []interface{}{
 		(*MetaData_Azure)(nil),
 		(*MetaData_Bitbucket)(nil),
 		(*MetaData_Circleci)(nil),
@@ -3792,7 +3602,6 @@ func file_source_metadata_proto_init() {
 		(*MetaData_PublicEventMonitoring)(nil),
 		(*MetaData_Sharepoint)(nil),
 		(*MetaData_GoogleDrive)(nil),
-		(*MetaData_AzureRepos)(nil),
 	}
 	type x struct{}
 	out := protoimpl.TypeBuilder{
@@ -3800,7 +3609,7 @@ func file_source_metadata_proto_init() {
 			GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
 			RawDescriptor: file_source_metadata_proto_rawDesc,
 			NumEnums:      1,
-			NumMessages:   28,
+			NumMessages:   27,
 			NumExtensions: 0,
 			NumServices:   0,
 		},

pkg/pb/source_metadatapb/source_metadata.pb.validate.go

@@ -2865,8 +2865,6 @@ func (m *GoogleDrive) validate(all bool) error {
 
 	// no validation rules for LastModifiedBy
 
-	// no validation rules for Path
-
 	if len(errors) > 0 {
 		return GoogleDriveMultiError(errors)
 	}
@@ -2944,127 +2942,6 @@ var _ interface {
 	ErrorName() string
 } = GoogleDriveValidationError{}
 
-// Validate checks the field values on AzureRepos with the rules defined in the
-// proto definition for this message. If any rules are violated, the first
-// error encountered is returned, or nil if there are no violations.
-func (m *AzureRepos) Validate() error {
-	return m.validate(false)
-}
-
-// ValidateAll checks the field values on AzureRepos with the rules defined in
-// the proto definition for this message. If any rules are violated, the
-// result is a list of violation errors wrapped in AzureReposMultiError, or
-// nil if none found.
-func (m *AzureRepos) ValidateAll() error {
-	return m.validate(true)
-}
-
-func (m *AzureRepos) validate(all bool) error {
-	if m == nil {
-		return nil
-	}
-
-	var errors []error
-
-	// no validation rules for Link
-
-	// no validation rules for Username
-
-	// no validation rules for Repository
-
-	// no validation rules for Commit
-
-	// no validation rules for Email
-
-	// no validation rules for File
-
-	// no validation rules for Timestamp
-
-	// no validation rules for Line
-
-	// no validation rules for Visibility
-
-	// no validation rules for Project
-
-	// no validation rules for Organization
-
-	if len(errors) > 0 {
-		return AzureReposMultiError(errors)
-	}
-
-	return nil
-}
-
-// AzureReposMultiError is an error wrapping multiple validation errors
-// returned by AzureRepos.ValidateAll() if the designated constraints aren't met.
-type AzureReposMultiError []error
-
-// Error returns a concatenation of all the error messages it wraps.
-func (m AzureReposMultiError) Error() string {
-	var msgs []string
-	for _, err := range m {
-		msgs = append(msgs, err.Error())
-	}
-	return strings.Join(msgs, "; ")
-}
-
-// AllErrors returns a list of validation violation errors.
-func (m AzureReposMultiError) AllErrors() []error { return m }
-
-// AzureReposValidationError is the validation error returned by
-// AzureRepos.Validate if the designated constraints aren't met.
-type AzureReposValidationError struct {
-	field  string
-	reason string
-	cause  error
-	key    bool
-}
-
-// Field function returns field value.
-func (e AzureReposValidationError) Field() string { return e.field }
-
-// Reason function returns reason value.
-func (e AzureReposValidationError) Reason() string { return e.reason }
-
-// Cause function returns cause value.
-func (e AzureReposValidationError) Cause() error { return e.cause }
-
-// Key function returns key value.
-func (e AzureReposValidationError) Key() bool { return e.key }
-
-// ErrorName returns error name.
-func (e AzureReposValidationError) ErrorName() string { return "AzureReposValidationError" }
-
-// Error satisfies the builtin error interface
-func (e AzureReposValidationError) Error() string {
-	cause := ""
-	if e.cause != nil {
-		cause = fmt.Sprintf(" | caused by: %v", e.cause)
-	}
-
-	key := ""
-	if e.key {
-		key = "key for "
-	}
-
-	return fmt.Sprintf(
-		"invalid %sAzureRepos.%s: %s%s",
-		key,
-		e.field,
-		e.reason,
-		cause)
-}
-
-var _ error = AzureReposValidationError{}
-
-var _ interface {
-	Field() string
-	Reason() string
-	Key() bool
-	Cause() error
-	ErrorName() string
-} = AzureReposValidationError{}
-
 // Validate checks the field values on MetaData with the rules defined in the
 // proto definition for this message. If any rules are violated, the first
 // error encountered is returned, or nil if there are no violations.
@@ -3895,37 +3772,6 @@ func (m *MetaData) validate(all bool) error {
 			}
 		}
 
-	case *MetaData_AzureRepos:
-
-		if all {
-			switch v := interface{}(m.GetAzureRepos()).(type) {
-			case interface{ ValidateAll() error }:
-				if err := v.ValidateAll(); err != nil {
-					errors = append(errors, MetaDataValidationError{
-						field:  "AzureRepos",
-						reason: "embedded message failed validation",
-						cause:  err,
-					})
-				}
-			case interface{ Validate() error }:
-				if err := v.Validate(); err != nil {
-					errors = append(errors, MetaDataValidationError{
-						field:  "AzureRepos",
-						reason: "embedded message failed validation",
-						cause:  err,
-					})
-				}
-			}
-		} else if v, ok := interface{}(m.GetAzureRepos()).(interface{ Validate() error }); ok {
-			if err := v.Validate(); err != nil {
-				return MetaDataValidationError{
-					field:  "AzureRepos",
-					reason: "embedded message failed validation",
-					cause:  err,
-				}
-			}
-		}
-
 	}
 
 	if len(errors) > 0 {

pkg/pb/sourcespb/sources.pb.go

@@ -59,7 +59,6 @@ const (
 	SourceType_SOURCE_TYPE_GOOGLE_DRIVE               SourceType = 28
 	SourceType_SOURCE_TYPE_SHAREPOINT                 SourceType = 29
 	SourceType_SOURCE_TYPE_GCS_UNAUTHED               SourceType = 30
-	SourceType_SOURCE_TYPE_AZURE_REPOS                SourceType = 31
 )
 
 // Enum value maps for SourceType.
@@ -96,7 +95,6 @@ var (
 		28: "SOURCE_TYPE_GOOGLE_DRIVE",
 		29: "SOURCE_TYPE_SHAREPOINT",
 		30: "SOURCE_TYPE_GCS_UNAUTHED",
-		31: "SOURCE_TYPE_AZURE_REPOS",
 	}
 	SourceType_value = map[string]int32{
 		"SOURCE_TYPE_AZURE_STORAGE":              0,
@@ -130,7 +128,6 @@ var (
 		"SOURCE_TYPE_GOOGLE_DRIVE":               28,
 		"SOURCE_TYPE_SHAREPOINT":                 29,
 		"SOURCE_TYPE_GCS_UNAUTHED":               30,
-		"SOURCE_TYPE_AZURE_REPOS":                31,
 	}
 )
 
@@ -1417,18 +1414,15 @@ type GitHub struct {
 	//	*GitHub_Token
 	//	*GitHub_Unauthenticated
 	//	*GitHub_BasicAuth
-	Credential                 isGitHub_Credential `protobuf_oneof:"credential"`
-	Repositories               []string            `protobuf:"bytes,5,rep,name=repositories,proto3" json:"repositories,omitempty"`
-	Organizations              []string            `protobuf:"bytes,6,rep,name=organizations,proto3" json:"organizations,omitempty"`
-	ScanUsers                  bool                `protobuf:"varint,7,opt,name=scanUsers,proto3" json:"scanUsers,omitempty"`
-	IncludeForks               bool                `protobuf:"varint,8,opt,name=includeForks,proto3" json:"includeForks,omitempty"`
-	Head                       string              `protobuf:"bytes,9,opt,name=head,proto3" json:"head,omitempty"`
-	Base                       string              `protobuf:"bytes,10,opt,name=base,proto3" json:"base,omitempty"`
-	IgnoreRepos                []string            `protobuf:"bytes,11,rep,name=ignoreRepos,proto3" json:"ignoreRepos,omitempty"`
-	IncludeRepos               []string            `protobuf:"bytes,12,rep,name=includeRepos,proto3" json:"includeRepos,omitempty"`
-	IncludePullRequestComments bool                `protobuf:"varint,14,opt,name=includePullRequestComments,proto3" json:"includePullRequestComments,omitempty"`
-	IncludeIssueComments       bool                `protobuf:"varint,15,opt,name=includeIssueComments,proto3" json:"includeIssueComments,omitempty"`
-	IncludeGistComments        bool                `protobuf:"varint,16,opt,name=includeGistComments,proto3" json:"includeGistComments,omitempty"`
+	Credential    isGitHub_Credential `protobuf_oneof:"credential"`
+	Repositories  []string            `protobuf:"bytes,5,rep,name=repositories,proto3" json:"repositories,omitempty"`
+	Organizations []string            `protobuf:"bytes,6,rep,name=organizations,proto3" json:"organizations,omitempty"`
+	ScanUsers     bool                `protobuf:"varint,7,opt,name=scanUsers,proto3" json:"scanUsers,omitempty"`
+	IncludeForks  bool                `protobuf:"varint,8,opt,name=includeForks,proto3" json:"includeForks,omitempty"`
+	Head          string              `protobuf:"bytes,9,opt,name=head,proto3" json:"head,omitempty"`
+	Base          string              `protobuf:"bytes,10,opt,name=base,proto3" json:"base,omitempty"`
+	IgnoreRepos   []string            `protobuf:"bytes,11,rep,name=ignoreRepos,proto3" json:"ignoreRepos,omitempty"`
+	IncludeRepos  []string            `protobuf:"bytes,12,rep,name=includeRepos,proto3" json:"includeRepos,omitempty"`
 }
 
 func (x *GitHub) Reset() {
@@ -1561,27 +1555,6 @@ func (x *GitHub) GetIncludeRepos() []string {
 	return nil
 }
 
-func (x *GitHub) GetIncludePullRequestComments() bool {
-	if x != nil {
-		return x.IncludePullRequestComments
-	}
-	return false
-}
-
-func (x *GitHub) GetIncludeIssueComments() bool {
-	if x != nil {
-		return x.IncludeIssueComments
-	}
-	return false
-}
-
-func (x *GitHub) GetIncludeGistComments() bool {
-	if x != nil {
-		return x.IncludeGistComments
-	}
-	return false
-}
-
 type isGitHub_Credential interface {
 	isGitHub_Credential()
 }
@@ -2988,158 +2961,6 @@ type Sharepoint_Oauth struct {
 
 func (*Sharepoint_Oauth) isSharepoint_Credential() {}
 
-type AzureRepos struct {
-	state         protoimpl.MessageState
-	sizeCache     protoimpl.SizeCache
-	unknownFields protoimpl.UnknownFields
-
-	Endpoint string `protobuf:"bytes,1,opt,name=endpoint,proto3" json:"endpoint,omitempty"`
-	// Types that are assignable to Credential:
-	//	*AzureRepos_Token
-	//	*AzureRepos_Oauth
-	Credential      isAzureRepos_Credential `protobuf_oneof:"credential"`
-	Repositories    []string                `protobuf:"bytes,4,rep,name=repositories,proto3" json:"repositories,omitempty"`
-	Organizations   []string                `protobuf:"bytes,5,rep,name=organizations,proto3" json:"organizations,omitempty"`
-	Projects        []string                `protobuf:"bytes,6,rep,name=projects,proto3" json:"projects,omitempty"`
-	IncludeForks    bool                    `protobuf:"varint,7,opt,name=includeForks,proto3" json:"includeForks,omitempty"`
-	IgnoreRepos     []string                `protobuf:"bytes,8,rep,name=ignoreRepos,proto3" json:"ignoreRepos,omitempty"`
-	IncludeRepos    []string                `protobuf:"bytes,9,rep,name=includeRepos,proto3" json:"includeRepos,omitempty"`
-	IncludeProjects []string                `protobuf:"bytes,10,rep,name=includeProjects,proto3" json:"includeProjects,omitempty"`
-	IgnoreProjects  []string                `protobuf:"bytes,11,rep,name=ignoreProjects,proto3" json:"ignoreProjects,omitempty"`
-}
-
-func (x *AzureRepos) Reset() {
-	*x = AzureRepos{}
-	if protoimpl.UnsafeEnabled {
-		mi := &file_sources_proto_msgTypes[28]
-		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
-		ms.StoreMessageInfo(mi)
-	}
-}
-
-func (x *AzureRepos) String() string {
-	return protoimpl.X.MessageStringOf(x)
-}
-
-func (*AzureRepos) ProtoMessage() {}
-
-func (x *AzureRepos) ProtoReflect() protoreflect.Message {
-	mi := &file_sources_proto_msgTypes[28]
-	if protoimpl.UnsafeEnabled && x != nil {
-		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
-		if ms.LoadMessageInfo() == nil {
-			ms.StoreMessageInfo(mi)
-		}
-		return ms
-	}
-	return mi.MessageOf(x)
-}
-
-// Deprecated: Use AzureRepos.ProtoReflect.Descriptor instead.
-func (*AzureRepos) Descriptor() ([]byte, []int) {
-	return file_sources_proto_rawDescGZIP(), []int{28}
-}
-
-func (x *AzureRepos) GetEndpoint() string {
-	if x != nil {
-		return x.Endpoint
-	}
-	return ""
-}
-
-func (m *AzureRepos) GetCredential() isAzureRepos_Credential {
-	if m != nil {
-		return m.Credential
-	}
-	return nil
-}
-
-func (x *AzureRepos) GetToken() string {
-	if x, ok := x.GetCredential().(*AzureRepos_Token); ok {
-		return x.Token
-	}
-	return ""
-}
-
-func (x *AzureRepos) GetOauth() *credentialspb.Oauth2 {
-	if x, ok := x.GetCredential().(*AzureRepos_Oauth); ok {
-		return x.Oauth
-	}
-	return nil
-}
-
-func (x *AzureRepos) GetRepositories() []string {
-	if x != nil {
-		return x.Repositories
-	}
-	return nil
-}
-
-func (x *AzureRepos) GetOrganizations() []string {
-	if x != nil {
-		return x.Organizations
-	}
-	return nil
-}
-
-func (x *AzureRepos) GetProjects() []string {
-	if x != nil {
-		return x.Projects
-	}
-	return nil
-}
-
-func (x *AzureRepos) GetIncludeForks() bool {
-	if x != nil {
-		return x.IncludeForks
-	}
-	return false
-}
-
-func (x *AzureRepos) GetIgnoreRepos() []string {
-	if x != nil {
-		return x.IgnoreRepos
-	}
-	return nil
-}
-
-func (x *AzureRepos) GetIncludeRepos() []string {
-	if x != nil {
-		return x.IncludeRepos
-	}
-	return nil
-}
-
-func (x *AzureRepos) GetIncludeProjects() []string {
-	if x != nil {
-		return x.IncludeProjects
-	}
-	return nil
-}
-
-func (x *AzureRepos) GetIgnoreProjects() []string {
-	if x != nil {
-		return x.IgnoreProjects
-	}
-	return nil
-}
-
-type isAzureRepos_Credential interface {
-	isAzureRepos_Credential()
-}
-
-type AzureRepos_Token struct {
-	Token string `protobuf:"bytes,2,opt,name=token,proto3,oneof"`
-}
-
-type AzureRepos_Oauth struct {
-	Oauth *credentialspb.Oauth2 `protobuf:"bytes,3,opt,name=oauth,proto3,oneof"`
-}
-
-func (*AzureRepos_Token) isAzureRepos_Credential() {}
-
-func (*AzureRepos_Oauth) isAzureRepos_Credential() {}
-
 var File_sources_proto protoreflect.FileDescriptor
 
 var file_sources_proto_rawDesc = []byte{
@@ -3336,7 +3157,7 @@ var file_sources_proto_rawDesc = []byte{
 	0x65, 0x70, 0x6f, 0x73, 0x69, 0x74, 0x6f, 0x72, 0x69, 0x65, 0x73, 0x12, 0x21, 0x0a, 0x0c, 0x69,
 	0x67, 0x6e, 0x6f, 0x72, 0x65, 0x5f, 0x72, 0x65, 0x70, 0x6f, 0x73, 0x18, 0x06, 0x20, 0x03, 0x28,
 	0x09, 0x52, 0x0b, 0x69, 0x67, 0x6e, 0x6f, 0x72, 0x65, 0x52, 0x65, 0x70, 0x6f, 0x73, 0x42, 0x0c,
-	0x0a, 0x0a, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x22, 0xb0, 0x05, 0x0a,
+	0x0a, 0x0a, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x22, 0x8a, 0x04, 0x0a,
 	0x06, 0x47, 0x69, 0x74, 0x48, 0x75, 0x62, 0x12, 0x24, 0x0a, 0x08, 0x65, 0x6e, 0x64, 0x70, 0x6f,
 	0x69, 0x6e, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x42, 0x08, 0xfa, 0x42, 0x05, 0x72, 0x03,
 	0x90, 0x01, 0x01, 0x52, 0x08, 0x65, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x12, 0x37, 0x0a,
@@ -3368,213 +3189,176 @@ var file_sources_proto_rawDesc = []byte{
 	0x65, 0x70, 0x6f, 0x73, 0x18, 0x0b, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0b, 0x69, 0x67, 0x6e, 0x6f,
 	0x72, 0x65, 0x52, 0x65, 0x70, 0x6f, 0x73, 0x12, 0x22, 0x0a, 0x0c, 0x69, 0x6e, 0x63, 0x6c, 0x75,
 	0x64, 0x65, 0x52, 0x65, 0x70, 0x6f, 0x73, 0x18, 0x0c, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0c, 0x69,
-	0x6e, 0x63, 0x6c, 0x75, 0x64, 0x65, 0x52, 0x65, 0x70, 0x6f, 0x73, 0x12, 0x3e, 0x0a, 0x1a, 0x69,
-	0x6e, 0x63, 0x6c, 0x75, 0x64, 0x65, 0x50, 0x75, 0x6c, 0x6c, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73,
-	0x74, 0x43, 0x6f, 0x6d, 0x6d, 0x65, 0x6e, 0x74, 0x73, 0x18, 0x0e, 0x20, 0x01, 0x28, 0x08, 0x52,
-	0x1a, 0x69, 0x6e, 0x63, 0x6c, 0x75, 0x64, 0x65, 0x50, 0x75, 0x6c, 0x6c, 0x52, 0x65, 0x71, 0x75,
-	0x65, 0x73, 0x74, 0x43, 0x6f, 0x6d, 0x6d, 0x65, 0x6e, 0x74, 0x73, 0x12, 0x32, 0x0a, 0x14, 0x69,
-	0x6e, 0x63, 0x6c, 0x75, 0x64, 0x65, 0x49, 0x73, 0x73, 0x75, 0x65, 0x43, 0x6f, 0x6d, 0x6d, 0x65,
-	0x6e, 0x74, 0x73, 0x18, 0x0f, 0x20, 0x01, 0x28, 0x08, 0x52, 0x14, 0x69, 0x6e, 0x63, 0x6c, 0x75,
-	0x64, 0x65, 0x49, 0x73, 0x73, 0x75, 0x65, 0x43, 0x6f, 0x6d, 0x6d, 0x65, 0x6e, 0x74, 0x73, 0x12,
-	0x30, 0x0a, 0x13, 0x69, 0x6e, 0x63, 0x6c, 0x75, 0x64, 0x65, 0x47, 0x69, 0x73, 0x74, 0x43, 0x6f,
-	0x6d, 0x6d, 0x65, 0x6e, 0x74, 0x73, 0x18, 0x10, 0x20, 0x01, 0x28, 0x08, 0x52, 0x13, 0x69, 0x6e,
-	0x63, 0x6c, 0x75, 0x64, 0x65, 0x47, 0x69, 0x73, 0x74, 0x43, 0x6f, 0x6d, 0x6d, 0x65, 0x6e, 0x74,
-	0x73, 0x42, 0x0c, 0x0a, 0x0a, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x22,
-	0x42, 0x0a, 0x0b, 0x47, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x44, 0x72, 0x69, 0x76, 0x65, 0x12, 0x25,
-	0x0a, 0x0d, 0x72, 0x65, 0x66, 0x72, 0x65, 0x73, 0x68, 0x5f, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x18,
-	0x01, 0x20, 0x01, 0x28, 0x09, 0x48, 0x00, 0x52, 0x0c, 0x72, 0x65, 0x66, 0x72, 0x65, 0x73, 0x68,
-	0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x42, 0x0c, 0x0a, 0x0a, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74,
-	0x69, 0x61, 0x6c, 0x22, 0xc7, 0x02, 0x0a, 0x04, 0x4a, 0x49, 0x52, 0x41, 0x12, 0x24, 0x0a, 0x08,
-	0x65, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x42, 0x08,
-	0xfa, 0x42, 0x05, 0x72, 0x03, 0x90, 0x01, 0x01, 0x52, 0x08, 0x65, 0x6e, 0x64, 0x70, 0x6f, 0x69,
-	0x6e, 0x74, 0x12, 0x37, 0x0a, 0x0a, 0x62, 0x61, 0x73, 0x69, 0x63, 0x5f, 0x61, 0x75, 0x74, 0x68,
-	0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x16, 0x2e, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74,
-	0x69, 0x61, 0x6c, 0x73, 0x2e, 0x42, 0x61, 0x73, 0x69, 0x63, 0x41, 0x75, 0x74, 0x68, 0x48, 0x00,
-	0x52, 0x09, 0x62, 0x61, 0x73, 0x69, 0x63, 0x41, 0x75, 0x74, 0x68, 0x12, 0x48, 0x0a, 0x0f, 0x75,
-	0x6e, 0x61, 0x75, 0x74, 0x68, 0x65, 0x6e, 0x74, 0x69, 0x63, 0x61, 0x74, 0x65, 0x64, 0x18, 0x03,
-	0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61,
-	0x6c, 0x73, 0x2e, 0x55, 0x6e, 0x61, 0x75, 0x74, 0x68, 0x65, 0x6e, 0x74, 0x69, 0x63, 0x61, 0x74,
-	0x65, 0x64, 0x48, 0x00, 0x52, 0x0f, 0x75, 0x6e, 0x61, 0x75, 0x74, 0x68, 0x65, 0x6e, 0x74, 0x69,
-	0x63, 0x61, 0x74, 0x65, 0x64, 0x12, 0x2b, 0x0a, 0x05, 0x6f, 0x61, 0x75, 0x74, 0x68, 0x18, 0x04,
-	0x20, 0x01, 0x28, 0x0b, 0x32, 0x13, 0x2e, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61,
-	0x6c, 0x73, 0x2e, 0x4f, 0x61, 0x75, 0x74, 0x68, 0x32, 0x48, 0x00, 0x52, 0x05, 0x6f, 0x61, 0x75,
-	0x74, 0x68, 0x12, 0x16, 0x0a, 0x05, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x18, 0x06, 0x20, 0x01, 0x28,
-	0x09, 0x48, 0x00, 0x52, 0x05, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x12, 0x1a, 0x0a, 0x08, 0x70, 0x72,
-	0x6f, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x18, 0x05, 0x20, 0x03, 0x28, 0x09, 0x52, 0x08, 0x70, 0x72,
-	0x6f, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x12, 0x27, 0x0a, 0x0f, 0x69, 0x67, 0x6e, 0x6f, 0x72, 0x65,
-	0x5f, 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x18, 0x07, 0x20, 0x03, 0x28, 0x09, 0x52,
-	0x0e, 0x69, 0x67, 0x6e, 0x6f, 0x72, 0x65, 0x50, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x42,
-	0x0c, 0x0a, 0x0a, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x22, 0x73, 0x0a,
-	0x19, 0x4e, 0x50, 0x4d, 0x55, 0x6e, 0x61, 0x75, 0x74, 0x68, 0x65, 0x6e, 0x74, 0x69, 0x63, 0x61,
+	0x6e, 0x63, 0x6c, 0x75, 0x64, 0x65, 0x52, 0x65, 0x70, 0x6f, 0x73, 0x42, 0x0c, 0x0a, 0x0a, 0x63,
+	0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x22, 0x42, 0x0a, 0x0b, 0x47, 0x6f, 0x6f,
+	0x67, 0x6c, 0x65, 0x44, 0x72, 0x69, 0x76, 0x65, 0x12, 0x25, 0x0a, 0x0d, 0x72, 0x65, 0x66, 0x72,
+	0x65, 0x73, 0x68, 0x5f, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x48,
+	0x00, 0x52, 0x0c, 0x72, 0x65, 0x66, 0x72, 0x65, 0x73, 0x68, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x42,
+	0x0c, 0x0a, 0x0a, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x22, 0xc7, 0x02,
+	0x0a, 0x04, 0x4a, 0x49, 0x52, 0x41, 0x12, 0x24, 0x0a, 0x08, 0x65, 0x6e, 0x64, 0x70, 0x6f, 0x69,
+	0x6e, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x42, 0x08, 0xfa, 0x42, 0x05, 0x72, 0x03, 0x90,
+	0x01, 0x01, 0x52, 0x08, 0x65, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x12, 0x37, 0x0a, 0x0a,
+	0x62, 0x61, 0x73, 0x69, 0x63, 0x5f, 0x61, 0x75, 0x74, 0x68, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b,
+	0x32, 0x16, 0x2e, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x73, 0x2e, 0x42,
+	0x61, 0x73, 0x69, 0x63, 0x41, 0x75, 0x74, 0x68, 0x48, 0x00, 0x52, 0x09, 0x62, 0x61, 0x73, 0x69,
+	0x63, 0x41, 0x75, 0x74, 0x68, 0x12, 0x48, 0x0a, 0x0f, 0x75, 0x6e, 0x61, 0x75, 0x74, 0x68, 0x65,
+	0x6e, 0x74, 0x69, 0x63, 0x61, 0x74, 0x65, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c,
+	0x2e, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x73, 0x2e, 0x55, 0x6e, 0x61,
+	0x75, 0x74, 0x68, 0x65, 0x6e, 0x74, 0x69, 0x63, 0x61, 0x74, 0x65, 0x64, 0x48, 0x00, 0x52, 0x0f,
+	0x75, 0x6e, 0x61, 0x75, 0x74, 0x68, 0x65, 0x6e, 0x74, 0x69, 0x63, 0x61, 0x74, 0x65, 0x64, 0x12,
+	0x2b, 0x0a, 0x05, 0x6f, 0x61, 0x75, 0x74, 0x68, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x13,
+	0x2e, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x73, 0x2e, 0x4f, 0x61, 0x75,
+	0x74, 0x68, 0x32, 0x48, 0x00, 0x52, 0x05, 0x6f, 0x61, 0x75, 0x74, 0x68, 0x12, 0x16, 0x0a, 0x05,
+	0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x48, 0x00, 0x52, 0x05, 0x74,
+	0x6f, 0x6b, 0x65, 0x6e, 0x12, 0x1a, 0x0a, 0x08, 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x73,
+	0x18, 0x05, 0x20, 0x03, 0x28, 0x09, 0x52, 0x08, 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x73,
+	0x12, 0x27, 0x0a, 0x0f, 0x69, 0x67, 0x6e, 0x6f, 0x72, 0x65, 0x5f, 0x70, 0x72, 0x6f, 0x6a, 0x65,
+	0x63, 0x74, 0x73, 0x18, 0x07, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0e, 0x69, 0x67, 0x6e, 0x6f, 0x72,
+	0x65, 0x50, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x42, 0x0c, 0x0a, 0x0a, 0x63, 0x72, 0x65,
+	0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x22, 0x73, 0x0a, 0x19, 0x4e, 0x50, 0x4d, 0x55, 0x6e,
+	0x61, 0x75, 0x74, 0x68, 0x65, 0x6e, 0x74, 0x69, 0x63, 0x61, 0x74, 0x65, 0x64, 0x50, 0x61, 0x63,
+	0x6b, 0x61, 0x67, 0x65, 0x12, 0x48, 0x0a, 0x0f, 0x75, 0x6e, 0x61, 0x75, 0x74, 0x68, 0x65, 0x6e,
+	0x74, 0x69, 0x63, 0x61, 0x74, 0x65, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e,
+	0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x73, 0x2e, 0x55, 0x6e, 0x61, 0x75,
+	0x74, 0x68, 0x65, 0x6e, 0x74, 0x69, 0x63, 0x61, 0x74, 0x65, 0x64, 0x48, 0x00, 0x52, 0x0f, 0x75,
+	0x6e, 0x61, 0x75, 0x74, 0x68, 0x65, 0x6e, 0x74, 0x69, 0x63, 0x61, 0x74, 0x65, 0x64, 0x42, 0x0c,
+	0x0a, 0x0a, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x22, 0x74, 0x0a, 0x1a,
+	0x50, 0x79, 0x50, 0x49, 0x55, 0x6e, 0x61, 0x75, 0x74, 0x68, 0x65, 0x6e, 0x74, 0x69, 0x63, 0x61,
 	0x74, 0x65, 0x64, 0x50, 0x61, 0x63, 0x6b, 0x61, 0x67, 0x65, 0x12, 0x48, 0x0a, 0x0f, 0x75, 0x6e,
 	0x61, 0x75, 0x74, 0x68, 0x65, 0x6e, 0x74, 0x69, 0x63, 0x61, 0x74, 0x65, 0x64, 0x18, 0x01, 0x20,
 	0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c,
 	0x73, 0x2e, 0x55, 0x6e, 0x61, 0x75, 0x74, 0x68, 0x65, 0x6e, 0x74, 0x69, 0x63, 0x61, 0x74, 0x65,
 	0x64, 0x48, 0x00, 0x52, 0x0f, 0x75, 0x6e, 0x61, 0x75, 0x74, 0x68, 0x65, 0x6e, 0x74, 0x69, 0x63,
 	0x61, 0x74, 0x65, 0x64, 0x42, 0x0c, 0x0a, 0x0a, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69,
-	0x61, 0x6c, 0x22, 0x74, 0x0a, 0x1a, 0x50, 0x79, 0x50, 0x49, 0x55, 0x6e, 0x61, 0x75, 0x74, 0x68,
-	0x65, 0x6e, 0x74, 0x69, 0x63, 0x61, 0x74, 0x65, 0x64, 0x50, 0x61, 0x63, 0x6b, 0x61, 0x67, 0x65,
+	0x61, 0x6c, 0x22, 0xf0, 0x02, 0x0a, 0x02, 0x53, 0x33, 0x12, 0x37, 0x0a, 0x0a, 0x61, 0x63, 0x63,
+	0x65, 0x73, 0x73, 0x5f, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x16, 0x2e,
+	0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x73, 0x2e, 0x4b, 0x65, 0x79, 0x53,
+	0x65, 0x63, 0x72, 0x65, 0x74, 0x48, 0x00, 0x52, 0x09, 0x61, 0x63, 0x63, 0x65, 0x73, 0x73, 0x4b,
+	0x65, 0x79, 0x12, 0x48, 0x0a, 0x0f, 0x75, 0x6e, 0x61, 0x75, 0x74, 0x68, 0x65, 0x6e, 0x74, 0x69,
+	0x63, 0x61, 0x74, 0x65, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x63, 0x72,
+	0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x73, 0x2e, 0x55, 0x6e, 0x61, 0x75, 0x74, 0x68,
+	0x65, 0x6e, 0x74, 0x69, 0x63, 0x61, 0x74, 0x65, 0x64, 0x48, 0x00, 0x52, 0x0f, 0x75, 0x6e, 0x61,
+	0x75, 0x74, 0x68, 0x65, 0x6e, 0x74, 0x69, 0x63, 0x61, 0x74, 0x65, 0x64, 0x12, 0x4c, 0x0a, 0x11,
+	0x63, 0x6c, 0x6f, 0x75, 0x64, 0x5f, 0x65, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e,
+	0x74, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1d, 0x2e, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e,
+	0x74, 0x69, 0x61, 0x6c, 0x73, 0x2e, 0x43, 0x6c, 0x6f, 0x75, 0x64, 0x45, 0x6e, 0x76, 0x69, 0x72,
+	0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x48, 0x00, 0x52, 0x10, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x45,
+	0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x12, 0x49, 0x0a, 0x0d, 0x73, 0x65,
+	0x73, 0x73, 0x69, 0x6f, 0x6e, 0x5f, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x18, 0x05, 0x20, 0x01, 0x28,
+	0x0b, 0x32, 0x22, 0x2e, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x73, 0x2e,
+	0x41, 0x57, 0x53, 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x53,
+	0x65, 0x63, 0x72, 0x65, 0x74, 0x48, 0x00, 0x52, 0x0c, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e,
+	0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x12, 0x18, 0x0a, 0x07, 0x62, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x73,
+	0x18, 0x03, 0x20, 0x03, 0x28, 0x09, 0x52, 0x07, 0x62, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x73, 0x12,
+	0x26, 0x0a, 0x0f, 0x6d, 0x61, 0x78, 0x5f, 0x6f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x5f, 0x73, 0x69,
+	0x7a, 0x65, 0x18, 0x06, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0d, 0x6d, 0x61, 0x78, 0x4f, 0x62, 0x6a,
+	0x65, 0x63, 0x74, 0x53, 0x69, 0x7a, 0x65, 0x42, 0x0c, 0x0a, 0x0a, 0x63, 0x72, 0x65, 0x64, 0x65,
+	0x6e, 0x74, 0x69, 0x61, 0x6c, 0x22, 0xc3, 0x01, 0x0a, 0x05, 0x53, 0x6c, 0x61, 0x63, 0x6b, 0x12,
+	0x24, 0x0a, 0x08, 0x65, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28,
+	0x09, 0x42, 0x08, 0xfa, 0x42, 0x05, 0x72, 0x03, 0x90, 0x01, 0x01, 0x52, 0x08, 0x65, 0x6e, 0x64,
+	0x70, 0x6f, 0x69, 0x6e, 0x74, 0x12, 0x16, 0x0a, 0x05, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x18, 0x02,
+	0x20, 0x01, 0x28, 0x09, 0x48, 0x00, 0x52, 0x05, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x12, 0x32, 0x0a,
+	0x06, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x73, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x18, 0x2e,
+	0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x73, 0x2e, 0x53, 0x6c, 0x61, 0x63,
+	0x6b, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x73, 0x48, 0x00, 0x52, 0x06, 0x74, 0x6f, 0x6b, 0x65, 0x6e,
+	0x73, 0x12, 0x1a, 0x0a, 0x08, 0x63, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x73, 0x18, 0x03, 0x20,
+	0x03, 0x28, 0x09, 0x52, 0x08, 0x63, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x73, 0x12, 0x1e, 0x0a,
+	0x0a, 0x69, 0x67, 0x6e, 0x6f, 0x72, 0x65, 0x4c, 0x69, 0x73, 0x74, 0x18, 0x04, 0x20, 0x03, 0x28,
+	0x09, 0x52, 0x0a, 0x69, 0x67, 0x6e, 0x6f, 0x72, 0x65, 0x4c, 0x69, 0x73, 0x74, 0x42, 0x0c, 0x0a,
+	0x0a, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x22, 0x06, 0x0a, 0x04, 0x54,
+	0x65, 0x73, 0x74, 0x22, 0x31, 0x0a, 0x09, 0x42, 0x75, 0x69, 0x6c, 0x64, 0x6b, 0x69, 0x74, 0x65,
+	0x12, 0x16, 0x0a, 0x05, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x48,
+	0x00, 0x52, 0x05, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x42, 0x0c, 0x0a, 0x0a, 0x63, 0x72, 0x65, 0x64,
+	0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x22, 0xdb, 0x01, 0x0a, 0x06, 0x47, 0x65, 0x72, 0x72, 0x69,
+	0x74, 0x12, 0x24, 0x0a, 0x08, 0x65, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x18, 0x01, 0x20,
+	0x01, 0x28, 0x09, 0x42, 0x08, 0xfa, 0x42, 0x05, 0x72, 0x03, 0x90, 0x01, 0x01, 0x52, 0x08, 0x65,
+	0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x12, 0x37, 0x0a, 0x0a, 0x62, 0x61, 0x73, 0x69, 0x63,
+	0x5f, 0x61, 0x75, 0x74, 0x68, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x16, 0x2e, 0x63, 0x72,
+	0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x73, 0x2e, 0x42, 0x61, 0x73, 0x69, 0x63, 0x41,
+	0x75, 0x74, 0x68, 0x48, 0x00, 0x52, 0x09, 0x62, 0x61, 0x73, 0x69, 0x63, 0x41, 0x75, 0x74, 0x68,
 	0x12, 0x48, 0x0a, 0x0f, 0x75, 0x6e, 0x61, 0x75, 0x74, 0x68, 0x65, 0x6e, 0x74, 0x69, 0x63, 0x61,
-	0x74, 0x65, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x63, 0x72, 0x65, 0x64,
+	0x74, 0x65, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x63, 0x72, 0x65, 0x64,
 	0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x73, 0x2e, 0x55, 0x6e, 0x61, 0x75, 0x74, 0x68, 0x65, 0x6e,
 	0x74, 0x69, 0x63, 0x61, 0x74, 0x65, 0x64, 0x48, 0x00, 0x52, 0x0f, 0x75, 0x6e, 0x61, 0x75, 0x74,
-	0x68, 0x65, 0x6e, 0x74, 0x69, 0x63, 0x61, 0x74, 0x65, 0x64, 0x42, 0x0c, 0x0a, 0x0a, 0x63, 0x72,
-	0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x22, 0xf0, 0x02, 0x0a, 0x02, 0x53, 0x33, 0x12,
-	0x37, 0x0a, 0x0a, 0x61, 0x63, 0x63, 0x65, 0x73, 0x73, 0x5f, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20,
-	0x01, 0x28, 0x0b, 0x32, 0x16, 0x2e, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c,
-	0x73, 0x2e, 0x4b, 0x65, 0x79, 0x53, 0x65, 0x63, 0x72, 0x65, 0x74, 0x48, 0x00, 0x52, 0x09, 0x61,
-	0x63, 0x63, 0x65, 0x73, 0x73, 0x4b, 0x65, 0x79, 0x12, 0x48, 0x0a, 0x0f, 0x75, 0x6e, 0x61, 0x75,
-	0x74, 0x68, 0x65, 0x6e, 0x74, 0x69, 0x63, 0x61, 0x74, 0x65, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28,
-	0x0b, 0x32, 0x1c, 0x2e, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x73, 0x2e,
-	0x55, 0x6e, 0x61, 0x75, 0x74, 0x68, 0x65, 0x6e, 0x74, 0x69, 0x63, 0x61, 0x74, 0x65, 0x64, 0x48,
-	0x00, 0x52, 0x0f, 0x75, 0x6e, 0x61, 0x75, 0x74, 0x68, 0x65, 0x6e, 0x74, 0x69, 0x63, 0x61, 0x74,
-	0x65, 0x64, 0x12, 0x4c, 0x0a, 0x11, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x5f, 0x65, 0x6e, 0x76, 0x69,
-	0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1d, 0x2e,
-	0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x73, 0x2e, 0x43, 0x6c, 0x6f, 0x75,
-	0x64, 0x45, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x48, 0x00, 0x52, 0x10,
-	0x63, 0x6c, 0x6f, 0x75, 0x64, 0x45, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74,
-	0x12, 0x49, 0x0a, 0x0d, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x5f, 0x74, 0x6f, 0x6b, 0x65,
-	0x6e, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x22, 0x2e, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e,
-	0x74, 0x69, 0x61, 0x6c, 0x73, 0x2e, 0x41, 0x57, 0x53, 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e,
-	0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x53, 0x65, 0x63, 0x72, 0x65, 0x74, 0x48, 0x00, 0x52, 0x0c, 0x73,
-	0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x12, 0x18, 0x0a, 0x07, 0x62,
-	0x75, 0x63, 0x6b, 0x65, 0x74, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x09, 0x52, 0x07, 0x62, 0x75,
-	0x63, 0x6b, 0x65, 0x74, 0x73, 0x12, 0x26, 0x0a, 0x0f, 0x6d, 0x61, 0x78, 0x5f, 0x6f, 0x62, 0x6a,
-	0x65, 0x63, 0x74, 0x5f, 0x73, 0x69, 0x7a, 0x65, 0x18, 0x06, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0d,
-	0x6d, 0x61, 0x78, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x53, 0x69, 0x7a, 0x65, 0x42, 0x0c, 0x0a,
-	0x0a, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x22, 0xc3, 0x01, 0x0a, 0x05,
-	0x53, 0x6c, 0x61, 0x63, 0x6b, 0x12, 0x24, 0x0a, 0x08, 0x65, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e,
-	0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x42, 0x08, 0xfa, 0x42, 0x05, 0x72, 0x03, 0x90, 0x01,
-	0x01, 0x52, 0x08, 0x65, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x12, 0x16, 0x0a, 0x05, 0x74,
-	0x6f, 0x6b, 0x65, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x48, 0x00, 0x52, 0x05, 0x74, 0x6f,
-	0x6b, 0x65, 0x6e, 0x12, 0x32, 0x0a, 0x06, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x73, 0x18, 0x05, 0x20,
-	0x01, 0x28, 0x0b, 0x32, 0x18, 0x2e, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c,
-	0x73, 0x2e, 0x53, 0x6c, 0x61, 0x63, 0x6b, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x73, 0x48, 0x00, 0x52,
-	0x06, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x73, 0x12, 0x1a, 0x0a, 0x08, 0x63, 0x68, 0x61, 0x6e, 0x6e,
-	0x65, 0x6c, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x09, 0x52, 0x08, 0x63, 0x68, 0x61, 0x6e, 0x6e,
-	0x65, 0x6c, 0x73, 0x12, 0x1e, 0x0a, 0x0a, 0x69, 0x67, 0x6e, 0x6f, 0x72, 0x65, 0x4c, 0x69, 0x73,
-	0x74, 0x18, 0x04, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0a, 0x69, 0x67, 0x6e, 0x6f, 0x72, 0x65, 0x4c,
-	0x69, 0x73, 0x74, 0x42, 0x0c, 0x0a, 0x0a, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61,
-	0x6c, 0x22, 0x06, 0x0a, 0x04, 0x54, 0x65, 0x73, 0x74, 0x22, 0x31, 0x0a, 0x09, 0x42, 0x75, 0x69,
-	0x6c, 0x64, 0x6b, 0x69, 0x74, 0x65, 0x12, 0x16, 0x0a, 0x05, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x18,
-	0x01, 0x20, 0x01, 0x28, 0x09, 0x48, 0x00, 0x52, 0x05, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x42, 0x0c,
-	0x0a, 0x0a, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x22, 0xdb, 0x01, 0x0a,
-	0x06, 0x47, 0x65, 0x72, 0x72, 0x69, 0x74, 0x12, 0x24, 0x0a, 0x08, 0x65, 0x6e, 0x64, 0x70, 0x6f,
-	0x69, 0x6e, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x42, 0x08, 0xfa, 0x42, 0x05, 0x72, 0x03,
-	0x90, 0x01, 0x01, 0x52, 0x08, 0x65, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x12, 0x37, 0x0a,
-	0x0a, 0x62, 0x61, 0x73, 0x69, 0x63, 0x5f, 0x61, 0x75, 0x74, 0x68, 0x18, 0x02, 0x20, 0x01, 0x28,
-	0x0b, 0x32, 0x16, 0x2e, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x73, 0x2e,
-	0x42, 0x61, 0x73, 0x69, 0x63, 0x41, 0x75, 0x74, 0x68, 0x48, 0x00, 0x52, 0x09, 0x62, 0x61, 0x73,
-	0x69, 0x63, 0x41, 0x75, 0x74, 0x68, 0x12, 0x48, 0x0a, 0x0f, 0x75, 0x6e, 0x61, 0x75, 0x74, 0x68,
-	0x65, 0x6e, 0x74, 0x69, 0x63, 0x61, 0x74, 0x65, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32,
-	0x1c, 0x2e, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x73, 0x2e, 0x55, 0x6e,
-	0x61, 0x75, 0x74, 0x68, 0x65, 0x6e, 0x74, 0x69, 0x63, 0x61, 0x74, 0x65, 0x64, 0x48, 0x00, 0x52,
-	0x0f, 0x75, 0x6e, 0x61, 0x75, 0x74, 0x68, 0x65, 0x6e, 0x74, 0x69, 0x63, 0x61, 0x74, 0x65, 0x64,
-	0x12, 0x1a, 0x0a, 0x08, 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x18, 0x04, 0x20, 0x03,
-	0x28, 0x09, 0x52, 0x08, 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x42, 0x0c, 0x0a, 0x0a,
-	0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x22, 0xa5, 0x01, 0x0a, 0x07, 0x4a,
-	0x65, 0x6e, 0x6b, 0x69, 0x6e, 0x73, 0x12, 0x24, 0x0a, 0x08, 0x65, 0x6e, 0x64, 0x70, 0x6f, 0x69,
-	0x6e, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x42, 0x08, 0xfa, 0x42, 0x05, 0x72, 0x03, 0x90,
-	0x01, 0x01, 0x52, 0x08, 0x65, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x12, 0x37, 0x0a, 0x0a,
-	0x62, 0x61, 0x73, 0x69, 0x63, 0x5f, 0x61, 0x75, 0x74, 0x68, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b,
-	0x32, 0x16, 0x2e, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x73, 0x2e, 0x42,
-	0x61, 0x73, 0x69, 0x63, 0x41, 0x75, 0x74, 0x68, 0x48, 0x00, 0x52, 0x09, 0x62, 0x61, 0x73, 0x69,
-	0x63, 0x41, 0x75, 0x74, 0x68, 0x12, 0x2d, 0x0a, 0x06, 0x68, 0x65, 0x61, 0x64, 0x65, 0x72, 0x18,
-	0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x13, 0x2e, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69,
-	0x61, 0x6c, 0x73, 0x2e, 0x48, 0x65, 0x61, 0x64, 0x65, 0x72, 0x48, 0x00, 0x52, 0x06, 0x68, 0x65,
-	0x61, 0x64, 0x65, 0x72, 0x42, 0x0c, 0x0a, 0x0a, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69,
-	0x61, 0x6c, 0x22, 0x9f, 0x02, 0x0a, 0x05, 0x54, 0x65, 0x61, 0x6d, 0x73, 0x12, 0x24, 0x0a, 0x08,
-	0x65, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x42, 0x08,
-	0xfa, 0x42, 0x05, 0x72, 0x03, 0x90, 0x01, 0x01, 0x52, 0x08, 0x65, 0x6e, 0x64, 0x70, 0x6f, 0x69,
-	0x6e, 0x74, 0x12, 0x16, 0x0a, 0x05, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28,
-	0x09, 0x48, 0x00, 0x52, 0x05, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x12, 0x46, 0x0a, 0x0d, 0x61, 0x75,
-	0x74, 0x68, 0x65, 0x6e, 0x74, 0x69, 0x63, 0x61, 0x74, 0x65, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28,
-	0x0b, 0x32, 0x1e, 0x2e, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x73, 0x2e,
-	0x43, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x43, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c,
-	0x73, 0x48, 0x00, 0x52, 0x0d, 0x61, 0x75, 0x74, 0x68, 0x65, 0x6e, 0x74, 0x69, 0x63, 0x61, 0x74,
-	0x65, 0x64, 0x12, 0x2b, 0x0a, 0x05, 0x6f, 0x61, 0x75, 0x74, 0x68, 0x18, 0x07, 0x20, 0x01, 0x28,
-	0x0b, 0x32, 0x13, 0x2e, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x73, 0x2e,
-	0x4f, 0x61, 0x75, 0x74, 0x68, 0x32, 0x48, 0x00, 0x52, 0x05, 0x6f, 0x61, 0x75, 0x74, 0x68, 0x12,
-	0x1a, 0x0a, 0x08, 0x63, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28,
-	0x09, 0x52, 0x08, 0x63, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x73, 0x12, 0x1e, 0x0a, 0x0a, 0x69,
-	0x67, 0x6e, 0x6f, 0x72, 0x65, 0x4c, 0x69, 0x73, 0x74, 0x18, 0x05, 0x20, 0x03, 0x28, 0x09, 0x52,
-	0x0a, 0x69, 0x67, 0x6e, 0x6f, 0x72, 0x65, 0x4c, 0x69, 0x73, 0x74, 0x12, 0x19, 0x0a, 0x08, 0x74,
-	0x65, 0x61, 0x6d, 0x5f, 0x69, 0x64, 0x73, 0x18, 0x06, 0x20, 0x03, 0x28, 0x09, 0x52, 0x07, 0x74,
-	0x65, 0x61, 0x6d, 0x49, 0x64, 0x73, 0x42, 0x0c, 0x0a, 0x0a, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e,
-	0x74, 0x69, 0x61, 0x6c, 0x22, 0xc3, 0x01, 0x0a, 0x0b, 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63,
-	0x74, 0x6f, 0x72, 0x79, 0x12, 0x24, 0x0a, 0x08, 0x65, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74,
-	0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x42, 0x08, 0xfa, 0x42, 0x05, 0x72, 0x03, 0x90, 0x01, 0x01,
-	0x52, 0x08, 0x65, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x12, 0x37, 0x0a, 0x0a, 0x62, 0x61,
-	0x73, 0x69, 0x63, 0x5f, 0x61, 0x75, 0x74, 0x68, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x16,
-	0x2e, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x73, 0x2e, 0x42, 0x61, 0x73,
-	0x69, 0x63, 0x41, 0x75, 0x74, 0x68, 0x48, 0x00, 0x52, 0x09, 0x62, 0x61, 0x73, 0x69, 0x63, 0x41,
-	0x75, 0x74, 0x68, 0x12, 0x23, 0x0a, 0x0c, 0x61, 0x63, 0x63, 0x65, 0x73, 0x73, 0x5f, 0x74, 0x6f,
-	0x6b, 0x65, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x48, 0x00, 0x52, 0x0b, 0x61, 0x63, 0x63,
-	0x65, 0x73, 0x73, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x12, 0x22, 0x0a, 0x0c, 0x72, 0x65, 0x70, 0x6f,
-	0x73, 0x69, 0x74, 0x6f, 0x72, 0x69, 0x65, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0c,
-	0x72, 0x65, 0x70, 0x6f, 0x73, 0x69, 0x74, 0x6f, 0x72, 0x69, 0x65, 0x73, 0x42, 0x0c, 0x0a, 0x0a,
-	0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x22, 0x94, 0x01, 0x0a, 0x06, 0x53,
-	0x79, 0x73, 0x6c, 0x6f, 0x67, 0x12, 0x1a, 0x0a, 0x08, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f,
-	0x6c, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f,
-	0x6c, 0x12, 0x24, 0x0a, 0x0d, 0x6c, 0x69, 0x73, 0x74, 0x65, 0x6e, 0x41, 0x64, 0x64, 0x72, 0x65,
-	0x73, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0d, 0x6c, 0x69, 0x73, 0x74, 0x65, 0x6e,
-	0x41, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x12, 0x18, 0x0a, 0x07, 0x74, 0x6c, 0x73, 0x43, 0x65,
-	0x72, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x74, 0x6c, 0x73, 0x43, 0x65, 0x72,
-	0x74, 0x12, 0x16, 0x0a, 0x06, 0x74, 0x6c, 0x73, 0x4b, 0x65, 0x79, 0x18, 0x04, 0x20, 0x01, 0x28,
-	0x09, 0x52, 0x06, 0x74, 0x6c, 0x73, 0x4b, 0x65, 0x79, 0x12, 0x16, 0x0a, 0x06, 0x66, 0x6f, 0x72,
-	0x6d, 0x61, 0x74, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x66, 0x6f, 0x72, 0x6d, 0x61,
-	0x74, 0x22, 0xd6, 0x01, 0x0a, 0x15, 0x50, 0x75, 0x62, 0x6c, 0x69, 0x63, 0x45, 0x76, 0x65, 0x6e,
-	0x74, 0x4d, 0x6f, 0x6e, 0x69, 0x74, 0x6f, 0x72, 0x69, 0x6e, 0x67, 0x12, 0x48, 0x0a, 0x0f, 0x75,
-	0x6e, 0x61, 0x75, 0x74, 0x68, 0x65, 0x6e, 0x74, 0x69, 0x63, 0x61, 0x74, 0x65, 0x64, 0x18, 0x01,
-	0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61,
-	0x6c, 0x73, 0x2e, 0x55, 0x6e, 0x61, 0x75, 0x74, 0x68, 0x65, 0x6e, 0x74, 0x69, 0x63, 0x61, 0x74,
-	0x65, 0x64, 0x48, 0x00, 0x52, 0x0f, 0x75, 0x6e, 0x61, 0x75, 0x74, 0x68, 0x65, 0x6e, 0x74, 0x69,
-	0x63, 0x61, 0x74, 0x65, 0x64, 0x12, 0x16, 0x0a, 0x06, 0x64, 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x18,
-	0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x64, 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x12, 0x1b, 0x0a,
-	0x09, 0x6d, 0x61, 0x78, 0x5f, 0x64, 0x65, 0x70, 0x74, 0x68, 0x18, 0x03, 0x20, 0x01, 0x28, 0x03,
-	0x52, 0x08, 0x6d, 0x61, 0x78, 0x44, 0x65, 0x70, 0x74, 0x68, 0x12, 0x30, 0x0a, 0x05, 0x73, 0x69,
-	0x6e, 0x63, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67,
-	0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65,
-	0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x05, 0x73, 0x69, 0x6e, 0x63, 0x65, 0x42, 0x0c, 0x0a, 0x0a,
-	0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x22, 0x51, 0x0a, 0x0d, 0x53, 0x6c,
-	0x61, 0x63, 0x6b, 0x52, 0x65, 0x61, 0x6c, 0x74, 0x69, 0x6d, 0x65, 0x12, 0x32, 0x0a, 0x06, 0x74,
-	0x6f, 0x6b, 0x65, 0x6e, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x18, 0x2e, 0x63, 0x72,
-	0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x73, 0x2e, 0x53, 0x6c, 0x61, 0x63, 0x6b, 0x54,
-	0x6f, 0x6b, 0x65, 0x6e, 0x73, 0x48, 0x00, 0x52, 0x06, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x73, 0x42,
-	0x0c, 0x0a, 0x0a, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x22, 0x62, 0x0a,
-	0x0a, 0x53, 0x68, 0x61, 0x72, 0x65, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x12, 0x2b, 0x0a, 0x05, 0x6f,
-	0x61, 0x75, 0x74, 0x68, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x13, 0x2e, 0x63, 0x72, 0x65,
-	0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x73, 0x2e, 0x4f, 0x61, 0x75, 0x74, 0x68, 0x32, 0x48,
-	0x00, 0x52, 0x05, 0x6f, 0x61, 0x75, 0x74, 0x68, 0x12, 0x19, 0x0a, 0x08, 0x73, 0x69, 0x74, 0x65,
-	0x5f, 0x75, 0x72, 0x6c, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x73, 0x69, 0x74, 0x65,
-	0x55, 0x72, 0x6c, 0x42, 0x0c, 0x0a, 0x0a, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61,
-	0x6c, 0x22, 0xa7, 0x03, 0x0a, 0x0a, 0x41, 0x7a, 0x75, 0x72, 0x65, 0x52, 0x65, 0x70, 0x6f, 0x73,
+	0x68, 0x65, 0x6e, 0x74, 0x69, 0x63, 0x61, 0x74, 0x65, 0x64, 0x12, 0x1a, 0x0a, 0x08, 0x70, 0x72,
+	0x6f, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x09, 0x52, 0x08, 0x70, 0x72,
+	0x6f, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x42, 0x0c, 0x0a, 0x0a, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e,
+	0x74, 0x69, 0x61, 0x6c, 0x22, 0xa5, 0x01, 0x0a, 0x07, 0x4a, 0x65, 0x6e, 0x6b, 0x69, 0x6e, 0x73,
 	0x12, 0x24, 0x0a, 0x08, 0x65, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x18, 0x01, 0x20, 0x01,
 	0x28, 0x09, 0x42, 0x08, 0xfa, 0x42, 0x05, 0x72, 0x03, 0x90, 0x01, 0x01, 0x52, 0x08, 0x65, 0x6e,
-	0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x12, 0x16, 0x0a, 0x05, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x18,
-	0x02, 0x20, 0x01, 0x28, 0x09, 0x48, 0x00, 0x52, 0x05, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x12, 0x2b,
-	0x0a, 0x05, 0x6f, 0x61, 0x75, 0x74, 0x68, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x13, 0x2e,
-	0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x73, 0x2e, 0x4f, 0x61, 0x75, 0x74,
-	0x68, 0x32, 0x48, 0x00, 0x52, 0x05, 0x6f, 0x61, 0x75, 0x74, 0x68, 0x12, 0x22, 0x0a, 0x0c, 0x72,
-	0x65, 0x70, 0x6f, 0x73, 0x69, 0x74, 0x6f, 0x72, 0x69, 0x65, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28,
-	0x09, 0x52, 0x0c, 0x72, 0x65, 0x70, 0x6f, 0x73, 0x69, 0x74, 0x6f, 0x72, 0x69, 0x65, 0x73, 0x12,
-	0x24, 0x0a, 0x0d, 0x6f, 0x72, 0x67, 0x61, 0x6e, 0x69, 0x7a, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73,
-	0x18, 0x05, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0d, 0x6f, 0x72, 0x67, 0x61, 0x6e, 0x69, 0x7a, 0x61,
-	0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x1a, 0x0a, 0x08, 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74,
-	0x73, 0x18, 0x06, 0x20, 0x03, 0x28, 0x09, 0x52, 0x08, 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74,
-	0x73, 0x12, 0x22, 0x0a, 0x0c, 0x69, 0x6e, 0x63, 0x6c, 0x75, 0x64, 0x65, 0x46, 0x6f, 0x72, 0x6b,
-	0x73, 0x18, 0x07, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0c, 0x69, 0x6e, 0x63, 0x6c, 0x75, 0x64, 0x65,
-	0x46, 0x6f, 0x72, 0x6b, 0x73, 0x12, 0x20, 0x0a, 0x0b, 0x69, 0x67, 0x6e, 0x6f, 0x72, 0x65, 0x52,
-	0x65, 0x70, 0x6f, 0x73, 0x18, 0x08, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0b, 0x69, 0x67, 0x6e, 0x6f,
-	0x72, 0x65, 0x52, 0x65, 0x70, 0x6f, 0x73, 0x12, 0x22, 0x0a, 0x0c, 0x69, 0x6e, 0x63, 0x6c, 0x75,
-	0x64, 0x65, 0x52, 0x65, 0x70, 0x6f, 0x73, 0x18, 0x09, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0c, 0x69,
-	0x6e, 0x63, 0x6c, 0x75, 0x64, 0x65, 0x52, 0x65, 0x70, 0x6f, 0x73, 0x12, 0x28, 0x0a, 0x0f, 0x69,
-	0x6e, 0x63, 0x6c, 0x75, 0x64, 0x65, 0x50, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x18, 0x0a,
-	0x20, 0x03, 0x28, 0x09, 0x52, 0x0f, 0x69, 0x6e, 0x63, 0x6c, 0x75, 0x64, 0x65, 0x50, 0x72, 0x6f,
-	0x6a, 0x65, 0x63, 0x74, 0x73, 0x12, 0x26, 0x0a, 0x0e, 0x69, 0x67, 0x6e, 0x6f, 0x72, 0x65, 0x50,
-	0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x18, 0x0b, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0e, 0x69,
-	0x67, 0x6e, 0x6f, 0x72, 0x65, 0x50, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x42, 0x0c, 0x0a,
-	0x0a, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x2a, 0x84, 0x07, 0x0a, 0x0a,
+	0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x12, 0x37, 0x0a, 0x0a, 0x62, 0x61, 0x73, 0x69, 0x63, 0x5f,
+	0x61, 0x75, 0x74, 0x68, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x16, 0x2e, 0x63, 0x72, 0x65,
+	0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x73, 0x2e, 0x42, 0x61, 0x73, 0x69, 0x63, 0x41, 0x75,
+	0x74, 0x68, 0x48, 0x00, 0x52, 0x09, 0x62, 0x61, 0x73, 0x69, 0x63, 0x41, 0x75, 0x74, 0x68, 0x12,
+	0x2d, 0x0a, 0x06, 0x68, 0x65, 0x61, 0x64, 0x65, 0x72, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32,
+	0x13, 0x2e, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x73, 0x2e, 0x48, 0x65,
+	0x61, 0x64, 0x65, 0x72, 0x48, 0x00, 0x52, 0x06, 0x68, 0x65, 0x61, 0x64, 0x65, 0x72, 0x42, 0x0c,
+	0x0a, 0x0a, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x22, 0x9f, 0x02, 0x0a,
+	0x05, 0x54, 0x65, 0x61, 0x6d, 0x73, 0x12, 0x24, 0x0a, 0x08, 0x65, 0x6e, 0x64, 0x70, 0x6f, 0x69,
+	0x6e, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x42, 0x08, 0xfa, 0x42, 0x05, 0x72, 0x03, 0x90,
+	0x01, 0x01, 0x52, 0x08, 0x65, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x12, 0x16, 0x0a, 0x05,
+	0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x48, 0x00, 0x52, 0x05, 0x74,
+	0x6f, 0x6b, 0x65, 0x6e, 0x12, 0x46, 0x0a, 0x0d, 0x61, 0x75, 0x74, 0x68, 0x65, 0x6e, 0x74, 0x69,
+	0x63, 0x61, 0x74, 0x65, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1e, 0x2e, 0x63, 0x72,
+	0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x73, 0x2e, 0x43, 0x6c, 0x69, 0x65, 0x6e, 0x74,
+	0x43, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x73, 0x48, 0x00, 0x52, 0x0d, 0x61,
+	0x75, 0x74, 0x68, 0x65, 0x6e, 0x74, 0x69, 0x63, 0x61, 0x74, 0x65, 0x64, 0x12, 0x2b, 0x0a, 0x05,
+	0x6f, 0x61, 0x75, 0x74, 0x68, 0x18, 0x07, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x13, 0x2e, 0x63, 0x72,
+	0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x73, 0x2e, 0x4f, 0x61, 0x75, 0x74, 0x68, 0x32,
+	0x48, 0x00, 0x52, 0x05, 0x6f, 0x61, 0x75, 0x74, 0x68, 0x12, 0x1a, 0x0a, 0x08, 0x63, 0x68, 0x61,
+	0x6e, 0x6e, 0x65, 0x6c, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x09, 0x52, 0x08, 0x63, 0x68, 0x61,
+	0x6e, 0x6e, 0x65, 0x6c, 0x73, 0x12, 0x1e, 0x0a, 0x0a, 0x69, 0x67, 0x6e, 0x6f, 0x72, 0x65, 0x4c,
+	0x69, 0x73, 0x74, 0x18, 0x05, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0a, 0x69, 0x67, 0x6e, 0x6f, 0x72,
+	0x65, 0x4c, 0x69, 0x73, 0x74, 0x12, 0x19, 0x0a, 0x08, 0x74, 0x65, 0x61, 0x6d, 0x5f, 0x69, 0x64,
+	0x73, 0x18, 0x06, 0x20, 0x03, 0x28, 0x09, 0x52, 0x07, 0x74, 0x65, 0x61, 0x6d, 0x49, 0x64, 0x73,
+	0x42, 0x0c, 0x0a, 0x0a, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x22, 0xc3,
+	0x01, 0x0a, 0x0b, 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x6f, 0x72, 0x79, 0x12, 0x24,
+	0x0a, 0x08, 0x65, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09,
+	0x42, 0x08, 0xfa, 0x42, 0x05, 0x72, 0x03, 0x90, 0x01, 0x01, 0x52, 0x08, 0x65, 0x6e, 0x64, 0x70,
+	0x6f, 0x69, 0x6e, 0x74, 0x12, 0x37, 0x0a, 0x0a, 0x62, 0x61, 0x73, 0x69, 0x63, 0x5f, 0x61, 0x75,
+	0x74, 0x68, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x16, 0x2e, 0x63, 0x72, 0x65, 0x64, 0x65,
+	0x6e, 0x74, 0x69, 0x61, 0x6c, 0x73, 0x2e, 0x42, 0x61, 0x73, 0x69, 0x63, 0x41, 0x75, 0x74, 0x68,
+	0x48, 0x00, 0x52, 0x09, 0x62, 0x61, 0x73, 0x69, 0x63, 0x41, 0x75, 0x74, 0x68, 0x12, 0x23, 0x0a,
+	0x0c, 0x61, 0x63, 0x63, 0x65, 0x73, 0x73, 0x5f, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x18, 0x03, 0x20,
+	0x01, 0x28, 0x09, 0x48, 0x00, 0x52, 0x0b, 0x61, 0x63, 0x63, 0x65, 0x73, 0x73, 0x54, 0x6f, 0x6b,
+	0x65, 0x6e, 0x12, 0x22, 0x0a, 0x0c, 0x72, 0x65, 0x70, 0x6f, 0x73, 0x69, 0x74, 0x6f, 0x72, 0x69,
+	0x65, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0c, 0x72, 0x65, 0x70, 0x6f, 0x73, 0x69,
+	0x74, 0x6f, 0x72, 0x69, 0x65, 0x73, 0x42, 0x0c, 0x0a, 0x0a, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e,
+	0x74, 0x69, 0x61, 0x6c, 0x22, 0x94, 0x01, 0x0a, 0x06, 0x53, 0x79, 0x73, 0x6c, 0x6f, 0x67, 0x12,
+	0x1a, 0x0a, 0x08, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x18, 0x01, 0x20, 0x01, 0x28,
+	0x09, 0x52, 0x08, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x12, 0x24, 0x0a, 0x0d, 0x6c,
+	0x69, 0x73, 0x74, 0x65, 0x6e, 0x41, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x18, 0x02, 0x20, 0x01,
+	0x28, 0x09, 0x52, 0x0d, 0x6c, 0x69, 0x73, 0x74, 0x65, 0x6e, 0x41, 0x64, 0x64, 0x72, 0x65, 0x73,
+	0x73, 0x12, 0x18, 0x0a, 0x07, 0x74, 0x6c, 0x73, 0x43, 0x65, 0x72, 0x74, 0x18, 0x03, 0x20, 0x01,
+	0x28, 0x09, 0x52, 0x07, 0x74, 0x6c, 0x73, 0x43, 0x65, 0x72, 0x74, 0x12, 0x16, 0x0a, 0x06, 0x74,
+	0x6c, 0x73, 0x4b, 0x65, 0x79, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x74, 0x6c, 0x73,
+	0x4b, 0x65, 0x79, 0x12, 0x16, 0x0a, 0x06, 0x66, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x18, 0x05, 0x20,
+	0x01, 0x28, 0x09, 0x52, 0x06, 0x66, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x22, 0xd6, 0x01, 0x0a, 0x15,
+	0x50, 0x75, 0x62, 0x6c, 0x69, 0x63, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x4d, 0x6f, 0x6e, 0x69, 0x74,
+	0x6f, 0x72, 0x69, 0x6e, 0x67, 0x12, 0x48, 0x0a, 0x0f, 0x75, 0x6e, 0x61, 0x75, 0x74, 0x68, 0x65,
+	0x6e, 0x74, 0x69, 0x63, 0x61, 0x74, 0x65, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c,
+	0x2e, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x73, 0x2e, 0x55, 0x6e, 0x61,
+	0x75, 0x74, 0x68, 0x65, 0x6e, 0x74, 0x69, 0x63, 0x61, 0x74, 0x65, 0x64, 0x48, 0x00, 0x52, 0x0f,
+	0x75, 0x6e, 0x61, 0x75, 0x74, 0x68, 0x65, 0x6e, 0x74, 0x69, 0x63, 0x61, 0x74, 0x65, 0x64, 0x12,
+	0x16, 0x0a, 0x06, 0x64, 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52,
+	0x06, 0x64, 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x12, 0x1b, 0x0a, 0x09, 0x6d, 0x61, 0x78, 0x5f, 0x64,
+	0x65, 0x70, 0x74, 0x68, 0x18, 0x03, 0x20, 0x01, 0x28, 0x03, 0x52, 0x08, 0x6d, 0x61, 0x78, 0x44,
+	0x65, 0x70, 0x74, 0x68, 0x12, 0x30, 0x0a, 0x05, 0x73, 0x69, 0x6e, 0x63, 0x65, 0x18, 0x04, 0x20,
+	0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f,
+	0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52,
+	0x05, 0x73, 0x69, 0x6e, 0x63, 0x65, 0x42, 0x0c, 0x0a, 0x0a, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e,
+	0x74, 0x69, 0x61, 0x6c, 0x22, 0x51, 0x0a, 0x0d, 0x53, 0x6c, 0x61, 0x63, 0x6b, 0x52, 0x65, 0x61,
+	0x6c, 0x74, 0x69, 0x6d, 0x65, 0x12, 0x32, 0x0a, 0x06, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x73, 0x18,
+	0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x18, 0x2e, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69,
+	0x61, 0x6c, 0x73, 0x2e, 0x53, 0x6c, 0x61, 0x63, 0x6b, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x73, 0x48,
+	0x00, 0x52, 0x06, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x73, 0x42, 0x0c, 0x0a, 0x0a, 0x63, 0x72, 0x65,
+	0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x22, 0x62, 0x0a, 0x0a, 0x53, 0x68, 0x61, 0x72, 0x65,
+	0x70, 0x6f, 0x69, 0x6e, 0x74, 0x12, 0x2b, 0x0a, 0x05, 0x6f, 0x61, 0x75, 0x74, 0x68, 0x18, 0x01,
+	0x20, 0x01, 0x28, 0x0b, 0x32, 0x13, 0x2e, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61,
+	0x6c, 0x73, 0x2e, 0x4f, 0x61, 0x75, 0x74, 0x68, 0x32, 0x48, 0x00, 0x52, 0x05, 0x6f, 0x61, 0x75,
+	0x74, 0x68, 0x12, 0x19, 0x0a, 0x08, 0x73, 0x69, 0x74, 0x65, 0x5f, 0x75, 0x72, 0x6c, 0x18, 0x02,
+	0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x73, 0x69, 0x74, 0x65, 0x55, 0x72, 0x6c, 0x42, 0x0c, 0x0a,
+	0x0a, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x2a, 0xe7, 0x06, 0x0a, 0x0a,
 	0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x54, 0x79, 0x70, 0x65, 0x12, 0x1d, 0x0a, 0x19, 0x53, 0x4f,
 	0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x41, 0x5a, 0x55, 0x52, 0x45, 0x5f,
 	0x53, 0x54, 0x4f, 0x52, 0x41, 0x47, 0x45, 0x10, 0x00, 0x12, 0x19, 0x0a, 0x15, 0x53, 0x4f, 0x55,
@@ -3629,13 +3413,11 @@ var file_sources_proto_rawDesc = []byte{
 	0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x53, 0x48, 0x41, 0x52, 0x45,
 	0x50, 0x4f, 0x49, 0x4e, 0x54, 0x10, 0x1d, 0x12, 0x1c, 0x0a, 0x18, 0x53, 0x4f, 0x55, 0x52, 0x43,
 	0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x47, 0x43, 0x53, 0x5f, 0x55, 0x4e, 0x41, 0x55, 0x54,
-	0x48, 0x45, 0x44, 0x10, 0x1e, 0x12, 0x1b, 0x0a, 0x17, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f,
-	0x54, 0x59, 0x50, 0x45, 0x5f, 0x41, 0x5a, 0x55, 0x52, 0x45, 0x5f, 0x52, 0x45, 0x50, 0x4f, 0x53,
-	0x10, 0x1f, 0x42, 0x3b, 0x5a, 0x39, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d,
-	0x2f, 0x74, 0x72, 0x75, 0x66, 0x66, 0x6c, 0x65, 0x73, 0x65, 0x63, 0x75, 0x72, 0x69, 0x74, 0x79,
-	0x2f, 0x74, 0x72, 0x75, 0x66, 0x66, 0x6c, 0x65, 0x68, 0x6f, 0x67, 0x2f, 0x76, 0x33, 0x2f, 0x70,
-	0x6b, 0x67, 0x2f, 0x70, 0x62, 0x2f, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x73, 0x70, 0x62, 0x62,
-	0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
+	0x48, 0x45, 0x44, 0x10, 0x1e, 0x42, 0x3b, 0x5a, 0x39, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e,
+	0x63, 0x6f, 0x6d, 0x2f, 0x74, 0x72, 0x75, 0x66, 0x66, 0x6c, 0x65, 0x73, 0x65, 0x63, 0x75, 0x72,
+	0x69, 0x74, 0x79, 0x2f, 0x74, 0x72, 0x75, 0x66, 0x66, 0x6c, 0x65, 0x68, 0x6f, 0x67, 0x2f, 0x76,
+	0x33, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x70, 0x62, 0x2f, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x73,
+	0x70, 0x62, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
 }
 
 var (
@@ -3651,7 +3433,7 @@ func file_sources_proto_rawDescGZIP() []byte {
 }
 
 var file_sources_proto_enumTypes = make([]protoimpl.EnumInfo, 2)
-var file_sources_proto_msgTypes = make([]protoimpl.MessageInfo, 29)
+var file_sources_proto_msgTypes = make([]protoimpl.MessageInfo, 28)
 var file_sources_proto_goTypes = []interface{}{
 	(SourceType)(0),                             // 0: sources.SourceType
 	(Confluence_GetAllSpacesScope)(0),           // 1: sources.Confluence.GetAllSpacesScope
@@ -3683,73 +3465,71 @@ var file_sources_proto_goTypes = []interface{}{
 	(*PublicEventMonitoring)(nil),               // 27: sources.PublicEventMonitoring
 	(*SlackRealtime)(nil),                       // 28: sources.SlackRealtime
 	(*Sharepoint)(nil),                          // 29: sources.Sharepoint
-	(*AzureRepos)(nil),                          // 30: sources.AzureRepos
-	(*durationpb.Duration)(nil),                 // 31: google.protobuf.Duration
-	(*anypb.Any)(nil),                           // 32: google.protobuf.Any
-	(*credentialspb.BasicAuth)(nil),             // 33: credentials.BasicAuth
-	(*credentialspb.Unauthenticated)(nil),       // 34: credentials.Unauthenticated
-	(*credentialspb.Oauth2)(nil),                // 35: credentials.Oauth2
-	(*credentialspb.KeySecret)(nil),             // 36: credentials.KeySecret
-	(*credentialspb.CloudEnvironment)(nil),      // 37: credentials.CloudEnvironment
-	(*credentialspb.SSHAuth)(nil),               // 38: credentials.SSHAuth
-	(*credentialspb.GitHubApp)(nil),             // 39: credentials.GitHubApp
-	(*credentialspb.AWSSessionTokenSecret)(nil), // 40: credentials.AWSSessionTokenSecret
-	(*credentialspb.SlackTokens)(nil),           // 41: credentials.SlackTokens
-	(*credentialspb.Header)(nil),                // 42: credentials.Header
-	(*credentialspb.ClientCredentials)(nil),     // 43: credentials.ClientCredentials
-	(*timestamppb.Timestamp)(nil),               // 44: google.protobuf.Timestamp
+	(*durationpb.Duration)(nil),                 // 30: google.protobuf.Duration
+	(*anypb.Any)(nil),                           // 31: google.protobuf.Any
+	(*credentialspb.BasicAuth)(nil),             // 32: credentials.BasicAuth
+	(*credentialspb.Unauthenticated)(nil),       // 33: credentials.Unauthenticated
+	(*credentialspb.Oauth2)(nil),                // 34: credentials.Oauth2
+	(*credentialspb.KeySecret)(nil),             // 35: credentials.KeySecret
+	(*credentialspb.CloudEnvironment)(nil),      // 36: credentials.CloudEnvironment
+	(*credentialspb.SSHAuth)(nil),               // 37: credentials.SSHAuth
+	(*credentialspb.GitHubApp)(nil),             // 38: credentials.GitHubApp
+	(*credentialspb.AWSSessionTokenSecret)(nil), // 39: credentials.AWSSessionTokenSecret
+	(*credentialspb.SlackTokens)(nil),           // 40: credentials.SlackTokens
+	(*credentialspb.Header)(nil),                // 41: credentials.Header
+	(*credentialspb.ClientCredentials)(nil),     // 42: credentials.ClientCredentials
+	(*timestamppb.Timestamp)(nil),               // 43: google.protobuf.Timestamp
 }
 var file_sources_proto_depIdxs = []int32{
-	31, // 0: sources.LocalSource.scan_interval:type_name -> google.protobuf.Duration
-	32, // 1: sources.LocalSource.connection:type_name -> google.protobuf.Any
-	33, // 2: sources.AzureStorage.basic_auth:type_name -> credentials.BasicAuth
-	34, // 3: sources.AzureStorage.unauthenticated:type_name -> credentials.Unauthenticated
-	35, // 4: sources.Bitbucket.oauth:type_name -> credentials.Oauth2
-	33, // 5: sources.Bitbucket.basic_auth:type_name -> credentials.BasicAuth
-	34, // 6: sources.Confluence.unauthenticated:type_name -> credentials.Unauthenticated
-	33, // 7: sources.Confluence.basic_auth:type_name -> credentials.BasicAuth
+	30, // 0: sources.LocalSource.scan_interval:type_name -> google.protobuf.Duration
+	31, // 1: sources.LocalSource.connection:type_name -> google.protobuf.Any
+	32, // 2: sources.AzureStorage.basic_auth:type_name -> credentials.BasicAuth
+	33, // 3: sources.AzureStorage.unauthenticated:type_name -> credentials.Unauthenticated
+	34, // 4: sources.Bitbucket.oauth:type_name -> credentials.Oauth2
+	32, // 5: sources.Bitbucket.basic_auth:type_name -> credentials.BasicAuth
+	33, // 6: sources.Confluence.unauthenticated:type_name -> credentials.Unauthenticated
+	32, // 7: sources.Confluence.basic_auth:type_name -> credentials.BasicAuth
 	1,  // 8: sources.Confluence.spaces_scope:type_name -> sources.Confluence.GetAllSpacesScope
-	34, // 9: sources.Docker.unauthenticated:type_name -> credentials.Unauthenticated
-	33, // 10: sources.Docker.basic_auth:type_name -> credentials.BasicAuth
-	36, // 11: sources.ECR.access_key:type_name -> credentials.KeySecret
-	34, // 12: sources.GCS.unauthenticated:type_name -> credentials.Unauthenticated
-	37, // 13: sources.GCS.adc:type_name -> credentials.CloudEnvironment
-	35, // 14: sources.GCS.oauth:type_name -> credentials.Oauth2
-	33, // 15: sources.Git.basic_auth:type_name -> credentials.BasicAuth
-	34, // 16: sources.Git.unauthenticated:type_name -> credentials.Unauthenticated
-	38, // 17: sources.Git.ssh_auth:type_name -> credentials.SSHAuth
-	35, // 18: sources.GitLab.oauth:type_name -> credentials.Oauth2
-	33, // 19: sources.GitLab.basic_auth:type_name -> credentials.BasicAuth
-	39, // 20: sources.GitHub.github_app:type_name -> credentials.GitHubApp
-	34, // 21: sources.GitHub.unauthenticated:type_name -> credentials.Unauthenticated
-	33, // 22: sources.GitHub.basic_auth:type_name -> credentials.BasicAuth
-	33, // 23: sources.JIRA.basic_auth:type_name -> credentials.BasicAuth
-	34, // 24: sources.JIRA.unauthenticated:type_name -> credentials.Unauthenticated
-	35, // 25: sources.JIRA.oauth:type_name -> credentials.Oauth2
-	34, // 26: sources.NPMUnauthenticatedPackage.unauthenticated:type_name -> credentials.Unauthenticated
-	34, // 27: sources.PyPIUnauthenticatedPackage.unauthenticated:type_name -> credentials.Unauthenticated
-	36, // 28: sources.S3.access_key:type_name -> credentials.KeySecret
-	34, // 29: sources.S3.unauthenticated:type_name -> credentials.Unauthenticated
-	37, // 30: sources.S3.cloud_environment:type_name -> credentials.CloudEnvironment
-	40, // 31: sources.S3.session_token:type_name -> credentials.AWSSessionTokenSecret
-	41, // 32: sources.Slack.tokens:type_name -> credentials.SlackTokens
-	33, // 33: sources.Gerrit.basic_auth:type_name -> credentials.BasicAuth
-	34, // 34: sources.Gerrit.unauthenticated:type_name -> credentials.Unauthenticated
-	33, // 35: sources.Jenkins.basic_auth:type_name -> credentials.BasicAuth
-	42, // 36: sources.Jenkins.header:type_name -> credentials.Header
-	43, // 37: sources.Teams.authenticated:type_name -> credentials.ClientCredentials
-	35, // 38: sources.Teams.oauth:type_name -> credentials.Oauth2
-	33, // 39: sources.Artifactory.basic_auth:type_name -> credentials.BasicAuth
-	34, // 40: sources.PublicEventMonitoring.unauthenticated:type_name -> credentials.Unauthenticated
-	44, // 41: sources.PublicEventMonitoring.since:type_name -> google.protobuf.Timestamp
-	41, // 42: sources.SlackRealtime.tokens:type_name -> credentials.SlackTokens
-	35, // 43: sources.Sharepoint.oauth:type_name -> credentials.Oauth2
-	35, // 44: sources.AzureRepos.oauth:type_name -> credentials.Oauth2
-	45, // [45:45] is the sub-list for method output_type
-	45, // [45:45] is the sub-list for method input_type
-	45, // [45:45] is the sub-list for extension type_name
-	45, // [45:45] is the sub-list for extension extendee
-	0,  // [0:45] is the sub-list for field type_name
+	33, // 9: sources.Docker.unauthenticated:type_name -> credentials.Unauthenticated
+	32, // 10: sources.Docker.basic_auth:type_name -> credentials.BasicAuth
+	35, // 11: sources.ECR.access_key:type_name -> credentials.KeySecret
+	33, // 12: sources.GCS.unauthenticated:type_name -> credentials.Unauthenticated
+	36, // 13: sources.GCS.adc:type_name -> credentials.CloudEnvironment
+	34, // 14: sources.GCS.oauth:type_name -> credentials.Oauth2
+	32, // 15: sources.Git.basic_auth:type_name -> credentials.BasicAuth
+	33, // 16: sources.Git.unauthenticated:type_name -> credentials.Unauthenticated
+	37, // 17: sources.Git.ssh_auth:type_name -> credentials.SSHAuth
+	34, // 18: sources.GitLab.oauth:type_name -> credentials.Oauth2
+	32, // 19: sources.GitLab.basic_auth:type_name -> credentials.BasicAuth
+	38, // 20: sources.GitHub.github_app:type_name -> credentials.GitHubApp
+	33, // 21: sources.GitHub.unauthenticated:type_name -> credentials.Unauthenticated
+	32, // 22: sources.GitHub.basic_auth:type_name -> credentials.BasicAuth
+	32, // 23: sources.JIRA.basic_auth:type_name -> credentials.BasicAuth
+	33, // 24: sources.JIRA.unauthenticated:type_name -> credentials.Unauthenticated
+	34, // 25: sources.JIRA.oauth:type_name -> credentials.Oauth2
+	33, // 26: sources.NPMUnauthenticatedPackage.unauthenticated:type_name -> credentials.Unauthenticated
+	33, // 27: sources.PyPIUnauthenticatedPackage.unauthenticated:type_name -> credentials.Unauthenticated
+	35, // 28: sources.S3.access_key:type_name -> credentials.KeySecret
+	33, // 29: sources.S3.unauthenticated:type_name -> credentials.Unauthenticated
+	36, // 30: sources.S3.cloud_environment:type_name -> credentials.CloudEnvironment
+	39, // 31: sources.S3.session_token:type_name -> credentials.AWSSessionTokenSecret
+	40, // 32: sources.Slack.tokens:type_name -> credentials.SlackTokens
+	32, // 33: sources.Gerrit.basic_auth:type_name -> credentials.BasicAuth
+	33, // 34: sources.Gerrit.unauthenticated:type_name -> credentials.Unauthenticated
+	32, // 35: sources.Jenkins.basic_auth:type_name -> credentials.BasicAuth
+	41, // 36: sources.Jenkins.header:type_name -> credentials.Header
+	42, // 37: sources.Teams.authenticated:type_name -> credentials.ClientCredentials
+	34, // 38: sources.Teams.oauth:type_name -> credentials.Oauth2
+	32, // 39: sources.Artifactory.basic_auth:type_name -> credentials.BasicAuth
+	33, // 40: sources.PublicEventMonitoring.unauthenticated:type_name -> credentials.Unauthenticated
+	43, // 41: sources.PublicEventMonitoring.since:type_name -> google.protobuf.Timestamp
+	40, // 42: sources.SlackRealtime.tokens:type_name -> credentials.SlackTokens
+	34, // 43: sources.Sharepoint.oauth:type_name -> credentials.Oauth2
+	44, // [44:44] is the sub-list for method output_type
+	44, // [44:44] is the sub-list for method input_type
+	44, // [44:44] is the sub-list for extension type_name
+	44, // [44:44] is the sub-list for extension extendee
+	0,  // [0:44] is the sub-list for field type_name
 }
 
 func init() { file_sources_proto_init() }
@@ -4094,18 +3874,6 @@ func file_sources_proto_init() {
 				return nil
 			}
 		}
-		file_sources_proto_msgTypes[28].Exporter = func(v interface{}, i int) interface{} {
-			switch v := v.(*AzureRepos); i {
-			case 0:
-				return &v.state
-			case 1:
-				return &v.sizeCache
-			case 2:
-				return &v.unknownFields
-			default:
-				return nil
-			}
-		}
 	}
 	file_sources_proto_msgTypes[1].OneofWrappers = []interface{}{
 		(*AzureStorage_ConnectionString)(nil),
@@ -4213,17 +3981,13 @@ func file_sources_proto_init() {
 	file_sources_proto_msgTypes[27].OneofWrappers = []interface{}{
 		(*Sharepoint_Oauth)(nil),
 	}
-	file_sources_proto_msgTypes[28].OneofWrappers = []interface{}{
-		(*AzureRepos_Token)(nil),
-		(*AzureRepos_Oauth)(nil),
-	}
 	type x struct{}
 	out := protoimpl.TypeBuilder{
 		File: protoimpl.DescBuilder{
 			GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
 			RawDescriptor: file_sources_proto_rawDesc,
 			NumEnums:      2,
-			NumMessages:   29,
+			NumMessages:   28,
 			NumExtensions: 0,
 			NumServices:   0,
 		},

pkg/pb/sourcespb/sources.pb.validate.go

@@ -1880,12 +1880,6 @@ func (m *GitHub) validate(all bool) error {
 
 	// no validation rules for Base
 
-	// no validation rules for IncludePullRequestComments
-
-	// no validation rules for IncludeIssueComments
-
-	// no validation rules for IncludeGistComments
-
 	switch m.Credential.(type) {
 
 	case *GitHub_GithubApp:
@@ -4461,154 +4455,3 @@ var _ interface {
 	Cause() error
 	ErrorName() string
 } = SharepointValidationError{}
-
-// Validate checks the field values on AzureRepos with the rules defined in the
-// proto definition for this message. If any rules are violated, the first
-// error encountered is returned, or nil if there are no violations.
-func (m *AzureRepos) Validate() error {
-	return m.validate(false)
-}
-
-// ValidateAll checks the field values on AzureRepos with the rules defined in
-// the proto definition for this message. If any rules are violated, the
-// result is a list of violation errors wrapped in AzureReposMultiError, or
-// nil if none found.
-func (m *AzureRepos) ValidateAll() error {
-	return m.validate(true)
-}
-
-func (m *AzureRepos) validate(all bool) error {
-	if m == nil {
-		return nil
-	}
-
-	var errors []error
-
-	if _, err := url.Parse(m.GetEndpoint()); err != nil {
-		err = AzureReposValidationError{
-			field:  "Endpoint",
-			reason: "value must be a valid URI",
-			cause:  err,
-		}
-		if !all {
-			return err
-		}
-		errors = append(errors, err)
-	}
-
-	// no validation rules for IncludeForks
-
-	switch m.Credential.(type) {
-
-	case *AzureRepos_Token:
-		// no validation rules for Token
-
-	case *AzureRepos_Oauth:
-
-		if all {
-			switch v := interface{}(m.GetOauth()).(type) {
-			case interface{ ValidateAll() error }:
-				if err := v.ValidateAll(); err != nil {
-					errors = append(errors, AzureReposValidationError{
-						field:  "Oauth",
-						reason: "embedded message failed validation",
-						cause:  err,
-					})
-				}
-			case interface{ Validate() error }:
-				if err := v.Validate(); err != nil {
-					errors = append(errors, AzureReposValidationError{
-						field:  "Oauth",
-						reason: "embedded message failed validation",
-						cause:  err,
-					})
-				}
-			}
-		} else if v, ok := interface{}(m.GetOauth()).(interface{ Validate() error }); ok {
-			if err := v.Validate(); err != nil {
-				return AzureReposValidationError{
-					field:  "Oauth",
-					reason: "embedded message failed validation",
-					cause:  err,
-				}
-			}
-		}
-
-	}
-
-	if len(errors) > 0 {
-		return AzureReposMultiError(errors)
-	}
-
-	return nil
-}
-
-// AzureReposMultiError is an error wrapping multiple validation errors
-// returned by AzureRepos.ValidateAll() if the designated constraints aren't met.
-type AzureReposMultiError []error
-
-// Error returns a concatenation of all the error messages it wraps.
-func (m AzureReposMultiError) Error() string {
-	var msgs []string
-	for _, err := range m {
-		msgs = append(msgs, err.Error())
-	}
-	return strings.Join(msgs, "; ")
-}
-
-// AllErrors returns a list of validation violation errors.
-func (m AzureReposMultiError) AllErrors() []error { return m }
-
-// AzureReposValidationError is the validation error returned by
-// AzureRepos.Validate if the designated constraints aren't met.
-type AzureReposValidationError struct {
-	field  string
-	reason string
-	cause  error
-	key    bool
-}
-
-// Field function returns field value.
-func (e AzureReposValidationError) Field() string { return e.field }
-
-// Reason function returns reason value.
-func (e AzureReposValidationError) Reason() string { return e.reason }
-
-// Cause function returns cause value.
-func (e AzureReposValidationError) Cause() error { return e.cause }
-
-// Key function returns key value.
-func (e AzureReposValidationError) Key() bool { return e.key }
-
-// ErrorName returns error name.
-func (e AzureReposValidationError) ErrorName() string { return "AzureReposValidationError" }
-
-// Error satisfies the builtin error interface
-func (e AzureReposValidationError) Error() string {
-	cause := ""
-	if e.cause != nil {
-		cause = fmt.Sprintf(" | caused by: %v", e.cause)
-	}
-
-	key := ""
-	if e.key {
-		key = "key for "
-	}
-
-	return fmt.Sprintf(
-		"invalid %sAzureRepos.%s: %s%s",
-		key,
-		e.field,
-		e.reason,
-		cause)
-}
-
-var _ error = AzureReposValidationError{}
-
-var _ interface {
-	Field() string
-	Reason() string
-	Key() bool
-	Cause() error
-	ErrorName() string
-} = AzureReposValidationError{}

pkg/sources/chunker.go

@@ -12,8 +12,6 @@ const (
 	ChunkSize = 10 * 1024
 	// PeekSize is the size of the peek into the previous chunk.
 	PeekSize = 3 * 1024
-	// TotalChunkSize is the total size of a chunk with peek data.
-	TotalChunkSize = ChunkSize + PeekSize
 )
 
 // Chunker takes a chunk and splits it into chunks of ChunkSize.
@@ -21,31 +19,27 @@ func Chunker(originalChunk *Chunk) chan *Chunk {
 	chunkChan := make(chan *Chunk)
 	go func() {
 		defer close(chunkChan)
-		if len(originalChunk.Data) <= TotalChunkSize {
+		if len(originalChunk.Data) <= ChunkSize+PeekSize {
 			chunkChan <- originalChunk
 			return
 		}
 		r := bytes.NewReader(originalChunk.Data)
 		reader := bufio.NewReaderSize(bufio.NewReader(r), ChunkSize)
 		for {
-			chunkBytes := make([]byte, TotalChunkSize)
+			chunkBytes := make([]byte, ChunkSize)
 			chunk := *originalChunk
-			chunkBytes = chunkBytes[:ChunkSize]
 			n, err := reader.Read(chunkBytes)
 			if err != nil && !errors.Is(err, io.EOF) {
 				break
 			}
-			if n == 0 {
-				if errors.Is(err, io.EOF) {
-					break
-				}
-				continue
-			}
 			peekData, _ := reader.Peek(PeekSize)
-			copy(chunkBytes[n:], peekData)
-			chunk.Data = chunkBytes[:n+len(peekData)]
-
-			chunkChan <- &chunk
+			chunk.Data = append(chunkBytes[:n], peekData...)
+			if n > 0 {
+				chunkChan <- &chunk
+			}
+			if errors.Is(err, io.EOF) {
+				break
+			}
 		}
 	}()
 	return chunkChan

pkg/sources/chunker_test.go

@@ -72,15 +72,3 @@ func TestChunker(t *testing.T) {
 	}
 
 }
-
-func BenchmarkChunker(b *testing.B) {
-	data := bytes.Repeat([]byte("a"), ChunkSize*100)
-	chunk := &Chunk{
-		Data: data,
-	}
-	b.ResetTimer()
-	for i := 0; i < b.N; i++ {
-		for range Chunker(chunk) {
-		}
-	}
-}

pkg/sources/errors.go

@@ -1,7 +1,6 @@
 package sources
 
 import (
-	"errors"
 	"fmt"
 	"sync"
 )
@@ -33,13 +32,5 @@ func (s *ScanErrors) Count() uint64 {
 }
 
 func (s *ScanErrors) String() string {
-	s.mu.RLock()
-	defer s.mu.RUnlock()
 	return fmt.Sprintf("%v", s.errors)
 }
-
-func (s *ScanErrors) Errors() error {
-	s.mu.RLock()
-	defer s.mu.RUnlock()
-	return errors.Join(s.errors...)
-}

pkg/sources/filesystem/filesystem.go

@@ -45,8 +45,6 @@ type Source struct {
 // Ensure the Source satisfies the interfaces at compile time
 var _ sources.Source = (*Source)(nil)
 var _ sources.SourceUnitUnmarshaller = (*Source)(nil)
-var _ sources.SourceUnitEnumerator = (*Source)(nil)
-var _ sources.SourceUnitChunker = (*Source)(nil)
 
 // Type returns the type of source.
 // It is used for matching source types in configuration and job input.
@@ -217,57 +215,3 @@ func (s *Source) scanFile(ctx context.Context, path string, chunksChan chan *sou
 	}
 	return nil
 }
-
-// Enumerate implements SourceUnitEnumerator interface. This implementation simply
-// passes the configured paths as the source unit, whether it be a single
-// filepath or a directory.
-func (s *Source) Enumerate(ctx context.Context, reporter sources.UnitReporter) error {
-	for _, path := range s.paths {
-		item := sources.CommonSourceUnit{ID: path}
-		if err := reporter.UnitOk(ctx, item); err != nil {
-			return err
-		}
-	}
-	return nil
-}
-
-// ChunkUnit implements SourceUnitChunker interface.
-func (s *Source) ChunkUnit(ctx context.Context, unit sources.SourceUnit, reporter sources.ChunkReporter) error {
-	path := unit.SourceUnitID()
-	logger := ctx.Logger().WithValues("path", path)
-
-	cleanPath := filepath.Clean(path)
-	fileInfo, err := os.Stat(cleanPath)
-	if err != nil {
-		return reporter.ChunkErr(ctx, fmt.Errorf("unable to get file info: %w", err))
-	}
-
-	ch := make(chan *sources.Chunk)
-	var scanErr error
-	go func() {
-		defer close(ch)
-		if fileInfo.IsDir() {
-			// TODO: Finer grain error tracking of individual chunks.
-			scanErr = s.scanDir(ctx, cleanPath, ch)
-		} else {
-			// TODO: Finer grain error tracking of individual
-			// chunks (in the case of archives).
-			scanErr = s.scanFile(ctx, cleanPath, ch)
-		}
-	}()
-
-	for chunk := range ch {
-		if chunk == nil {
-			continue
-		}
-		if err := reporter.ChunkOk(ctx, *chunk); err != nil {
-			return err
-		}
-	}
-
-	if scanErr != nil && scanErr != io.EOF {
-		logger.Info("error scanning filesystem", "error", scanErr)
-		return reporter.ChunkErr(ctx, scanErr)
-	}
-	return nil
-}

pkg/sources/git/git.go

@@ -10,9 +10,7 @@ import (
 	"os/exec"
 	"path/filepath"
 	"runtime"
-	"strconv"
 	"strings"
-	"sync/atomic"
 	"time"
 
 	diskbufferreader "github.com/bill-rich/disk-buffer-reader"
@@ -42,11 +40,7 @@ type Source struct {
 	verify   bool
 	git      *Git
 	sources.Progress
-	conn        *sourcespb.Git
-	scanOptions *ScanOptions
-	// Kludge to preserve engine.ScanGit functionality which doesn't expect
-	// the scanning to clean up the directory.
-	preserveTempDirs bool
+	conn *sourcespb.Git
 }
 
 type Git struct {
@@ -56,14 +50,9 @@ type Git struct {
 	jobID              int64
 	sourceMetadataFunc func(file, email, commit, timestamp, repository string, line int64) *source_metadatapb.MetaData
 	verify             bool
-	metrics            metrics
 	concurrency        *semaphore.Weighted
 }
 
-type metrics struct {
-	commitsScanned uint64
-}
-
 func NewGit(sourceType sourcespb.SourceType, jobID, sourceID int64, sourceName string, verify bool, concurrency int,
 	sourceMetadataFunc func(file, email, commit, timestamp, repository string, line int64) *source_metadatapb.MetaData,
 ) *Git {
@@ -96,19 +85,6 @@ func (s *Source) JobID() int64 {
 	return s.jobId
 }
 
-// WithScanOptions sets the scan options.
-func (s *Source) WithScanOptions(scanOptions *ScanOptions) {
-	s.scanOptions = scanOptions
-}
-
-// WithPreserveTempDirs sets whether to preserve temp directories when scanning
-// the provided list of s.conn.Directories. NOTE: This is *only* for
-// s.conn.Directories, not all temp directories created. This is also a kludge
-// and should be refactored away.
-func (s *Source) WithPreserveTempDirs(preserve bool) {
-	s.preserveTempDirs = preserve
-}
-
 // Init returns an initialized GitHub source.
 func (s *Source) Init(aCtx context.Context, name string, jobId, sourceId int64, verify bool, connection *anypb.Any, concurrency int) error {
 	s.name = name
@@ -152,29 +128,8 @@ func (s *Source) Init(aCtx context.Context, name string, jobId, sourceId int64,
 
 // Chunks emits chunks of bytes over a channel.
 func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) error {
-	if err := s.scanRepos(ctx, chunksChan); err != nil {
-		return err
-	}
-	if err := s.scanDirs(ctx, chunksChan); err != nil {
-		return err
-	}
-
-	totalRepos := len(s.conn.Repositories) + len(s.conn.Directories)
-	ctx.Logger().V(1).Info("Git source finished scanning", "repo_count", totalRepos)
-	s.SetProgressComplete(
-		totalRepos, totalRepos,
-		fmt.Sprintf("Completed scanning source %s", s.name), "",
-	)
-	return nil
-}
-
-// scanRepos scans the configured repositories in s.conn.Repositories.
-func (s *Source) scanRepos(ctx context.Context, chunksChan chan *sources.Chunk) error {
-	if len(s.conn.Repositories) == 0 {
-		return nil
-	}
-	totalRepos := len(s.conn.Repositories) + len(s.conn.Directories)
 	// TODO: refactor to remove duplicate code
+	totalRepos := len(s.conn.Repositories) + len(s.conn.Directories)
 	switch cred := s.conn.GetCredential().(type) {
 	case *sourcespb.Git_BasicAuth:
 		user := cred.BasicAuth.Username
@@ -191,7 +146,7 @@ func (s *Source) scanRepos(ctx context.Context, chunksChan chan *sources.Chunk)
 				if err != nil {
 					return err
 				}
-				return s.git.ScanRepo(ctx, repo, path, s.scanOptions, chunksChan)
+				return s.git.ScanRepo(ctx, repo, path, NewScanOptions(), chunksChan)
 			}(repoURI)
 			if err != nil {
 				ctx.Logger().Info("error scanning repository", "repo", repoURI, "error", err)
@@ -210,7 +165,7 @@ func (s *Source) scanRepos(ctx context.Context, chunksChan chan *sources.Chunk)
 				if err != nil {
 					return err
 				}
-				return s.git.ScanRepo(ctx, repo, path, s.scanOptions, chunksChan)
+				return s.git.ScanRepo(ctx, repo, path, NewScanOptions(), chunksChan)
 			}(repoURI)
 			if err != nil {
 				ctx.Logger().Info("error scanning repository", "repo", repoURI, "error", err)
@@ -229,7 +184,7 @@ func (s *Source) scanRepos(ctx context.Context, chunksChan chan *sources.Chunk)
 				if err != nil {
 					return err
 				}
-				return s.git.ScanRepo(ctx, repo, path, s.scanOptions, chunksChan)
+				return s.git.ScanRepo(ctx, repo, path, NewScanOptions(), chunksChan)
 			}(repoURI)
 			if err != nil {
 				ctx.Logger().Info("error scanning repository", "repo", repoURI, "error", err)
@@ -239,42 +194,41 @@ func (s *Source) scanRepos(ctx context.Context, chunksChan chan *sources.Chunk)
 	default:
 		return errors.New("invalid connection type for git source")
 	}
-	return nil
-}
 
-// scanDirs scans the configured directories in s.conn.Directories.
-func (s *Source) scanDirs(ctx context.Context, chunksChan chan *sources.Chunk) error {
-	totalRepos := len(s.conn.Repositories) + len(s.conn.Directories)
 	for i, gitDir := range s.conn.Directories {
 		s.SetProgressComplete(len(s.conn.Repositories)+i, totalRepos, fmt.Sprintf("Repo: %s", gitDir), "")
 
 		if len(gitDir) == 0 {
 			continue
 		}
-		if strings.HasSuffix(gitDir, "git") {
-			// TODO: Figure out why we skip directories ending in "git".
-			continue
-		}
-		// try paths instead of url
-		repo, err := RepoFromPath(gitDir)
-		if err != nil {
-			ctx.Logger().Info("error scanning repository", "repo", gitDir, "error", err)
-			continue
-		}
-
-		err = func(repoPath string) error {
-			if !s.preserveTempDirs && strings.HasPrefix(repoPath, filepath.Join(os.TempDir(), "trufflehog")) {
-				defer os.RemoveAll(repoPath)
+		if !strings.HasSuffix(gitDir, "git") {
+			// try paths instead of url
+			repo, err := RepoFromPath(gitDir)
+			if err != nil {
+				ctx.Logger().Info("error scanning repository", "repo", gitDir, "error", err)
+				continue
 			}
 
-			return s.git.ScanRepo(ctx, repo, repoPath, s.scanOptions, chunksChan)
-		}(gitDir)
-		if err != nil {
-			ctx.Logger().Info("error scanning repository", "repo", gitDir, "error", err)
-			continue
+			err = func(repoPath string) error {
+				if strings.HasPrefix(repoPath, filepath.Join(os.TempDir(), "trufflehog")) {
+					defer os.RemoveAll(repoPath)
+				}
+
+				return s.git.ScanRepo(ctx, repo, repoPath, NewScanOptions(), chunksChan)
+			}(gitDir)
+			if err != nil {
+				ctx.Logger().Info("error scanning repository", "repo", gitDir, "error", err)
+				continue
+			}
 		}
 
 	}
+
+	ctx.Logger().V(1).Info("Git source finished scanning", "repo-count", totalRepos)
+	s.SetProgressComplete(
+		totalRepos, totalRepos,
+		fmt.Sprintf("Completed scanning source %s", s.name), "",
+	)
 	return nil
 }
 
@@ -292,7 +246,7 @@ func CleanOnError(err *error, path string) {
 	}
 }
 
-func GitURLParse(gitURL string) (*url.URL, error) {
+func gitURLParse(gitURL string) (*url.URL, error) {
 	parsedURL, originalError := url.Parse(gitURL)
 	if originalError != nil {
 		var err error
@@ -316,7 +270,7 @@ func CloneRepo(ctx context.Context, userInfo *url.Userinfo, gitUrl string, args
 		return "", nil, err
 	}
 	defer CleanOnError(&err, clonePath)
-	cloneURL, err := GitURLParse(gitUrl)
+	cloneURL, err := gitURLParse(gitUrl)
 	if err != nil {
 		return "", nil, err
 	}
@@ -384,10 +338,6 @@ func CloneRepoUsingSSH(ctx context.Context, gitUrl string, args ...string) (stri
 	return CloneRepo(ctx, userInfo, gitUrl, args...)
 }
 
-func (s *Git) CommitsScanned() uint64 {
-	return atomic.LoadUint64(&s.metrics.commitsScanned)
-}
-
 func (s *Git) ScanCommits(ctx context.Context, repo *git.Repository, path string, scanOptions *ScanOptions, chunksChan chan *sources.Chunk) error {
 	if err := GitCmdCheck(); err != nil {
 		return err
@@ -420,7 +370,6 @@ func (s *Git) ScanCommits(ctx context.Context, repo *git.Repository, path string
 			break
 		}
 		depth++
-		atomic.AddUint64(&s.metrics.commitsScanned, 1)
 		logger.V(5).Info("scanning commit", "commit", commit.Hash)
 		for _, diff := range commit.Diffs {
 			if !scanOptions.Filter.Pass(diff.PathB) {
@@ -688,23 +637,15 @@ func normalizeConfig(scanOptions *ScanOptions, repo *git.Repository) (err error)
 }
 
 // GenerateLink crafts a link to the specific file from a commit. This works in most major git providers (Github/Gitlab)
-func GenerateLink(repo, commit, file string, line int64) string {
+func GenerateLink(repo, commit, file string) string {
 	// bitbucket links are commits not commit...
 	if strings.Contains(repo, "bitbucket.org/") {
 		return repo[:len(repo)-4] + "/commits/" + commit
 	}
-	var link string
+	link := repo[:len(repo)-4] + "/blob/" + commit + "/" + file
+
 	if file == "" {
 		link = repo[:len(repo)-4] + "/commit/" + commit
-	} else {
-		link = repo[:len(repo)-4] + "/blob/" + commit + "/" + file
-
-		// Both GitHub and Gitlab support hyperlinking to a specific line with #L<number>, e.g.:
-		// https://github.com/trufflesecurity/trufflehog/blob/e856a6890d0da5a218f4f9283500b80043884641/go.mod#L169
-		// https://gitlab.com/pdftk-java/pdftk/-/blob/88559a08f34175b6fae76c40a88f0377f64a12d7/java/com/gitlab/pdftk_java/report.java#L893
-		if line > 0 && (strings.Contains(repo, "github") || strings.Contains(repo, "gitlab")) {
-			link += "#L" + strconv.FormatInt(line, 10)
-		}
 	}
 	return link
 }
@@ -757,7 +698,7 @@ func PrepareRepoSinceCommit(ctx context.Context, uriString, commitHash string) (
 	// the uriString is github.com, then we query the API for the timestamp of the
 	// hash and use that to clone.
 
-	uri, err := GitURLParse(uriString)
+	uri, err := gitURLParse(uriString)
 	if err != nil {
 		return "", false, fmt.Errorf("unable to parse Git URI: %s", err)
 	}
@@ -805,7 +746,7 @@ func PrepareRepoSinceCommit(ctx context.Context, uriString, commitHash string) (
 		}
 		path, _, err = CloneRepoUsingToken(ctx, password, remotePath, uri.User.Username(), "--shallow-since", timestamp)
 		if err != nil {
-			return path, true, fmt.Errorf("failed to clone authenticated Git repo (%s): %s", uri.Redacted(), err)
+			return path, true, fmt.Errorf("failed to clone authenticated Git repo (%s): %s", remotePath, err)
 		}
 	default:
 		ctx.Logger().V(1).Info("cloning repo without authentication", "uri", uri)
@@ -822,7 +763,7 @@ func PrepareRepoSinceCommit(ctx context.Context, uriString, commitHash string) (
 // PrepareRepo clones a repo if possible and returns the cloned repo path.
 func PrepareRepo(ctx context.Context, uriString string) (string, bool, error) {
 	var path string
-	uri, err := GitURLParse(uriString)
+	uri, err := gitURLParse(uriString)
 	if err != nil {
 		return "", false, fmt.Errorf("unable to parse Git URI: %s", err)
 	}
@@ -843,7 +784,7 @@ func PrepareRepo(ctx context.Context, uriString string) (string, bool, error) {
 			}
 			path, _, err = CloneRepoUsingToken(ctx, password, remotePath, uri.User.Username())
 			if err != nil {
-				return path, remote, fmt.Errorf("failed to clone authenticated Git repo (%s): %s", uri.Redacted(), err)
+				return path, remote, fmt.Errorf("failed to clone authenticated Git repo (%s): %s", remotePath, err)
 			}
 		default:
 			ctx.Logger().V(1).Info("cloning repo without authentication", "uri", uri)

pkg/sources/git/git_test.go

@@ -156,7 +156,6 @@ func Test_generateLink(t *testing.T) {
 		repo   string
 		commit string
 		file   string
-		line   int64
 	}
 	tests := []struct {
 		name string
@@ -172,16 +171,6 @@ func Test_generateLink(t *testing.T) {
 			},
 			want: "https://github.com/trufflesec-julian/confluence-go-api/blob/047b4a2ba42fc5b6c0bd535c5307434a666db5ec/.gitignore",
 		},
-		{
-			name: "test link gen",
-			args: args{
-				repo:   "https://github.com/trufflesec-julian/confluence-go-api.git",
-				commit: "047b4a2ba42fc5b6c0bd535c5307434a666db5ec",
-				file:   ".gitignore",
-				line:   int64(4),
-			},
-			want: "https://github.com/trufflesec-julian/confluence-go-api/blob/047b4a2ba42fc5b6c0bd535c5307434a666db5ec/.gitignore#L4",
-		},
 		{
 			name: "test link gen - no file",
 			args: args{
@@ -193,7 +182,7 @@ func Test_generateLink(t *testing.T) {
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			if got := GenerateLink(tt.args.repo, tt.args.commit, tt.args.file, tt.args.line); got != tt.want {
+			if got := GenerateLink(tt.args.repo, tt.args.commit, tt.args.file); got != tt.want {
 				t.Errorf("generateLink() = %v, want %v", got, tt.want)
 			}
 		})
@@ -540,7 +529,7 @@ func TestGitURLParse(t *testing.T) {
 			"ssh",
 		},
 	} {
-		u, err := GitURLParse(tt.url)
+		u, err := gitURLParse(tt.url)
 		if err != nil {
 			t.Fatal(err)
 		}

pkg/sources/github/github.go

@@ -44,12 +44,9 @@ const (
 )
 
 type Source struct {
-	name string
-	// Protects the user and token.
-	userMu      sync.Mutex
-	githubUser  string
-	githubToken string
-
+	name              string
+	githubUser        string
+	githubToken       string
 	sourceID          int64
 	jobID             int64
 	verify            bool
@@ -61,24 +58,16 @@ type Source struct {
 	repoSizes         repoSize
 	totalRepoSize     int // total size in bytes of all repos
 	git               *git.Git
-
-	scanOptMu   sync.Mutex // protects the scanOptions
-	scanOptions *git.ScanOptions
-
-	httpClient      *http.Client
-	log             logr.Logger
-	conn            *sourcespb.GitHub
-	jobPool         *errgroup.Group
-	resumeInfoMutex sync.Mutex
-	resumeInfoSlice []string
-	apiClient       *github.Client
-
-	mu        sync.Mutex // protects the visibility maps
-	publicMap map[string]source_metadatapb.Visibility
-
-	includePRComments    bool
-	includeIssueComments bool
-	includeGistComments  bool
+	scanOptions       *git.ScanOptions
+	httpClient        *http.Client
+	log               logr.Logger
+	conn              *sourcespb.GitHub
+	jobPool           *errgroup.Group
+	resumeInfoMutex   sync.Mutex
+	resumeInfoSlice   []string
+	apiClient         *github.Client
+	mu                sync.Mutex
+	publicMap         map[string]source_metadatapb.Visibility
 	sources.Progress
 	sources.CommonSourceUnitUnmarshaller
 }
@@ -87,13 +76,6 @@ func (s *Source) WithScanOptions(scanOptions *git.ScanOptions) {
 	s.scanOptions = scanOptions
 }
 
-func (s *Source) setScanOptions(base, head string) {
-	s.scanOptMu.Lock()
-	defer s.scanOptMu.Unlock()
-	s.scanOptions.BaseHash = base
-	s.scanOptions.HeadHash = head
-}
-
 // Ensure the Source satisfies the interfaces at compile time
 var _ sources.Source = (*Source)(nil)
 var _ sources.SourceUnitUnmarshaller = (*Source)(nil)
@@ -231,10 +213,6 @@ func (s *Source) Init(aCtx context.Context, name string, jobID, sourceID int64,
 		s.filteredRepoCache.Set(r, r)
 	}
 
-	s.includeIssueComments = s.conn.IncludeIssueComments
-	s.includePRComments = s.conn.IncludePullRequestComments
-	s.includeGistComments = s.conn.IncludeGistComments
-
 	s.orgsCache = memory.New()
 	for _, org := range s.conn.Organizations {
 		s.orgsCache.Set(org, org)
@@ -261,7 +239,7 @@ func (s *Source) Init(aCtx context.Context, name string, jobID, sourceID int64,
 						File:       sanitizer.UTF8(file),
 						Email:      sanitizer.UTF8(email),
 						Repository: sanitizer.UTF8(repository),
-						Link:       git.GenerateLink(repository, commit, file, line),
+						Link:       git.GenerateLink(repository, commit, file),
 						Timestamp:  sanitizer.UTF8(timestamp),
 						Line:       line,
 						Visibility: s.visibilityOf(aCtx, repository),
@@ -373,11 +351,11 @@ func (s *Source) enumerate(ctx context.Context, apiEndpoint string) (*github.Cli
 
 	switch cred := s.conn.GetCredential().(type) {
 	case *sourcespb.GitHub_BasicAuth:
-		if err = s.enumerateBasicAuth(ctx, apiEndpoint, cred.BasicAuth); err != nil {
+		if err = s.enumerateBasicAuth(ctx, cred.BasicAuth); err != nil {
 			return nil, err
 		}
 	case *sourcespb.GitHub_Unauthenticated:
-		s.enumerateUnauthenticated(ctx, apiEndpoint)
+		s.enumerateUnauthenticated(ctx)
 	case *sourcespb.GitHub_Token:
 		if err = s.enumerateWithToken(ctx, apiEndpoint, cred.Token); err != nil {
 			return nil, err
@@ -399,16 +377,11 @@ func (s *Source) enumerate(ctx context.Context, apiEndpoint string) (*github.Cli
 	return installationClient, nil
 }
 
-func (s *Source) enumerateBasicAuth(ctx context.Context, apiEndpoint string, basicAuth *credentialspb.BasicAuth) error {
-	s.httpClient.Transport = &github.BasicAuthTransport{
+func (s *Source) enumerateBasicAuth(ctx context.Context, basicAuth *credentialspb.BasicAuth) error {
+	s.apiClient = github.NewClient(&http.Client{Transport: &github.BasicAuthTransport{
 		Username: basicAuth.Username,
 		Password: basicAuth.Password,
-	}
-	ghClient, err := createGitHubClient(s.httpClient, apiEndpoint)
-	if err != nil {
-		s.log.Error(err, "error creating GitHub client")
-	}
-	s.apiClient = ghClient
+	}})
 
 	for _, org := range s.orgsCache.Keys() {
 		if err := s.getReposByOrg(ctx, org); err != nil {
@@ -419,12 +392,8 @@ func (s *Source) enumerateBasicAuth(ctx context.Context, apiEndpoint string, bas
 	return nil
 }
 
-func (s *Source) enumerateUnauthenticated(ctx context.Context, apiEndpoint string) {
-	ghClient, err := createGitHubClient(s.httpClient, apiEndpoint)
-	if err != nil {
-		s.log.Error(err, "error creating GitHub client")
-	}
-	s.apiClient = ghClient
+func (s *Source) enumerateUnauthenticated(ctx context.Context) {
+	s.apiClient = github.NewClient(s.httpClient)
 	if s.orgsCache.Count() > unauthGithubOrgRateLimt {
 		s.log.Info("You may experience rate limiting when using the unauthenticated GitHub api. Consider using an authenticated scan instead.")
 	}
@@ -458,14 +427,19 @@ func (s *Source) enumerateWithToken(ctx context.Context, apiEndpoint, token stri
 		Source: oauth2.ReuseTokenSource(nil, ts),
 	}
 
-	// If we're using public GitHub, make a regular client.
+	var err error
+	// If we're using public Github, make a regular client.
 	// Otherwise, make an enterprise client.
-	var isGHE bool = apiEndpoint != "https://api.github.com"
-	ghClient, err := createGitHubClient(s.httpClient, apiEndpoint)
-	if err != nil {
-		s.log.Error(err, "error creating GitHub client")
+	var isGHE bool
+	if apiEndpoint == "https://api.github.com" {
+		s.apiClient = github.NewClient(s.httpClient)
+	} else {
+		isGHE = true
+		s.apiClient, err = github.NewEnterpriseClient(apiEndpoint, apiEndpoint, s.httpClient)
+		if err != nil {
+			return errors.New(err)
+		}
 	}
-	s.apiClient = ghClient
 
 	// TODO: this should support scanning users too
 
@@ -571,28 +545,6 @@ func (s *Source) enumerateWithApp(ctx context.Context, apiEndpoint string, app *
 		return nil, errors.New(err)
 	}
 
-	// This client is required to create installation tokens for cloning.
-	// Otherwise, the required JWT is not in the request for the token :/
-	// This client uses the source's original HTTP transport, so make sure
-	// to build it before modifying that transport (such as is done during
-	// the creation of the other API client below).
-	appItr, err := ghinstallation.NewAppsTransport(
-		s.httpClient.Transport,
-		appID,
-		[]byte(app.PrivateKey))
-	if err != nil {
-		return nil, errors.New(err)
-	}
-	appItr.BaseURL = apiEndpoint
-
-	// Does this need to be separate from |s.httpClient|?
-	instHttpClient := common.RetryableHttpClientTimeout(60)
-	instHttpClient.Transport = appItr
-	installationClient, err = github.NewEnterpriseClient(apiEndpoint, apiEndpoint, instHttpClient)
-	if err != nil {
-		return nil, errors.New(err)
-	}
-
 	// This client is used for most APIs.
 	itr, err := ghinstallation.New(
 		s.httpClient.Transport,
@@ -603,9 +555,22 @@ func (s *Source) enumerateWithApp(ctx context.Context, apiEndpoint string, app *
 		return nil, errors.New(err)
 	}
 	itr.BaseURL = apiEndpoint
+	s.apiClient, err = github.NewEnterpriseClient(apiEndpoint, apiEndpoint, &http.Client{Transport: itr})
+	if err != nil {
+		return nil, errors.New(err)
+	}
 
-	s.httpClient.Transport = itr
-	s.apiClient, err = github.NewEnterpriseClient(apiEndpoint, apiEndpoint, s.httpClient)
+	// This client is required to create installation tokens for cloning.
+	// Otherwise, the required JWT is not in the request for the token :/
+	appItr, err := ghinstallation.NewAppsTransport(
+		s.httpClient.Transport,
+		appID,
+		[]byte(app.PrivateKey))
+	if err != nil {
+		return nil, errors.New(err)
+	}
+	appItr.BaseURL = apiEndpoint
+	installationClient, err = github.NewEnterpriseClient(apiEndpoint, apiEndpoint, &http.Client{Transport: appItr})
 	if err != nil {
 		return nil, errors.New(err)
 	}
@@ -638,21 +603,6 @@ func (s *Source) enumerateWithApp(ctx context.Context, apiEndpoint string, app *
 	return installationClient, nil
 }
 
-func createGitHubClient(httpClient *http.Client, apiEndpoint string) (ghClient *github.Client, err error) {
-	// If we're using public GitHub, make a regular client.
-	// Otherwise, make an enterprise client.
-	if apiEndpoint == "https://api.github.com" {
-		ghClient = github.NewClient(httpClient)
-	} else {
-		ghClient, err = github.NewEnterpriseClient(apiEndpoint, apiEndpoint, httpClient)
-		if err != nil {
-			return nil, errors.New(err)
-		}
-	}
-
-	return ghClient, err
-}
-
 func (s *Source) scan(ctx context.Context, installationClient *github.Client, chunksChan chan *sources.Chunk) error {
 	var scanned uint64
 
@@ -698,7 +648,6 @@ func (s *Source) scan(ctx context.Context, installationClient *github.Client, ch
 			path, repo, err = s.cloneRepo(ctx, repoURL, installationClient)
 			if err != nil {
 				scanErrs.Add(err)
-				return nil
 			}
 
 			defer os.RemoveAll(path)
@@ -707,7 +656,8 @@ func (s *Source) scan(ctx context.Context, installationClient *github.Client, ch
 				return nil
 			}
 
-			s.setScanOptions(s.conn.Base, s.conn.Head)
+			s.scanOptions.BaseHash = s.conn.Base
+			s.scanOptions.HeadHash = s.conn.Head
 
 			repoSize := s.repoSizes.getRepo(repoURL)
 			logger.V(2).Info(fmt.Sprintf("scanning repo %d/%d", i, len(s.repos)), "repo_size", repoSize)
@@ -716,17 +666,10 @@ func (s *Source) scan(ctx context.Context, installationClient *github.Client, ch
 			defer func(start time.Time) {
 				logger.V(2).Info(fmt.Sprintf("scanned %d/%d repos", scanned, len(s.repos)), "repo_size", repoSize, "duration_seconds", time.Since(start).Seconds())
 			}(now)
-
 			if err = s.git.ScanRepo(ctx, repo, path, s.scanOptions, chunksChan); err != nil {
 				scanErrs.Add(fmt.Errorf("error scanning repo %s: %w", repoURL, err))
 				return nil
 			}
-
-			if err = s.scanComments(ctx, repoURL, chunksChan); err != nil {
-				scanErrs.Add(fmt.Errorf("error scanning comments in repo %s: %w", repoURL, err))
-				return nil
-			}
-
 			atomic.AddUint64(&scanned, 1)
 
 			return nil
@@ -976,237 +919,3 @@ func (s *Source) setProgressCompleteWithRepo(index int, offset int, repoURL stri
 
 	s.SetProgressComplete(index+offset, len(s.repos)+offset, fmt.Sprintf("Repo: %s", repoURL), encodedResumeInfo)
 }
-
-func (s *Source) scanComments(ctx context.Context, repoPath string, chunksChan chan *sources.Chunk) error {
-	s.log.Info("scanning comments", "repository", repoPath)
-
-	// Support ssh and https URLs
-	repoURL, err := git.GitURLParse(repoPath)
-	if err != nil {
-		return err
-	}
-
-	trimmedURL := removeURLAndSplit(repoURL.String())
-	if repoURL.Host == "gist.github.com" && s.includeGistComments {
-		// GitHub Gist URL.
-		var gistId string
-		if len(trimmedURL) == 2 {
-			// https://gist.github.com/<id>
-			gistId = trimmedURL[1]
-		} else if len(trimmedURL) == 3 {
-			// https://gist.github.com/<owner>/<id>
-			gistId = trimmedURL[2]
-		} else {
-			return fmt.Errorf("failed to parse Gist URL: '%s'", repoURL.String())
-		}
-
-		options := &github.ListOptions{
-			PerPage: defaultPagination,
-			Page:    1,
-		}
-		for {
-			comments, resp, err := s.apiClient.Gists.ListComments(ctx, gistId, options)
-			if s.handleRateLimit(err, resp) {
-				break
-			}
-			if err != nil {
-				return err
-			}
-
-			err = s.chunkGistComments(ctx, repoURL.String(), comments, chunksChan)
-			if err != nil {
-				return err
-			}
-
-			options.Page++
-			if len(comments) < options.PerPage {
-				break
-			}
-		}
-	} else {
-		// Normal repository URL (https://github.com/<owner>/<repo>).
-		owner := trimmedURL[1]
-		repo := trimmedURL[2]
-
-		var (
-			sortType      = "created"
-			directionType = "desc"
-			allComments   = 0
-		)
-
-		if s.includeIssueComments {
-
-			issueOpts := &github.IssueListCommentsOptions{
-				Sort:      &sortType,
-				Direction: &directionType,
-				ListOptions: github.ListOptions{
-					PerPage: defaultPagination,
-					Page:    1,
-				},
-			}
-
-			for {
-				issueComments, resp, err := s.apiClient.Issues.ListComments(ctx, owner, repo, allComments, issueOpts)
-				if s.handleRateLimit(err, resp) {
-					break
-				}
-
-				if err != nil {
-					return err
-				}
-
-				err = s.chunkIssueComments(ctx, repo, issueComments, chunksChan, repoPath)
-				if err != nil {
-					return err
-				}
-
-				issueOpts.ListOptions.Page++
-
-				if len(issueComments) < defaultPagination {
-					break
-				}
-			}
-
-		}
-
-		if s.includePRComments {
-			prOpts := &github.PullRequestListCommentsOptions{
-				Sort:      sortType,
-				Direction: directionType,
-				ListOptions: github.ListOptions{
-					PerPage: defaultPagination,
-					Page:    1,
-				},
-			}
-
-			for {
-				prComments, resp, err := s.apiClient.PullRequests.ListComments(ctx, owner, repo, allComments, prOpts)
-				if s.handleRateLimit(err, resp) {
-					break
-				}
-
-				if err != nil {
-					return err
-				}
-
-				err = s.chunkPullRequestComments(ctx, repo, prComments, chunksChan, repoPath)
-				if err != nil {
-					return err
-				}
-
-				prOpts.ListOptions.Page++
-
-				if len(prComments) < defaultPagination {
-					break
-				}
-			}
-		}
-	}
-
-	return nil
-}
-
-func (s *Source) chunkIssueComments(ctx context.Context, repo string, comments []*github.IssueComment, chunksChan chan *sources.Chunk, repoPath string) error {
-	for _, comment := range comments {
-		// Create chunk and send it to the channel.
-		chunk := &sources.Chunk{
-			SourceName: s.name,
-			SourceID:   s.SourceID(),
-			SourceType: s.Type(),
-			SourceMetadata: &source_metadatapb.MetaData{
-				Data: &source_metadatapb.MetaData_Github{
-					Github: &source_metadatapb.Github{
-						Link:       sanitizer.UTF8(comment.GetHTMLURL()),
-						Username:   sanitizer.UTF8(comment.GetUser().GetLogin()),
-						Email:      sanitizer.UTF8(comment.GetUser().GetEmail()),
-						Repository: sanitizer.UTF8(repo),
-						Timestamp:  sanitizer.UTF8(comment.GetCreatedAt().String()),
-						Visibility: s.visibilityOf(ctx, repoPath),
-					},
-				},
-			},
-			Data:   []byte(sanitizer.UTF8(comment.GetBody())),
-			Verify: s.verify,
-		}
-
-		select {
-		case <-ctx.Done():
-			return ctx.Err()
-		case chunksChan <- chunk:
-		}
-	}
-	return nil
-}
-
-func (s *Source) chunkPullRequestComments(ctx context.Context, repo string, comments []*github.PullRequestComment, chunksChan chan *sources.Chunk, repoPath string) error {
-	for _, comment := range comments {
-		// Create chunk and send it to the channel.
-		chunk := &sources.Chunk{
-			SourceName: s.name,
-			SourceID:   s.SourceID(),
-			SourceType: s.Type(),
-			SourceMetadata: &source_metadatapb.MetaData{
-				Data: &source_metadatapb.MetaData_Github{
-					Github: &source_metadatapb.Github{
-						Link:       sanitizer.UTF8(comment.GetHTMLURL()),
-						Username:   sanitizer.UTF8(comment.GetUser().GetLogin()),
-						Email:      sanitizer.UTF8(comment.GetUser().GetEmail()),
-						Repository: sanitizer.UTF8(repo),
-						Timestamp:  sanitizer.UTF8(comment.GetCreatedAt().String()),
-					},
-				},
-			},
-			Data:   []byte(sanitizer.UTF8(comment.GetBody())),
-			Verify: s.verify,
-		}
-
-		select {
-		case <-ctx.Done():
-			return ctx.Err()
-		case chunksChan <- chunk:
-		}
-	}
-	return nil
-}
-
-func (s *Source) chunkGistComments(ctx context.Context, gistUrl string, comments []*github.GistComment, chunksChan chan *sources.Chunk) error {
-	for _, comment := range comments {
-		// Create chunk and send it to the channel.
-		chunk := &sources.Chunk{
-			SourceName: s.name,
-			SourceID:   s.SourceID(),
-			SourceType: s.Type(),
-			SourceMetadata: &source_metadatapb.MetaData{
-				Data: &source_metadatapb.MetaData_Github{
-					Github: &source_metadatapb.Github{
-						Link:       sanitizer.UTF8(comment.GetURL()),
-						Username:   sanitizer.UTF8(comment.GetUser().GetLogin()),
-						Email:      sanitizer.UTF8(comment.GetUser().GetEmail()),
-						Repository: sanitizer.UTF8(gistUrl),
-						Timestamp:  sanitizer.UTF8(comment.GetCreatedAt().String()),
-						// Fetching this information requires making an additional API call.
-						// We may want to include this in the future.
-						//Visibility: s.visibilityOf(ctx, repoPath),
-					},
-				},
-			},
-			Data:   []byte(sanitizer.UTF8(comment.GetBody())),
-			Verify: s.verify,
-		}
-
-		select {
-		case <-ctx.Done():
-			return ctx.Err()
-		case chunksChan <- chunk:
-		}
-	}
-	return nil
-}
-
-func removeURLAndSplit(url string) []string {
-	trimmedURL := strings.TrimPrefix(url, "https://")
-	trimmedURL = strings.TrimSuffix(trimmedURL, ".git")
-	splitURL := strings.Split(trimmedURL, "/")
-
-	return splitURL
-}

pkg/sources/github/github_integration_test.go

@@ -80,220 +80,6 @@ func TestSource_Token(t *testing.T) {
 	assert.NoError(t, err)
 }
 
-func TestSource_ScanComments(t *testing.T) {
-	ctx, cancel := context.WithTimeout(context.Background(), time.Second*3)
-	defer cancel()
-
-	secret, err := common.GetTestSecret(ctx)
-	if err != nil {
-		t.Fatal(fmt.Errorf("failed to access secret: %v", err))
-	}
-
-	// For the personal access token test
-	githubToken := secret.MustGetField("GITHUB_TOKEN")
-
-	const totalPRChunks = 2
-	const totalIssueChunks = 1
-
-	type init struct {
-		name       string
-		verify     bool
-		connection *sourcespb.GitHub
-	}
-	tests := []struct {
-		name              string
-		init              init
-		wantChunk         *sources.Chunk
-		wantErr           bool
-		minRepo           int
-		minOrg            int
-		numExpectedChunks int
-	}{
-		{
-			name: "token authenticated, single repo, single issue comment",
-			init: init{
-				name: "test source",
-				connection: &sourcespb.GitHub{
-					Repositories:               []string{"https://github.com/truffle-test-integration-org/another-test-repo.git"},
-					IncludeIssueComments:       true,
-					IncludePullRequestComments: false,
-					Credential: &sourcespb.GitHub_Token{
-						Token: githubToken,
-					},
-				},
-			},
-			numExpectedChunks: totalIssueChunks,
-			wantChunk: &sources.Chunk{
-				SourceType: sourcespb.SourceType_SOURCE_TYPE_GITHUB,
-				SourceName: "test source",
-				SourceMetadata: &source_metadatapb.MetaData{
-					Data: &source_metadatapb.MetaData_Github{
-						Github: &source_metadatapb.Github{
-							Link:      "https://github.com/truffle-test-integration-org/another-test-repo/issues/1#issuecomment-1603436833",
-							Username:  "truffle-sandbox",
-							Timestamp: "2023-06-22 23:33:46 +0000 UTC",
-						},
-					},
-				},
-				Verify: false,
-			},
-			wantErr: false,
-		},
-		{
-			name: "token authenticated, single repo, pull request comment",
-			init: init{
-				name: "test source",
-				connection: &sourcespb.GitHub{
-					Repositories:               []string{"https://github.com/truffle-test-integration-org/another-test-repo.git"},
-					IncludePullRequestComments: true,
-					IncludeIssueComments:       false,
-					Credential: &sourcespb.GitHub_Token{
-						Token: githubToken,
-					},
-				},
-			},
-			numExpectedChunks: totalPRChunks,
-			wantChunk: &sources.Chunk{
-				SourceType: sourcespb.SourceType_SOURCE_TYPE_GITHUB,
-				SourceName: "test source",
-				SourceMetadata: &source_metadatapb.MetaData{
-					Data: &source_metadatapb.MetaData_Github{
-						Github: &source_metadatapb.Github{
-							Link:      "https://github.com/truffle-test-integration-org/another-test-repo/pull/2#discussion_r1242763304",
-							Username:  "truffle-sandbox",
-							Timestamp: "2023-06-26 21:00:11 +0000 UTC",
-						},
-					},
-				},
-				Verify: false,
-			},
-			wantErr: false,
-		},
-	}
-
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			s := Source{}
-
-			conn, err := anypb.New(tt.init.connection)
-			if err != nil {
-				t.Fatal(err)
-			}
-
-			err = s.Init(ctx, tt.init.name, 0, 0, tt.init.verify, conn, 4)
-			if (err != nil) != tt.wantErr {
-				t.Errorf("Source.Init() error = %v, wantErr %v", err, tt.wantErr)
-				return
-			}
-
-			if tt.wantErr {
-				return
-			}
-
-			chunksCh := make(chan *sources.Chunk, 5)
-			go func() {
-				// Close the channel
-				defer close(chunksCh)
-				err = s.Chunks(ctx, chunksCh)
-				if (err != nil) != tt.wantErr {
-					if ctx.Err() != nil {
-						return
-					}
-					t.Errorf("Source.Chunks() error = %v, wantErr %v", err, tt.wantErr)
-					return
-				}
-			}()
-
-			i := 0
-			for gotChunk := range chunksCh {
-				// Skip chunks that are not comments.
-				if gotChunk.SourceMetadata.GetGithub().GetCommit() != "" {
-					continue
-				}
-				i++
-				githubCommentCheckFunc(gotChunk, tt.wantChunk, i, t, tt.name)
-			}
-
-			// Confirm all comments were processed.
-			if i != tt.numExpectedChunks {
-				t.Errorf("did not complete all chunks, got %d, want %d", i, tt.numExpectedChunks)
-			}
-
-		})
-	}
-}
-
-func TestSource_ScanChunks(t *testing.T) {
-	ctx, cancel := context.WithTimeout(context.Background(), time.Second*30)
-	defer cancel()
-
-	secret, err := common.GetTestSecret(ctx)
-	if err != nil {
-		t.Fatal(fmt.Errorf("failed to access secret: %v", err))
-	}
-
-	// For the personal access token test.
-	githubToken := secret.MustGetField("GITHUB_TOKEN")
-
-	type init struct {
-		name       string
-		verify     bool
-		connection *sourcespb.GitHub
-	}
-	tests := []struct {
-		name       string
-		init       init
-		wantChunks int
-	}{
-		{
-			name: "token authenticated, 4 repos",
-			init: init{
-				name: "test source",
-				connection: &sourcespb.GitHub{
-					Repositories: []string{
-						"https://github.com/truffle-test-integration-org/another-test-repo.git",
-						"https://github.com/trufflesecurity/trufflehog.git",
-						"https://github.com/Akash-goyal-github/Inventory-Management-System.git",
-						"https://github.com/R1ck404/Crypto-Exchange-Example.git",
-						"https://github.com/Stability-AI/generative-models.git",
-						"https://github.com/bloomberg/blazingmq.git",
-						"https://github.com/Kong/kong.git",
-					},
-					Credential: &sourcespb.GitHub_Token{Token: githubToken},
-				},
-			},
-			wantChunks: 20000,
-		},
-	}
-
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			s := Source{}
-
-			conn, err := anypb.New(tt.init.connection)
-			if err != nil {
-				t.Fatal(err)
-			}
-
-			err = s.Init(ctx, tt.init.name, 0, 0, tt.init.verify, conn, 8)
-			assert.Nil(t, err)
-
-			chunksCh := make(chan *sources.Chunk, 1)
-			go func() {
-				defer close(chunksCh)
-				err = s.Chunks(ctx, chunksCh)
-				assert.Nil(t, err)
-			}()
-
-			i := 0
-			for range chunksCh {
-				i++
-			}
-			assert.GreaterOrEqual(t, i, tt.wantChunks)
-		})
-	}
-}
-
 func TestSource_Scan(t *testing.T) {
 	ctx, cancel := context.WithTimeout(context.Background(), time.Second*300)
 	defer cancel()
@@ -793,23 +579,6 @@ func basicCheckFunc(minOrg, minRepo int, wantChunk *sources.Chunk, s *Source) so
 	}
 }
 
-func githubCommentCheckFunc(gotChunk, wantChunk *sources.Chunk, i int, t *testing.T, name string) {
-	if gotChunk.SourceType != wantChunk.SourceType {
-		t.Errorf("want SourceType %v, got %v", wantChunk.SourceType, gotChunk.SourceType)
-	}
-
-	assert.NotEmpty(t, gotChunk.SourceMetadata.Data, "SourceMetadata.Data should not be empty")
-
-	// First Chunk should be a Issue Comment, Second Chunk should be a PR Comment.
-	if i == 1 && name == "token authenticated, single repo, single issue comment" &&
-		wantChunk.SourceMetadata.GetGithub().GetLink() != gotChunk.SourceMetadata.GetGithub().GetLink() {
-		t.Errorf("want %+v \n got %+v \n", wantChunk.SourceMetadata.GetGithub().GetLink(), gotChunk.SourceMetadata.GetGithub().GetLink())
-	} else if i == 2 && name == "token authenticated, single repo, single pr comment" &&
-		wantChunk.SourceMetadata.GetGithub().GetLink() != gotChunk.SourceMetadata.GetGithub().GetLink() {
-		t.Errorf("want %+v \n got %+v \n", wantChunk.SourceMetadata.GetGithub().GetLink(), gotChunk.SourceMetadata.GetGithub().GetLink())
-	}
-}
-
 // func TestSource_paginateRepos(t *testing.T) {
 // 	type args struct {
 // 		ctx       context.Context

pkg/sources/github/github_test.go

@@ -339,8 +339,7 @@ func TestHandleRateLimit(t *testing.T) {
 func TestEnumerateUnauthenticated(t *testing.T) {
 	defer gock.Off()
 
-	apiEndpoint := "https://api.github.com"
-	gock.New(apiEndpoint).
+	gock.New("https://api.github.com").
 		Get("/orgs/super-secret-org/repos").
 		Reply(200).
 		JSON([]map[string]string{{"clone_url": "https://github.com/super-secret-repo.git", "full_name": "super-secret-repo"}})
@@ -348,7 +347,7 @@ func TestEnumerateUnauthenticated(t *testing.T) {
 	s := initTestSource(nil)
 	s.orgsCache = memory.New()
 	s.orgsCache.Set("super-secret-org", "super-secret-org")
-	s.enumerateUnauthenticated(context.Background(), apiEndpoint)
+	s.enumerateUnauthenticated(context.Background())
 	assert.Equal(t, 1, s.filteredRepoCache.Count())
 	ok := s.filteredRepoCache.Exists("super-secret-repo")
 	assert.True(t, ok)

pkg/sources/github/repo.go

@@ -49,8 +49,12 @@ func (s *Source) cloneRepo(
 		}
 
 	case *sourcespb.GitHub_Token:
-		if err := s.getUserAndToken(ctx, repoURL, installationClient); err != nil {
-			return "", nil, fmt.Errorf("error getting token for repo %s: %w", repoURL, err)
+		// We never refresh user provided tokens, so if we already have them, we never need to try and fetch them again.
+		if s.githubUser == "" || s.githubToken == "" {
+			s.githubUser, s.githubToken, err = s.userAndToken(ctx, installationClient)
+			if err != nil {
+				return "", nil, fmt.Errorf("error getting token for repo %s: %w", repoURL, err)
+			}
 		}
 		path, repo, err = git.CloneRepoUsingToken(ctx, s.githubToken, repoURL, s.githubUser)
 		if err != nil {
@@ -62,20 +66,6 @@ func (s *Source) cloneRepo(
 	return path, repo, nil
 }
 
-func (s *Source) getUserAndToken(ctx context.Context, repoURL string, installationClient *github.Client) error {
-	// We never refresh user provided tokens, so if we already have them, we never need to try and fetch them again.
-	s.userMu.Lock()
-	defer s.userMu.Unlock()
-	if s.githubUser == "" || s.githubToken == "" {
-		var err error
-		s.githubUser, s.githubToken, err = s.userAndToken(ctx, installationClient)
-		if err != nil {
-			return fmt.Errorf("error getting token for repo %s: %w", repoURL, err)
-		}
-	}
-	return nil
-}
-
 func (s *Source) userAndToken(ctx context.Context, installationClient *github.Client) (string, string, error) {
 	switch cred := s.conn.GetCredential().(type) {
 	case *sourcespb.GitHub_BasicAuth:

pkg/sources/gitlab/gitlab.go

@@ -127,7 +127,7 @@ func (s *Source) Init(_ context.Context, name string, jobId, sourceId int64, ver
 						File:       sanitizer.UTF8(file),
 						Email:      sanitizer.UTF8(email),
 						Repository: sanitizer.UTF8(repository),
-						Link:       git.GenerateLink(repository, commit, file, line),
+						Link:       git.GenerateLink(repository, commit, file),
 						Timestamp:  sanitizer.UTF8(timestamp),
 						Line:       line,
 					},

pkg/sources/job_progress.go

@@ -1,272 +0,0 @@
-package sources
-
-//go:generate mockgen --source=./job_progress.go --destination=mock_job_progress_test.go --package=sources
-
-import (
-	"context"
-	"errors"
-	"fmt"
-	"sync"
-	"sync/atomic"
-	"time"
-)
-
-type JobProgressHook interface {
-	// Start and End marks the overall start and end time for this job.
-	Start(JobProgressRef, time.Time)
-	End(JobProgressRef, time.Time)
-	// StartEnumerating and EndEnumerating marks the start and end time for
-	// calling the source's Enumerate method. If the source does not
-	// support enumeration these methods will never be called.
-	StartEnumerating(JobProgressRef, time.Time)
-	EndEnumerating(JobProgressRef, time.Time)
-	// StartUnitChunking and EndUnitChunking marks the start and end time
-	// for calling the source's ChunkUnit method for a given unit. If the
-	// source does not support enumeration these methods will never be
-	// called.
-	StartUnitChunking(JobProgressRef, SourceUnit, time.Time)
-	EndUnitChunking(JobProgressRef, SourceUnit, time.Time)
-	// ReportError is called when any general error is encountered, usually
-	// from enumeration.
-	ReportError(JobProgressRef, error)
-	// ReportUnit is called when a unit has been enumerated. If the source
-	// does not support enumeration this method will never be called.
-	ReportUnit(JobProgressRef, SourceUnit)
-	// ReportChunk is called when a chunk has been produced for the given
-	// unit. The unit will be nil if the source does not support
-	// enumeration.
-	ReportChunk(JobProgressRef, SourceUnit, *Chunk)
-	// Finish marks the job as done.
-	Finish(JobProgressRef)
-}
-
-// JobProgressRef is a wrapper of a JobProgress for read-only access to its state.
-type JobProgressRef struct {
-	SourceID    int64
-	JobID       int64
-	jobProgress *JobProgress
-}
-
-// Snapshot returns a snapshot of the job's current metrics.
-func (r *JobProgressRef) Snapshot() JobProgressMetrics {
-	if r.jobProgress == nil {
-		return JobProgressMetrics{}
-	}
-	return r.jobProgress.Snapshot()
-}
-
-// Done returns a channel that will block until the job has completed.
-func (r *JobProgressRef) Done() <-chan struct{} {
-	if r.jobProgress == nil {
-		// Return a closed channel so it does not block.
-		ch := make(chan struct{})
-		close(ch)
-		return ch
-	}
-	return r.jobProgress.Done()
-}
-
-// Fatal is a wrapper around error to differentiate non-fatal errors from fatal
-// ones. A fatal error is typically from a finished context or any error
-// returned from a source's Init, Chunks, Enumerate, or ChunkUnit methods.
-type Fatal struct{ error }
-
-func (f Fatal) Error() string { return fmt.Sprintf("fatal: %s", f.error.Error()) }
-func (f Fatal) Unwrap() error { return f.error }
-
-// ChunkError is a custom error type for errors encountered during chunking of
-// a specific unit.
-type ChunkError struct {
-	unit SourceUnit
-	err  error
-}
-
-func (f ChunkError) Error() string {
-	return fmt.Sprintf("error chunking unit %q: %s", f.unit.SourceUnitID(), f.err.Error())
-}
-func (f ChunkError) Unwrap() error { return f.err }
-
-// JobProgress aggregates information about a run of a Source.
-type JobProgress struct {
-	// Unique identifiers for this job.
-	SourceID int64
-	JobID    int64
-	// Tracks whether the job is finished or not.
-	ctx    context.Context
-	cancel context.CancelFunc
-	// Metrics.
-	metrics     JobProgressMetrics
-	metricsLock sync.Mutex
-	// Coarse grained hooks for adding extra functionality when events trigger.
-	hooks []JobProgressHook
-}
-
-// JobProgressMetrics tracks the metrics of a job.
-type JobProgressMetrics struct {
-	StartTime       time.Time
-	EndTime         time.Time
-	TotalUnits      uint64
-	FinishedUnits   uint64
-	TotalChunks     uint64
-	Errors          []error
-	DoneEnumerating bool
-}
-
-// WithHooks adds hooks to be called when an event triggers.
-func WithHooks(hooks ...JobProgressHook) func(*JobProgress) {
-	return func(jp *JobProgress) { jp.hooks = append(jp.hooks, hooks...) }
-}
-
-// NewJobProgress creates a new job report for the given source and job ID.
-func NewJobProgress(sourceID, jobID int64, opts ...func(*JobProgress)) *JobProgress {
-	ctx, cancel := context.WithCancel(context.Background())
-	jp := &JobProgress{
-		SourceID: sourceID,
-		JobID:    jobID,
-		ctx:      ctx,
-		cancel:   cancel,
-	}
-	for _, opt := range opts {
-		opt(jp)
-	}
-	return jp
-}
-
-// executeHooks is a helper method to execute all the hooks for the given
-// closure.
-func (jp *JobProgress) executeHooks(todo func(hook JobProgressHook)) {
-	for _, hook := range jp.hooks {
-		// TODO: Non-blocking?
-		todo(hook)
-	}
-}
-
-// TODO: Comment all this mess. They are mostly implementing JobProgressHook but
-// without the JobProgressRef parameter.
-func (jp *JobProgress) Start(start time.Time) {
-	jp.metricsLock.Lock()
-	jp.metrics.StartTime = start
-	jp.metricsLock.Unlock()
-
-	jp.executeHooks(func(hook JobProgressHook) { hook.Start(jp.Ref(), start) })
-}
-func (jp *JobProgress) End(end time.Time) {
-	jp.metricsLock.Lock()
-	jp.metrics.EndTime = end
-	jp.metricsLock.Unlock()
-
-	jp.executeHooks(func(hook JobProgressHook) { hook.End(jp.Ref(), end) })
-}
-func (jp *JobProgress) Finish() {
-	jp.cancel()
-	jp.executeHooks(func(hook JobProgressHook) { hook.Finish(jp.Ref()) })
-}
-func (jp *JobProgress) Done() <-chan struct{} { return jp.ctx.Done() }
-func (jp *JobProgress) ReportUnit(unit SourceUnit) {
-	atomic.AddUint64(&jp.metrics.TotalUnits, 1)
-	jp.executeHooks(func(hook JobProgressHook) { hook.ReportUnit(jp.Ref(), unit) })
-}
-func (jp *JobProgress) ReportChunk(unit SourceUnit, chunk *Chunk) {
-	atomic.AddUint64(&jp.metrics.TotalChunks, 1)
-	jp.executeHooks(func(hook JobProgressHook) { hook.ReportChunk(jp.Ref(), unit, chunk) })
-}
-func (jp *JobProgress) StartUnitChunking(unit SourceUnit, start time.Time) {
-	// TODO: Record time.
-	jp.executeHooks(func(hook JobProgressHook) { hook.StartUnitChunking(jp.Ref(), unit, start) })
-}
-func (jp *JobProgress) EndUnitChunking(unit SourceUnit, end time.Time) {
-	// TODO: Record time.
-	atomic.AddUint64(&jp.metrics.FinishedUnits, 1)
-	jp.executeHooks(func(hook JobProgressHook) { hook.EndUnitChunking(jp.Ref(), unit, end) })
-}
-func (jp *JobProgress) StartEnumerating(start time.Time) {
-	// TODO: Record time.
-	jp.executeHooks(func(hook JobProgressHook) { hook.StartEnumerating(jp.Ref(), start) })
-}
-func (jp *JobProgress) EndEnumerating(end time.Time) {
-	jp.metricsLock.Lock()
-	// TODO: Record time.
-	jp.metrics.DoneEnumerating = true
-	jp.metricsLock.Unlock()
-
-	jp.executeHooks(func(hook JobProgressHook) { hook.EndEnumerating(jp.Ref(), end) })
-}
-
-// Snapshot safely gets the job's current metrics.
-func (jp *JobProgress) Snapshot() JobProgressMetrics {
-	jp.metricsLock.Lock()
-	defer jp.metricsLock.Unlock()
-	return jp.metrics
-}
-
-// ReportError adds a non-nil error to the aggregate of errors
-// encountered during scanning.
-func (jp *JobProgress) ReportError(err error) {
-	if err == nil {
-		return
-	}
-	jp.metricsLock.Lock()
-	jp.metrics.Errors = append(jp.metrics.Errors, err)
-	jp.metricsLock.Unlock()
-
-	jp.executeHooks(func(hook JobProgressHook) { hook.ReportError(jp.Ref(), err) })
-}
-
-// Ref provides a read-only reference to the JobProgress.
-func (jp *JobProgress) Ref() JobProgressRef {
-	return JobProgressRef{
-		SourceID:    jp.SourceID,
-		JobID:       jp.JobID,
-		jobProgress: jp,
-	}
-}
-
-// EnumerationErrors joins all errors encountered during initialization or
-// enumeration.
-func (m JobProgressMetrics) EnumerationError() error {
-	return errors.Join(m.Errors...)
-}
-
-// ChunkErrors joins all errors encountered during chunking.
-func (m JobProgressMetrics) ChunkError() error {
-	var aggregate []error
-	for _, err := range m.Errors {
-		var chunkErr ChunkError
-		if ok := errors.As(err, &chunkErr); ok {
-			aggregate = append(aggregate, err)
-		}
-	}
-	return errors.Join(aggregate...)
-}
-
-// FatalError returns the first Fatal error, if any, encountered in the scan.
-func (m JobProgressMetrics) FatalError() error {
-	for _, err := range m.Errors {
-		var fatalErr Fatal
-		if found := errors.As(err, &fatalErr); found {
-			return fatalErr
-		}
-	}
-	return nil
-}
-
-// FatalErrors returns all of the encountered fatal errors joined together.
-func (m JobProgressMetrics) FatalErrors() error {
-	var aggregate []error
-	for _, err := range m.Errors {
-		var fatalErr Fatal
-		if found := errors.As(err, &fatalErr); found {
-			aggregate = append(aggregate, fatalErr)
-		}
-	}
-	return errors.Join(aggregate...)
-}
-
-func (m JobProgressMetrics) PercentComplete() int {
-	num := m.FinishedUnits
-	den := m.TotalUnits
-	if num == 0 && den == 0 {
-		return 0
-	}
-	return int(num * 100 / den)
-}

pkg/sources/job_progress_test.go

@@ -1,117 +0,0 @@
-package sources
-
-import (
-	"errors"
-	"fmt"
-	"testing"
-	"time"
-
-	"github.com/stretchr/testify/assert"
-	"go.uber.org/mock/gomock"
-)
-
-func TestJobProgressFatalErrors(t *testing.T) {
-	var jp JobProgress
-
-	// Add a non-fatal error.
-	jp.ReportError(fmt.Errorf("oh no"))
-	assert.Greater(t, len(jp.Snapshot().Errors), 0)
-	assert.NoError(t, jp.Snapshot().FatalError())
-	assert.NoError(t, jp.Snapshot().ChunkError())
-
-	// Add a fatal error and make sure we can test comparison.
-	err := fmt.Errorf("fatal error")
-	jp.ReportError(Fatal{err})
-	assert.Greater(t, len(jp.Snapshot().Errors), 0)
-	assert.Error(t, jp.Snapshot().FatalError())
-	assert.NoError(t, jp.Snapshot().ChunkError())
-	assert.True(t, errors.Is(jp.Snapshot().FatalError(), err))
-
-	// Add another fatal error and test we still return the first.
-	jp.ReportError(Fatal{fmt.Errorf("second fatal error")})
-	assert.Greater(t, len(jp.Snapshot().Errors), 0)
-	assert.Error(t, jp.Snapshot().FatalError())
-	assert.NoError(t, jp.Snapshot().ChunkError())
-	assert.True(t, errors.Is(jp.Snapshot().FatalError(), err))
-}
-
-func TestJobProgressRef(t *testing.T) {
-	jp := NewJobProgress(123, 456)
-	ref := jp.Ref()
-	assert.Equal(t, int64(123), ref.SourceID)
-	assert.Equal(t, int64(456), ref.JobID)
-
-	// Test Done() blocks until Finish() is called.
-	select {
-	case <-jp.Done():
-		assert.FailNow(t, "job should not be finished")
-	default:
-	}
-
-	jp.Finish()
-	select {
-	case <-jp.Done():
-	default:
-		assert.FailNow(t, "job should be finished")
-	}
-}
-
-func TestJobProgressHook(t *testing.T) {
-	ctrl := gomock.NewController(t)
-	defer ctrl.Finish()
-
-	hook := NewMockJobProgressHook(ctrl)
-	jp := NewJobProgress(123, 456, WithHooks(hook))
-
-	// Start(JobProgressRef, time.Time)
-	// End(JobProgressRef, time.Time)
-	// StartEnumerating(JobProgressRef, time.Time)
-	// EndEnumerating(JobProgressRef, time.Time)
-	// StartUnitChunking(JobProgressRef, SourceUnit, time.Time)
-	// EndUnitChunking(JobProgressRef, SourceUnit, time.Time)
-	// ReportError(JobProgressRef, error)
-	// ReportUnit(JobProgressRef, SourceUnit)
-	// ReportChunk(JobProgressRef, SourceUnit, *Chunk)
-	// Finish(JobProgressRef)
-
-	startTime := time.Now()
-	endTime := time.Now().Add(10 * time.Second)
-	startEnum := time.Now().Add(20 * time.Second)
-	endEnum := time.Now().Add(30 * time.Second)
-	startChunk := time.Now().Add(40 * time.Second)
-	endChunk := time.Now().Add(50 * time.Second)
-	reportErr := fmt.Errorf("reporting error")
-	reportUnit := CommonSourceUnit{"reporting unit"}
-	reportChunk := &Chunk{Data: []byte("reporting chunk")}
-
-	hook.EXPECT().Start(gomock.Any(), startTime)
-	hook.EXPECT().End(gomock.Any(), endTime)
-	hook.EXPECT().StartEnumerating(gomock.Any(), startEnum)
-	hook.EXPECT().EndEnumerating(gomock.Any(), endEnum)
-	hook.EXPECT().StartUnitChunking(gomock.Any(), reportUnit, startChunk)
-	hook.EXPECT().EndUnitChunking(gomock.Any(), reportUnit, endChunk)
-	hook.EXPECT().ReportError(gomock.Any(), reportErr)
-	hook.EXPECT().ReportUnit(gomock.Any(), reportUnit)
-	hook.EXPECT().ReportChunk(gomock.Any(), reportUnit, reportChunk)
-	hook.EXPECT().Finish(gomock.Any())
-
-	jp.Start(startTime)
-	jp.End(endTime)
-	jp.StartEnumerating(startEnum)
-	jp.EndEnumerating(endEnum)
-	jp.StartUnitChunking(reportUnit, startChunk)
-	jp.EndUnitChunking(reportUnit, endChunk)
-	jp.ReportError(reportErr)
-	jp.ReportUnit(reportUnit)
-	jp.ReportChunk(reportUnit, reportChunk)
-	jp.Finish()
-}
-
-func TestJobProgressDone(t *testing.T) {
-	ref := JobProgressRef{}
-	select {
-	case <-ref.Done():
-	default:
-		assert.FailNow(t, "done should not block for a nil job")
-	}
-}

pkg/sources/mock_job_progress_test.go

@@ -1,155 +0,0 @@
-// Code generated by MockGen. DO NOT EDIT.
-// Source: ./job_progress.go
-
-// Package sources is a generated GoMock package.
-package sources
-
-import (
-	reflect "reflect"
-	time "time"
-
-	gomock "go.uber.org/mock/gomock"
-)
-
-// MockJobProgressHook is a mock of JobProgressHook interface.
-type MockJobProgressHook struct {
-	ctrl     *gomock.Controller
-	recorder *MockJobProgressHookMockRecorder
-}
-
-// MockJobProgressHookMockRecorder is the mock recorder for MockJobProgressHook.
-type MockJobProgressHookMockRecorder struct {
-	mock *MockJobProgressHook
-}
-
-// NewMockJobProgressHook creates a new mock instance.
-func NewMockJobProgressHook(ctrl *gomock.Controller) *MockJobProgressHook {
-	mock := &MockJobProgressHook{ctrl: ctrl}
-	mock.recorder = &MockJobProgressHookMockRecorder{mock}
-	return mock
-}
-
-// EXPECT returns an object that allows the caller to indicate expected use.
-func (m *MockJobProgressHook) EXPECT() *MockJobProgressHookMockRecorder {
-	return m.recorder
-}
-
-// End mocks base method.
-func (m *MockJobProgressHook) End(arg0 JobProgressRef, arg1 time.Time) {
-	m.ctrl.T.Helper()
-	m.ctrl.Call(m, "End", arg0, arg1)
-}
-
-// End indicates an expected call of End.
-func (mr *MockJobProgressHookMockRecorder) End(arg0, arg1 interface{}) *gomock.Call {
-	mr.mock.ctrl.T.Helper()
-	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "End", reflect.TypeOf((*MockJobProgressHook)(nil).End), arg0, arg1)
-}
-
-// EndEnumerating mocks base method.
-func (m *MockJobProgressHook) EndEnumerating(arg0 JobProgressRef, arg1 time.Time) {
-	m.ctrl.T.Helper()
-	m.ctrl.Call(m, "EndEnumerating", arg0, arg1)
-}
-
-// EndEnumerating indicates an expected call of EndEnumerating.
-func (mr *MockJobProgressHookMockRecorder) EndEnumerating(arg0, arg1 interface{}) *gomock.Call {
-	mr.mock.ctrl.T.Helper()
-	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "EndEnumerating", reflect.TypeOf((*MockJobProgressHook)(nil).EndEnumerating), arg0, arg1)
-}
-
-// EndUnitChunking mocks base method.
-func (m *MockJobProgressHook) EndUnitChunking(arg0 JobProgressRef, arg1 SourceUnit, arg2 time.Time) {
-	m.ctrl.T.Helper()
-	m.ctrl.Call(m, "EndUnitChunking", arg0, arg1, arg2)
-}
-
-// EndUnitChunking indicates an expected call of EndUnitChunking.
-func (mr *MockJobProgressHookMockRecorder) EndUnitChunking(arg0, arg1, arg2 interface{}) *gomock.Call {
-	mr.mock.ctrl.T.Helper()
-	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "EndUnitChunking", reflect.TypeOf((*MockJobProgressHook)(nil).EndUnitChunking), arg0, arg1, arg2)
-}
-
-// Finish mocks base method.
-func (m *MockJobProgressHook) Finish(arg0 JobProgressRef) {
-	m.ctrl.T.Helper()
-	m.ctrl.Call(m, "Finish", arg0)
-}
-
-// Finish indicates an expected call of Finish.
-func (mr *MockJobProgressHookMockRecorder) Finish(arg0 interface{}) *gomock.Call {
-	mr.mock.ctrl.T.Helper()
-	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Finish", reflect.TypeOf((*MockJobProgressHook)(nil).Finish), arg0)
-}
-
-// ReportChunk mocks base method.
-func (m *MockJobProgressHook) ReportChunk(arg0 JobProgressRef, arg1 SourceUnit, arg2 *Chunk) {
-	m.ctrl.T.Helper()
-	m.ctrl.Call(m, "ReportChunk", arg0, arg1, arg2)
-}
-
-// ReportChunk indicates an expected call of ReportChunk.
-func (mr *MockJobProgressHookMockRecorder) ReportChunk(arg0, arg1, arg2 interface{}) *gomock.Call {
-	mr.mock.ctrl.T.Helper()
-	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ReportChunk", reflect.TypeOf((*MockJobProgressHook)(nil).ReportChunk), arg0, arg1, arg2)
-}
-
-// ReportError mocks base method.
-func (m *MockJobProgressHook) ReportError(arg0 JobProgressRef, arg1 error) {
-	m.ctrl.T.Helper()
-	m.ctrl.Call(m, "ReportError", arg0, arg1)
-}
-
-// ReportError indicates an expected call of ReportError.
-func (mr *MockJobProgressHookMockRecorder) ReportError(arg0, arg1 interface{}) *gomock.Call {
-	mr.mock.ctrl.T.Helper()
-	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ReportError", reflect.TypeOf((*MockJobProgressHook)(nil).ReportError), arg0, arg1)
-}
-
-// ReportUnit mocks base method.
-func (m *MockJobProgressHook) ReportUnit(arg0 JobProgressRef, arg1 SourceUnit) {
-	m.ctrl.T.Helper()
-	m.ctrl.Call(m, "ReportUnit", arg0, arg1)
-}
-
-// ReportUnit indicates an expected call of ReportUnit.
-func (mr *MockJobProgressHookMockRecorder) ReportUnit(arg0, arg1 interface{}) *gomock.Call {
-	mr.mock.ctrl.T.Helper()
-	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ReportUnit", reflect.TypeOf((*MockJobProgressHook)(nil).ReportUnit), arg0, arg1)
-}
-
-// Start mocks base method.
-func (m *MockJobProgressHook) Start(arg0 JobProgressRef, arg1 time.Time) {
-	m.ctrl.T.Helper()
-	m.ctrl.Call(m, "Start", arg0, arg1)
-}
-
-// Start indicates an expected call of Start.
-func (mr *MockJobProgressHookMockRecorder) Start(arg0, arg1 interface{}) *gomock.Call {
-	mr.mock.ctrl.T.Helper()
-	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Start", reflect.TypeOf((*MockJobProgressHook)(nil).Start), arg0, arg1)
-}
-
-// StartEnumerating mocks base method.
-func (m *MockJobProgressHook) StartEnumerating(arg0 JobProgressRef, arg1 time.Time) {
-	m.ctrl.T.Helper()
-	m.ctrl.Call(m, "StartEnumerating", arg0, arg1)
-}
-
-// StartEnumerating indicates an expected call of StartEnumerating.
-func (mr *MockJobProgressHookMockRecorder) StartEnumerating(arg0, arg1 interface{}) *gomock.Call {
-	mr.mock.ctrl.T.Helper()
-	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "StartEnumerating", reflect.TypeOf((*MockJobProgressHook)(nil).StartEnumerating), arg0, arg1)
-}
-
-// StartUnitChunking mocks base method.
-func (m *MockJobProgressHook) StartUnitChunking(arg0 JobProgressRef, arg1 SourceUnit, arg2 time.Time) {
-	m.ctrl.T.Helper()
-	m.ctrl.Call(m, "StartUnitChunking", arg0, arg1, arg2)
-}
-
-// StartUnitChunking indicates an expected call of StartUnitChunking.
-func (mr *MockJobProgressHookMockRecorder) StartUnitChunking(arg0, arg1, arg2 interface{}) *gomock.Call {
-	mr.mock.ctrl.T.Helper()
-	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "StartUnitChunking", reflect.TypeOf((*MockJobProgressHook)(nil).StartUnitChunking), arg0, arg1, arg2)
-}

pkg/sources/s3/s3.go

@@ -229,19 +229,19 @@ func (s *Source) pageChunker(ctx context.Context, client *s3.S3, chunksChan chan
 		// ignore large files
 		if *obj.Size > s.maxObjectSize {
 			s.log.V(3).Info("Skipping %d byte file (over maxObjectSize limit)", "object", *obj.Key)
-			continue
+			return
 		}
 
 		// file empty file
 		if *obj.Size == 0 {
 			s.log.V(5).Info("Skipping 0 byte file", "object", *obj.Key)
-			continue
+			return
 		}
 
 		// skip incompatible extensions
 		if common.SkipFile(*obj.Key) {
 			s.log.V(5).Info("Skipping file with incompatible extension", "object", *obj.Key)
-			continue
+			return
 		}
 
 		s.jobPool.Go(func() error {

pkg/sources/source_manager.go

@@ -1,377 +0,0 @@
-package sources
-
-import (
-	"fmt"
-	"sync"
-	"sync/atomic"
-	"time"
-
-	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
-	"github.com/trufflesecurity/trufflehog/v3/pkg/context"
-	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
-	"golang.org/x/sync/errgroup"
-)
-
-// handle uniquely identifies a Source given to the manager to manage. If the
-// SourceManager is connected to the API, it will be equivalent to the unique
-// source ID, otherwise it behaves as a counter.
-type handle int64
-
-// SourceInitFunc is a function that takes a source and job ID and returns an
-// initialized Source.
-type SourceInitFunc func(ctx context.Context, sourceID int64, jobID int64) (Source, error)
-
-type SourceManager struct {
-	api   apiClient
-	hooks []JobProgressHook
-	// Map of handle to source initializer.
-	handles     map[handle]SourceInitFunc
-	handlesLock sync.Mutex
-	// Pool limiting the amount of concurrent sources running.
-	pool            errgroup.Group
-	concurrentUnits int
-	// Run the sources using source unit enumeration / chunking if available.
-	useSourceUnits bool
-	// Downstream chunks channel to be scanned.
-	outputChunks chan *Chunk
-	// Set when Wait() returns.
-	done bool
-}
-
-// apiClient is an interface for optionally communicating with an external API.
-type apiClient interface {
-	// RegisterSource lets the API know we have a source and it returns a unique source ID for it.
-	RegisterSource(ctx context.Context, name string, kind sourcespb.SourceType) (int64, error)
-	// GetJobID queries the API for an existing job or new job ID.
-	GetJobID(ctx context.Context, id int64) (int64, error)
-}
-
-// WithAPI adds an API client to the manager for tracking jobs and progress. If
-// the API is also a JobProgressHook, it will be added to the list of event hooks.
-func WithAPI(api apiClient) func(*SourceManager) {
-	return func(mgr *SourceManager) { mgr.api = api }
-}
-
-func WithReportHook(hook JobProgressHook) func(*SourceManager) {
-	return func(mgr *SourceManager) {
-		mgr.hooks = append(mgr.hooks, hook)
-	}
-}
-
-// WithConcurrency limits the concurrent number of sources a manager can run.
-func WithConcurrency(concurrency int) func(*SourceManager) {
-	return func(mgr *SourceManager) { mgr.pool.SetLimit(concurrency) }
-}
-
-// WithBufferedOutput sets the size of the buffer used for the Chunks() channel.
-func WithBufferedOutput(size int) func(*SourceManager) {
-	return func(mgr *SourceManager) { mgr.outputChunks = make(chan *Chunk, size) }
-}
-
-// WithSourceUnits enables using source unit enumeration and chunking if the
-// source supports it.
-func WithSourceUnits() func(*SourceManager) {
-	return func(mgr *SourceManager) { mgr.useSourceUnits = true }
-}
-
-// WithConcurrentUnits limits the number of units to be scanned concurrently.
-// The default is unlimited.
-func WithConcurrentUnits(n int) func(*SourceManager) {
-	return func(mgr *SourceManager) { mgr.concurrentUnits = n }
-}
-
-// NewManager creates a new manager with the provided options.
-func NewManager(opts ...func(*SourceManager)) *SourceManager {
-	mgr := SourceManager{
-		// Default to the headless API. Can be overwritten by the WithAPI option.
-		api:          &headlessAPI{},
-		handles:      make(map[handle]SourceInitFunc),
-		outputChunks: make(chan *Chunk),
-	}
-	for _, opt := range opts {
-		opt(&mgr)
-	}
-	return &mgr
-}
-
-// Enroll informs the SourceManager to track and manage a Source.
-func (s *SourceManager) Enroll(ctx context.Context, name string, kind sourcespb.SourceType, f SourceInitFunc) (handle, error) {
-	if s.done {
-		return 0, fmt.Errorf("manager is done")
-	}
-	id, err := s.api.RegisterSource(ctx, name, kind)
-	if err != nil {
-		return 0, err
-	}
-	handleID := handle(id)
-	s.handlesLock.Lock()
-	defer s.handlesLock.Unlock()
-	if _, ok := s.handles[handleID]; ok {
-		// TODO: smartly handle this?
-		return 0, fmt.Errorf("handle ID '%d' already in use", handleID)
-	}
-	s.handles[handleID] = f
-	return handleID, nil
-}
-
-// Run blocks until a resource is available to run the source, then
-// synchronously runs it. The first fatal error, if any, will be returned.
-func (s *SourceManager) Run(ctx context.Context, handle handle) (JobProgressRef, error) {
-	report, err := s.asyncRun(ctx, handle)
-	if err != nil {
-		return report, err
-	}
-	<-report.Done()
-	return report, report.Snapshot().FatalError()
-}
-
-// ScheduleRun blocks until a resource is available to run the source, then
-// asynchronously runs it. Error information is stored and accessible via the
-// JobProgressRef as it becomes available.
-func (s *SourceManager) ScheduleRun(ctx context.Context, handle handle) (JobProgressRef, error) {
-	return s.asyncRun(ctx, handle)
-}
-
-// asyncRun is a helper method to asynchronously run the Source. It calls out
-// to the API to get a job ID for this run, creates a report, then waits for an
-// available goroutine to asynchronously run it.
-func (s *SourceManager) asyncRun(ctx context.Context, handle handle) (JobProgressRef, error) {
-	// Do preflight checks before waiting on the pool.
-	if err := s.preflightChecks(ctx, handle); err != nil {
-		return JobProgressRef{}, err
-	}
-	// Get a Job ID.
-	jobID, err := s.api.GetJobID(ctx, int64(handle))
-	if err != nil {
-		return JobProgressRef{SourceID: int64(handle)}, err
-	}
-	// Start a report for this job.
-	report := NewJobProgress(int64(handle), jobID, WithHooks(s.hooks...))
-	s.pool.Go(func() error {
-		defer common.Recover(ctx)
-		_ = s.run(ctx, handle, jobID, report)
-		return nil
-	})
-	return report.Ref(), nil
-}
-
-// Chunks returns the read only channel of all the chunks produced by all of
-// the sources managed by this manager.
-func (s *SourceManager) Chunks() <-chan *Chunk {
-	return s.outputChunks
-}
-
-// Wait blocks until all running sources are completed and closes the channel
-// returned by Chunks(). The manager should not be reused after calling this
-// method. This current implementation is not thread safe and should only be
-// called by one thread.
-func (s *SourceManager) Wait() {
-	// Check if the manager has been Waited.
-	if s.done {
-		return
-	}
-	defer close(s.outputChunks)
-	defer func() { s.done = true }()
-
-	// We are only using the errgroup for limiting concurrency.
-	// TODO: Maybe switch to using a semaphore.Weighted.
-	_ = s.pool.Wait()
-}
-
-// preflightChecks is a helper method to check the Manager or the context isn't
-// done and that the handle is valid.
-func (s *SourceManager) preflightChecks(ctx context.Context, handle handle) error {
-	// Check if the manager has been Waited.
-	if s.done {
-		return fmt.Errorf("manager is done")
-	}
-	// Check the handle is valid.
-	if _, ok := s.getInitFunc(handle); !ok {
-		return fmt.Errorf("unrecognized handle")
-	}
-	return ctx.Err()
-}
-
-// run is a helper method to sychronously run the source. It does not check for
-// acquired resources. An error is returned if there was a fatal error during
-// the run. This information is also recorded in the JobProgress.
-func (s *SourceManager) run(ctx context.Context, handle handle, jobID int64, report *JobProgress) error {
-	defer report.Finish()
-	report.Start(time.Now())
-	defer func() { report.End(time.Now()) }()
-
-	// Initialize the source.
-	initFunc, ok := s.getInitFunc(handle)
-	if !ok {
-		// Shouldn't happen due to preflight checks.
-		err := fmt.Errorf("unrecognized handle")
-		report.ReportError(Fatal{err})
-		return Fatal{err}
-	}
-	source, err := initFunc(ctx, int64(handle), jobID)
-	if err != nil {
-		report.ReportError(Fatal{err})
-		return Fatal{err}
-	}
-	// Check for the preferred method of tracking source units.
-	if enumChunker, ok := source.(SourceUnitEnumChunker); ok && s.useSourceUnits {
-		return s.runWithUnits(ctx, handle, enumChunker, report)
-	}
-	return s.runWithoutUnits(ctx, handle, source, report)
-}
-
-// runWithoutUnits is a helper method to run a Source. It has coarse-grained
-// job reporting.
-func (s *SourceManager) runWithoutUnits(ctx context.Context, handle handle, source Source, report *JobProgress) error {
-	// Introspect on the chunks we get from the Chunks method.
-	ch := make(chan *Chunk)
-	var wg sync.WaitGroup
-	// Consume chunks and export chunks.
-	wg.Add(1)
-	go func() {
-		defer wg.Done()
-		for chunk := range ch {
-			report.ReportChunk(nil, chunk)
-			_ = common.CancellableWrite(ctx, s.outputChunks, chunk)
-		}
-	}()
-	// Don't return from this function until the goroutine has finished
-	// outputting chunks to the downstream channel. Closing the channel
-	// will stop the goroutine, so that needs to happen first in the defer
-	// stack.
-	defer wg.Wait()
-	defer close(ch)
-	if err := source.Chunks(ctx, ch); err != nil {
-		report.ReportError(Fatal{err})
-		return Fatal{err}
-	}
-	return nil
-}
-
-// runWithUnits is a helper method to run a Source that is also a
-// SourceUnitEnumChunker. This allows better introspection of what is getting
-// scanned and any errors encountered.
-func (s *SourceManager) runWithUnits(ctx context.Context, handle handle, source SourceUnitEnumChunker, report *JobProgress) error {
-	unitReporter := &mgrUnitReporter{
-		unitCh: make(chan SourceUnit),
-		report: report,
-	}
-	// Create a function that will save the first error encountered (if
-	// any) and discard the rest.
-	fatalErr := make(chan error, 1)
-	catchFirstFatal := func(err error) {
-		select {
-		case fatalErr <- err:
-		default:
-		}
-	}
-	// Produce units.
-	go func() {
-		// TODO: Catch panics and add to report.
-		report.StartEnumerating(time.Now())
-		defer func() { report.EndEnumerating(time.Now()) }()
-		defer close(unitReporter.unitCh)
-		if err := source.Enumerate(ctx, unitReporter); err != nil {
-			report.ReportError(Fatal{err})
-			catchFirstFatal(Fatal{err})
-		}
-	}()
-	var wg sync.WaitGroup
-	// TODO: Maybe switch to using a semaphore.Weighted.
-	var unitPool errgroup.Group
-	if s.concurrentUnits != 0 {
-		// Negative values indicated no limit.
-		unitPool.SetLimit(s.concurrentUnits)
-	}
-	for unit := range unitReporter.unitCh {
-		unit := unit
-		chunkReporter := &mgrChunkReporter{
-			unit:    unit,
-			chunkCh: make(chan *Chunk),
-			report:  report,
-		}
-		// Consume units and produce chunks.
-		unitPool.Go(func() error {
-			report.StartUnitChunking(unit, time.Now())
-			// TODO: Catch panics and add to report.
-			defer close(chunkReporter.chunkCh)
-			if err := source.ChunkUnit(ctx, unit, chunkReporter); err != nil {
-				report.ReportError(Fatal{err})
-				catchFirstFatal(Fatal{err})
-			}
-			return nil
-		})
-		// Consume chunks and export chunks.
-		wg.Add(1)
-		go func() {
-			defer wg.Done()
-			defer func() { report.EndUnitChunking(unit, time.Now()) }()
-			for chunk := range chunkReporter.chunkCh {
-				report.ReportChunk(chunkReporter.unit, chunk)
-				_ = common.CancellableWrite(ctx, s.outputChunks, chunk)
-			}
-		}()
-	}
-	wg.Wait()
-	select {
-	case err := <-fatalErr:
-		return err
-	default:
-		return nil
-	}
-}
-
-// getInitFunc is a helper method for safe concurrent access to the
-// map[handle]SourceInitFunc map.
-func (s *SourceManager) getInitFunc(handle handle) (SourceInitFunc, bool) {
-	s.handlesLock.Lock()
-	defer s.handlesLock.Unlock()
-	f, ok := s.handles[handle]
-	return f, ok
-}
-
-// headlessAPI implements the apiClient interface locally.
-type headlessAPI struct {
-	// Counters for assigning handle and job IDs.
-	sourceIDCounter int64
-	jobIDCounter    int64
-}
-
-func (api *headlessAPI) RegisterSource(ctx context.Context, name string, kind sourcespb.SourceType) (int64, error) {
-	return atomic.AddInt64(&api.sourceIDCounter, 1), nil
-}
-
-func (api *headlessAPI) GetJobID(ctx context.Context, id int64) (int64, error) {
-	return atomic.AddInt64(&api.jobIDCounter, 1), nil
-}
-
-// mgrUnitReporter implements the UnitReporter interface.
-type mgrUnitReporter struct {
-	unitCh chan SourceUnit
-	report *JobProgress
-}
-
-func (s *mgrUnitReporter) UnitOk(ctx context.Context, unit SourceUnit) error {
-	return common.CancellableWrite(ctx, s.unitCh, unit)
-}
-
-func (s *mgrUnitReporter) UnitErr(ctx context.Context, err error) error {
-	s.report.ReportError(err)
-	return nil
-}
-
-// mgrChunkReporter implements the ChunkReporter interface.
-type mgrChunkReporter struct {
-	unit    SourceUnit
-	chunkCh chan *Chunk
-	report  *JobProgress
-}
-
-func (s *mgrChunkReporter) ChunkOk(ctx context.Context, chunk Chunk) error {
-	return common.CancellableWrite(ctx, s.chunkCh, &chunk)
-}
-
-func (s *mgrChunkReporter) ChunkErr(ctx context.Context, err error) error {
-	s.report.ReportError(ChunkError{s.unit, err})
-	return nil
-}

pkg/sources/source_manager_test.go

@@ -1,191 +0,0 @@
-package sources
-
-import (
-	"fmt"
-	"testing"
-
-	"github.com/trufflesecurity/trufflehog/v3/pkg/context"
-	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
-	"google.golang.org/protobuf/types/known/anypb"
-)
-
-// DummySource implements Source and is used for testing a SourceManager.
-type DummySource struct {
-	sourceID int64
-	jobID    int64
-	chunker
-}
-
-func (d *DummySource) Type() sourcespb.SourceType { return 1337 }
-func (d *DummySource) SourceID() int64            { return d.sourceID }
-func (d *DummySource) JobID() int64               { return d.jobID }
-func (d *DummySource) Init(_ context.Context, _ string, jobID, sourceID int64, _ bool, _ *anypb.Any, _ int) error {
-	d.sourceID = sourceID
-	d.jobID = jobID
-	return nil
-}
-func (d *DummySource) GetProgress() *Progress { return nil }
-
-// Interface to easily test different chunking methods.
-type chunker interface {
-	Chunks(context.Context, chan *Chunk) error
-	ChunkUnit(ctx context.Context, unit SourceUnit, reporter ChunkReporter) error
-	Enumerate(ctx context.Context, reporter UnitReporter) error
-}
-
-// Chunk method that writes count bytes to the channel before returning.
-type counterChunker struct {
-	chunkCounter byte
-	count        int
-}
-
-func (c *counterChunker) Chunks(_ context.Context, ch chan *Chunk) error {
-	for i := 0; i < c.count; i++ {
-		ch <- &Chunk{Data: []byte{c.chunkCounter}}
-		c.chunkCounter++
-	}
-	return nil
-}
-
-// countChunk implements SourceUnit.
-type countChunk byte
-
-func (c countChunk) SourceUnitID() string { return fmt.Sprintf("countChunk(%d)", c) }
-
-func (c *counterChunker) Enumerate(ctx context.Context, reporter UnitReporter) error {
-	for i := 0; i < c.count; i++ {
-		if err := reporter.UnitOk(ctx, countChunk(byte(i))); err != nil {
-			return err
-		}
-	}
-	return nil
-}
-
-func (c *counterChunker) ChunkUnit(ctx context.Context, unit SourceUnit, reporter ChunkReporter) error {
-	return reporter.ChunkOk(ctx, Chunk{Data: []byte{byte(unit.(countChunk))}})
-}
-
-// Chunk method that always returns an error.
-type errorChunker struct{ error }
-
-func (c errorChunker) Chunks(context.Context, chan *Chunk) error                  { return c }
-func (c errorChunker) Enumerate(context.Context, UnitReporter) error              { return c }
-func (c errorChunker) ChunkUnit(context.Context, SourceUnit, ChunkReporter) error { return c }
-
-// enrollDummy is a helper function to enroll a DummySource with a SourceManager.
-func enrollDummy(mgr *SourceManager, chunkMethod chunker) (handle, error) {
-	return mgr.Enroll(context.Background(), "dummy", 1337,
-		func(ctx context.Context, jobID, sourceID int64) (Source, error) {
-			source := &DummySource{chunker: chunkMethod}
-			if err := source.Init(ctx, "dummy", jobID, sourceID, true, nil, 42); err != nil {
-				return nil, err
-			}
-			return source, nil
-		})
-}
-
-// tryRead is a helper function that will try to read from a channel and return
-// an error if it cannot.
-func tryRead(ch <-chan *Chunk) (*Chunk, error) {
-	select {
-	case chunk := <-ch:
-		return chunk, nil
-	default:
-		return nil, fmt.Errorf("no chunk available")
-	}
-}
-
-func TestSourceManagerRun(t *testing.T) {
-	mgr := NewManager(WithBufferedOutput(8))
-	handle, err := enrollDummy(mgr, &counterChunker{count: 1})
-	if err != nil {
-		t.Fatalf("unexpected error enrolling source: %v", err)
-	}
-	for i := 0; i < 3; i++ {
-		if _, err := mgr.Run(context.Background(), handle); err != nil {
-			t.Fatalf("unexpected error running source: %v", err)
-		}
-		chunk, err := tryRead(mgr.Chunks())
-		if err != nil {
-			t.Fatalf("reading chunk failed: %v", err)
-		}
-		if chunk.Data[0] != byte(i) {
-			t.Fatalf("unexpected chunk value, wanted %v, got: %v", chunk.Data[0], i)
-		}
-
-		// The Chunks channel should be empty now.
-		if chunk, err := tryRead(mgr.Chunks()); err == nil {
-			t.Fatalf("unexpected chunk found: %+v", chunk)
-		}
-	}
-}
-
-func TestSourceManagerWait(t *testing.T) {
-	mgr := NewManager()
-	handle, err := enrollDummy(mgr, &counterChunker{count: 1})
-	if err != nil {
-		t.Fatalf("unexpected error enrolling source: %v", err)
-	}
-	// Asynchronously run the source.
-	if _, err := mgr.ScheduleRun(context.Background(), handle); err != nil {
-		t.Fatalf("unexpected error scheduling run: %v", err)
-	}
-	// Read the 1 chunk we're expecting so Waiting completes.
-	<-mgr.Chunks()
-	// Wait for all resources to complete.
-	mgr.Wait()
-	// Enroll and run should return an error now.
-	if _, err := enrollDummy(mgr, &counterChunker{count: 1}); err == nil {
-		t.Fatalf("expected enroll to fail")
-	}
-	if _, err := mgr.ScheduleRun(context.Background(), handle); err == nil {
-		t.Fatalf("expected scheduling run to fail")
-	}
-}
-
-func TestSourceManagerError(t *testing.T) {
-	mgr := NewManager()
-	handle, err := enrollDummy(mgr, errorChunker{fmt.Errorf("oops")})
-	if err != nil {
-		t.Fatalf("unexpected error enrolling source: %v", err)
-	}
-	// A synchronous run should fail.
-	if _, err := mgr.Run(context.Background(), handle); err == nil {
-		t.Fatalf("expected run to fail")
-	}
-	// Scheduling a run should not fail, but the error should surface in
-	// Wait().
-	ref, err := mgr.ScheduleRun(context.Background(), handle)
-	if err != nil {
-		t.Fatalf("unexpected error scheduling run: %v", err)
-	}
-	<-ref.Done()
-	if err := ref.Snapshot().FatalError(); err == nil {
-		t.Fatalf("expected wait to fail")
-	}
-}
-
-func TestSourceManagerReport(t *testing.T) {
-	for _, opts := range [][]func(*SourceManager){
-		{WithBufferedOutput(8)},
-		{WithBufferedOutput(8), WithSourceUnits()},
-		{WithBufferedOutput(8), WithSourceUnits(), WithConcurrentUnits(1)},
-	} {
-		mgr := NewManager(opts...)
-		handle, err := enrollDummy(mgr, &counterChunker{count: 4})
-		if err != nil {
-			t.Fatalf("unexpected error enrolling source: %v", err)
-		}
-		// Synchronously run the source.
-		ref, err := mgr.Run(context.Background(), handle)
-		if err != nil {
-			t.Fatalf("unexpected error running source: %v", err)
-		}
-		if errs := ref.Snapshot().Errors; len(errs) > 0 {
-			t.Fatalf("unexpected errors in report: %v", errs)
-		}
-		if count := ref.Snapshot().TotalChunks; count != 4 {
-			t.Fatalf("expected report to have 4 chunks, got: %d", count)
-		}
-	}
-}

pkg/sources/sources.go

@@ -44,59 +44,12 @@ type Source interface {
 	GetProgress() *Progress
 }
 
-// SourceUnitEnumChunker are the two required interfaces to support enumerating
-// and chunking of units.
-type SourceUnitEnumChunker interface {
-	SourceUnitEnumerator
-	SourceUnitChunker
-}
-
 // SourceUnitUnmarshaller defines an optional interface a Source can implement
 // to support units coming from an external source.
 type SourceUnitUnmarshaller interface {
 	UnmarshalSourceUnit(data []byte) (SourceUnit, error)
 }
 
-// SourceUnitEnumerator defines an optional interface a Source can implement to
-// support enumerating an initialized Source into SourceUnits.
-type SourceUnitEnumerator interface {
-	// Enumerate creates 0 or more units from an initialized source,
-	// reporting them or any errors to the UnitReporter. This method is
-	// synchronous but can be called in a goroutine to support concurrent
-	// enumeration and chunking. An error should only be returned from this
-	// method in the case of context cancellation, fatal source errors, or
-	// errors returned by the reporter All other errors related to unit
-	// enumeration are tracked by the UnitReporter.
-	Enumerate(ctx context.Context, reporter UnitReporter) error
-}
-
-// UnitReporter defines the interface a source will use to report whether a
-// unit was found during enumeration. Either method may be called any number of
-// times. Implementors of this interface should allow for concurrent calls.
-type UnitReporter interface {
-	UnitOk(ctx context.Context, unit SourceUnit) error
-	UnitErr(ctx context.Context, err error) error
-}
-
-// SourceUnitChunker defines an optional interface a Source can implement to
-// support chunking a single SourceUnit.
-type SourceUnitChunker interface {
-	// ChunkUnit creates 0 or more chunks from a unit, reporting them or
-	// any errors to the ChunkReporter. An error should only be returned
-	// from this method in the case of context cancellation, fatal source
-	// errors, or errors returned by the reporter. All other errors related
-	// to unit chunking are tracked by the ChunkReporter.
-	ChunkUnit(ctx context.Context, unit SourceUnit, reporter ChunkReporter) error
-}
-
-// ChunkReporter defines the interface a source will use to report whether a
-// chunk was found during unit chunking. Either method may be called any number
-// of times. Implementors of this interface should allow for concurrent calls.
-type ChunkReporter interface {
-	ChunkOk(ctx context.Context, chunk Chunk) error
-	ChunkErr(ctx context.Context, err error) error
-}
-
 // SourceUnit is an object that represents a Source's unit of work. This is
 // used as the output of enumeration, progress reporting, and job distribution.
 type SourceUnit interface {