Set total count during enumeration.

* added new detectors and fixed mesibo detector

* added bscscan.com API detector

* added coinmarketcap detector

* update alchemy

* update blocknative

* update bscscan test

* update cmc test

* update tests

---------

Co-authored-by: Dustin Decker <dustin@trufflesec.com>

.github/workflows/release.yml

@@ -1,16 +1,9 @@
 name: Release
 
-# on:
-#   push:
-#     tags:
-#       - 'v*'
-
 on:
   push:
     tags:
       - v*
-    branches:
-      - main
 
 permissions:
   contents: write

.gitignore

@@ -1,4 +1,7 @@
 .idea
 dist
 .env
-*.test
+*.test
+
+# binary
+trufflehog

.pre-commit-hooks.yaml

@@ -0,0 +1,6 @@
+- id: trufflehog
+  name: TruffleHog
+  description: Detect secrets in your data with TruffleHog.
+  entry: trufflehog git file://. --since-commit HEAD --only-verified --fail
+  language: golang
+  pass_filenames: false

README.md

@@ -84,6 +84,24 @@ Expected output:
 trufflehog s3 --bucket=<bucket name> --only-verified
 ```
 
+### Example 5: Scan a Github Repo using SSH authentication in docker
+
+```bash
+docker run --rm -v "$HOME/.ssh:/root/.ssh:ro" trufflesecurity/trufflehog:latest git ssh://github.com/trufflesecurity/test_keys
+```
+
+### Example 6: Scan individual files or directories
+
+```bash
+trufflehog filesystem path/to/file1.txt path/to/file2.txt path/to/dir
+```
+
+### Example 7: Scan GCS buckets for verified secrets.
+
+```bash
+trufflehog gcs --project-id=<project-ID> --cloud-environment --only-verified
+```
+
 ### FAQ
 
 + All I see is `🐷🔑🐷  TruffleHog. Unearth your secrets. 🐷🔑🐷` and the program exits, what gives?
@@ -94,12 +112,6 @@ trufflehog s3 --bucket=<bucket name> --only-verified
   + Check out our Driftwood blog post to learn how to do this, in short we've confirmed the key can be used live for SSH or SSL [Blog post](https://trufflesecurity.com/blog/driftwood-know-if-private-keys-are-sensitive/)
 
 
-### Example 5: Scan a Github Repo using SSH authentication in docker
-
-```bash
-docker run --rm -v "$HOME/.ssh:/root/.ssh:ro" trufflesecurity/trufflehog:latest git ssh://github.com/trufflesecurity/test_keys
-```
-
 # What's new in v3?
 
 TruffleHog v3 is a complete rewrite in Go with many new powerful features.
@@ -167,10 +179,11 @@ TruffleHog has a sub-command for each source of data that you may want to scan:
 - github
 - gitlab
 - S3
-- filesystem
+- filesystem (files and directories)
 - syslog
 - circleci
-- file and stdin (coming soon)
+- GCS (Google Cloud Storage)
+- stdin (coming soon)
 
 Each subcommand can have options that you can see with the `--help` flag provided to the sub command:
 
@@ -184,6 +197,7 @@ Flags:
       --help                     Show context-sensitive help (also try --help-long and --help-man).
       --debug                    Run in debug mode.
       --trace                    Run in trace mode.
+      --profile                  Enables profiling and sets a pprof and fgprof server on :18066.
   -j, --json                     Output in JSON format.
       --json-legacy              Use the pre-v3.0 JSON format. Only works with git, gitlab, and github sources.
       --concurrency=10           Number of concurrent workers.
@@ -311,7 +325,7 @@ detectors:
 ```
 
 ```
-$ trufflehog filesystem --directory /tmp --config config.yaml --only-verified
+$ trufflehog filesystem /tmp --config config.yaml --only-verified
 🐷🔑🐷  TruffleHog. Unearth your secrets. 🐷🔑🐷
 
 Found verified result 🐷🔑

go.mod

@@ -10,21 +10,22 @@ replace github.com/gitleaks/go-gitdiff => github.com/trufflesecurity/go-gitdiff
 
 require (
 	cloud.google.com/go/secretmanager v1.10.0
+	cloud.google.com/go/storage v1.29.0
 	github.com/Azure/go-autorest/autorest/azure/auth v0.5.11
-	github.com/TheZeroSlave/zapsentry v1.12.0
+	github.com/TheZeroSlave/zapsentry v1.14.0
 	github.com/aws/aws-sdk-go v1.44.83
 	github.com/bill-rich/disk-buffer-reader v0.1.7
 	github.com/bill-rich/go-syslog v0.0.0-20220413021637-49edb52a574c
 	github.com/bitfinexcom/bitfinex-api-go v0.0.0-20210608095005-9e0b26f200fb
-	github.com/bradleyfalzon/ghinstallation/v2 v2.1.0
+	github.com/bradleyfalzon/ghinstallation/v2 v2.2.0
 	github.com/crewjam/rfc5424 v0.1.0
 	github.com/denisenkom/go-mssqldb v0.12.3
 	github.com/envoyproxy/protoc-gen-validate v0.9.1
-	github.com/fatih/color v1.13.0
+	github.com/fatih/color v1.15.0
 	github.com/felixge/fgprof v0.9.3
-	github.com/getsentry/sentry-go v0.18.0
+	github.com/getsentry/sentry-go v0.19.0
 	github.com/go-errors/errors v1.4.2
-	github.com/go-git/go-git/v5 v5.5.2
+	github.com/go-git/go-git/v5 v5.6.0
 	github.com/go-ldap/ldap/v3 v3.4.4
 	github.com/go-logr/logr v1.2.3
 	github.com/go-logr/zapr v1.2.3
@@ -34,6 +35,7 @@ require (
 	github.com/golang-jwt/jwt v3.2.2+incompatible
 	github.com/google/go-cmp v0.5.9
 	github.com/google/go-github/v42 v42.0.0
+	github.com/googleapis/gax-go/v2 v2.7.1
 	github.com/gorilla/mux v1.8.0
 	github.com/hashicorp/go-retryablehttp v0.7.2
 	github.com/jlaffaye/ftp v0.1.0
@@ -44,30 +46,32 @@ require (
 	github.com/mattn/go-sqlite3 v1.14.16
 	github.com/mholt/archiver/v4 v4.0.0-alpha.7
 	github.com/paulbellamy/ratecounter v0.2.0
+	github.com/petar-dambovaliev/aho-corasick v0.0.0-20211021192214-5ab2d9280aa9
 	github.com/pkg/errors v0.9.1
-	github.com/rabbitmq/amqp091-go v1.6.0
+	github.com/rabbitmq/amqp091-go v1.7.0
 	github.com/sergi/go-diff v1.3.1
-	github.com/stretchr/testify v1.8.1
+	github.com/stretchr/testify v1.8.2
 	github.com/tailscale/depaware v0.0.0-20210622194025-720c4b409502
-	github.com/xanzy/go-gitlab v0.78.0
-	go.mongodb.org/mongo-driver v1.11.1
+	github.com/xanzy/go-gitlab v0.81.0
+	go.mongodb.org/mongo-driver v1.11.2
 	go.uber.org/zap v1.24.0
-	golang.org/x/crypto v0.6.0
+	golang.org/x/crypto v0.7.0
 	golang.org/x/exp v0.0.0-20221018205818-5c77f4b2bbd7
-	golang.org/x/oauth2 v0.4.0
+	golang.org/x/oauth2 v0.6.0
 	golang.org/x/sync v0.1.0
-	golang.org/x/text v0.7.0
-	google.golang.org/genproto v0.0.0-20221201164419-0e50fba7f41c
-	google.golang.org/protobuf v1.28.1
+	golang.org/x/text v0.8.0
+	google.golang.org/api v0.111.0
+	google.golang.org/protobuf v1.29.1
 	gopkg.in/alecthomas/kingpin.v2 v2.2.6
 	gopkg.in/h2non/gock.v1 v1.1.2
 	sigs.k8s.io/yaml v1.3.0
 )
 
 require (
-	cloud.google.com/go/compute v1.12.1 // indirect
-	cloud.google.com/go/compute/metadata v0.2.1 // indirect
-	cloud.google.com/go/iam v0.8.0 // indirect
+	cloud.google.com/go v0.110.0 // indirect
+	cloud.google.com/go/compute v1.18.0 // indirect
+	cloud.google.com/go/compute/metadata v0.2.3 // indirect
+	cloud.google.com/go/iam v0.12.0 // indirect
 	github.com/Azure/go-autorest v14.2.0+incompatible // indirect
 	github.com/Azure/go-autorest/autorest v0.11.24 // indirect
 	github.com/Azure/go-autorest/autorest/adal v0.9.18 // indirect
@@ -92,17 +96,17 @@ require (
 	github.com/go-git/gcfg v1.5.0 // indirect
 	github.com/go-git/go-billy/v5 v5.4.0 // indirect
 	github.com/go-ole/go-ole v1.2.6 // indirect
-	github.com/golang-jwt/jwt/v4 v4.4.1 // indirect
+	github.com/golang-jwt/jwt/v4 v4.5.0 // indirect
 	github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe // indirect
 	github.com/golang-sql/sqlexp v0.1.0 // indirect
 	github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e // indirect
-	github.com/golang/protobuf v1.5.2 // indirect
+	github.com/golang/protobuf v1.5.3 // indirect
 	github.com/golang/snappy v0.0.4 // indirect
-	github.com/google/go-github/v45 v45.2.0 // indirect
+	github.com/google/go-github/v50 v50.1.0 // indirect
 	github.com/google/go-querystring v1.1.0 // indirect
 	github.com/google/pprof v0.0.0-20211214055906-6f57359322fd // indirect
-	github.com/googleapis/enterprise-certificate-proxy v0.2.0 // indirect
-	github.com/googleapis/gax-go/v2 v2.7.0 // indirect
+	github.com/google/uuid v1.3.0 // indirect
+	github.com/googleapis/enterprise-certificate-proxy v0.2.3 // indirect
 	github.com/h2non/parth v0.0.0-20190131123155-b4df798d6542 // indirect
 	github.com/hashicorp/errwrap v1.0.0 // indirect
 	github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
@@ -115,14 +119,14 @@ require (
 	github.com/klauspost/compress v1.15.11 // indirect
 	github.com/klauspost/pgzip v1.2.5 // indirect
 	github.com/mattn/go-colorable v0.1.13 // indirect
-	github.com/mattn/go-isatty v0.0.16 // indirect
+	github.com/mattn/go-isatty v0.0.17 // indirect
 	github.com/mitchellh/go-homedir v1.1.0 // indirect
 	github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe // indirect
 	github.com/nwaples/rardecode/v2 v2.0.0-beta.2 // indirect
 	github.com/onsi/ginkgo v1.16.5 // indirect
 	github.com/onsi/gomega v1.23.0 // indirect
 	github.com/pierrec/lz4/v4 v4.1.14 // indirect
-	github.com/pjbgf/sha1cd v0.2.3 // indirect
+	github.com/pjbgf/sha1cd v0.3.0 // indirect
 	github.com/pkg/diff v0.0.0-20200914180035-5b29258ca4f7 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/skeema/knownhosts v1.1.0 // indirect
@@ -137,14 +141,15 @@ require (
 	go.opencensus.io v0.24.0 // indirect
 	go.uber.org/atomic v1.7.0 // indirect
 	go.uber.org/multierr v1.6.0 // indirect
-	golang.org/x/mod v0.7.0 // indirect
-	golang.org/x/net v0.6.0 // indirect
-	golang.org/x/sys v0.5.0 // indirect
+	golang.org/x/mod v0.8.0 // indirect
+	golang.org/x/net v0.8.0 // indirect
+	golang.org/x/sys v0.6.0 // indirect
 	golang.org/x/time v0.3.0 // indirect
-	golang.org/x/tools v0.3.0 // indirect
-	google.golang.org/api v0.103.0 // indirect
+	golang.org/x/tools v0.6.0 // indirect
+	golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect
 	google.golang.org/appengine v1.6.7 // indirect
-	google.golang.org/grpc v1.51.0 // indirect
+	google.golang.org/genproto v0.0.0-20230303212802-e74f57abe488 // indirect
+	google.golang.org/grpc v1.53.0 // indirect
 	gopkg.in/warnings.v0 v0.1.2 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect

go.sum

@@ -1,14 +1,17 @@
 cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
-cloud.google.com/go v0.105.0 h1:DNtEKRBAAzeS4KyIory52wWHuClNaXJ5x1F7xa4q+5Y=
-cloud.google.com/go/compute v1.12.1 h1:gKVJMEyqV5c/UnpzjjQbo3Rjvvqpr9B1DFSbJC4OXr0=
-cloud.google.com/go/compute v1.12.1/go.mod h1:e8yNOBcBONZU1vJKCvCoDw/4JQsA0dpM4x/6PIIOocU=
-cloud.google.com/go/compute/metadata v0.2.1 h1:efOwf5ymceDhK6PKMnnrTHP4pppY5L22mle96M1yP48=
-cloud.google.com/go/compute/metadata v0.2.1/go.mod h1:jgHgmJd2RKBGzXqF5LR2EZMGxBkeanZ9wwa75XHJgOM=
-cloud.google.com/go/iam v0.8.0 h1:E2osAkZzxI/+8pZcxVLcDtAQx/u+hZXVryUaYQ5O0Kk=
-cloud.google.com/go/iam v0.8.0/go.mod h1:lga0/y3iH6CX7sYqypWJ33hf7kkfXJag67naqGESjkE=
-cloud.google.com/go/longrunning v0.3.0 h1:NjljC+FYPV3uh5/OwWT6pVU+doBqMg2x/rZlE+CamDs=
+cloud.google.com/go v0.110.0 h1:Zc8gqp3+a9/Eyph2KDmcGaPtbKRIoqq4YTlL4NMD0Ys=
+cloud.google.com/go v0.110.0/go.mod h1:SJnCLqQ0FCFGSZMUNUf84MV3Aia54kn7pi8st7tMzaY=
+cloud.google.com/go/compute v1.18.0 h1:FEigFqoDbys2cvFkZ9Fjq4gnHBP55anJ0yQyau2f9oY=
+cloud.google.com/go/compute v1.18.0/go.mod h1:1X7yHxec2Ga+Ss6jPyjxRxpu2uu7PLgsOVXvgU0yacs=
+cloud.google.com/go/compute/metadata v0.2.3 h1:mg4jlk7mCAj6xXp9UJ4fjI9VUI5rubuGBW5aJ7UnBMY=
+cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA=
+cloud.google.com/go/iam v0.12.0 h1:DRtTY29b75ciH6Ov1PHb4/iat2CLCvrOm40Q0a6DFpE=
+cloud.google.com/go/iam v0.12.0/go.mod h1:knyHGviacl11zrtZUoDuYpDgLjvr28sLQaG0YB2GYAY=
+cloud.google.com/go/longrunning v0.4.1 h1:v+yFJOfKC3yZdY6ZUI933pIYdhyhV8S3NpWrXWmg7jM=
 cloud.google.com/go/secretmanager v1.10.0 h1:pu03bha7ukxF8otyPKTFdDz+rr9sE3YauS5PliDXK60=
 cloud.google.com/go/secretmanager v1.10.0/go.mod h1:MfnrdvKMPNra9aZtQFvBcvRU54hbPD8/HayQdlUgJpU=
+cloud.google.com/go/storage v1.29.0 h1:6weCgzRvMg7lzuUurI4697AqIRPU1SvzHhynwpW31jI=
+cloud.google.com/go/storage v1.29.0/go.mod h1:4puEjyTKnku6gfKoTfNOU/W+a9JyuVNxjpS5GBrB8h4=
 github.com/Azure/azure-sdk-for-go/sdk/azcore v0.19.0/go.mod h1:h6H6c8enJmmocHUbLiiGY6sx7f9i+X3m1CHdd5c6Rdw=
 github.com/Azure/azure-sdk-for-go/sdk/azidentity v0.11.0/go.mod h1:HcM1YX14R7CJcghJGOYCgdezslRSVzqwLf/q+4Y2r/0=
 github.com/Azure/azure-sdk-for-go/sdk/internal v0.7.0/go.mod h1:yqy467j36fJxcRV2TzfVZ1pCb5vxm4BtZPUdYWe/Xo8=
@@ -37,8 +40,8 @@ github.com/Microsoft/go-winio v0.5.2 h1:a9IhgEQBCUEk6QCdml9CiJGhAws+YwffDHEMp1VM
 github.com/Microsoft/go-winio v0.5.2/go.mod h1:WpS1mjBmmwHBEWmogvA2mj8546UReBk4v8QkMxJ6pZY=
 github.com/ProtonMail/go-crypto v0.0.0-20221026131551-cf6655e29de4 h1:ra2OtmuW0AE5csawV4YXMNGNQQXvLRps3z2Z59OPO+I=
 github.com/ProtonMail/go-crypto v0.0.0-20221026131551-cf6655e29de4/go.mod h1:UBYPn8k0D56RtnR8RFQMjmh4KrZzWJ5o7Z9SYjossQ8=
-github.com/TheZeroSlave/zapsentry v1.12.0 h1:hhfwvD7pQbnCIOvAEGM4NXLiaX4b6W6ZWNtL6awSBGc=
-github.com/TheZeroSlave/zapsentry v1.12.0/go.mod h1:00uO/VpPrSJG/XigAfTi0F4WMFIw2DmP/IDVUhPBvNw=
+github.com/TheZeroSlave/zapsentry v1.14.0 h1:z6H0lIyKVpdXsP5K8J11b4/+V+QCDLDslZmtXsnduMk=
+github.com/TheZeroSlave/zapsentry v1.14.0/go.mod h1:0hUR1Et5tSroEYhWOZcxE2iBqcFkuG7jGGlUmOA7XAs=
 github.com/acomagu/bufpipe v1.0.3 h1:fxAGrHZTgQ9w5QqVItgzwj235/uYZYgbXitB+dLupOk=
 github.com/acomagu/bufpipe v1.0.3/go.mod h1:mxdxdup/WdsKVreO5GpW4+M/1CE2sMG4jeGJ2sYmHc4=
 github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 h1:JYp7IbQjafoB+tBA3gMyHYHrpOtNuDiK/uB5uXxq5wM=
@@ -61,8 +64,8 @@ github.com/bill-rich/go-syslog v0.0.0-20220413021637-49edb52a574c h1:tSME5FDS02q
 github.com/bill-rich/go-syslog v0.0.0-20220413021637-49edb52a574c/go.mod h1:+sCc6hztur+oZCLOsNk6wCCy+GLrnSNHSRmTnnL+8iQ=
 github.com/bitfinexcom/bitfinex-api-go v0.0.0-20210608095005-9e0b26f200fb h1:9v7Bzlg+1EBYi2IYcUmOwHReBEfqBbYIj3ZCi9cIe1Q=
 github.com/bitfinexcom/bitfinex-api-go v0.0.0-20210608095005-9e0b26f200fb/go.mod h1:EkOqCuelvo7DY8vCOoZ09p7pHvAK9B1PHI9GeM4Rdxc=
-github.com/bradleyfalzon/ghinstallation/v2 v2.1.0 h1:5+NghM1Zred9Z078QEZtm28G/kfDfZN/92gkDlLwGVA=
-github.com/bradleyfalzon/ghinstallation/v2 v2.1.0/go.mod h1:Xg3xPRN5Mcq6GDqeUVhFbjEWMb4JHCyWEeeBGEYQoTU=
+github.com/bradleyfalzon/ghinstallation/v2 v2.2.0 h1:AVvVU33rE8wdTS1aNnenwpigEBA9mvzI5OhjhZfH/LU=
+github.com/bradleyfalzon/ghinstallation/v2 v2.2.0/go.mod h1:xo3iIfK0lDKECe0s19nbxT0KKvk7LsrGc4NxR5ckKMA=
 github.com/bwesterb/go-ristretto v1.2.0/go.mod h1:fUIoIZaG73pV5biE2Blr2xEzDoMj7NFEuV9ekS419A0=
 github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
 github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
@@ -94,15 +97,15 @@ github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1m
 github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
 github.com/envoyproxy/protoc-gen-validate v0.9.1 h1:PS7VIOgmSVhWUEeZwTe7z7zouA22Cr590PzXKbZHOVY=
 github.com/envoyproxy/protoc-gen-validate v0.9.1/go.mod h1:OKNgG7TCp5pF4d6XftA0++PMirau2/yoOwVac3AbF2w=
-github.com/fatih/color v1.13.0 h1:8LOYc1KYPPmyKMuN8QV2DNRWNbLo6LZ0iLs8+mlH53w=
-github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk=
+github.com/fatih/color v1.15.0 h1:kOqh6YHBtK8aywxGerMG2Eq3H6Qgoqeo13Bk2Mv/nBs=
+github.com/fatih/color v1.15.0/go.mod h1:0h5ZqXfHYED7Bhv2ZJamyIOUej9KtShiJESRwBDUSsw=
 github.com/felixge/fgprof v0.9.3 h1:VvyZxILNuCiUCSXtPtYmmtGvb65nqXh2QFWc0Wpf2/g=
 github.com/felixge/fgprof v0.9.3/go.mod h1:RdbpDgzqYVh/T9fPELJyV7EYJuHB55UTEULNun8eiPw=
 github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
 github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4=
 github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
-github.com/getsentry/sentry-go v0.18.0 h1:MtBW5H9QgdcJabtZcuJG80BMOwaBpkRDZkxRkNC1sN0=
-github.com/getsentry/sentry-go v0.18.0/go.mod h1:Kgon4Mby+FJ7ZWHFUAZgVaIa8sxHtnRJRLTXZr51aKQ=
+github.com/getsentry/sentry-go v0.19.0 h1:BcCH3CN5tXt5aML+gwmbFwVptLLQA+eT866fCO9wVOM=
+github.com/getsentry/sentry-go v0.19.0/go.mod h1:y3+lGEFEFexZtpbG1GUE2WD/f9zGyKYwpEqryTOC/nE=
 github.com/gliderlabs/ssh v0.3.5 h1:OcaySEmAQJgyYcArR+gGGTHCyE7nvhEMTlYY+Dp8CpY=
 github.com/gliderlabs/ssh v0.3.5/go.mod h1:8XB4KraRrX39qHhT6yxPsHedjA08I/uBVwj4xC+/+z4=
 github.com/go-asn1-ber/asn1-ber v1.5.4 h1:vXT6d/FNDiELJnLb6hGNa309LMsrCoYFvpwHDF0+Y1A=
@@ -116,8 +119,8 @@ github.com/go-git/go-billy/v5 v5.4.0 h1:Vaw7LaSTRJOUric7pe4vnzBSgyuf2KrLsu2Y4ZpQ
 github.com/go-git/go-billy/v5 v5.4.0/go.mod h1:vjbugF6Fz7JIflbVpl1hJsGjSHNltrSw45YK/ukIvQg=
 github.com/go-git/go-git-fixtures/v4 v4.3.1 h1:y5z6dd3qi8Hl+stezc8p3JxDkoTRqMAlKnXHuzrfjTQ=
 github.com/go-git/go-git-fixtures/v4 v4.3.1/go.mod h1:8LHG1a3SRW71ettAD/jW13h8c6AqjVSeL11RAdgaqpo=
-github.com/go-git/go-git/v5 v5.5.2 h1:v8lgZa5k9ylUw+OR/roJHTxR4QItsNFI5nKtAXFuynw=
-github.com/go-git/go-git/v5 v5.5.2/go.mod h1:BE5hUJ5yaV2YMxhmaP4l6RBQ08kMxKSPD4BlxtH7OjI=
+github.com/go-git/go-git/v5 v5.6.0 h1:JvBdYfcttd+0kdpuWO7KTu0FYgCf5W0t5VwkWGobaa4=
+github.com/go-git/go-git/v5 v5.6.0/go.mod h1:6nmJ0tJ3N4noMV1Omv7rC5FG3/o8Cm51TB4CJp7mRmE=
 github.com/go-ldap/ldap/v3 v3.4.4 h1:qPjipEpt+qDa6SI/h1fzuGWoRUY+qqQ9sOZq67/PYUs=
 github.com/go-ldap/ldap/v3 v3.4.4/go.mod h1:fe1MsuN5eJJ1FeLT/LEBVdWfNWKh459R7aXgXtJC+aI=
 github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
@@ -141,8 +144,8 @@ github.com/golang-jwt/jwt v3.2.2+incompatible h1:IfV12K8xAKAnZqdXVzCZ+TOjboZ2keL
 github.com/golang-jwt/jwt v3.2.2+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I=
 github.com/golang-jwt/jwt/v4 v4.0.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg=
 github.com/golang-jwt/jwt/v4 v4.2.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg=
-github.com/golang-jwt/jwt/v4 v4.4.1 h1:pC5DB52sCeK48Wlb9oPcdhnjkz1TKt1D/P7WKJ0kUcQ=
-github.com/golang-jwt/jwt/v4 v4.4.1/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0=
+github.com/golang-jwt/jwt/v4 v4.5.0 h1:7cYmW1XlMY7h7ii7UhUyChSgS5wUJEnm9uZVTGqOWzg=
+github.com/golang-jwt/jwt/v4 v4.5.0/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0=
 github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe h1:lXe2qZdvpiX5WZkZR4hgp4KJVfY3nMkvmwbVkpv1rVY=
 github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe/go.mod h1:8vg3r2VgvsThLBIFL93Qb5yWzgyZWhEmBwUJWevAkK0=
 github.com/golang-sql/sqlexp v0.1.0 h1:ZCD6MBpcuOVfGVqsEmY5/4FtYiKz6tSyUv9LPEDei6A=
@@ -163,8 +166,8 @@ github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QD
 github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
 github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
 github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
-github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw=
-github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
+github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg=
+github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
 github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
 github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM=
 github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
@@ -176,22 +179,24 @@ github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/
 github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
 github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
 github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
 github.com/google/go-github/v42 v42.0.0 h1:YNT0FwjPrEysRkLIiKuEfSvBPCGKphW5aS5PxwaoLec=
 github.com/google/go-github/v42 v42.0.0/go.mod h1:jgg/jvyI0YlDOM1/ps6XYh04HNQ3vKf0CVko62/EhRg=
-github.com/google/go-github/v45 v45.2.0 h1:5oRLszbrkvxDDqBCNj2hjDZMKmvexaZ1xw/FCD+K3FI=
-github.com/google/go-github/v45 v45.2.0/go.mod h1:FObaZJEDSTa/WGCzZ2Z3eoCDXWJKMenWWTrd8jrta28=
+github.com/google/go-github/v50 v50.1.0 h1:hMUpkZjklC5GJ+c3GquSqOP/T4BNsB7XohaPhtMOzRk=
+github.com/google/go-github/v50 v50.1.0/go.mod h1:Ev4Tre8QoKiolvbpOSG3FIi4Mlon3S2Nt9W5JYqKiwA=
 github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8=
 github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU=
+github.com/google/martian/v3 v3.3.2 h1:IqNFLAmvJOgVlpdEBiQbDc2EwKW77amAycfTuWKdfvw=
 github.com/google/pprof v0.0.0-20211214055906-6f57359322fd h1:1FjCyPC+syAzJ5/2S8fqdZK1R22vvA0J7JZKcuOIQ7Y=
 github.com/google/pprof v0.0.0-20211214055906-6f57359322fd/go.mod h1:KgnwoLYCZ8IQu3XUZ8Nc/bM9CCZFOyjUNOSygVozoDg=
 github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
-github.com/googleapis/enterprise-certificate-proxy v0.2.0 h1:y8Yozv7SZtlU//QXbezB6QkpuE6jMD2/gfzk4AftXjs=
-github.com/googleapis/enterprise-certificate-proxy v0.2.0/go.mod h1:8C0jb7/mgJe/9KK8Lm7X9ctZC2t60YyIpYEI16jx0Qg=
-github.com/googleapis/gax-go/v2 v2.7.0 h1:IcsPKeInNvYi7eqSaDjiZqDDKu5rsmunY0Y1YupQSSQ=
-github.com/googleapis/gax-go/v2 v2.7.0/go.mod h1:TEop28CZZQ2y+c0VxMUmu1lV+fQx57QpBWsYpwqHJx8=
+github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I=
+github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/googleapis/enterprise-certificate-proxy v0.2.3 h1:yk9/cqRKtT9wXZSsRH9aurXEpJX+U6FLtpYTdC3R06k=
+github.com/googleapis/enterprise-certificate-proxy v0.2.3/go.mod h1:AwSRAtLfXpU5Nm3pW+v7rGDHp09LsPtGY9MduiEsR9k=
+github.com/googleapis/gax-go/v2 v2.7.1 h1:gF4c0zjUP2H/s/hEGyLA3I0fA2ZWjzYiONAD6cvPr8A=
+github.com/googleapis/gax-go/v2 v2.7.1/go.mod h1:4orTrqY6hXxxaUL4LHIPl6lGo8vAE38/qKbhSAKP6QI=
 github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI=
 github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So=
 github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
@@ -246,19 +251,18 @@ github.com/lib/pq v1.10.7 h1:p7ZhMD+KsSRozJr34udlUrhboJwWAgCg34+/ZZNvZZw=
 github.com/lib/pq v1.10.7/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
 github.com/matryer/is v1.2.0 h1:92UTHpy8CDwaJ08GqLDzhhuixiBUUD1p3AU6PHddz4A=
 github.com/matryer/is v1.2.0/go.mod h1:2fLPjFQM9rhQ15aVEtbuwhJinnOqrmgXPNdZsdwlWXA=
-github.com/mattn/go-colorable v0.1.9/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc=
 github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=
 github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg=
-github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
-github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94=
-github.com/mattn/go-isatty v0.0.16 h1:bq3VjFmv/sOjHtdEhmkEV4x1AJtvUvOJ2PFAZ5+peKQ=
 github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
+github.com/mattn/go-isatty v0.0.17 h1:BTarxUcIeDqL27Mc+vyvdWYSL28zpIhv3RoTdsLMPng=
+github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
 github.com/mattn/go-sqlite3 v1.14.16 h1:yOQRA0RpS5PFz/oikGwBEqvAWhWg5ufRz4ETLjwpU1Y=
 github.com/mattn/go-sqlite3 v1.14.16/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg=
 github.com/mholt/archiver/v4 v4.0.0-alpha.7 h1:xzByj8G8tj0Oq7ZYYU4+ixL/CVb5ruWCm0EZQ1PjOkE=
 github.com/mholt/archiver/v4 v4.0.0-alpha.7/go.mod h1:Fs8qUkO74HHaidabihzYephJH8qmGD/nCP6tE5xC9BM=
 github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y=
 github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
+github.com/mmcloughlin/avo v0.5.0/go.mod h1:ChHFdoV7ql95Wi7vuq2YT1bwCJqiWdZrQ1im3VujLYM=
 github.com/modocache/gover v0.0.0-20171022184752-b58185e213c5/go.mod h1:caMODM3PzxT8aQXRPkAt8xlV/e7d7w8GM5g0fa5F0D8=
 github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe h1:iruDEfMl2E6fbMZ9s0scYfZQ84/6SPL6zC8ACM2oIL0=
 github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc=
@@ -281,11 +285,13 @@ github.com/onsi/gomega v1.23.0/go.mod h1:Z/NWtiqwBrwUt4/2loMmHL63EDLnYHmVbuBpDr2
 github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk=
 github.com/paulbellamy/ratecounter v0.2.0 h1:2L/RhJq+HA8gBQImDXtLPrDXK5qAj6ozWVK/zFXVJGs=
 github.com/paulbellamy/ratecounter v0.2.0/go.mod h1:Hfx1hDpSGoqxkVVpBi/IlYD7kChlfo5C6hzIHwPqfFE=
+github.com/petar-dambovaliev/aho-corasick v0.0.0-20211021192214-5ab2d9280aa9 h1:lL+y4Xv20pVlCGyLzNHRC0I0rIHhIL1lTvHizoS/dU8=
+github.com/petar-dambovaliev/aho-corasick v0.0.0-20211021192214-5ab2d9280aa9/go.mod h1:EHPiTAKtiFmrMldLUNswFwfZ2eJIYBHktdaUTZxYWRw=
 github.com/pierrec/lz4/v4 v4.1.14 h1:+fL8AQEZtz/ijeNnpduH0bROTu0O3NZAlPjQxGn8LwE=
 github.com/pierrec/lz4/v4 v4.1.14/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
 github.com/pingcap/errors v0.11.4 h1:lFuQV/oaUMGcD2tqt+01ROSmJs75VG1ToEOkZIZ4nE4=
-github.com/pjbgf/sha1cd v0.2.3 h1:uKQP/7QOzNtKYH7UTohZLcjF5/55EnTw0jO/Ru4jZwI=
-github.com/pjbgf/sha1cd v0.2.3/go.mod h1:HOK9QrgzdHpbc2Kzip0Q1yi3M2MFGPADtR6HjG65m5M=
+github.com/pjbgf/sha1cd v0.3.0 h1:4D5XXmUUBUl/xQ6IjCkEAbqXskkq/4O7LmGn0AqMDs4=
+github.com/pjbgf/sha1cd v0.3.0/go.mod h1:nZ1rrWOcGJ5uZgEEVL1VUM9iRQiZvWdbZjkKyFzPPsI=
 github.com/pkg/browser v0.0.0-20180916011732-0a3d74bf9ce4/go.mod h1:4OwLy04Bl9Ef3GJJCoec+30X3LQs/0/m4HFRt/2LUSA=
 github.com/pkg/diff v0.0.0-20200914180035-5b29258ca4f7 h1:+/+DxvQaYifJ+grD4klzrS5y+KJXldn/2YTl5JG+vZ8=
 github.com/pkg/diff v0.0.0-20200914180035-5b29258ca4f7/go.mod h1:zO8QMzTeZd5cpnIkz/Gn6iK0jDfGicM1nynOkkPIl28=
@@ -295,8 +301,8 @@ github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
-github.com/rabbitmq/amqp091-go v1.6.0 h1:fiFHcMcP+5/A+35MXniiagho1lEXX4+jpHGR+q+LeBU=
-github.com/rabbitmq/amqp091-go v1.6.0/go.mod h1:wfClAtY0C7bOHxd3GjmF26jEHn+rR/0B3+YV+Vn9/NI=
+github.com/rabbitmq/amqp091-go v1.7.0 h1:V5CF5qPem5OGSnEo8BoSbsDGwejg6VUJsKEdneaoTUo=
+github.com/rabbitmq/amqp091-go v1.7.0/go.mod h1:wfClAtY0C7bOHxd3GjmF26jEHn+rR/0B3+YV+Vn9/NI=
 github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
 github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM=
 github.com/sergi/go-diff v1.3.1 h1:xkr+Oxo4BOQKmkn/B9eMK0g5Kg/983T9DqqPHwYqD+8=
@@ -320,8 +326,9 @@ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/
 github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.7.2/go.mod h1:R6va5+xMeoiuVRoj+gSkQ7d3FALtqAAGI1FQKckRals=
 github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
-github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk=
 github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
+github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8=
+github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
 github.com/tailscale/depaware v0.0.0-20210622194025-720c4b409502 h1:34icjjmqJ2HPjrSuJYEkdZ+0ItmGQAQ75cRHIiftIyE=
 github.com/tailscale/depaware v0.0.0-20210622194025-720c4b409502/go.mod h1:p9lPsd+cx33L3H9nNoecRRxPssFKUwwI50I3pZ0yT+8=
 github.com/therootcompany/xz v1.0.1 h1:CmOtsn1CbtmyYiusbfmhmkpAAETj0wBIH6kCYaX+xzw=
@@ -333,8 +340,8 @@ github.com/trufflesecurity/overseer v1.1.7-custom5/go.mod h1:nT9w37AiO1Nop2VhVhN
 github.com/ulikunitz/xz v0.5.6/go.mod h1:2bypXElzHzzJZwzH67Y6wb67pO62Rzfn7BSiF4ABRW8=
 github.com/ulikunitz/xz v0.5.10 h1:t92gobL9l3HE202wg3rlk19F6X+JOxl9BBrCCMYEYd8=
 github.com/ulikunitz/xz v0.5.10/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14=
-github.com/xanzy/go-gitlab v0.78.0 h1:8jUHfQVAprG04Av5g0PxVd3CNsZ5hCbojIax7Hba1mE=
-github.com/xanzy/go-gitlab v0.78.0/go.mod h1:DlByVTSXhPsJMYL6+cm8e8fTJjeBmhrXdC/yvkKKt6M=
+github.com/xanzy/go-gitlab v0.81.0 h1:ofbhZ5ZY9AjHATWQie4qd2JfncdUmvcSA/zfQB767Dk=
+github.com/xanzy/go-gitlab v0.81.0/go.mod h1:VMbY3JIWdZ/ckvHbQqkyd3iYk2aViKrNIQ23IbFMQDo=
 github.com/xanzy/ssh-agent v0.3.3 h1:+/15pJfg/RsTxqYcX6fHqOXZwwMP+2VyYWJeWM2qQFM=
 github.com/xanzy/ssh-agent v0.3.3/go.mod h1:6dzNDKs0J9rVPHPhaGCukekBHKqfl+L3KghI1Bc68Uw=
 github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c=
@@ -350,8 +357,8 @@ github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1
 github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
 github.com/yusufpapurcu/wmi v1.2.2 h1:KBNDSne4vP5mbSWnJbO+51IMOXJB67QiYCSBrubbPRg=
 github.com/yusufpapurcu/wmi v1.2.2/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0=
-go.mongodb.org/mongo-driver v1.11.1 h1:QP0znIRTuL0jf1oBQoAoM0C6ZJfBK4kx0Uumtv1A7w8=
-go.mongodb.org/mongo-driver v1.11.1/go.mod h1:s7p5vEtfbeR1gYi6pnj3c3/urpbLv2T5Sfd6Rp2HBB8=
+go.mongodb.org/mongo-driver v1.11.2 h1:+1v2rDQUWNcGW7/7E0Jvdz51V38XXxJfhzbV17aNHCw=
+go.mongodb.org/mongo-driver v1.11.2/go.mod h1:s7p5vEtfbeR1gYi6pnj3c3/urpbLv2T5Sfd6Rp2HBB8=
 go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0=
 go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo=
 go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw=
@@ -364,6 +371,7 @@ go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9i
 go.uber.org/zap v1.19.0/go.mod h1:xg/QME4nWcxGxrpdeYfq7UvYrLh66cuVKdrbD1XF/NI=
 go.uber.org/zap v1.24.0 h1:FiJd5l1UOLj0wCgbSE0rwwXHzEdAZS6hiiSnxJN/D60=
 go.uber.org/zap v1.24.0/go.mod h1:2kMP+WWQ8aoFoedH3T2sq6iJ2yDWpHbP0f6MQbS9Gkg=
+golang.org/x/arch v0.1.0/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
@@ -374,9 +382,10 @@ golang.org/x/crypto v0.0.0-20211215153901-e495a2d5b3d3/go.mod h1:IxCIyHEi3zRg3s0
 golang.org/x/crypto v0.0.0-20220525230936-793ad666bf5e/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
 golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
 golang.org/x/crypto v0.0.0-20220826181053-bd7e27e6170d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
+golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw=
 golang.org/x/crypto v0.3.0/go.mod h1:hebNnKkNXi2UzZN1eVRvBB7co0a+JxK6XbPiWVs/3J4=
-golang.org/x/crypto v0.6.0 h1:qfktjS5LUO+fFKeJXZ+ikTRijMmljikvG68fpMMruSc=
-golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58=
+golang.org/x/crypto v0.7.0 h1:AvwMYaRytfdeVt3u6mLaxYtErKYjxA2OXjJ1HHq6t3A=
+golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20221018205818-5c77f4b2bbd7 h1:lhh9BuEmjIBOdOhMIGSQzdW5LbTtFbUV1k9O/Rlq3SE=
 golang.org/x/exp v0.0.0-20221018205818-5c77f4b2bbd7/go.mod h1:cyybsKvd6eL0RnXn6p/Grxp8F5bW7iYuBgsNCOHpMYE=
@@ -388,8 +397,9 @@ golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
-golang.org/x/mod v0.7.0 h1:LapD9S96VoQRhi/GrNTqeBJFrUjs5UHCAtTlgwA5oZA=
-golang.org/x/mod v0.7.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
+golang.org/x/mod v0.6.0/go.mod h1:4mET923SAdbXp2ki8ey+zGs1SLqsuM2Y0uvdZR/fUNI=
+golang.org/x/mod v0.8.0 h1:LUYupSeNrTNCGzR/hVBk2NHZO4hXcVaW1k4Qx7rjPx8=
+golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
 golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@@ -408,12 +418,13 @@ golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qx
 golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
 golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
 golang.org/x/net v0.0.0-20220826154423-83b083e8dc8b/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk=
+golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco=
 golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY=
-golang.org/x/net v0.6.0 h1:L4ZwwTvKW9gr0ZMS1yrHD9GZhIuVjOBBnaKH+SPQK0Q=
-golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
+golang.org/x/net v0.8.0 h1:Zrh2ngAOFYneWTAIAPethzeaQLuHwhuBkuV6ZiRnUaQ=
+golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
-golang.org/x/oauth2 v0.4.0 h1:NF0gk8LVPg1Ml7SSbGyySuoxdsXitj7TvgvuRxIMc/M=
-golang.org/x/oauth2 v0.4.0/go.mod h1:RznEsdpjGAINPTOF0UH/t+xJ75L18YO3Ho6Pyn+uRec=
+golang.org/x/oauth2 v0.6.0 h1:Lh8GPgSKBfWSwFvtuWOfeI3aAAnbXTSutYxJiOJFgIw=
+golang.org/x/oauth2 v0.6.0/go.mod h1:ycmewcwgD4Rpr3eZJLSB4Kyyljb3qDh40vJ8STE5HKw=
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@@ -431,8 +442,6 @@ golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -445,7 +454,6 @@ golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
@@ -454,23 +462,25 @@ golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220825204002-c680a09ffe64/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.5.0 h1:MUK/U/4lj1t1oPg0HfuXDN/Z1wv31ZJ/YcPiGccS4DU=
-golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.6.0 h1:MVltZSvRTcU2ljQOhs94SXPftV6DCNnZViHeQps87pQ=
+golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/term v0.0.0-20220722155259-a9ba230a4035/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
+golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc=
-golang.org/x/term v0.5.0 h1:n2a8QNdAb0sZNpU9R1ALUXBbY+w51fCQDN+7EdxNBsY=
+golang.org/x/term v0.6.0 h1:clScbb1cHjoCkyRbWwBEUZ5H/tIFu5TAXIqaZD0Gcjw=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
 golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
-golang.org/x/text v0.7.0 h1:4BRB4x83lYWy72KwLD/qYDuTu7q9PjSagHvijDw7cLo=
-golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
+golang.org/x/text v0.8.0 h1:57P1ETyNKtuIjB4SRd15iJxuhj8Gc416Y78H3qgMh68=
+golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
 golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4=
 golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
@@ -484,14 +494,17 @@ golang.org/x/tools v0.0.0-20201211185031-d93e913c1a58/go.mod h1:emZCQorbCU4vsT4f
 golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
 golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
 golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
-golang.org/x/tools v0.3.0 h1:SrNbZl6ECOS1qFzgTdQfWXZM9XBkiA6tkFrH9YSTPHM=
-golang.org/x/tools v0.3.0/go.mod h1:/rWhSS2+zyEVwoJf8YAX6L2f0ntZ7Kn/mGgAWcipA5k=
+golang.org/x/tools v0.2.0/go.mod h1:y4OqIKeOV/fWJetJ8bXPU1sEVniLMIyDAZWeHdV+NTA=
+golang.org/x/tools v0.6.0 h1:BOw41kyTf3PuCW1pVQf8+Cyg8pMlkYB1oo9iJ6D/lKM=
+golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
-google.golang.org/api v0.103.0 h1:9yuVqlu2JCvcLg9p8S3fcFLZij8EPSyvODIY1rkMizQ=
-google.golang.org/api v0.103.0/go.mod h1:hGtW6nK1AC+d9si/UBhw8Xli+QMOf6xyNAyJw4qU9w0=
+golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 h1:H2TDz8ibqkAF6YGhCdN3jS9O0/s90v0rJh3X/OLHEUk=
+golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8=
+google.golang.org/api v0.111.0 h1:bwKi+z2BsdwYFRKrqwutM+axAlYLz83gt5pDSXCJT+0=
+google.golang.org/api v0.111.0/go.mod h1:qtFHvU9mhgTJegR31csQ+rwxyUTHOKFqCKWp1J0fdw0=
 google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
 google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
 google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c=
@@ -499,15 +512,15 @@ google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCID
 google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
 google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
 google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=
-google.golang.org/genproto v0.0.0-20221201164419-0e50fba7f41c h1:S34D59DS2GWOEwWNt4fYmTcFrtlOgukG2k9WsomZ7tg=
-google.golang.org/genproto v0.0.0-20221201164419-0e50fba7f41c/go.mod h1:rZS5c/ZVYMaOGBfO68GWtjOw/eLaZM1X6iVtgjZ+EWg=
+google.golang.org/genproto v0.0.0-20230303212802-e74f57abe488 h1:QQF+HdiI4iocoxUjjpLgvTYDHKm99C/VtTBFnfiCJos=
+google.golang.org/genproto v0.0.0-20230303212802-e74f57abe488/go.mod h1:TvhZT5f700eVlTNwND1xoEZQeWTB2RY/65kplwl/bFA=
 google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
 google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
 google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY=
 google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
 google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc=
-google.golang.org/grpc v1.51.0 h1:E1eGv1FTqoLIdnBCZufiSHgKjlqG6fKFf6pPWtMTh8U=
-google.golang.org/grpc v1.51.0/go.mod h1:wgNDFcnuBGmxLKI/qn4T+m5BtEBYXJPvibbUPsAIPww=
+google.golang.org/grpc v1.53.0 h1:LAv2ds7cmFV/XTS3XG1NneeENYrXGmorPxsBbptIjNc=
+google.golang.org/grpc v1.53.0/go.mod h1:OnIrk0ipVdj4N5d9IUoFUx72/VlD7+jUsHwZgwSMQpw=
 google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
 google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
 google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
@@ -519,8 +532,8 @@ google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpAD
 google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
 google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
 google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
-google.golang.org/protobuf v1.28.1 h1:d0NfwRgPtno5B1Wa6L2DAG+KivqkdutMf1UhdNx175w=
-google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
+google.golang.org/protobuf v1.29.1 h1:7QBf+IK2gx70Ap/hDsOmam3GE0v9HicjfEdAxE62UoM=
+google.golang.org/protobuf v1.29.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
 gopkg.in/alecthomas/kingpin.v2 v2.2.6 h1:jMFz6MfLP0/4fUyZle81rXUoxOBFi19VUFKVDOQfozc=
 gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
@@ -549,5 +562,6 @@ gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4=
 sigs.k8s.io/yaml v1.3.0 h1:a2VclLzOGrwOHDiV8EfBGhvjHvP46CtW5j6POvhYGGo=
 sigs.k8s.io/yaml v1.3.0/go.mod h1:GeOyir5tyXNByN85N/dRIT9es5UQNerPYEKK56eTBm8=

hack/generate/generate.go

@@ -9,6 +9,8 @@ import (
 	"text/template"
 
 	"github.com/go-errors/errors"
+	"golang.org/x/text/cases"
+	"golang.org/x/text/language"
 	"gopkg.in/alecthomas/kingpin.v2"
 )
 
@@ -24,7 +26,7 @@ func main() {
 	log.SetPrefix("😲 [generate] ")
 
 	kingpin.MustParse(app.Parse(os.Args[1:]))
-	nameTitle = strings.Title(*name)
+	nameTitle = cases.Title(language.AmericanEnglish).String(*name)
 	nameLower = strings.ToLower(*name)
 	nameUpper = strings.ToUpper(*name)
 
@@ -65,7 +67,7 @@ type templateJob struct {
 }
 
 func mustWriteTemplates(jobs []templateJob) {
-	log.Printf("Generating %s %s\n", strings.Title(*kind), nameTitle)
+	log.Printf("Generating %s %s\n", cases.Title(language.AmericanEnglish).String(*kind), nameTitle)
 
 	// Make the folder.
 	log.Printf("Creating folder %s\n", folderPath())
@@ -84,7 +86,7 @@ func mustWriteTemplates(jobs []templateJob) {
 
 		for _, rplString := range job.ReplaceString {
 			tmplRaw = strings.ReplaceAll(tmplRaw, strings.ToLower(rplString), "<<.NameLower>>")
-			tmplRaw = strings.ReplaceAll(tmplRaw, strings.Title(rplString), "<<.NameTitle>>")
+			tmplRaw = strings.ReplaceAll(tmplRaw, cases.Title(language.AmericanEnglish).String(rplString), "<<.NameTitle>>")
 			tmplRaw = strings.ReplaceAll(tmplRaw, strings.ToUpper(rplString), "<<.NameUpper>>")
 		}
 

hack/snifftest/main.go

@@ -173,7 +173,9 @@ func main() {
 		}
 
 		for _, repo := range strings.Split(*scanCmdRepo, ",") {
-			sem.Acquire(ctx, 1)
+			if err := sem.Acquire(ctx, 1); err != nil {
+				logFatal(err, "timed out waiting for semaphore")
+			}
 			wgChunkers.Add(1)
 			go func(r string) {
 				defer sem.Release(1)

main.go

@@ -21,10 +21,12 @@ import (
 	"github.com/trufflesecurity/trufflehog/v3/pkg/config"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/context"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/decoders"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/engine"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/handlers"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/log"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/output"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/sources/git"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/updater"
@@ -36,6 +38,7 @@ var (
 	cmd              string
 	debug            = cli.Flag("debug", "Run in debug mode.").Bool()
 	trace            = cli.Flag("trace", "Run in trace mode.").Bool()
+	profile          = cli.Flag("profile", "Enables profiling and sets a pprof and fgprof server on :18066.").Bool()
 	jsonOut          = cli.Flag("json", "Output in JSON format.").Short('j').Bool()
 	jsonLegacy       = cli.Flag("json-legacy", "Use the pre-v3.0 JSON format. Only works with git, gitlab, and github sources.").Bool()
 	concurrency      = cli.Flag("concurrency", "Number of concurrent workers.").Default(strconv.Itoa(runtime.NumCPU())).Int()
@@ -50,6 +53,8 @@ var (
 	archiveMaxSize       = cli.Flag("archive-max-size", "Maximum size of archive to scan. (Byte units eg. 512B, 2KB, 4MB)").Bytes()
 	archiveMaxDepth      = cli.Flag("archive-max-depth", "Maximum depth of archive to scan.").Int()
 	archiveTimeout       = cli.Flag("archive-timeout", "Maximum time to spend extracting an archive.").Duration()
+	includeDetectors     = cli.Flag("include-detectors", "Comma separated list of detector types to include. Protobuf name or IDs may be used, as well as ranges.").Default("all").String()
+	excludeDetectors     = cli.Flag("exclude-detectors", "Comma separated list of detector types to exclude. Protobuf name or IDs may be used, as well as ranges. IDs defined here take precedence over the include list.").String()
 
 	gitScan             = cli.Command("git", "Find credentials in git repositories.")
 	gitScanURI          = gitScan.Arg("uri", "Git repository URL. https://, file://, or ssh:// schema expected.").Required().String()
@@ -82,8 +87,10 @@ var (
 	gitlabScanIncludePaths = gitlabScan.Flag("include-paths", "Path to file with newline separated regexes for files to include in scan.").Short('i').String()
 	gitlabScanExcludePaths = gitlabScan.Flag("exclude-paths", "Path to file with newline separated regexes for files to exclude in scan.").Short('x').String()
 
-	filesystemScan        = cli.Command("filesystem", "Find credentials in a filesystem.")
-	filesystemDirectories = filesystemScan.Flag("directory", "Path to directory to scan. You can repeat this flag.").Required().Strings()
+	filesystemScan  = cli.Command("filesystem", "Find credentials in a filesystem.")
+	filesystemPaths = filesystemScan.Arg("path", "Path to file or directory to scan.").Strings()
+	// DEPRECATED: --directory is deprecated in favor of arguments.
+	filesystemDirectories = filesystemScan.Flag("directory", "Path to directory to scan. You can repeat this flag.").Strings()
 	// TODO: Add more filesystem scan options. Currently only supports scanning a list of directories.
 	// filesystemScanRecursive = filesystemScan.Flag("recursive", "Scan recursively.").Short('r').Bool()
 	filesystemScanIncludePaths = filesystemScan.Flag("include-paths", "Path to file with newline separated regexes for files to include in scan.").Short('i').String()
@@ -95,6 +102,18 @@ var (
 	s3ScanCloudEnv = s3Scan.Flag("cloud-environment", "Use IAM credentials in cloud environment.").Bool()
 	s3ScanBuckets  = s3Scan.Flag("bucket", "Name of S3 bucket to scan. You can repeat this flag.").Strings()
 
+	gcsScan           = cli.Command("gcs", "Find credentials in GCS buckets.")
+	gcsProjectID      = gcsScan.Flag("project-id", "GCS project ID used to authenticate. Can NOT be used with unauth scan. Can be provided with environment variable GOOGLE_CLOUD_PROJECT.").Envar("GOOGLE_CLOUD_PROJECT").String()
+	gcsCloudEnv       = gcsScan.Flag("cloud-environment", "Use Application Default Credentials, IAM credentials to authenticate.").Bool()
+	gcsServiceAccount = gcsScan.Flag("service-account", "Path to GCS service account JSON file.").ExistingFile()
+	gcsWithoutAuth    = gcsScan.Flag("without-auth", "Scan GCS buckets without authentication. This will only work for public buckets").Bool()
+	gcsAPIKey         = gcsScan.Flag("api-key", "GCS API key used to authenticate. Can be provided with environment variable GOOGLE_API_KEY.").Envar("GOOGLE_API_KEY").String()
+	gcsIncludeBuckets = gcsScan.Flag("include-buckets", "Buckets to scan. Comma seperated list of buckets. You can repeat this flag. Globs are supported").Short('I').Strings()
+	gcsExcludeBuckets = gcsScan.Flag("exclude-buckets", "Buckets to exclude from scan. Comma separated list of buckets. Globs are supported").Short('X').Strings()
+	gcsIncludeObjects = gcsScan.Flag("include-objects", "Objects to scan. Comma separated list of objects. you can repeat this flag. Globs are supported").Short('i').Strings()
+	gcsExcludeObjects = gcsScan.Flag("exclude-objects", "Objects to exclude from scan. Comma separated list of objects. You can repeat this flag. Globs are supported").Short('x').Strings()
+	gcsMaxObjectSize  = gcsScan.Flag("max-object-size", "Maximum size of objects to scan. Objects larger than this will be skipped. (Byte units eg. 512B, 2KB, 4MB)").Default("10MB").Bytes()
+
 	syslogScan     = cli.Command("syslog", "Scan syslog")
 	syslogAddress  = syslogScan.Flag("address", "Address and port to listen on for syslog. Example: 127.0.0.1:514").String()
 	syslogProtocol = syslogScan.Flag("protocol", "Protocol to listen on. udp or tcp").String()
@@ -177,7 +196,7 @@ func run(state overseer.State) {
 		*concurrency = 1
 	}
 
-	if *debug {
+	if *profile {
 		go func() {
 			router := mux.NewRouter()
 			router.PathPrefix("/debug/pprof").Handler(http.DefaultServeMux)
@@ -208,11 +227,62 @@ func run(state overseer.State) {
 		handlers.SetArchiveMaxTimeout(*archiveTimeout)
 	}
 
+	// Build include and exclude detector filter sets.
+	var includeDetectorTypes, excludeDetectorTypes map[detectorspb.DetectorType]config.DetectorID
+	{
+		includeList, err := config.ParseDetectors(*includeDetectors)
+		if err != nil {
+			// Exit if there was an error to inform the user of the misconfiguration.
+			logFatal(err, "invalid include list detector configuration")
+		}
+		excludeList, err := config.ParseDetectors(*excludeDetectors)
+		if err != nil {
+			// Exit if there was an error to inform the user of the misconfiguration.
+			logFatal(err, "invalid exclude list detector configuration")
+		}
+		includeDetectorTypes = detectorTypeToMap(includeList)
+		excludeDetectorTypes = detectorTypeToMap(excludeList)
+	}
+	includeFilter := func(d detectors.Detector) bool {
+		id, ok := includeDetectorTypes[d.Type()]
+		if id.Version == 0 {
+			return ok
+		}
+		versionD, ok := d.(detectors.Versioner)
+		if !ok {
+			// Error: version provided but not a detectors.Versioner
+			logFatal(
+				fmt.Errorf("version provided but detector does not have a version"),
+				"invalid include list detector configuration",
+				"detector", id,
+			)
+		}
+		return versionD.Version() == id.Version
+	}
+	excludeFilter := func(d detectors.Detector) bool {
+		id, ok := excludeDetectorTypes[d.Type()]
+		if id.Version == 0 {
+			return !ok
+		}
+		versionD, ok := d.(detectors.Versioner)
+		if !ok {
+			// Error: version provided but not a detectors.Versioner
+			logFatal(
+				fmt.Errorf("version provided but detector does not have a version"),
+				"invalid exclude list detector configuration",
+				"detector", id,
+			)
+		}
+		return versionD.Version() != id.Version
+	}
+
 	e := engine.Start(ctx,
 		engine.WithConcurrency(*concurrency),
 		engine.WithDecoders(decoders.DefaultDecoders()...),
 		engine.WithDetectors(!*noVerification, engine.DefaultDetectors()...),
 		engine.WithDetectors(!*noVerification, conf.Detectors...),
+		engine.WithFilterDetectors(includeFilter),
+		engine.WithFilterDetectors(excludeFilter),
 		engine.WithFilterUnverified(*filterUnverified),
 	)
 
@@ -286,10 +356,15 @@ func run(state overseer.State) {
 		if err != nil {
 			logFatal(err, "could not create filter")
 		}
-
+		if len(*filesystemDirectories) > 0 {
+			ctx.Logger().Info("--directory flag is deprecated, please pass directories as arguments")
+		}
+		paths := make([]string, 0, len(*filesystemPaths)+len(*filesystemDirectories))
+		paths = append(paths, *filesystemPaths...)
+		paths = append(paths, *filesystemDirectories...)
 		cfg := sources.FilesystemConfig{
-			Directories: *filesystemDirectories,
-			Filter:      filter,
+			Paths:  paths,
+			Filter: filter,
 		}
 		if err = e.ScanFileSystem(ctx, cfg); err != nil {
 			logFatal(err, "Failed to scan filesystem")
@@ -320,6 +395,23 @@ func run(state overseer.State) {
 		if err := e.ScanCircleCI(ctx, *circleCiScanToken); err != nil {
 			logFatal(err, "Failed to scan CircleCI.")
 		}
+	case gcsScan.FullCommand():
+		cfg := sources.GCSConfig{
+			ProjectID:      *gcsProjectID,
+			CloudCred:      *gcsCloudEnv,
+			ServiceAccount: *gcsServiceAccount,
+			WithoutAuth:    *gcsWithoutAuth,
+			ApiKey:         *gcsAPIKey,
+			IncludeBuckets: commaSeperatedToSlice(*gcsIncludeBuckets),
+			ExcludeBuckets: commaSeperatedToSlice(*gcsExcludeBuckets),
+			IncludeObjects: commaSeperatedToSlice(*gcsIncludeObjects),
+			ExcludeObjects: commaSeperatedToSlice(*gcsExcludeObjects),
+			Concurrency:    *concurrency,
+			MaxObjectSize:  int64(*gcsMaxObjectSize),
+		}
+		if err := e.ScanGCS(ctx, cfg); err != nil {
+			logFatal(err, "Failed to scan GCS.")
+		}
 	}
 	// asynchronously wait for scanning to finish and cleanup
 	go e.Finish(ctx)
@@ -365,6 +457,20 @@ func run(state overseer.State) {
 	}
 }
 
+func commaSeperatedToSlice(s []string) []string {
+	var result []string
+	for _, items := range s {
+		for _, item := range strings.Split(items, ",") {
+			item = strings.TrimSpace(item)
+			if item == "" {
+				continue
+			}
+			result = append(result, item)
+		}
+	}
+	return result
+}
+
 func printAverageDetectorTime(e *engine.Engine) {
 	fmt.Fprintln(os.Stderr, "Average detector time is the measurement of average time spent on each detector when results are returned.")
 	for detectorName, durations := range e.DetectorAvgTime() {
@@ -389,3 +495,11 @@ func logFatalFunc(logger logr.Logger) func(error, string, ...any) {
 		os.Exit(0)
 	}
 }
+
+func detectorTypeToMap(detectors []config.DetectorID) map[detectorspb.DetectorType]config.DetectorID {
+	output := make(map[detectorspb.DetectorType]config.DetectorID, len(detectors))
+	for _, d := range detectors {
+		output[d.ID] = d
+	}
+	return output
+}

pkg/common/secrets.go

@@ -7,9 +7,9 @@ import (
 	"time"
 
 	secretmanager "cloud.google.com/go/secretmanager/apiv1"
+	"cloud.google.com/go/secretmanager/apiv1/secretmanagerpb"
 	"github.com/joho/godotenv"
 	"github.com/pkg/errors"
-	secretmanagerpb "google.golang.org/genproto/googleapis/cloud/secretmanager/v1"
 )
 
 type Secret struct{ kv map[string]string }

pkg/common/vars.go

@@ -2,16 +2,17 @@ package common
 
 import (
 	"path/filepath"
+	"strings"
 )
 
 var (
 	KB, MB, GB, TB, PB = 1e3, 1e6, 1e9, 1e12, 1e15
-	IGNORED_EXTENSIONS = []string{"mp4", "avi", "mpeg", "mpg", "mov", "wmv", "m4p", "swf", "mp2", "flv", "vob", "webm", "hdv", "3gp", "ogg", "mp3", "wav", "flac", "webp"}
+	IgnoredExtensions  = []string{"mp4", "avi", "mpeg", "mpg", "mov", "wmv", "m4p", "swf", "mp2", "flv", "vob", "webm", "hdv", "3gp", "ogg", "mp3", "wav", "flac", "webp"}
 )
 
 func SkipFile(filename string) bool {
-	for _, ext := range IGNORED_EXTENSIONS {
-		if filepath.Ext(filename) == ext {
+	for _, ext := range IgnoredExtensions {
+		if strings.TrimPrefix(filepath.Ext(filename), ".") == ext {
 			return true
 		}
 	}

pkg/common/vars_test.go

@@ -0,0 +1,36 @@
+package common
+
+import "testing"
+
+func TestSkipFile(t *testing.T) {
+	type testCase struct {
+		file string
+		want bool
+	}
+
+	// Add a test case for each ignored extension.
+	testCases := make([]testCase, 0, len(IgnoredExtensions)+1)
+	for _, ext := range IgnoredExtensions {
+		testCases = append(testCases, testCase{
+			file: "test." + ext,
+			want: true,
+		})
+	}
+
+	// Add a test case for a file that should not be skipped.
+	testCases = append(testCases, testCase{file: "test.txt", want: false})
+
+	for _, tt := range testCases {
+		t.Run(tt.file, func(t *testing.T) {
+			if got := SkipFile(tt.file); got != tt.want {
+				t.Errorf("SkipFile(%v) got %v, want %v", tt.file, got, tt.want)
+			}
+		})
+	}
+}
+
+func BenchmarkSkipFile(b *testing.B) {
+	for i := 0; i < b.N; i++ {
+		SkipFile("test.mp4")
+	}
+}

pkg/config/detectors.go

@@ -0,0 +1,183 @@
+package config
+
+import (
+	"fmt"
+	"sort"
+	"strconv"
+	"strings"
+
+	dpb "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+var (
+	specialGroups = map[string][]DetectorID{
+		"all": allDetectors(),
+	}
+
+	detectorTypeValue = make(map[string]dpb.DetectorType, len(dpb.DetectorType_value))
+	validDetectors    = make(map[dpb.DetectorType]struct{}, len(dpb.DetectorType_value))
+	maxDetectorType   dpb.DetectorType
+)
+
+// Setup package local global variables.
+func init() {
+	for k, v := range dpb.DetectorType_value {
+		dt := dpb.DetectorType(v)
+		detectorTypeValue[strings.ToLower(k)] = dt
+		validDetectors[dt] = struct{}{}
+		if dt > maxDetectorType {
+			maxDetectorType = dt
+		}
+	}
+}
+
+type DetectorID struct {
+	ID      dpb.DetectorType
+	Version int
+}
+
+// ParseDetectors parses user supplied string into a list of detectors types.
+// "all" will return the list of all available detectors. The input is comma
+// separated and may use the case-insensitive detector name defined in the
+// protobuf, or the protobuf enum number. A range may be used as well in the
+// form "start-end". Order is preserved and duplicates are ignored.
+func ParseDetectors(input string) ([]DetectorID, error) {
+	var output []DetectorID
+	seenDetector := map[DetectorID]struct{}{}
+	for _, item := range strings.Split(input, ",") {
+		item = strings.TrimSpace(item)
+		if item == "" {
+			continue
+		}
+		detectors, ok := specialGroups[strings.ToLower(item)]
+		if !ok {
+			var err error
+			detectors, err = asRange(item)
+			if err != nil {
+				return nil, err
+			}
+		}
+		for _, d := range detectors {
+			if _, ok := seenDetector[d]; ok {
+				continue
+			}
+			seenDetector[d] = struct{}{}
+			output = append(output, d)
+		}
+	}
+	return output, nil
+}
+
+func (id DetectorID) String() string {
+	name := dpb.DetectorType_name[int32(id.ID)]
+	if name == "" {
+		name = "<invalid ID>"
+	}
+	if id.Version == 0 {
+		return name
+	}
+	return fmt.Sprintf("%s.v%d", name, id.Version)
+}
+
+// allDetectors reutrns an ordered slice of all detector types.
+func allDetectors() []DetectorID {
+	all := make([]DetectorID, 0, len(dpb.DetectorType_name))
+	for id := range dpb.DetectorType_name {
+		all = append(all, DetectorID{ID: dpb.DetectorType(id)})
+	}
+	sort.Slice(all, func(i, j int) bool { return all[i].ID < all[j].ID })
+	return all
+}
+
+// asRange converts a single input into a slice of detector types. If the input
+// is not in range format, a slice of length 1 is returned. Unbounded ranges
+// are allowed.
+func asRange(input string) ([]DetectorID, error) {
+	// Check if it's a single detector type.
+	dt, err := asDetectorID(input)
+	if err == nil {
+		return []DetectorID{dt}, nil
+	}
+
+	// Check if it's a range; if not return the error from above.
+	start, end, found := strings.Cut(input, "-")
+	if !found {
+		return nil, err
+	}
+	start, end = strings.TrimSpace(start), strings.TrimSpace(end)
+
+	// Convert the range start and end to a DetectorType.
+	dtStart, err := asDetectorID(start)
+	if err != nil {
+		return nil, err
+	}
+	dtEnd, err := asDetectorID(end)
+	// If end is empty it's an unbounded range.
+	if err != nil && end != "" {
+		return nil, err
+	}
+	if end == "" {
+		dtEnd.ID = maxDetectorType
+	}
+
+	// Ensure these ranges don't have versions.
+	if dtEnd.Version != 0 || dtStart.Version != 0 {
+		return nil, fmt.Errorf("versions within ranges are not supported: %s", input)
+	}
+
+	step := dpb.DetectorType(1)
+	if dtStart.ID > dtEnd.ID {
+		step = -1
+	}
+	var output []DetectorID
+	for dt := dtStart.ID; dt != dtEnd.ID; dt += step {
+		if _, ok := validDetectors[dt]; !ok {
+			continue
+		}
+		output = append(output, DetectorID{ID: dt})
+	}
+	return append(output, dtEnd), nil
+}
+
+// asDetectorID converts the case-insensitive input into a DetectorID.
+// Name or ID may be used.
+func asDetectorID(input string) (DetectorID, error) {
+	if input == "" {
+		return DetectorID{}, fmt.Errorf("empty detector")
+	}
+	var detectorID DetectorID
+	// Separate the version if there is one.
+	if detector, version, hasVersion := strings.Cut(input, "."); hasVersion {
+		parsedVersion, err := parseVersion(version)
+		if err != nil {
+			return DetectorID{}, fmt.Errorf("invalid version for input: %q error: %w", input, err)
+		}
+		detectorID.Version = parsedVersion
+		// Because there was a version, the detector type input is the part before the '.'
+		input = detector
+	}
+
+	// Check if it's a named detector.
+	if dt, ok := detectorTypeValue[strings.ToLower(input)]; ok {
+		detectorID.ID = dt
+		return detectorID, nil
+	}
+	// Check if it's a detector ID.
+	if i, err := strconv.ParseInt(input, 10, 32); err == nil {
+		dt := dpb.DetectorType(i)
+		if _, ok := validDetectors[dt]; !ok {
+			return DetectorID{}, fmt.Errorf("invalid detector ID: %s", input)
+		}
+		detectorID.ID = dt
+		return detectorID, nil
+	}
+	return DetectorID{}, fmt.Errorf("unrecognized detector type: %s", input)
+}
+
+func parseVersion(v string) (int, error) {
+	if !strings.HasPrefix(strings.ToLower(v), "v") {
+		return 0, fmt.Errorf("version must start with 'v'")
+	}
+	version := strings.TrimLeft(v, "vV")
+	return strconv.Atoi(version)
+}

pkg/config/detectors_test.go

@@ -0,0 +1,47 @@
+package config
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	dpb "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestDetectorParsing(t *testing.T) {
+	tests := map[string]struct {
+		input    string
+		expected []DetectorID
+	}{
+		"all":                       {"AlL", allDetectors()},
+		"trailing range":            {"0-", allDetectors()},
+		"all after 1":               {"1-", allDetectors()[1:]},
+		"named and valid range":     {"aWs,8-9", []DetectorID{{ID: dpb.DetectorType_AWS}, {ID: dpb.DetectorType_Github}, {ID: dpb.DetectorType_Gitlab}}},
+		"duplicate order preserved": {"9, 8, 9", []DetectorID{{ID: 9}, {ID: 8}}},
+		"named range":               {"github - gitlab", []DetectorID{{ID: dpb.DetectorType_Github}, {ID: dpb.DetectorType_Gitlab}}},
+		"range preserved":           {"8-9, 7-10", []DetectorID{{ID: 8}, {ID: 9}, {ID: 7}, {ID: 10}}},
+		"reverse range":             {"9-8", []DetectorID{{ID: 9}, {ID: 8}}},
+		"range preserved with all":  {"10-,all", append(allDetectors()[10:], allDetectors()[:10]...)},
+		"empty list item":           {"8, ,9", []DetectorID{{ID: 8}, {ID: 9}}},
+		"invalid end range":         {"0-1337", nil},
+		"invalid name":              {"foo", nil},
+		"negative":                  {"-1", nil},
+		"github.v1":                 {"github.v1", []DetectorID{{ID: dpb.DetectorType_Github, Version: 1}}},
+		"gitlab.v100":               {"gitlab.v100", []DetectorID{{ID: dpb.DetectorType_Gitlab, Version: 100}}},
+		"range with versions":       {"github.v2 - gitlab.v1", nil},
+		"invalid version no v":      {"gitlab.2", nil},
+		"invalid version no number": {"gitlab.github", nil},
+		"capital V is fine":         {"GiTlAb.V2", []DetectorID{{ID: dpb.DetectorType_Gitlab, Version: 2}}},
+		"id number with version":    {"8.v2", []DetectorID{{ID: 8, Version: 2}}},
+	}
+
+	for name, tt := range tests {
+		t.Run(name, func(t *testing.T) {
+			got, gotErr := ParseDetectors(tt.input)
+			if tt.expected == nil {
+				assert.Error(t, gotErr)
+				return
+			}
+			assert.Equal(t, tt.expected, got)
+		})
+	}
+}

pkg/custom_detectors/custom_detectors.go

@@ -12,6 +12,7 @@ import (
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/custom_detectorspb"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+	"golang.org/x/sync/errgroup"
 )
 
 // The maximum number of matches from one chunk. This const is used when
@@ -82,68 +83,98 @@ func (c *customRegexWebhook) FromData(ctx context.Context, verify bool, data []b
 	// ]
 	matches := permutateMatches(regexMatches)
 
-	// Create result object and test for verification.
-	for _, match := range matches {
-		if common.IsDone(ctx) {
-			// TODO: Log we're possibly leaving out results.
-			return results, nil
-		}
-		var raw string
-		for _, values := range match {
-			// values[0] contains the entire regex match.
-			raw += values[0]
-		}
-		result := detectors.Result{
-			DetectorType: detectorspb.DetectorType_CustomRegex,
-			Raw:          []byte(raw),
-		}
+	g := new(errgroup.Group)
 
-		if !verify {
-			results = append(results, result)
-			continue
-		}
-		// Verify via webhook.
-		jsonBody, err := json.Marshal(map[string]map[string][]string{
-			c.GetName(): match,
+	// Create result object and test for verification.
+	resultsCh := make(chan detectors.Result, maxTotalMatches)
+	for _, match := range matches {
+		match := match
+		g.Go(func() error {
+			return c.createResults(ctx, match, verify, resultsCh)
 		})
-		if err != nil {
-			continue
-		}
-		// Try each config until we successfully verify.
-		for _, verifyConfig := range c.GetVerify() {
-			if common.IsDone(ctx) {
-				// TODO: Log we're possibly leaving out results.
-				return results, nil
-			}
-			req, err := http.NewRequestWithContext(ctx, "POST", verifyConfig.GetEndpoint(), bytes.NewReader(jsonBody))
-			if err != nil {
-				continue
-			}
-			for _, header := range verifyConfig.GetHeaders() {
-				key, value, found := strings.Cut(header, ":")
-				if !found {
-					// Should be unreachable due to validation.
-					continue
-				}
-				req.Header.Add(key, strings.TrimLeft(value, "\t\n\v\f\r "))
-			}
-			res, err := httpClient.Do(req)
-			if err != nil {
-				continue
-			}
-			// TODO: Read response body.
-			res.Body.Close()
-			if res.StatusCode == http.StatusOK {
-				result.Verified = true
-				break
-			}
-		}
+	}
+
+	// Ignore any errors and collect as many of the results as we can.
+	_ = g.Wait()
+	close(resultsCh)
+
+	for result := range resultsCh {
 		results = append(results, result)
 	}
 
 	return results, nil
 }
 
+func (c *customRegexWebhook) createResults(ctx context.Context, match map[string][]string, verify bool, results chan<- detectors.Result) error {
+	if common.IsDone(ctx) {
+		// TODO: Log we're possibly leaving out results.
+		return ctx.Err()
+	}
+	var raw string
+	for _, values := range match {
+		// values[0] contains the entire regex match.
+		raw += values[0]
+	}
+	result := detectors.Result{
+		DetectorType: detectorspb.DetectorType_CustomRegex,
+		Raw:          []byte(raw),
+	}
+
+	if !verify {
+		select {
+		case <-ctx.Done():
+			return ctx.Err()
+		case results <- result:
+			return nil
+		}
+	}
+	// Verify via webhook.
+	jsonBody, err := json.Marshal(map[string]map[string][]string{
+		c.GetName(): match,
+	})
+	if err != nil {
+		// This should never happen, but if it does, return nil to not
+		// disrupt other verification.
+		return nil
+	}
+	// Try each config until we successfully verify.
+	for _, verifyConfig := range c.GetVerify() {
+		if common.IsDone(ctx) {
+			// TODO: Log we're possibly leaving out results.
+			return ctx.Err()
+		}
+		req, err := http.NewRequestWithContext(ctx, "POST", verifyConfig.GetEndpoint(), bytes.NewReader(jsonBody))
+		if err != nil {
+			continue
+		}
+		for _, header := range verifyConfig.GetHeaders() {
+			key, value, found := strings.Cut(header, ":")
+			if !found {
+				// Should be unreachable due to validation.
+				continue
+			}
+			req.Header.Add(key, strings.TrimLeft(value, "\t\n\v\f\r "))
+		}
+		res, err := httpClient.Do(req)
+		if err != nil {
+			continue
+		}
+		// TODO: Read response body.
+		res.Body.Close()
+		if res.StatusCode == http.StatusOK {
+			result.Verified = true
+			break
+		}
+	}
+
+	select {
+	case <-ctx.Done():
+		return ctx.Err()
+	case results <- result:
+		return nil
+	}
+}
+
 func (c *customRegexWebhook) Keywords() []string {
 	return c.GetKeywords()
 }

pkg/detectors/abbysale/abbysale.go

@@ -2,7 +2,6 @@ package abbysale
 
 import (
 	"context"
-	"fmt"
 	"net/http"
 	"regexp"
 	"strings"
@@ -52,7 +51,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 			if err != nil {
 				continue
 			}
-			req.Header.Add("x-api-key", fmt.Sprintf("%s", resMatch))
+			req.Header.Add("x-api-key", resMatch)
 			res, err := client.Do(req)
 			if err == nil {
 				defer res.Body.Close()

pkg/detectors/alchemy/alchemy.go

@@ -0,0 +1,76 @@
+package alchemy
+
+import (
+	"context"
+	"net/http"
+	"regexp"
+	"strings"
+
+	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+type Scanner struct{}
+
+// Ensure the Scanner satisfies the interface at compile time.
+var _ detectors.Detector = (*Scanner)(nil)
+
+var (
+	client = common.SaneHttpClient()
+
+	// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
+	keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"alchemy"}) + `\b([0-9a-zA-Z]{23}_[0-9a-zA-Z]{8})\b`)
+)
+
+// Keywords are used for efficiently pre-filtering chunks.
+// Use identifiers in the secret preferably, or the provider name.
+func (s Scanner) Keywords() []string {
+	return []string{"alchemy"}
+}
+
+// FromData will find and optionally verify Alchemy secrets in a given set of bytes.
+func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
+	dataStr := string(data)
+
+	matches := keyPat.FindAllStringSubmatch(dataStr, -1)
+
+	for _, match := range matches {
+		if len(match) != 2 {
+			continue
+		}
+		resMatch := strings.TrimSpace(match[1])
+
+		s1 := detectors.Result{
+			DetectorType: detectorspb.DetectorType_Alchemy,
+			Raw:          []byte(resMatch),
+		}
+
+		if verify {
+			req, err := http.NewRequestWithContext(ctx, "GET", "https://eth-mainnet.g.alchemy.com/v2/"+resMatch+"/getNFTs/?owner=vitalik.eth", nil)
+			if err != nil {
+				continue
+			}
+			res, err := client.Do(req)
+			if err == nil {
+				defer res.Body.Close()
+				if res.StatusCode >= 200 && res.StatusCode < 300 {
+					s1.Verified = true
+				} else {
+					// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
+					if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
+						continue
+					}
+				}
+			}
+		}
+
+		results = append(results, s1)
+	}
+
+	return results, nil
+}
+
+func (s Scanner) Type() detectorspb.DetectorType {
+	return detectorspb.DetectorType_Alchemy
+}

pkg/detectors/alchemy/alchemy_test.go

@@ -0,0 +1,119 @@
+//go:build detectors
+// +build detectors
+
+package alchemy
+
+import (
+	"context"
+	"fmt"
+	"testing"
+	"time"
+
+	"github.com/kylelemons/godebug/pretty"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+
+	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestAlchemy_FromChunk(t *testing.T) {
+	ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+	defer cancel()
+	testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors4")
+	if err != nil {
+		t.Fatalf("could not get test secrets from GCP: %s", err)
+	}
+	secret := testSecrets.MustGetField("ALCHEMY")
+	inactiveSecret := testSecrets.MustGetField("ALCHEMY_INACTIVE")
+
+	type args struct {
+		ctx    context.Context
+		data   []byte
+		verify bool
+	}
+	tests := []struct {
+		name    string
+		s       Scanner
+		args    args
+		want    []detectors.Result
+		wantErr bool
+	}{
+		{
+			name: "found, verified",
+			s:    Scanner{},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte(fmt.Sprintf("You can find a alchemy secret %s within", secret)),
+				verify: true,
+			},
+			want: []detectors.Result{
+				{
+					DetectorType: detectorspb.DetectorType_Alchemy,
+					Verified:     true,
+				},
+			},
+			wantErr: false,
+		},
+		{
+			name: "found, unverified",
+			s:    Scanner{},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte(fmt.Sprintf("You can find a alchemy secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+				verify: true,
+			},
+			want: []detectors.Result{
+				{
+					DetectorType: detectorspb.DetectorType_Alchemy,
+					Verified:     false,
+				},
+			},
+			wantErr: false,
+		},
+		{
+			name: "not found",
+			s:    Scanner{},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte("You cannot find the secret within"),
+				verify: true,
+			},
+			want:    nil,
+			wantErr: false,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			s := Scanner{}
+			got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("Alchemy.FromData() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			for i := range got {
+				if len(got[i].Raw) == 0 {
+					t.Fatalf("no raw secret present: \n %+v", got[i])
+				}
+				got[i].Raw = nil
+			}
+			if diff := pretty.Compare(got, tt.want); diff != "" {
+				t.Errorf("Alchemy.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+			}
+		})
+	}
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+	ctx := context.Background()
+	s := Scanner{}
+	for name, data := range detectors.MustGetBenchmarkData() {
+		benchmark.Run(name, func(b *testing.B) {
+			for n := 0; n < b.N; n++ {
+				_, err := s.FromData(ctx, false, data)
+				if err != nil {
+					b.Fatal(err)
+				}
+			}
+		})
+	}
+}

pkg/detectors/bitfinex/bitfinex.go

@@ -86,7 +86,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 
 				s1.Verified = isValid
 				// If there is a valid one, we need to stop iterating now and return the valid result
-				if isValid == true {
+				if isValid {
 					break
 				}
 			}

pkg/detectors/blocknative/blocknative.go

@@ -0,0 +1,84 @@
+package blocknative
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"net/http"
+	"regexp"
+	"strings"
+
+	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+type Scanner struct{}
+
+// Ensure the Scanner satisfies the interface at compile time.
+var _ detectors.Detector = (*Scanner)(nil)
+
+var (
+	client = common.SaneHttpClient()
+
+	// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
+	keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"blocknative"}) + `\b([0-9Aa-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})\b`)
+)
+
+// Keywords are used for efficiently pre-filtering chunks.
+// Use identifiers in the secret preferably, or the provider name.
+func (s Scanner) Keywords() []string {
+	return []string{"blocknative"}
+}
+
+// FromData will find and optionally verify Blocknative secrets in a given set of bytes.
+func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
+	dataStr := string(data)
+
+	matches := keyPat.FindAllStringSubmatch(dataStr, -1)
+
+	for _, match := range matches {
+		if len(match) != 2 {
+			continue
+		}
+		resMatch := strings.TrimSpace(match[1])
+
+		s1 := detectors.Result{
+			DetectorType: detectorspb.DetectorType_BlockNative,
+			Raw:          []byte(resMatch),
+		}
+
+		if verify {
+			req, err := http.NewRequestWithContext(ctx, "GET", "https://api.blocknative.com/gasprices/blockprices", nil)
+			if err != nil {
+				continue
+			}
+			req.Header.Add("Authorization", fmt.Sprintf("%s", resMatch))
+			res, err := client.Do(req)
+			if err == nil {
+				defer res.Body.Close()
+				bodyBytes, err := io.ReadAll(res.Body)
+				if err != nil {
+					continue
+				}
+				body := string(bodyBytes)
+				if !strings.Contains(body, "valid") {
+					s1.Verified = true
+				} else {
+					// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
+					if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
+						continue
+					}
+				}
+			}
+		}
+
+		results = append(results, s1)
+	}
+
+	return results, nil
+}
+
+func (s Scanner) Type() detectorspb.DetectorType {
+	return detectorspb.DetectorType_BlockNative
+}

pkg/detectors/blocknative/blocknative_test.go

@@ -0,0 +1,119 @@
+//go:build detectors
+// +build detectors
+
+package blocknative
+
+import (
+	"context"
+	"fmt"
+	"testing"
+	"time"
+
+	"github.com/kylelemons/godebug/pretty"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+
+	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestBlocknative_FromChunk(t *testing.T) {
+	ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+	defer cancel()
+	testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors4")
+	if err != nil {
+		t.Fatalf("could not get test secrets from GCP: %s", err)
+	}
+	secret := testSecrets.MustGetField("BLOCKNATIVE")
+	inactiveSecret := testSecrets.MustGetField("BLOCKNATIVE_INACTIVE")
+
+	type args struct {
+		ctx    context.Context
+		data   []byte
+		verify bool
+	}
+	tests := []struct {
+		name    string
+		s       Scanner
+		args    args
+		want    []detectors.Result
+		wantErr bool
+	}{
+		{
+			name: "found, verified",
+			s:    Scanner{},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte(fmt.Sprintf("You can find a blocknative secret %s within", secret)),
+				verify: true,
+			},
+			want: []detectors.Result{
+				{
+					DetectorType: detectorspb.DetectorType_BlockNative,
+					Verified:     true,
+				},
+			},
+			wantErr: false,
+		},
+		{
+			name: "found, unverified",
+			s:    Scanner{},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte(fmt.Sprintf("You can find a blocknative secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+				verify: true,
+			},
+			want: []detectors.Result{
+				{
+					DetectorType: detectorspb.DetectorType_BlockNative,
+					Verified:     false,
+				},
+			},
+			wantErr: false,
+		},
+		{
+			name: "not found",
+			s:    Scanner{},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte("You cannot find the secret within"),
+				verify: true,
+			},
+			want:    nil,
+			wantErr: false,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			s := Scanner{}
+			got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("Blocknative.FromData() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			for i := range got {
+				if len(got[i].Raw) == 0 {
+					t.Fatalf("no raw secret present: \n %+v", got[i])
+				}
+				got[i].Raw = nil
+			}
+			if diff := pretty.Compare(got, tt.want); diff != "" {
+				t.Errorf("Blocknative.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+			}
+		})
+	}
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+	ctx := context.Background()
+	s := Scanner{}
+	for name, data := range detectors.MustGetBenchmarkData() {
+		benchmark.Run(name, func(b *testing.B) {
+			for n := 0; n < b.N; n++ {
+				_, err := s.FromData(ctx, false, data)
+				if err != nil {
+					b.Fatal(err)
+				}
+			}
+		})
+	}
+}

pkg/detectors/brandfetch/brandfetch.go

@@ -2,7 +2,6 @@ package brandfetch
 
 import (
 	"context"
-	"fmt"
 	"net/http"
 	"regexp"
 	"strings"
@@ -56,7 +55,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 				continue
 			}
 			req.Header.Add("Content-Type", "application/json")
-			req.Header.Add("x-api-key", fmt.Sprintf("%s", resMatch))
+			req.Header.Add("x-api-key", resMatch)
 			res, err := client.Do(req)
 			if err == nil {
 				defer res.Body.Close()

pkg/detectors/browserstack/browserstack.go

@@ -20,8 +20,8 @@ var (
 	client = common.SaneHttpClient()
 
 	// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
-	keyPat  = regexp.MustCompile(detectors.PrefixRegex([]string{"browserstack", "key", "automate", "local"}) + `\b([0-9a-zA-Z]{20})\b`)
-	userPat = regexp.MustCompile(detectors.PrefixRegex([]string{"browserstack", "user", "automate", "local"}) + `\b(^[a-zA-Z\d]+([._-]?[a-zA-Z\d]+)*[a-zA-Z\d]+$)\b`)
+	keyPat  = regexp.MustCompile(detectors.PrefixRegex([]string{"https://", "http://", "hub-cloud.browserstack.com", "accessKey", "\"access_Key\":", "ACCESS_KEY", "key", "browserstackKey", "BS_AUTHKEY", "BROWSERSTACK_ACCESS_KEY"}) + `\b([0-9a-zA-Z]{20})\b`)
+	userPat = regexp.MustCompile(detectors.PrefixRegex([]string{"https://", "http://", "hub-cloud.browserstack.com", "userName", "\"username\":", "USER_NAME", "user", "browserstackUser", "BS_USERNAME", "BROWSERSTACK_USERNAME"}) + `\b([a-zA-Z\d]{3,18}[._-]+[a-zA-Z\d]{6})\b`)
 )
 
 // Keywords are used for efficiently pre-filtering chunks.

pkg/detectors/browserstack/browserstack_test.go

@@ -51,6 +51,7 @@ func TestBrowserStack_FromChunk(t *testing.T) {
 				{
 					DetectorType: detectorspb.DetectorType_BrowserStack,
 					Verified:     true,
+					RawV2:        []byte(fmt.Sprintf("%s%s", secret, secretUser)),
 				},
 			},
 			wantErr: false,
@@ -67,6 +68,7 @@ func TestBrowserStack_FromChunk(t *testing.T) {
 				{
 					DetectorType: detectorspb.DetectorType_BrowserStack,
 					Verified:     false,
+					RawV2:        []byte(fmt.Sprintf("%s%s", inactiveSecret, secretUser)),
 				},
 			},
 			wantErr: false,

pkg/detectors/bscscan/bscscan.go

@@ -0,0 +1,83 @@
+package bscscan
+
+import (
+	"context"
+	"io"
+	"net/http"
+	"regexp"
+	"strings"
+
+	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+type Scanner struct{}
+
+// Ensure the Scanner satisfies the interface at compile time.
+var _ detectors.Detector = (*Scanner)(nil)
+
+var (
+	client = common.SaneHttpClient()
+
+	// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
+	keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"bscscan"}) + `\b([0-9A-Z]{34})\b`)
+)
+
+// Keywords are used for efficiently pre-filtering chunks.
+// Use identifiers in the secret preferably, or the provider name.
+func (s Scanner) Keywords() []string {
+	return []string{"bscscan"}
+}
+
+// FromData will find and optionally verify Bscscan secrets in a given set of bytes.
+func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
+	dataStr := string(data)
+
+	matches := keyPat.FindAllStringSubmatch(dataStr, -1)
+
+	for _, match := range matches {
+		if len(match) != 2 {
+			continue
+		}
+		resMatch := strings.TrimSpace(match[1])
+
+		s1 := detectors.Result{
+			DetectorType: detectorspb.DetectorType_BscScan,
+			Raw:          []byte(resMatch),
+		}
+
+		if verify {
+			req, err := http.NewRequestWithContext(ctx, "GET", "https://api.bscscan.com/api?module=account&action=balance&address=0x70F657164e5b75689b64B7fd1fA275F334f28e18&apikey="+resMatch, nil)
+			if err != nil {
+				continue
+			}
+			res, err := client.Do(req)
+			if err == nil {
+				defer res.Body.Close()
+                                bodyBytes, err := io.ReadAll(res.Body)
+                                if err != nil {
+                                        continue
+                                }
+                                body := string(bodyBytes)
+
+                                if !strings.Contains(body, "NOTOK") {
+                                        s1.Verified = true
+                                } else {
+                                        // This function will check false positives for common test words, but also it will make sur>
+                                        if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
+                                                continue
+                                        }
+                                }
+                        }
+                }
+
+		results = append(results, s1)
+	}
+
+	return results, nil
+}
+
+func (s Scanner) Type() detectorspb.DetectorType {
+	return detectorspb.DetectorType_BscScan
+}

pkg/detectors/bscscan/bscscan_test.go

@@ -0,0 +1,119 @@
+//go:build detectors
+// +build detectors
+
+package bscscan
+
+import (
+	"context"
+	"fmt"
+	"testing"
+	"time"
+
+	"github.com/kylelemons/godebug/pretty"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+
+	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestBscscan_FromChunk(t *testing.T) {
+	ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+	defer cancel()
+	testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors4")
+	if err != nil {
+		t.Fatalf("could not get test secrets from GCP: %s", err)
+	}
+	secret := testSecrets.MustGetField("BSCSCAN")
+	inactiveSecret := testSecrets.MustGetField("BSCSCAN_INACTIVE")
+
+	type args struct {
+		ctx    context.Context
+		data   []byte
+		verify bool
+	}
+	tests := []struct {
+		name    string
+		s       Scanner
+		args    args
+		want    []detectors.Result
+		wantErr bool
+	}{
+		{
+			name: "found, verified",
+			s:    Scanner{},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte(fmt.Sprintf("You can find a bscscan secret %s within", secret)),
+				verify: true,
+			},
+			want: []detectors.Result{
+				{
+					DetectorType: detectorspb.DetectorType_BscScan,
+					Verified:     true,
+				},
+			},
+			wantErr: false,
+		},
+		{
+			name: "found, unverified",
+			s:    Scanner{},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte(fmt.Sprintf("You can find a bscscan secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+				verify: true,
+			},
+			want: []detectors.Result{
+				{
+					DetectorType: detectorspb.DetectorType_BscScan,
+					Verified:     false,
+				},
+			},
+			wantErr: false,
+		},
+		{
+			name: "not found",
+			s:    Scanner{},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte("You cannot find the secret within"),
+				verify: true,
+			},
+			want:    nil,
+			wantErr: false,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			s := Scanner{}
+			got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("Bscscan.FromData() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			for i := range got {
+				if len(got[i].Raw) == 0 {
+					t.Fatalf("no raw secret present: \n %+v", got[i])
+				}
+				got[i].Raw = nil
+			}
+			if diff := pretty.Compare(got, tt.want); diff != "" {
+				t.Errorf("Bscscan.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+			}
+		})
+	}
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+	ctx := context.Background()
+	s := Scanner{}
+	for name, data := range detectors.MustGetBenchmarkData() {
+		benchmark.Run(name, func(b *testing.B) {
+			for n := 0; n < b.N; n++ {
+				_, err := s.FromData(ctx, false, data)
+				if err != nil {
+					b.Fatal(err)
+				}
+			}
+		})
+	}
+}

pkg/detectors/clockify/clockify.go

@@ -2,7 +2,6 @@ package clockify
 
 import (
 	"context"
-	"fmt"
 	"net/http"
 	"regexp"
 	"strings"
@@ -53,7 +52,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 				continue
 			}
 			req.Header.Add("content-type", "application/json")
-			req.Header.Add("X-Api-Key", fmt.Sprintf("%s", resMatch))
+			req.Header.Add("X-Api-Key", resMatch)
 			res, err := client.Do(req)
 			if err == nil {
 				defer res.Body.Close()

pkg/detectors/cloudflarecakey/cloudflarecakey.go

@@ -66,7 +66,6 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 				if res.StatusCode >= 200 && res.StatusCode < 300 {
 					s1.Verified = true
 				}
-			} else {
 			}
 		}
 

pkg/detectors/coinmarketcap/coinmarketcap.go

@@ -0,0 +1,79 @@
+package coinmarketcap
+
+import (
+	"context"
+	"fmt"
+	"net/http"
+	"regexp"
+	"strings"
+
+	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+type Scanner struct{}
+
+// Ensure the Scanner satisfies the interface at compile time.
+var _ detectors.Detector = (*Scanner)(nil)
+
+var (
+	client = common.SaneHttpClient()
+
+	// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
+	keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"coinmarketcap","CMC"}) + `\b([0-9Aa-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})\b`)
+)
+
+// Keywords are used for efficiently pre-filtering chunks.
+// Use identifiers in the secret preferably, or the provider name.
+func (s Scanner) Keywords() []string {
+	return []string{"coinmarketcap","CMC"}
+}
+
+// FromData will find and optionally verify Coinmarketcap secrets in a given set of bytes.
+func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
+	dataStr := string(data)
+
+	matches := keyPat.FindAllStringSubmatch(dataStr, -1)
+
+	for _, match := range matches {
+		if len(match) != 2 {
+			continue
+		}
+		resMatch := strings.TrimSpace(match[1])
+
+		s1 := detectors.Result{
+			DetectorType: detectorspb.DetectorType_CoinMarketCap,
+			Raw:          []byte(resMatch),
+		}
+
+		if verify {
+			req, err := http.NewRequestWithContext(ctx, "GET", "https://pro-api.coinmarketcap.com/v1/cryptocurrency/listings/latest?start=1&limit=5000&convert=USD", nil)
+			if err != nil {
+				continue
+			}
+			req.Header.Add("Accept", "application/json")
+			req.Header.Add("X-CMC_PRO_API_KEY", fmt.Sprintf("%s", resMatch))
+			res, err := client.Do(req)
+			if err == nil {
+				defer res.Body.Close()
+				if res.StatusCode >= 200 && res.StatusCode < 300 {
+					s1.Verified = true
+				} else {
+					// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
+					if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
+						continue
+					}
+				}
+			}
+		}
+
+		results = append(results, s1)
+	}
+
+	return results, nil
+}
+
+func (s Scanner) Type() detectorspb.DetectorType {
+	return detectorspb.DetectorType_CoinMarketCap
+}

pkg/detectors/coinmarketcap/coinmarketcap_test.go

@@ -0,0 +1,119 @@
+//go:build detectors
+// +build detectors
+
+package coinmarketcap
+
+import (
+	"context"
+	"fmt"
+	"testing"
+	"time"
+
+	"github.com/kylelemons/godebug/pretty"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+
+	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestCoinmarketcap_FromChunk(t *testing.T) {
+	ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+	defer cancel()
+	testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors4")
+	if err != nil {
+		t.Fatalf("could not get test secrets from GCP: %s", err)
+	}
+	secret := testSecrets.MustGetField("COINMARKETCAP")
+	inactiveSecret := testSecrets.MustGetField("COINMARKETCAP_INACTIVE")
+
+	type args struct {
+		ctx    context.Context
+		data   []byte
+		verify bool
+	}
+	tests := []struct {
+		name    string
+		s       Scanner
+		args    args
+		want    []detectors.Result
+		wantErr bool
+	}{
+		{
+			name: "found, verified",
+			s:    Scanner{},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte(fmt.Sprintf("You can find a coinmarketcap secret %s within", secret)),
+				verify: true,
+			},
+			want: []detectors.Result{
+				{
+					DetectorType: detectorspb.DetectorType_CoinMarketCap,
+					Verified:     true,
+				},
+			},
+			wantErr: false,
+		},
+		{
+			name: "found, unverified",
+			s:    Scanner{},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte(fmt.Sprintf("You can find a coinmarketcap secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+				verify: true,
+			},
+			want: []detectors.Result{
+				{
+					DetectorType: detectorspb.DetectorType_CoinMarketCap,
+					Verified:     false,
+				},
+			},
+			wantErr: false,
+		},
+		{
+			name: "not found",
+			s:    Scanner{},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte("You cannot find the secret within"),
+				verify: true,
+			},
+			want:    nil,
+			wantErr: false,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			s := Scanner{}
+			got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("Coinmarketcap.FromData() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			for i := range got {
+				if len(got[i].Raw) == 0 {
+					t.Fatalf("no raw secret present: \n %+v", got[i])
+				}
+				got[i].Raw = nil
+			}
+			if diff := pretty.Compare(got, tt.want); diff != "" {
+				t.Errorf("Coinmarketcap.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+			}
+		})
+	}
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+	ctx := context.Background()
+	s := Scanner{}
+	for name, data := range detectors.MustGetBenchmarkData() {
+		benchmark.Run(name, func(b *testing.B) {
+			for n := 0; n < b.N; n++ {
+				_, err := s.FromData(ctx, false, data)
+				if err != nil {
+					b.Fatal(err)
+				}
+			}
+		})
+	}
+}

pkg/detectors/detectors.go

@@ -25,6 +25,12 @@ type Detector interface {
 	Type() detectorspb.DetectorType
 }
 
+// Versioner is an optional interface that a detector can implement to
+// differentiate instances of the same detector type.
+type Versioner interface {
+	Version() int
+}
+
 type Result struct {
 	// DetectorType is the type of Detector.
 	DetectorType detectorspb.DetectorType

pkg/detectors/etherscan/etherscan.go

@@ -0,0 +1,83 @@
+package etherscan
+
+import (
+	"context"
+	"io"
+	"net/http"
+	"regexp"
+	"strings"
+
+	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+type Scanner struct{}
+
+// Ensure the Scanner satisfies the interface at compile time.
+var _ detectors.Detector = (*Scanner)(nil)
+
+var (
+	client = common.SaneHttpClient()
+
+	// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
+	keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"etherscan"}) + `\b([0-9A-Z]{34})\b`)
+)
+
+// Keywords are used for efficiently pre-filtering chunks.
+// Use identifiers in the secret preferably, or the provider name.
+func (s Scanner) Keywords() []string {
+	return []string{"etherscan"}
+}
+
+// FromData will find and optionally verify Etherscan secrets in a given set of bytes.
+func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
+	dataStr := string(data)
+
+	matches := keyPat.FindAllStringSubmatch(dataStr, -1)
+
+	for _, match := range matches {
+		if len(match) != 2 {
+			continue
+		}
+		resMatch := strings.TrimSpace(match[1])
+
+		s1 := detectors.Result{
+			DetectorType: detectorspb.DetectorType_Etherscan,
+			Raw:          []byte(resMatch),
+		}
+
+		if verify {
+			req, err := http.NewRequestWithContext(ctx, "GET", "https://api.etherscan.io/api?module=account&action=balance&address=0xde0b295669a9fd93d5f28d9ec85e40f4cb697bae&tag=latest&apikey="+resMatch, nil)
+			if err != nil {
+				continue
+			}
+			res, err := client.Do(req)
+			if err == nil {
+				defer res.Body.Close()
+				bodyBytes, err := io.ReadAll(res.Body)
+				if err != nil {
+					continue
+				}
+				body := string(bodyBytes)
+
+				if strings.Contains(body, `"OK"`) {
+					s1.Verified = true
+				} else {
+					// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
+					if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
+						continue
+					}
+				}
+			}
+		}
+
+		results = append(results, s1)
+	}
+
+	return results, nil
+}
+
+func (s Scanner) Type() detectorspb.DetectorType {
+	return detectorspb.DetectorType_Etherscan
+}

pkg/detectors/etherscan/etherscan_test.go

@@ -0,0 +1,119 @@
+//go:build detectors
+// +build detectors
+
+package etherscan
+
+import (
+	"context"
+	"fmt"
+	"testing"
+	"time"
+
+	"github.com/kylelemons/godebug/pretty"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+
+	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestEtherscan_FromChunk(t *testing.T) {
+	ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+	defer cancel()
+	testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors4")
+	if err != nil {
+		t.Fatalf("could not get test secrets from GCP: %s", err)
+	}
+	secret := testSecrets.MustGetField("ETHERSCAN")
+	inactiveSecret := testSecrets.MustGetField("ETHERSCAN_INACTIVE")
+
+	type args struct {
+		ctx    context.Context
+		data   []byte
+		verify bool
+	}
+	tests := []struct {
+		name    string
+		s       Scanner
+		args    args
+		want    []detectors.Result
+		wantErr bool
+	}{
+		{
+			name: "found, verified",
+			s:    Scanner{},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte(fmt.Sprintf("You can find a etherscan secret %s within", secret)),
+				verify: true,
+			},
+			want: []detectors.Result{
+				{
+					DetectorType: detectorspb.DetectorType_Etherscan,
+					Verified:     true,
+				},
+			},
+			wantErr: false,
+		},
+		{
+			name: "found, unverified",
+			s:    Scanner{},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte(fmt.Sprintf("You can find a etherscan secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+				verify: true,
+			},
+			want: []detectors.Result{
+				{
+					DetectorType: detectorspb.DetectorType_Etherscan,
+					Verified:     false,
+				},
+			},
+			wantErr: false,
+		},
+		{
+			name: "not found",
+			s:    Scanner{},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte("You cannot find the secret within"),
+				verify: true,
+			},
+			want:    nil,
+			wantErr: false,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			s := Scanner{}
+			got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("Etherscan.FromData() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			for i := range got {
+				if len(got[i].Raw) == 0 {
+					t.Fatalf("no raw secret present: \n %+v", got[i])
+				}
+				got[i].Raw = nil
+			}
+			if diff := pretty.Compare(got, tt.want); diff != "" {
+				t.Errorf("Etherscan.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+			}
+		})
+	}
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+	ctx := context.Background()
+	s := Scanner{}
+	for name, data := range detectors.MustGetBenchmarkData() {
+		benchmark.Run(name, func(b *testing.B) {
+			for n := 0; n < b.N; n++ {
+				_, err := s.FromData(ctx, false, data)
+				if err != nil {
+					b.Fatal(err)
+				}
+			}
+		})
+	}
+}

pkg/detectors/everhour/everhour.go

@@ -2,7 +2,6 @@ package everhour
 
 import (
 	"context"
-	"fmt"
 	"net/http"
 	"regexp"
 	"strings"
@@ -53,7 +52,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 				continue
 			}
 			req.Header.Add("Content-Type", "application/json")
-			req.Header.Add("X-Api-Key", fmt.Sprintf("%s", resMatch))
+			req.Header.Add("X-Api-Key", resMatch)
 			res, err := client.Do(req)
 			if err == nil {
 				defer res.Body.Close()

pkg/detectors/fastlypersonaltoken/fastlypersonaltoken.go

@@ -61,7 +61,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 			if err != nil {
 				continue
 			}
-			req.Header.Add("Fastly-Key", fmt.Sprintf("%s", resMatch))
+			req.Header.Add("Fastly-Key", resMatch)
 			res, err := client.Do(req)
 			if err == nil {
 				bodyBytes, err := io.ReadAll(res.Body)

pkg/detectors/figmapersonalaccesstoken/figmapersonalaccesstoken.go

@@ -2,7 +2,6 @@ package figmapersonalaccesstoken
 
 import (
 	"context"
-	"fmt"
 	"net/http"
 	"regexp"
 	"strings"
@@ -51,7 +50,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 			if err != nil {
 				continue
 			}
-			req.Header.Add("X-Figma-Token", fmt.Sprintf("%s", resMatch))
+			req.Header.Add("X-Figma-Token", resMatch)
 			res, err := client.Do(req)
 			if err == nil {
 				defer res.Body.Close()

pkg/detectors/fulcrum/fulcrum.go

@@ -2,7 +2,6 @@ package fulcrum
 
 import (
 	"context"
-	"fmt"
 	"net/http"
 	"regexp"
 	"strings"
@@ -53,7 +52,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 				continue
 			}
 			req.Header.Add("Accept", "application/json")
-			req.Header.Add("X-ApiToken", fmt.Sprintf("%s", resMatch))
+			req.Header.Add("X-ApiToken", resMatch)
 			res, err := client.Do(req)
 			if err == nil {
 				defer res.Body.Close()

pkg/detectors/generic/generic.go

@@ -44,10 +44,7 @@ func New() Scanner {
 }
 
 type Scanner struct {
-	wordList             []string
-	badList              []string
-	programmingBookWords []string
-	excludeMatchers      []*regexp.Regexp
+	excludeMatchers []*regexp.Regexp
 }
 
 // Ensure the Scanner satisfies the interface at compile time.

pkg/detectors/github/github.go

@@ -15,8 +15,11 @@ import (
 
 type Scanner struct{}
 
-// Ensure the Scanner satisfies the interface at compile time.
+// Ensure the Scanner satisfies the interfaces at compile time.
 var _ detectors.Detector = (*Scanner)(nil)
+var _ detectors.Versioner = (*Scanner)(nil)
+
+func (s Scanner) Version() int { return 2 }
 
 var (
 	// Oauth token

pkg/detectors/github_old/github_old.go

@@ -14,8 +14,11 @@ import (
 
 type Scanner struct{}
 
-// Ensure the Scanner satisfies the interface at compile time.
+// Ensure the Scanner satisfies the interfaces at compile time.
 var _ detectors.Detector = (*Scanner)(nil)
+var _ detectors.Versioner = (*Scanner)(nil)
+
+func (s Scanner) Version() int { return 1 }
 
 var (
 	// Oauth token
@@ -38,7 +41,7 @@ type userRes struct {
 // Keywords are used for efficiently pre-filtering chunks.
 // Use identifiers in the secret preferably, or the provider name.
 func (s Scanner) Keywords() []string {
-	return []string{"github","gh","pat"}
+	return []string{"github", "gh", "pat"}
 }
 
 // FromData will find and optionally verify GitHub secrets in a given set of bytes.

pkg/detectors/gitlab/gitlab.go

@@ -14,8 +14,11 @@ import (
 
 type Scanner struct{}
 
-// Ensure the Scanner satisfies the interface at compile time.
+// Ensure the Scanner satisfies the interfaces at compile time.
 var _ detectors.Detector = (*Scanner)(nil)
+var _ detectors.Versioner = (*Scanner)(nil)
+
+func (s Scanner) Version() int { return 1 }
 
 var (
 	keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"gitlab"}) + `\b((?:glpat|)[a-zA-Z0-9\-=_]{20,22})\b`)

pkg/detectors/gitlabv2/gitlab.go

@@ -13,8 +13,11 @@ import (
 
 type Scanner struct{}
 
-// Ensure the Scanner satisfies the interface at compile time.
+// Ensure the Scanner satisfies the interfaces at compile time.
 var _ detectors.Detector = (*Scanner)(nil)
+var _ detectors.Versioner = (*Scanner)(nil)
+
+func (*Scanner) Version() int { return 2 }
 
 var (
 	keyPat = regexp.MustCompile(`\b(glpat-[a-zA-Z0-9\-=_]{20,22})\b`)

pkg/detectors/impala/impala.go

@@ -2,7 +2,6 @@ package impala
 
 import (
 	"context"
-	"fmt"
 	"net/http"
 	"regexp"
 	"strings"
@@ -52,7 +51,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 			if err != nil {
 				continue
 			}
-			req.Header.Add("x-api-key", fmt.Sprintf("%s", resMatch))
+			req.Header.Add("x-api-key", resMatch)
 			res, err := client.Do(req)
 			if err == nil {
 				defer res.Body.Close()

pkg/detectors/infura/infura.go

@@ -0,0 +1,84 @@
+package infura
+
+import (
+	"context"
+	"io"
+	"net/http"
+	"regexp"
+	"strings"
+
+	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+type Scanner struct{}
+
+// Ensure the Scanner satisfies the interface at compile time.
+var _ detectors.Detector = (*Scanner)(nil)
+
+var (
+	client = common.SaneHttpClient()
+
+	// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
+	keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"infura"}) + `\b([0-9a-z]{32})\b`)
+)
+
+// Keywords are used for efficiently pre-filtering chunks.
+// Use identifiers in the secret preferably, or the provider name.
+func (s Scanner) Keywords() []string {
+	return []string{"infura"}
+}
+
+// FromData will find and optionally verify Infura secrets in a given set of bytes.
+func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
+	dataStr := string(data)
+
+	matches := keyPat.FindAllStringSubmatch(dataStr, -1)
+
+	for _, match := range matches {
+		if len(match) != 2 {
+			continue
+		}
+		resMatch := strings.TrimSpace(match[1])
+
+		s1 := detectors.Result{
+			DetectorType: detectorspb.DetectorType_Infura,
+			Raw:          []byte(resMatch),
+		}
+
+		if verify {
+			payload := strings.NewReader(`{"jsonrpc":"2.0","method":"eth_blockNumber","params":[],"id":1}`)
+			req, err := http.NewRequestWithContext(ctx, "POST", "https://mainnet.infura.io/v3/"+resMatch, payload)
+			if err != nil {
+				continue
+			}
+			req.Header.Add("Content-Type", "application/json")
+			res, err := client.Do(req)
+			if err == nil {
+				defer res.Body.Close()
+				bodyBytes, err := io.ReadAll(res.Body)
+				if err != nil {
+					continue
+				}
+				body := string(bodyBytes)
+				if strings.Contains(body, `"result"`) {
+					s1.Verified = true
+				} else {
+					// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
+					if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
+						continue
+					}
+				}
+			}
+		}
+
+		results = append(results, s1)
+	}
+
+	return results, nil
+}
+
+func (s Scanner) Type() detectorspb.DetectorType {
+	return detectorspb.DetectorType_Infura
+}

pkg/detectors/infura/infura_test.go

@@ -0,0 +1,119 @@
+//go:build detectors
+// +build detectors
+
+package infura
+
+import (
+	"context"
+	"fmt"
+	"testing"
+	"time"
+
+	"github.com/kylelemons/godebug/pretty"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+
+	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestInfura_FromChunk(t *testing.T) {
+	ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+	defer cancel()
+	testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors4")
+	if err != nil {
+		t.Fatalf("could not get test secrets from GCP: %s", err)
+	}
+	secret := testSecrets.MustGetField("INFURA")
+	inactiveSecret := testSecrets.MustGetField("INFURA_INACTIVE")
+
+	type args struct {
+		ctx    context.Context
+		data   []byte
+		verify bool
+	}
+	tests := []struct {
+		name    string
+		s       Scanner
+		args    args
+		want    []detectors.Result
+		wantErr bool
+	}{
+		{
+			name: "found, verified",
+			s:    Scanner{},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte(fmt.Sprintf("You can find a infura secret %s within", secret)),
+				verify: true,
+			},
+			want: []detectors.Result{
+				{
+					DetectorType: detectorspb.DetectorType_Infura,
+					Verified:     true,
+				},
+			},
+			wantErr: false,
+		},
+		{
+			name: "found, unverified",
+			s:    Scanner{},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte(fmt.Sprintf("You can find a infura secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+				verify: true,
+			},
+			want: []detectors.Result{
+				{
+					DetectorType: detectorspb.DetectorType_Infura,
+					Verified:     false,
+				},
+			},
+			wantErr: false,
+		},
+		{
+			name: "not found",
+			s:    Scanner{},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte("You cannot find the secret within"),
+				verify: true,
+			},
+			want:    nil,
+			wantErr: false,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			s := Scanner{}
+			got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("Infura.FromData() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			for i := range got {
+				if len(got[i].Raw) == 0 {
+					t.Fatalf("no raw secret present: \n %+v", got[i])
+				}
+				got[i].Raw = nil
+			}
+			if diff := pretty.Compare(got, tt.want); diff != "" {
+				t.Errorf("Infura.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+			}
+		})
+	}
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+	ctx := context.Background()
+	s := Scanner{}
+	for name, data := range detectors.MustGetBenchmarkData() {
+		benchmark.Run(name, func(b *testing.B) {
+			for n := 0; n < b.N; n++ {
+				_, err := s.FromData(ctx, false, data)
+				if err != nil {
+					b.Fatal(err)
+				}
+			}
+		})
+	}
+}

pkg/detectors/interseller/interseller.go

@@ -56,11 +56,6 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 				defer res.Body.Close()
 				if res.StatusCode >= 200 && res.StatusCode < 300 {
 					s1.Verified = true
-				} else {
-					// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key
-					// if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
-					// 	continue
-					// }
 				}
 			}
 		}

pkg/detectors/keenio/keenio.go

@@ -2,7 +2,6 @@ package keenio
 
 import (
 	"context"
-	"fmt"
 	"net/http"
 	"regexp"
 	"strings"
@@ -61,7 +60,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 					continue
 				}
 				req.Header.Add("Content-Type", "application/json")
-				req.Header.Add("Authorization", fmt.Sprintf("%s", resMatch))
+				req.Header.Add("Authorization", resMatch)
 				res, err := client.Do(req)
 				if err == nil {
 					defer res.Body.Close()

pkg/detectors/mesibo/mesibo.go

@@ -13,7 +13,6 @@ import (
 )
 
 type Scanner struct{}
-
 // Ensure the Scanner satisfies the interface at compile time.
 var _ detectors.Detector = (*Scanner)(nil)
 
@@ -61,7 +60,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 				}
 				body := string(bodyBytes)
 
-				if !strings.Contains(body, "INVALIDTOKEN") {
+				if !strings.Contains(body, "AUTHFAIL") {
 					s1.Verified = true
 				} else {
 					if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {

pkg/detectors/moralis/moralis.go

@@ -0,0 +1,79 @@
+package moralis
+
+import (
+	"context"
+	"fmt"
+	"net/http"
+	"regexp"
+	"strings"
+
+	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+type Scanner struct{}
+
+// Ensure the Scanner satisfies the interface at compile time.
+var _ detectors.Detector = (*Scanner)(nil)
+
+var (
+	client = common.SaneHttpClient()
+
+	// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
+	keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"moralis"}) + `\b([0-9a-zA-Z]{64})\b`)
+)
+
+// Keywords are used for efficiently pre-filtering chunks.
+// Use identifiers in the secret preferably, or the provider name.
+func (s Scanner) Keywords() []string {
+	return []string{"moralis"}
+}
+
+// FromData will find and optionally verify Moralis secrets in a given set of bytes.
+func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
+	dataStr := string(data)
+
+	matches := keyPat.FindAllStringSubmatch(dataStr, -1)
+
+	for _, match := range matches {
+		if len(match) != 2 {
+			continue
+		}
+		resMatch := strings.TrimSpace(match[1])
+
+		s1 := detectors.Result{
+			DetectorType: detectorspb.DetectorType_Moralis,
+			Raw:          []byte(resMatch),
+		}
+
+		if verify {
+			req, err := http.NewRequestWithContext(ctx, "GET", "https://deep-index.moralis.io/api/v2/0xd8da6bf26964af9d7eed9e03e53415d37aa96045/nft?chain=eth&format=decimal", nil)
+			if err != nil {
+				continue
+			}
+			req.Header.Add("Accept", "application/json")
+			req.Header.Add("X-API-Key", fmt.Sprintf("%s", resMatch))
+			res, err := client.Do(req)
+			if err == nil {
+				defer res.Body.Close()
+				if res.StatusCode >= 200 && res.StatusCode < 300 {
+					s1.Verified = true
+				} else {
+					// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
+					if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
+						continue
+					}
+				}
+			}
+		}
+
+		results = append(results, s1)
+	}
+
+	return results, nil
+}
+
+func (s Scanner) Type() detectorspb.DetectorType {
+	return detectorspb.DetectorType_Moralis
+}

pkg/detectors/moralis/moralis_test.go

@@ -0,0 +1,119 @@
+//go:build detectors
+// +build detectors
+
+package moralis
+
+import (
+	"context"
+	"fmt"
+	"testing"
+	"time"
+
+	"github.com/kylelemons/godebug/pretty"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+
+	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestMoralis_FromChunk(t *testing.T) {
+	ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+	defer cancel()
+	testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors4")
+	if err != nil {
+		t.Fatalf("could not get test secrets from GCP: %s", err)
+	}
+	secret := testSecrets.MustGetField("MORALIS")
+	inactiveSecret := testSecrets.MustGetField("MORALIS_INACTIVE")
+
+	type args struct {
+		ctx    context.Context
+		data   []byte
+		verify bool
+	}
+	tests := []struct {
+		name    string
+		s       Scanner
+		args    args
+		want    []detectors.Result
+		wantErr bool
+	}{
+		{
+			name: "found, verified",
+			s:    Scanner{},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte(fmt.Sprintf("You can find a moralis secret %s within", secret)),
+				verify: true,
+			},
+			want: []detectors.Result{
+				{
+					DetectorType: detectorspb.DetectorType_Moralis,
+					Verified:     true,
+				},
+			},
+			wantErr: false,
+		},
+		{
+			name: "found, unverified",
+			s:    Scanner{},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte(fmt.Sprintf("You can find a moralis secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+				verify: true,
+			},
+			want: []detectors.Result{
+				{
+					DetectorType: detectorspb.DetectorType_Moralis,
+					Verified:     false,
+				},
+			},
+			wantErr: false,
+		},
+		{
+			name: "not found",
+			s:    Scanner{},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte("You cannot find the secret within"),
+				verify: true,
+			},
+			want:    nil,
+			wantErr: false,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			s := Scanner{}
+			got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("Moralis.FromData() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			for i := range got {
+				if len(got[i].Raw) == 0 {
+					t.Fatalf("no raw secret present: \n %+v", got[i])
+				}
+				got[i].Raw = nil
+			}
+			if diff := pretty.Compare(got, tt.want); diff != "" {
+				t.Errorf("Moralis.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+			}
+		})
+	}
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+	ctx := context.Background()
+	s := Scanner{}
+	for name, data := range detectors.MustGetBenchmarkData() {
+		benchmark.Run(name, func(b *testing.B) {
+			for n := 0; n < b.N; n++ {
+				_, err := s.FromData(ctx, false, data)
+				if err != nil {
+					b.Fatal(err)
+				}
+			}
+		})
+	}
+}

pkg/detectors/newrelicpersonalapikey/newrelicpersonalapikey.go

@@ -2,7 +2,6 @@ package newrelicpersonalapikey
 
 import (
 	"context"
-	"fmt"
 	"net/http"
 	"regexp"
 	"strings"
@@ -52,7 +51,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 			if err != nil {
 				continue
 			}
-			req.Header.Add("X-Api-Key", fmt.Sprintf("%s", resMatch))
+			req.Header.Add("X-Api-Key", resMatch)
 			res, err := client.Do(req)
 			if err == nil {
 				defer res.Body.Close()

pkg/detectors/newscatcher/newscatcher.go

@@ -2,7 +2,6 @@ package newscatcher
 
 import (
 	"context"
-	"fmt"
 	"net/http"
 	"regexp"
 	"strings"
@@ -52,7 +51,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 			if err != nil {
 				continue
 			}
-			req.Header.Add("x-api-key", fmt.Sprintf("%s", resMatch))
+			req.Header.Add("x-api-key", resMatch)
 			res, err := client.Do(req)
 			if err == nil {
 				defer res.Body.Close()

pkg/detectors/nftport/nftport.go

@@ -2,7 +2,6 @@ package nftport
 
 import (
 	"context"
-	"fmt"
 	"net/http"
 	"regexp"
 	"strings"
@@ -53,7 +52,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 				continue
 			}
 			req.Header.Add("Content-Type", "application/json")
-			req.Header.Add("Authorization", fmt.Sprintf("%s", resMatch))
+			req.Header.Add("Authorization", resMatch)
 			res, err := client.Do(req)
 			if err == nil {
 				defer res.Body.Close()

pkg/detectors/npmtoken/npmtoken.go

@@ -14,8 +14,11 @@ import (
 
 type Scanner struct{}
 
-// Ensure the Scanner satisfies the interface at compile time.
+// Ensure the Scanner satisfies the interfaces at compile time.
 var _ detectors.Detector = (*Scanner)(nil)
+var _ detectors.Versioner = (*Scanner)(nil)
+
+func (s Scanner) Version() int { return 1 }
 
 var (
 	client = common.SaneHttpClient()
@@ -34,7 +37,7 @@ func (s Scanner) Keywords() []string {
 func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
 	dataStr := string(data)
 
-  	matches := keyPat.FindAllStringSubmatch(dataStr, -1)
+	matches := keyPat.FindAllStringSubmatch(dataStr, -1)
 	for _, match := range matches {
 		if len(match) != 2 {
 			continue

pkg/detectors/npmtokenv2/npmtokenv2.go

@@ -13,8 +13,11 @@ import (
 
 type Scanner struct{}
 
-// Ensure the Scanner satisfies the interface at compile time.
+// Ensure the Scanner satisfies the interfaces at compile time.
 var _ detectors.Detector = (*Scanner)(nil)
+var _ detectors.Versioner = (*Scanner)(nil)
+
+func (s Scanner) Version() int { return 2 }
 
 var (
 	client = common.SaneHttpClient()

pkg/detectors/omnisend/omnisend.go

@@ -2,7 +2,6 @@ package omnisend
 
 import (
 	"context"
-	"fmt"
 	"net/http"
 	"regexp"
 	"strings"
@@ -52,7 +51,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 			if err != nil {
 				continue
 			}
-			req.Header.Add("X-API-KEY", fmt.Sprintf("%s", resMatch))
+			req.Header.Add("X-API-KEY", resMatch)
 			res, err := client.Do(req)
 			if err == nil {
 				defer res.Body.Close()

pkg/detectors/openai/openai.go

@@ -0,0 +1,113 @@
+package openai
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"regexp"
+	"strconv"
+
+	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+
+	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
+)
+
+type Scanner struct{}
+
+// Ensure the Scanner satisfies the interface at compile time.
+var _ detectors.Detector = (*Scanner)(nil)
+
+var (
+	keyPat = regexp.MustCompile(`\b((?:sk)-[a-zA-Z0-9]{48})\b`)
+)
+
+// TODO: Add secret context?? Information about access, ownership etc
+type orgResponse struct {
+	Data []organization `json:"data"`
+}
+
+type organization struct {
+	Id          string `json:"id"`
+	Title       string `json:"title"`
+	User        string `json:"name"`
+	Description string `json:"description"`
+	Personal    bool   `json:"personal"`
+	Default     bool   `json:"is_default"`
+	Role        string `json:"role"`
+}
+
+// Keywords are used for efficiently pre-filtering chunks.
+// Use identifiers in the secret preferably, or the provider name.
+func (s Scanner) Keywords() []string {
+	return []string{"sk-"}
+}
+
+// FromData will find and optionally verify OpenAI secrets in a given set of bytes.
+func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
+	dataStr := string(data)
+
+	matches := keyPat.FindAllStringSubmatch(dataStr, -1)
+
+	for _, match := range matches {
+		// First match is entire regex, second is the first group.
+		if len(match) != 2 {
+			continue
+		}
+
+		token := match[1]
+
+		s1 := detectors.Result{
+			DetectorType: detectorspb.DetectorType_OpenAI,
+			Redacted:     token[:3] + "..." + token[47:],
+			Raw:          []byte(token),
+		}
+
+		if verify {
+			client := common.SaneHttpClient()
+			// Undocumented API
+			// https://api.openai.com/v1/organizations
+			req, err := http.NewRequestWithContext(ctx, "GET", "https://api.openai.com/v1/organizations", nil)
+			if err != nil {
+				continue
+			}
+			req.Header.Add("Content-Type", "application/json; charset=utf-8")
+			req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", token))
+			res, err := client.Do(req)
+			if err == nil {
+				if res.StatusCode >= 200 && res.StatusCode < 300 {
+					var orgs orgResponse
+					err = json.NewDecoder(res.Body).Decode(&orgs)
+					res.Body.Close()
+					if err == nil {
+						s1.Verified = true
+						org := orgs.Data[0]
+						s1.ExtraData = map[string]string{
+							"id":          org.Id,
+							"title":       org.Title,
+							"user":        org.User,
+							"description": org.Description,
+							"role":        org.Role,
+							"is_personal": strconv.FormatBool(org.Personal),
+							"is_default":  strconv.FormatBool(org.Default),
+							"total_orgs":  fmt.Sprintf("%d", len(orgs.Data)),
+						}
+					}
+				}
+			}
+		}
+
+		if !s1.Verified && detectors.IsKnownFalsePositive(string(s1.Raw), detectors.DefaultFalsePositives, true) {
+			continue
+		}
+
+		results = append(results, s1)
+	}
+
+	return
+}
+
+func (s Scanner) Type() detectorspb.DetectorType {
+	return detectorspb.DetectorType_OpenAI
+}

pkg/detectors/openai/openai_test.go

@@ -0,0 +1,132 @@
+//go:build detectors
+// +build detectors
+
+package openai
+
+import (
+	"context"
+	"fmt"
+	"testing"
+	"time"
+
+	"github.com/kylelemons/godebug/pretty"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestOpenAI_FromChunk(t *testing.T) {
+	ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+	defer cancel()
+	testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors4")
+	if err != nil {
+		t.Fatalf("could not get test secrets from GCP: %s", err)
+	}
+
+	oaiUnverified := testSecrets.MustGetField("OPENAI_UNVERIFIED")
+	oaiVerified := testSecrets.MustGetField("OPENAI_VERIFIED")
+
+	type args struct {
+		ctx    context.Context
+		data   []byte
+		verify bool
+	}
+
+	tests := []struct {
+		name    string
+		s       Scanner
+		args    args
+		want    []detectors.Result
+		wantErr bool
+	}{
+		{
+			name: "Found, unverified OpenAI token sk-",
+			s:    Scanner{},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte(fmt.Sprintf("You can find an OpenAI secret %s within", oaiUnverified)),
+				verify: true,
+			},
+			want: []detectors.Result{
+				{
+					DetectorType: detectorspb.DetectorType_OpenAI,
+					Redacted:     "sk-...xxxx",
+					Verified:     false,
+				},
+			},
+			wantErr: false,
+		},
+		{
+			name: "Found, verified OpenAI token sk-",
+			s:    Scanner{},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte(fmt.Sprintf("You can find an OpenAI secret %s within", oaiVerified)),
+				verify: true,
+			},
+			want: []detectors.Result{
+				{
+					DetectorType: detectorspb.DetectorType_OpenAI,
+					Verified:     true,
+					Redacted:     "sk-...xxxx",
+					ExtraData: map[string]string{
+						"description": "",
+						"id":          "",
+						"is_default":  "",
+						"is_personal": "",
+						"role":        "",
+						"title":       "",
+						"total_orgs":  "",
+						"user":        "",
+					},
+				},
+			},
+			wantErr: false,
+		},
+		{
+			name: "not found",
+			s:    Scanner{},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte("You cannot find the secret within"),
+				verify: true,
+			},
+			want:    nil,
+			wantErr: false,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			s := Scanner{}
+			got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("OpenAI.FromData() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			for i := range got {
+				if len(got[i].Raw) == 0 {
+					t.Fatal("no raw secret present")
+				}
+				got[i].Raw = nil
+			}
+			if diff := pretty.Compare(got, tt.want); diff != "" {
+				t.Errorf("OpenAI.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+			}
+		})
+	}
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+	ctx := context.Background()
+	s := Scanner{}
+	for name, data := range detectors.MustGetBenchmarkData() {
+		benchmark.Run(name, func(b *testing.B) {
+			for n := 0; n < b.N; n++ {
+				_, err := s.FromData(ctx, false, data)
+				if err != nil {
+					b.Fatal(err)
+				}
+			}
+		})
+	}
+}

pkg/detectors/openuv/openuv.go

@@ -2,7 +2,6 @@ package openuv
 
 import (
 	"context"
-	"fmt"
 	"net/http"
 	"regexp"
 	"strings"
@@ -52,7 +51,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 			if err != nil {
 				continue
 			}
-			req.Header.Add("x-access-token", fmt.Sprintf("%s", resMatch))
+			req.Header.Add("x-access-token", resMatch)
 			res, err := client.Do(req)
 			if err == nil {
 				defer res.Body.Close()

pkg/detectors/pepipost/pepipost.go

@@ -2,7 +2,6 @@ package pepipost
 
 import (
 	"context"
-	"fmt"
 	"net/http"
 	"regexp"
 	"strings"
@@ -52,7 +51,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 			if err != nil {
 				continue
 			}
-			req.Header.Add("api_key", fmt.Sprintf("%s", resMatch))
+			req.Header.Add("api_key", resMatch)
 			res, err := client.Do(req)
 			if err == nil {
 				defer res.Body.Close()

pkg/detectors/postman/postman.go

@@ -2,7 +2,6 @@ package postman
 
 import (
 	"context"
-	"fmt"
 	"net/http"
 	"regexp"
 	"strings"
@@ -52,7 +51,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 			if err != nil {
 				continue
 			}
-			req.Header.Add("x-api-key", fmt.Sprintf("%s", resMatch))
+			req.Header.Add("x-api-key", resMatch)
 			res, err := client.Do(req)
 			if err == nil {
 				defer res.Body.Close()

pkg/detectors/postmark/postmark.go

@@ -2,7 +2,6 @@ package postmark
 
 import (
 	"context"
-	"fmt"
 	"net/http"
 	"regexp"
 	"strings"
@@ -53,7 +52,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 			}
 			req.Header.Add("Accept", "application/json")
 			req.Header.Add("Content-Type", "application/json")
-			req.Header.Add("X-Postmark-Server-Token", fmt.Sprintf("%s", resMatch))
+			req.Header.Add("X-Postmark-Server-Token", resMatch)
 			res, err := client.Do(req)
 			if err == nil {
 				defer res.Body.Close()

pkg/detectors/prospectio/prospectio.go

@@ -2,7 +2,6 @@ package prospectio
 
 import (
 	"context"
-	"fmt"
 	"net/http"
 	"regexp"
 	"strings"
@@ -52,7 +51,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 			if err != nil {
 				continue
 			}
-			req.Header.Add("Authorization", fmt.Sprintf("%s", resMatch))
+			req.Header.Add("Authorization", resMatch)
 			req.Header.Add("Content-Type", "application/vnd.api+json; charset=utf-8")
 			res, err := client.Do(req)
 			if err == nil {

pkg/detectors/pushbulletapikey/pushbulletapikey.go

@@ -2,7 +2,6 @@ package pushbulletapikey
 
 import (
 	"context"
-	"fmt"
 	"net/http"
 	"regexp"
 	"strings"
@@ -52,7 +51,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 			if err != nil {
 				continue
 			}
-			req.Header.Add("Access-Token", fmt.Sprintf("%s", resMatch))
+			req.Header.Add("Access-Token", resMatch)
 			res, err := client.Do(req)
 			if err == nil {
 				defer res.Body.Close()

pkg/detectors/qubole/qubole.go

@@ -2,7 +2,6 @@ package qubole
 
 import (
 	"context"
-	"fmt"
 	"net/http"
 	"regexp"
 	"strings"
@@ -54,7 +53,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 			}
 			req.Header.Add("Content-Type", "application/json")
 			req.Header.Add("Accept", "application/json")
-			req.Header.Add("X-AUTH-TOKEN", fmt.Sprintf("%s", resMatch))
+			req.Header.Add("X-AUTH-TOKEN", resMatch)
 			res, err := client.Do(req)
 			if err == nil {
 				defer res.Body.Close()

pkg/detectors/rapidapi/rapidapi.go

@@ -2,7 +2,6 @@ package rapidapi
 
 import (
 	"context"
-	"fmt"
 	"net/http"
 	"regexp"
 	"strings"
@@ -51,7 +50,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 			if err != nil {
 				continue
 			}
-			req.Header.Add("x-rapidapi-key", fmt.Sprintf("%s", resMatch))
+			req.Header.Add("x-rapidapi-key", resMatch)
 			res, err := client.Do(req)
 			if err == nil {
 				defer res.Body.Close()

pkg/detectors/rechargepayments/rechargepayments.go

@@ -16,7 +16,6 @@ type Scanner struct{}
 var _ detectors.Detector = (*Scanner)(nil)
 
 var (
-	client    = common.SaneHttpClient()
 	verifyURL = "https://api.rechargeapps.com/token_information"
 
 	// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.

pkg/detectors/scalewaykey/scalewaykey.go

@@ -2,7 +2,6 @@ package scalewaykey
 
 import (
 	"context"
-	"fmt"
 	"net/http"
 	"regexp"
 	"strings"
@@ -51,7 +50,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 			if err != nil {
 				continue
 			}
-			req.Header.Add("X-Auth-Token", fmt.Sprintf("%s", resMatch))
+			req.Header.Add("X-Auth-Token", resMatch)
 			res, err := client.Do(req)
 			if err == nil {
 				defer res.Body.Close()

pkg/detectors/sendinbluev2/sendinbluev2.go

@@ -2,7 +2,6 @@ package sendinbluev2
 
 import (
 	"context"
-	"fmt"
 	"net/http"
 	"regexp"
 	"strings"
@@ -51,7 +50,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 			if err != nil {
 				continue
 			}
-			req.Header.Add("api-key", fmt.Sprintf("%s", resMatch))
+			req.Header.Add("api-key", resMatch)
 			res, err := client.Do(req)
 			if err == nil {
 				defer res.Body.Close()

pkg/detectors/shortcut/shortcut.go

@@ -65,19 +65,11 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 
 				if strings.Contains(body, "app_url") {
 					s1.Verified = true
-				} else {
-					// if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
-					// 	continue
-					// }
 				}
-
 			}
-
 		}
-
 		results = append(results, s1)
 	}
-
 	return results, nil
 }
 

pkg/detectors/skrappio/skrappio.go

@@ -2,7 +2,6 @@ package skrappio
 
 import (
 	"context"
-	"fmt"
 	"net/http"
 	"regexp"
 	"strings"
@@ -53,7 +52,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 				continue
 			}
 			req.Header.Add("Content-Type", "application/json")
-			req.Header.Add("X-Access-Key", fmt.Sprintf("%s", resMatch))
+			req.Header.Add("X-Access-Key", resMatch)
 			res, err := client.Do(req)
 			if err == nil {
 				defer res.Body.Close()

pkg/detectors/slackwebhook/slackwebhook.go

@@ -21,7 +21,7 @@ var (
 	client = common.SaneHttpClient()
 
 	// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
-	keyPat = regexp.MustCompile(`(https://hooks\.slack\.com/services/[A-Za-z0-9+/]{44,46})`)
+	keyPat = regexp.MustCompile(`(https://hooks\.slack\.com/services/T[A-Z0-9]+/B[A-Z0-9]+/[A-Za-z0-9]{23,25})`)
 )
 
 // Keywords are used for efficiently pre-filtering chunks.
@@ -62,7 +62,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 					continue
 				}
 				body := string(bodyBytes)
-				if (res.StatusCode >= 200 && res.StatusCode < 300) || (res.StatusCode == 400 && strings.Contains(body, "text")) {
+				if (res.StatusCode >= 200 && res.StatusCode < 300) || (res.StatusCode == 400 && strings.Contains(body, "no_text")) {
 					s1.Verified = true
 				}
 			}

pkg/detectors/splunkobservabilitytoken/splunkobservabilitytoken.go

@@ -56,6 +56,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 			req.Header.Set("X-Sf-Token", resMatch)
 			res, err := client.Do(req)
 			if err == nil {
+				defer res.Body.Close()
 				if res.StatusCode >= 200 && res.StatusCode < 300 {
 					s1.Verified = true
 				} else {

pkg/detectors/stormglass/stormglass.go

@@ -2,7 +2,6 @@ package stormglass
 
 import (
 	"context"
-	"fmt"
 	"net/http"
 	"regexp"
 	"strings"
@@ -52,7 +51,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 			if err != nil {
 				continue
 			}
-			req.Header.Add("Authorization", fmt.Sprintf("%s", resMatch))
+			req.Header.Add("Authorization", resMatch)
 			res, err := client.Do(req)
 			if err == nil {
 				defer res.Body.Close()

pkg/detectors/tatumio/tatumio.go

@@ -2,7 +2,6 @@ package tatumio
 
 import (
 	"context"
-	"fmt"
 	"net/http"
 	"regexp"
 	"strings"
@@ -53,7 +52,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 				continue
 			}
 			req.Header.Add("Accept", "application/vnd.tatumio+json; version=3")
-			req.Header.Add("x-api-key", fmt.Sprintf("%s", resMatch))
+			req.Header.Add("x-api-key", resMatch)
 			res, err := client.Do(req)
 			if err == nil {
 				defer res.Body.Close()

pkg/detectors/uri/uri.go

@@ -26,10 +26,6 @@ var (
 	client = common.SaneHttpClient()
 )
 
-type proxyRes struct {
-	Verified bool `json:"verified"`
-}
-
 // Keywords are used for efficiently pre-filtering chunks.
 // Use identifiers in the secret preferably, or the provider name.
 func (s Scanner) Keywords() []string {

pkg/detectors/websitepulse/websitepulse.go

@@ -61,6 +61,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 				}
 				res, err := client.Do(req)
 				if err == nil {
+					defer res.Body.Close()
 					bodyBytes, err := io.ReadAll(res.Body)
 					if err != nil {
 						continue

pkg/detectors/zonkafeedback/zonkafeedback.go

@@ -2,7 +2,6 @@ package zonkafeedback
 
 import (
 	"context"
-	"fmt"
 	"net/http"
 	"regexp"
 	"strings"
@@ -52,7 +51,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 			if err != nil {
 				continue
 			}
-			req.Header.Add("Z-API-TOKEN", fmt.Sprintf("%s", resMatch))
+			req.Header.Add("Z-API-TOKEN", resMatch)
 			res, err := client.Do(req)
 			if err == nil {
 				defer res.Body.Close()

pkg/engine/defaults.go

@@ -442,6 +442,7 @@ import (
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/onesignal"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/onwaterio"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/oopspam"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/openai"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/opencagedata"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/opengraphr"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/openuv"
@@ -730,6 +731,13 @@ import (
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/zipcodebase"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/zonkafeedback"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/zulipchat"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/etherscan"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/infura"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/alchemy"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/blocknative"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/moralis"
+        "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/bscscan"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/coinmarketcap"
 )
 
 func DefaultDetectors() []detectors.Detector {
@@ -1166,6 +1174,7 @@ func DefaultDetectors() []detectors.Detector {
 		signupgenius.Scanner{},
 		streak.Scanner{},
 		route4me.Scanner{},
+		openai.Scanner{},
 		opencagedata.Scanner{},
 		positionstack.Scanner{},
 		upcdatabase.Scanner{},
@@ -1488,5 +1497,12 @@ func DefaultDetectors() []detectors.Detector {
 		ftp.Scanner{},
 		ldap.Scanner{},
 		shopify.Scanner{},
+		etherscan.Scanner{},
+		infura.Scanner{},
+		alchemy.Scanner{},
+		blocknative.Scanner{},
+		moralis.Scanner{},
+		bscscan.Scanner{},
+		coinmarketcap.Scanner{},
 	}
 }

pkg/engine/engine.go

@@ -9,6 +9,7 @@ import (
 	"sync/atomic"
 	"time"
 
+	ahocorasick "github.com/petar-dambovaliev/aho-corasick"
 	"google.golang.org/protobuf/proto"
 
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
@@ -36,6 +37,10 @@ type Engine struct {
 	// If there are multiple unverified results for the same chunk for the same detector,
 	// only the first one will be kept.
 	filterUnverified bool
+
+	// prefilter is a ahocorasick struct used for doing efficient string
+	// matching given a set of words (keywords from the rules in the config)
+	prefilter ahocorasick.AhoCorasick
 }
 
 type EngineOption func(*Engine)
@@ -73,6 +78,31 @@ func WithFilterUnverified(filter bool) EngineOption {
 	}
 }
 
+// WithFilterDetectors applies a filter to the configured list of detectors. If
+// the filterFunc returns true, the detector will be included for scanning.
+// This option applies to the existing list of detectors configured, so the
+// order this option appears matters. All filtering happens before scanning.
+func WithFilterDetectors(filterFunc func(detectors.Detector) bool) EngineOption {
+	return func(e *Engine) {
+		// If no detectors are configured, do nothing.
+		if e.detectors == nil {
+			return
+		}
+		e.detectors[true] = filterDetectors(filterFunc, e.detectors[true])
+		e.detectors[false] = filterDetectors(filterFunc, e.detectors[false])
+	}
+}
+
+func filterDetectors(filterFunc func(detectors.Detector) bool, input []detectors.Detector) []detectors.Detector {
+	var output []detectors.Detector
+	for _, detector := range input {
+		if filterFunc(detector) {
+			output = append(output, detector)
+		}
+	}
+	return output
+}
+
 func Start(ctx context.Context, options ...EngineOption) *Engine {
 	e := &Engine{
 		chunks:          make(chan *sources.Chunk),
@@ -85,7 +115,6 @@ func Start(ctx context.Context, options ...EngineOption) *Engine {
 	}
 
 	// Set defaults.
-
 	if e.concurrency == 0 {
 		numCPU := runtime.NumCPU()
 		ctx.Logger().Info("No concurrency specified, defaulting to max", "cpu", numCPU)
@@ -103,6 +132,23 @@ func Start(ctx context.Context, options ...EngineOption) *Engine {
 		e.detectors[false] = []detectors.Detector{}
 	}
 
+	// build ahocorasick prefilter for efficient string matching
+	// on keywords
+	keywords := []string{}
+	for _, d := range e.detectors[false] {
+		keywords = append(keywords, d.Keywords()...)
+	}
+	for _, d := range e.detectors[true] {
+		keywords = append(keywords, d.Keywords()...)
+	}
+	builder := ahocorasick.NewAhoCorasickBuilder(ahocorasick.Opts{
+		AsciiCaseInsensitive: true,
+		MatchOnlyWholeWords:  false,
+		MatchKind:            ahocorasick.LeftMostLongestMatch,
+		DFA:                  true,
+	})
+	e.prefilter = builder.Build(keywords)
+
 	ctx.Logger().V(2).Info("loaded decoders", "count", len(e.decoders))
 	ctx.Logger().V(2).Info("loaded detectors",
 		"total", len(e.detectors[true])+len(e.detectors[false]),
@@ -183,6 +229,7 @@ func (e *Engine) DetectorAvgTime() map[string][]time.Duration {
 func (e *Engine) detectorWorker(ctx context.Context) {
 	for originalChunk := range e.chunks {
 		for chunk := range sources.Chunker(originalChunk) {
+			matchedKeywords := make(map[string]struct{})
 			atomic.AddUint64(&e.bytesScanned, uint64(len(chunk.Data)))
 			for _, decoder := range e.decoders {
 				var decoderType detectorspb.DecoderType
@@ -199,21 +246,30 @@ func (e *Engine) detectorWorker(ctx context.Context) {
 				if decoded == nil {
 					continue
 				}
+
 				dataLower := strings.ToLower(string(decoded.Data))
+				matches := e.prefilter.FindAll(dataLower)
+
+				for _, m := range matches {
+					matchedKeywords[dataLower[m.Start():m.End()]] = struct{}{}
+				}
+
 				for verify, detectorsSet := range e.detectors {
 					for _, detector := range detectorsSet {
-						start := time.Now()
-						foundKeyword := false
+						chunkContainsKeyword := false
 						for _, kw := range detector.Keywords() {
-							if strings.Contains(dataLower, strings.ToLower(kw)) {
-								foundKeyword = true
+							if _, ok := matchedKeywords[strings.ToLower(kw)]; ok {
+								chunkContainsKeyword = true
 								break
 							}
 						}
-						if !foundKeyword {
+
+						if !chunkContainsKeyword {
 							continue
 						}
 
+						start := time.Now()
+
 						results, err := func() ([]detectors.Result, error) {
 							ctx, cancel := context.WithTimeout(ctx, time.Second*10)
 							defer cancel()

pkg/engine/filesystem.go

@@ -17,7 +17,7 @@ import (
 // ScanFileSystem scans a given file system.
 func (e *Engine) ScanFileSystem(ctx context.Context, c sources.FilesystemConfig) error {
 	connection := &sourcespb.Filesystem{
-		Directories: c.Directories,
+		Paths: c.Paths,
 	}
 	var conn anypb.Any
 	err := anypb.MarshalFrom(&conn, connection, proto.MarshalOptions{})

pkg/engine/gcs.go

@@ -0,0 +1,106 @@
+package engine
+
+import (
+	"fmt"
+
+	"google.golang.org/protobuf/proto"
+	"google.golang.org/protobuf/types/known/anypb"
+
+	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/context"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/sources/gcs"
+)
+
+// ScanGCS with the provided options.
+func (e *Engine) ScanGCS(ctx context.Context, c sources.GCSConfig) error {
+	// Project ID is required if using any authenticated access.
+	if c.ProjectID == "" && !c.WithoutAuth {
+		return fmt.Errorf("project ID is required")
+	}
+
+	// If using unauthenticated access, the project ID is not used.
+	if c.ProjectID != "" && c.WithoutAuth {
+		c.ProjectID = ""
+		ctx.Logger().Info("project ID is not used when using unauthenticated access, ignoring provided project ID")
+	}
+
+	connection := &sourcespb.GCS{
+		ProjectId:      c.ProjectID,
+		IncludeBuckets: c.IncludeBuckets,
+		ExcludeBuckets: c.ExcludeBuckets,
+		IncludeObjects: c.IncludeObjects,
+		ExcludeObjects: c.ExcludeObjects,
+	}
+
+	// Make sure only one auth method is selected.
+	if !isAuthValid(ctx, c, connection) {
+		return fmt.Errorf("multiple auth methods selected, please select only one")
+	}
+
+	var conn anypb.Any
+	err := anypb.MarshalFrom(&conn, connection, proto.MarshalOptions{})
+	if err != nil {
+		return fmt.Errorf("failed to marshal GCS connection: %w", err)
+	}
+
+	source := gcs.Source{}
+	ctx = context.WithValues(ctx,
+		"source_type", source.Type().String(),
+		"source_name", "gcs",
+	)
+	if err = source.Init(ctx, "trufflehog - GCS", 0, 0, true, &conn, int(c.Concurrency)); err != nil {
+		return fmt.Errorf("failed to initialize GCS source: %w", err)
+	}
+
+	e.sourcesWg.Add(1)
+	go func() {
+		defer common.RecoverWithExit(ctx)
+		defer e.sourcesWg.Done()
+		if err := source.Chunks(ctx, e.ChunksChan()); err != nil {
+			ctx.Logger().Error(err, "could not scan GCS")
+		}
+	}()
+	return nil
+}
+
+func isAuthValid(ctx context.Context, c sources.GCSConfig, connection *sourcespb.GCS) bool {
+	var isAuthSelected bool
+
+	if c.WithoutAuth {
+		isAuthSelected = true
+		connection.Credential = &sourcespb.GCS_Unauthenticated{}
+	}
+	if c.CloudCred {
+		if isAuthSelected {
+			return false
+		}
+		isAuthSelected = true
+		connection.Credential = &sourcespb.GCS_Adc{}
+	}
+	if c.ServiceAccount != "" {
+		if isAuthSelected {
+			return false
+		}
+		isAuthSelected = true
+		connection.Credential = &sourcespb.GCS_ServiceAccountFile{
+			ServiceAccountFile: c.ServiceAccount,
+		}
+	}
+	if c.ApiKey != "" {
+		if isAuthSelected {
+			return false
+		}
+		isAuthSelected = true
+		connection.Credential = &sourcespb.GCS_ApiKey{
+			ApiKey: c.ApiKey,
+		}
+	}
+	if !isAuthSelected {
+		ctx.Logger().Info("no auth method selected, using unauthenticated")
+		connection.Credential = &sourcespb.GCS_Unauthenticated{}
+	}
+
+	return true
+}

pkg/engine/gcs_test.go

@@ -0,0 +1,70 @@
+package engine
+
+import (
+	"strings"
+	"testing"
+
+	"github.com/trufflesecurity/trufflehog/v3/pkg/context"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
+)
+
+func TestScanGCS(t *testing.T) {
+	tests := []struct {
+		name      string
+		gcsConfig sources.GCSConfig
+		wantErr   bool
+	}{
+		{
+			name: "scanned GCS",
+			gcsConfig: sources.GCSConfig{
+				ApiKey:         "abc123",
+				ProjectID:      "test-project",
+				CloudCred:      false,
+				WithoutAuth:    false,
+				ServiceAccount: "",
+			},
+		},
+		{
+			name:      "missing project ID, with auth",
+			gcsConfig: sources.GCSConfig{ApiKey: "abc123"},
+			wantErr:   true,
+		},
+		{
+			name:      "missing project ID, without auth, public scan",
+			gcsConfig: sources.GCSConfig{WithoutAuth: true},
+		},
+		{
+			name: "multiple selected auth methods",
+			gcsConfig: sources.GCSConfig{
+				ApiKey:         "abc123",
+				ProjectID:      "test-project",
+				CloudCred:      true,
+				WithoutAuth:    false,
+				ServiceAccount: "",
+			},
+			wantErr: true,
+		},
+		{
+			name: "no auth method selected",
+			gcsConfig: sources.GCSConfig{
+				ProjectID:     "test-project",
+				MaxObjectSize: 10 * 1024 * 1024,
+			},
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			e := &Engine{}
+			err := e.ScanGCS(context.Background(), test.gcsConfig)
+			if err != nil && !test.wantErr && !strings.Contains(err.Error(), "googleapi: Error 400: Bad Request") {
+				t.Errorf("ScanGCS() got: %v, want: %v", err, nil)
+				return
+			}
+
+			if err == nil && test.wantErr {
+				t.Errorf("ScanGCS() got: %v, want: %v", err, "error")
+			}
+		})
+	}
+}

pkg/gitparse/gitparse.go

@@ -139,7 +139,8 @@ func (c *Parser) RepoPath(ctx context.Context, source string, head string, abbre
 
 // Unstaged parses the output of the `git diff` command for the `source` path.
 func (c *Parser) Unstaged(ctx context.Context, source string) (chan Commit, error) {
-	args := []string{"-C", source, "diff", "-p", "-U5", "--full-history", "--diff-filter=AM", "--date=format:%a %b %d %H:%M:%S %Y %z", "HEAD"}
+	// Provide the --cached flag to diff to get the diff of the staged changes.
+	args := []string{"-C", source, "diff", "-p", "-U5", "--cached", "--full-history", "--diff-filter=AM", "--date=format:%a %b %d %H:%M:%S %Y %z", "HEAD"}
 
 	cmd := exec.Command("git", args...)
 
@@ -241,14 +242,17 @@ func (c *Parser) fromReader(ctx context.Context, stdOut io.Reader, commitChan ch
 					totalSize += diff.Content.Len()
 				}
 				if totalSize > c.maxCommitSize {
+					oldCommit := currentCommit
 					commitChan <- *currentCommit
 					currentCommit = &Commit{
 						Hash:    currentCommit.Hash,
 						Author:  currentCommit.Author,
 						Date:    currentCommit.Date,
-						Message: currentCommit.Message,
+						Message: strings.Builder{},
 						Diffs:   []Diff{},
 					}
+					// Message needs to be recreated here otherwise writing to it again will result in a panic.
+					currentCommit.Message.WriteString(oldCommit.Message.String())
 				}
 			}
 			currentDiff = &Diff{}

pkg/pb/detectorspb/detectors.pb.go

@@ -973,6 +973,13 @@ const (
 	DetectorType_Shopify                       DetectorType = 902
 	DetectorType_RabbitMQ                      DetectorType = 903
 	DetectorType_CustomRegex                   DetectorType = 904
+	DetectorType_Etherscan                     DetectorType = 905
+	DetectorType_Infura                        DetectorType = 906
+	DetectorType_Alchemy                       DetectorType = 907
+	DetectorType_BlockNative                   DetectorType = 908
+	DetectorType_Moralis                       DetectorType = 909
+	DetectorType_BscScan                       DetectorType = 910
+	DetectorType_CoinMarketCap                 DetectorType = 911
 )
 
 // Enum value maps for DetectorType.
@@ -1879,6 +1886,13 @@ var (
 		902: "Shopify",
 		903: "RabbitMQ",
 		904: "CustomRegex",
+		905: "Etherscan",
+		906: "Infura",
+		907: "Alchemy",
+		908: "BlockNative",
+		909: "Moralis",
+		910: "BscScan",
+		911: "CoinMarketCap",
 	}
 	DetectorType_value = map[string]int32{
 		"Alibaba":                       0,
@@ -2782,6 +2796,13 @@ var (
 		"Shopify":                       902,
 		"RabbitMQ":                      903,
 		"CustomRegex":                   904,
+		"Etherscan":                     905,
+		"Infura":                        906,
+		"Alchemy":                       907,
+		"BlockNative":                   908,
+		"Moralis":                       909,
+		"BscScan":                       910,
+		"CoinMarketCap":                 911,
 	}
 )
 
@@ -3146,7 +3167,7 @@ var file_detectors_proto_rawDesc = []byte{
 	0x0a, 0x0b, 0x44, 0x65, 0x63, 0x6f, 0x64, 0x65, 0x72, 0x54, 0x79, 0x70, 0x65, 0x12, 0x0b, 0x0a,
 	0x07, 0x55, 0x4e, 0x4b, 0x4e, 0x4f, 0x57, 0x4e, 0x10, 0x00, 0x12, 0x09, 0x0a, 0x05, 0x50, 0x4c,
 	0x41, 0x49, 0x4e, 0x10, 0x01, 0x12, 0x0a, 0x0a, 0x06, 0x42, 0x41, 0x53, 0x45, 0x36, 0x34, 0x10,
-	0x02, 0x2a, 0x91, 0x71, 0x0a, 0x0c, 0x44, 0x65, 0x74, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x54, 0x79,
+	0x02, 0x2a, 0xfe, 0x71, 0x0a, 0x0c, 0x44, 0x65, 0x74, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x54, 0x79,
 	0x70, 0x65, 0x12, 0x0b, 0x0a, 0x07, 0x41, 0x6c, 0x69, 0x62, 0x61, 0x62, 0x61, 0x10, 0x00, 0x12,
 	0x08, 0x0a, 0x04, 0x41, 0x4d, 0x51, 0x50, 0x10, 0x01, 0x12, 0x07, 0x0a, 0x03, 0x41, 0x57, 0x53,
 	0x10, 0x02, 0x12, 0x09, 0x0a, 0x05, 0x41, 0x7a, 0x75, 0x72, 0x65, 0x10, 0x03, 0x12, 0x0a, 0x0a,
@@ -4051,11 +4072,18 @@ var file_detectors_proto_rawDesc = []byte{
 	0x44, 0x41, 0x50, 0x10, 0x85, 0x07, 0x12, 0x0c, 0x0a, 0x07, 0x53, 0x68, 0x6f, 0x70, 0x69, 0x66,
 	0x79, 0x10, 0x86, 0x07, 0x12, 0x0d, 0x0a, 0x08, 0x52, 0x61, 0x62, 0x62, 0x69, 0x74, 0x4d, 0x51,
 	0x10, 0x87, 0x07, 0x12, 0x10, 0x0a, 0x0b, 0x43, 0x75, 0x73, 0x74, 0x6f, 0x6d, 0x52, 0x65, 0x67,
-	0x65, 0x78, 0x10, 0x88, 0x07, 0x42, 0x3d, 0x5a, 0x3b, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e,
-	0x63, 0x6f, 0x6d, 0x2f, 0x74, 0x72, 0x75, 0x66, 0x66, 0x6c, 0x65, 0x73, 0x65, 0x63, 0x75, 0x72,
-	0x69, 0x74, 0x79, 0x2f, 0x74, 0x72, 0x75, 0x66, 0x66, 0x6c, 0x65, 0x68, 0x6f, 0x67, 0x2f, 0x76,
-	0x33, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x70, 0x62, 0x2f, 0x64, 0x65, 0x74, 0x65, 0x63, 0x74, 0x6f,
-	0x72, 0x73, 0x70, 0x62, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
+	0x65, 0x78, 0x10, 0x88, 0x07, 0x12, 0x0e, 0x0a, 0x09, 0x45, 0x74, 0x68, 0x65, 0x72, 0x73, 0x63,
+	0x61, 0x6e, 0x10, 0x89, 0x07, 0x12, 0x0b, 0x0a, 0x06, 0x49, 0x6e, 0x66, 0x75, 0x72, 0x61, 0x10,
+	0x8a, 0x07, 0x12, 0x0c, 0x0a, 0x07, 0x41, 0x6c, 0x63, 0x68, 0x65, 0x6d, 0x79, 0x10, 0x8b, 0x07,
+	0x12, 0x10, 0x0a, 0x0b, 0x42, 0x6c, 0x6f, 0x63, 0x6b, 0x4e, 0x61, 0x74, 0x69, 0x76, 0x65, 0x10,
+	0x8c, 0x07, 0x12, 0x0c, 0x0a, 0x07, 0x4d, 0x6f, 0x72, 0x61, 0x6c, 0x69, 0x73, 0x10, 0x8d, 0x07,
+	0x12, 0x0c, 0x0a, 0x07, 0x42, 0x73, 0x63, 0x53, 0x63, 0x61, 0x6e, 0x10, 0x8e, 0x07, 0x12, 0x12,
+	0x0a, 0x0d, 0x43, 0x6f, 0x69, 0x6e, 0x4d, 0x61, 0x72, 0x6b, 0x65, 0x74, 0x43, 0x61, 0x70, 0x10,
+	0x8f, 0x07, 0x42, 0x3d, 0x5a, 0x3b, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d,
+	0x2f, 0x74, 0x72, 0x75, 0x66, 0x66, 0x6c, 0x65, 0x73, 0x65, 0x63, 0x75, 0x72, 0x69, 0x74, 0x79,
+	0x2f, 0x74, 0x72, 0x75, 0x66, 0x66, 0x6c, 0x65, 0x68, 0x6f, 0x67, 0x2f, 0x76, 0x33, 0x2f, 0x70,
+	0x6b, 0x67, 0x2f, 0x70, 0x62, 0x2f, 0x64, 0x65, 0x74, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x73, 0x70,
+	0x62, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
 }
 
 var (

pkg/pb/source_metadatapb/source_metadata.pb.validate.go

@@ -1267,6 +1267,117 @@ var _ interface {
 	ErrorName() string
 } = GitlabValidationError{}
 
+// Validate checks the field values on GoogleDrive with the rules defined in
+// the proto definition for this message. If any rules are violated, the first
+// error encountered is returned, or nil if there are no violations.
+func (m *GoogleDrive) Validate() error {
+	return m.validate(false)
+}
+
+// ValidateAll checks the field values on GoogleDrive with the rules defined in
+// the proto definition for this message. If any rules are violated, the
+// result is a list of violation errors wrapped in GoogleDriveMultiError, or
+// nil if none found.
+func (m *GoogleDrive) ValidateAll() error {
+	return m.validate(true)
+}
+
+func (m *GoogleDrive) validate(all bool) error {
+	if m == nil {
+		return nil
+	}
+
+	var errors []error
+
+	// no validation rules for File
+
+	// no validation rules for Link
+
+	// no validation rules for Email
+
+	// no validation rules for Timestamp
+
+	// no validation rules for Shared
+
+	// no validation rules for LastModifiedBy
+
+	if len(errors) > 0 {
+		return GoogleDriveMultiError(errors)
+	}
+
+	return nil
+}
+
+// GoogleDriveMultiError is an error wrapping multiple validation errors
+// returned by GoogleDrive.ValidateAll() if the designated constraints aren't met.
+type GoogleDriveMultiError []error
+
+// Error returns a concatenation of all the error messages it wraps.
+func (m GoogleDriveMultiError) Error() string {
+	var msgs []string
+	for _, err := range m {
+		msgs = append(msgs, err.Error())
+	}
+	return strings.Join(msgs, "; ")
+}
+
+// AllErrors returns a list of validation violation errors.
+func (m GoogleDriveMultiError) AllErrors() []error { return m }
+
+// GoogleDriveValidationError is the validation error returned by
+// GoogleDrive.Validate if the designated constraints aren't met.
+type GoogleDriveValidationError struct {
+	field  string
+	reason string
+	cause  error
+	key    bool
+}
+
+// Field function returns field value.
+func (e GoogleDriveValidationError) Field() string { return e.field }
+
+// Reason function returns reason value.
+func (e GoogleDriveValidationError) Reason() string { return e.reason }
+
+// Cause function returns cause value.
+func (e GoogleDriveValidationError) Cause() error { return e.cause }
+
+// Key function returns key value.
+func (e GoogleDriveValidationError) Key() bool { return e.key }
+
+// ErrorName returns error name.
+func (e GoogleDriveValidationError) ErrorName() string { return "GoogleDriveValidationError" }
+
+// Error satisfies the builtin error interface
+func (e GoogleDriveValidationError) Error() string {
+	cause := ""
+	if e.cause != nil {
+		cause = fmt.Sprintf(" | caused by: %v", e.cause)
+	}
+
+	key := ""
+	if e.key {
+		key = "key for "
+	}
+
+	return fmt.Sprintf(
+		"invalid %sGoogleDrive.%s: %s%s",
+		key,
+		e.field,
+		e.reason,
+		cause)
+}
+
+var _ error = GoogleDriveValidationError{}
+
+var _ interface {
+	Field() string
+	Reason() string
+	Key() bool
+	Cause() error
+	ErrorName() string
+} = GoogleDriveValidationError{}
+
 // Validate checks the field values on GCS with the rules defined in the proto
 // definition for this message. If any rules are violated, the first error
 // encountered is returned, or nil if there are no violations.
@@ -1290,12 +1401,18 @@ func (m *GCS) validate(all bool) error {
 
 	// no validation rules for Bucket
 
-	// no validation rules for File
+	// no validation rules for Filename
 
 	// no validation rules for Link
 
 	// no validation rules for Email
 
+	// no validation rules for CreatedAt
+
+	// no validation rules for UpdatedAt
+
+	// no validation rules for ContentType
+
 	if len(errors) > 0 {
 		return GCSMultiError(errors)
 	}

pkg/pb/sourcespb/sources.pb.validate.go

@@ -1242,10 +1242,82 @@ func (m *GCS) validate(all bool) error {
 
 	var errors []error
 
+	// no validation rules for ProjectId
+
+	// no validation rules for MaxObjectSize
+
 	switch m.Credential.(type) {
 
-	case *GCS_JsonSa:
-		// no validation rules for JsonSa
+	case *GCS_JsonServiceAccount:
+		// no validation rules for JsonServiceAccount
+
+	case *GCS_ApiKey:
+		// no validation rules for ApiKey
+
+	case *GCS_Unauthenticated:
+
+		if all {
+			switch v := interface{}(m.GetUnauthenticated()).(type) {
+			case interface{ ValidateAll() error }:
+				if err := v.ValidateAll(); err != nil {
+					errors = append(errors, GCSValidationError{
+						field:  "Unauthenticated",
+						reason: "embedded message failed validation",
+						cause:  err,
+					})
+				}
+			case interface{ Validate() error }:
+				if err := v.Validate(); err != nil {
+					errors = append(errors, GCSValidationError{
+						field:  "Unauthenticated",
+						reason: "embedded message failed validation",
+						cause:  err,
+					})
+				}
+			}
+		} else if v, ok := interface{}(m.GetUnauthenticated()).(interface{ Validate() error }); ok {
+			if err := v.Validate(); err != nil {
+				return GCSValidationError{
+					field:  "Unauthenticated",
+					reason: "embedded message failed validation",
+					cause:  err,
+				}
+			}
+		}
+
+	case *GCS_Adc:
+
+		if all {
+			switch v := interface{}(m.GetAdc()).(type) {
+			case interface{ ValidateAll() error }:
+				if err := v.ValidateAll(); err != nil {
+					errors = append(errors, GCSValidationError{
+						field:  "Adc",
+						reason: "embedded message failed validation",
+						cause:  err,
+					})
+				}
+			case interface{ Validate() error }:
+				if err := v.Validate(); err != nil {
+					errors = append(errors, GCSValidationError{
+						field:  "Adc",
+						reason: "embedded message failed validation",
+						cause:  err,
+					})
+				}
+			}
+		} else if v, ok := interface{}(m.GetAdc()).(interface{ Validate() error }); ok {
+			if err := v.Validate(); err != nil {
+				return GCSValidationError{
+					field:  "Adc",
+					reason: "embedded message failed validation",
+					cause:  err,
+				}
+			}
+		}
+
+	case *GCS_ServiceAccountFile:
+		// no validation rules for ServiceAccountFile
 
 	}
 
@@ -1887,6 +1959,112 @@ var _ interface {
 	ErrorName() string
 } = GitHubValidationError{}
 
+// Validate checks the field values on GoogleDrive with the rules defined in
+// the proto definition for this message. If any rules are violated, the first
+// error encountered is returned, or nil if there are no violations.
+func (m *GoogleDrive) Validate() error {
+	return m.validate(false)
+}
+
+// ValidateAll checks the field values on GoogleDrive with the rules defined in
+// the proto definition for this message. If any rules are violated, the
+// result is a list of violation errors wrapped in GoogleDriveMultiError, or
+// nil if none found.
+func (m *GoogleDrive) ValidateAll() error {
+	return m.validate(true)
+}
+
+func (m *GoogleDrive) validate(all bool) error {
+	if m == nil {
+		return nil
+	}
+
+	var errors []error
+
+	switch m.Credential.(type) {
+
+	case *GoogleDrive_RefreshToken:
+		// no validation rules for RefreshToken
+
+	}
+
+	if len(errors) > 0 {
+		return GoogleDriveMultiError(errors)
+	}
+
+	return nil
+}
+
+// GoogleDriveMultiError is an error wrapping multiple validation errors
+// returned by GoogleDrive.ValidateAll() if the designated constraints aren't met.
+type GoogleDriveMultiError []error
+
+// Error returns a concatenation of all the error messages it wraps.
+func (m GoogleDriveMultiError) Error() string {
+	var msgs []string
+	for _, err := range m {
+		msgs = append(msgs, err.Error())
+	}
+	return strings.Join(msgs, "; ")
+}
+
+// AllErrors returns a list of validation violation errors.
+func (m GoogleDriveMultiError) AllErrors() []error { return m }
+
+// GoogleDriveValidationError is the validation error returned by
+// GoogleDrive.Validate if the designated constraints aren't met.
+type GoogleDriveValidationError struct {
+	field  string
+	reason string
+	cause  error
+	key    bool
+}
+
+// Field function returns field value.
+func (e GoogleDriveValidationError) Field() string { return e.field }
+
+// Reason function returns reason value.
+func (e GoogleDriveValidationError) Reason() string { return e.reason }
+
+// Cause function returns cause value.
+func (e GoogleDriveValidationError) Cause() error { return e.cause }
+
+// Key function returns key value.
+func (e GoogleDriveValidationError) Key() bool { return e.key }
+
+// ErrorName returns error name.
+func (e GoogleDriveValidationError) ErrorName() string { return "GoogleDriveValidationError" }
+
+// Error satisfies the builtin error interface
+func (e GoogleDriveValidationError) Error() string {
+	cause := ""
+	if e.cause != nil {
+		cause = fmt.Sprintf(" | caused by: %v", e.cause)
+	}
+
+	key := ""
+	if e.key {
+		key = "key for "
+	}
+
+	return fmt.Sprintf(
+		"invalid %sGoogleDrive.%s: %s%s",
+		key,
+		e.field,
+		e.reason,
+		cause)
+}
+
+var _ error = GoogleDriveValidationError{}
+
+var _ interface {
+	Field() string
+	Reason() string
+	Key() bool
+	Cause() error
+	ErrorName() string
+} = GoogleDriveValidationError{}
+
 // Validate checks the field values on JIRA with the rules defined in the proto
 // definition for this message. If any rules are violated, the first error
 // encountered is returned, or nil if there are no violations.

pkg/sources/errors.go

@@ -1,6 +1,7 @@
 package sources
 
 import (
+	"fmt"
 	"sync"
 )
 
@@ -29,3 +30,7 @@ func (s *ScanErrors) Count() uint64 {
 	defer s.mu.RUnlock()
 	return uint64(len(s.errors))
 }
+
+func (s *ScanErrors) String() string {
+	return fmt.Sprintf("%v", s.errors)
+}

pkg/sources/errors_test.go

@@ -1,6 +1,7 @@
 package sources
 
 import (
+	"fmt"
 	"reflect"
 	"sync"
 	"testing"
@@ -147,3 +148,12 @@ func TestScanErrorsCount(t *testing.T) {
 		})
 	}
 }
+
+func TestScanErrorsString(t *testing.T) {
+	se := NewScanErrors()
+	se.Add(nil)
+	want := "[<nil>]"
+	if got := fmt.Sprintf("%v", se); got != want {
+		t.Errorf("got %q, want %q", got, want)
+	}
+}

pkg/sources/filesystem/filesystem.go

@@ -70,8 +70,7 @@ func (s *Source) Init(aCtx context.Context, name string, jobId, sourceId int64,
 	if err := anypb.UnmarshalTo(connection, &conn, proto.UnmarshalOptions{}); err != nil {
 		return errors.WrapPrefix(err, "error unmarshalling connection", 0)
 	}
-
-	s.paths = conn.Directories
+	s.paths = append(conn.Paths, conn.Directories...)
 
 	return nil
 }
@@ -83,100 +82,122 @@ func (s *Source) WithFilter(filter *common.Filter) {
 // Chunks emits chunks of bytes over a channel.
 func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) error {
 	for i, path := range s.paths {
+		logger := ctx.Logger().WithValues("path", path)
+		if common.IsDone(ctx) {
+			return nil
+		}
 		s.SetProgressComplete(i, len(s.paths), fmt.Sprintf("Path: %s", path), "")
 
 		cleanPath := filepath.Clean(path)
-		done := false
-		go func() {
-			<-ctx.Done()
-			done = true
-		}()
+		fileInfo, err := os.Stat(cleanPath)
+		if err != nil {
+			logger.Error(err, "unable to get file info")
+			continue
+		}
 
-		err := fs.WalkDir(os.DirFS(cleanPath), ".", func(relativePath string, d fs.DirEntry, err error) error {
-			if err != nil {
-				return nil
-			}
-
-			path := filepath.Join(cleanPath, relativePath)
-			logger := ctx.Logger().WithValues("file", path)
-
-			fileStat, err := os.Stat(path)
-			if err != nil {
-				logger.Error(err, "unable to stat file")
-				return nil
-			}
-			if !fileStat.Mode().IsRegular() {
-				return nil
-			}
-
-			if s.filter != nil && !s.filter.Pass(path) {
-				return nil
-			}
-
-			inputFile, err := os.Open(path)
-			if err != nil {
-				logger.V(2).Info("error opening file", "error", err)
-				return nil
-			}
-			defer inputFile.Close()
-			logger.V(5).Info("scanning file")
-
-			reReader, err := diskbufferreader.New(inputFile)
-			if err != nil {
-				logger.Error(err, "Could not create re-readable reader.")
-			}
-			defer reReader.Close()
-
-			chunkSkel := &sources.Chunk{
-				SourceType: s.Type(),
-				SourceName: s.name,
-				SourceID:   s.SourceID(),
-				SourceMetadata: &source_metadatapb.MetaData{
-					Data: &source_metadatapb.MetaData_Filesystem{
-						Filesystem: &source_metadatapb.Filesystem{
-							File: sanitizer.UTF8(path),
-						},
-					},
-				},
-				Verify: s.verify,
-			}
-			if handlers.HandleFile(ctx, reReader, chunkSkel, chunksChan) {
-				return nil
-			}
-
-			if err := reReader.Reset(); err != nil {
-				return err
-			}
-			reReader.Stop()
-			data, err := io.ReadAll(reReader)
-			if err != nil {
-				return err
-			}
-			chunksChan <- &sources.Chunk{
-				SourceType: s.Type(),
-				SourceName: s.name,
-				SourceID:   s.SourceID(),
-				Data:       data,
-				SourceMetadata: &source_metadatapb.MetaData{
-					Data: &source_metadatapb.MetaData_Filesystem{
-						Filesystem: &source_metadatapb.Filesystem{
-							File: sanitizer.UTF8(path),
-						},
-					},
-				},
-				Verify: s.verify,
-			}
-			return nil
-		})
+		if fileInfo.IsDir() {
+			err = s.scanDir(ctx, cleanPath, chunksChan)
+		} else {
+			err = s.scanFile(ctx, cleanPath, chunksChan)
+		}
 
 		if err != nil && err != io.EOF {
-			return errors.New(err)
+			logger.Info("error scanning filesystem", "error", err)
 		}
-
-		if done {
-			return nil
-		}
-
+	}
+	return nil
+}
+
+func (s *Source) scanDir(ctx context.Context, path string, chunksChan chan *sources.Chunk) error {
+	return fs.WalkDir(os.DirFS(path), ".", func(relativePath string, d fs.DirEntry, err error) error {
+		if err != nil {
+			return nil
+		}
+		fullPath := filepath.Join(path, relativePath)
+
+		// Skip over non-regular files. We do this check here to suppress noisy
+		// logs for trying to scan directories and other non-regular files in
+		// our traversal.
+		fileStat, err := os.Stat(fullPath)
+		if err != nil {
+			ctx.Logger().Info("unable to stat file", "path", fullPath, "error", err)
+			return nil
+		}
+		if !fileStat.Mode().IsRegular() {
+			return nil
+		}
+		if s.filter != nil && !s.filter.Pass(fullPath) {
+			return nil
+		}
+
+		if err = s.scanFile(ctx, fullPath, chunksChan); err != nil {
+			ctx.Logger().Info("error scanning file", "path", fullPath, "error", err)
+		}
+		return nil
+	})
+}
+
+func (s *Source) scanFile(ctx context.Context, path string, chunksChan chan *sources.Chunk) error {
+	logger := ctx.Logger().WithValues("path", path)
+	fileStat, err := os.Stat(path)
+	if err != nil {
+		return fmt.Errorf("unable to stat file: %w", err)
+	}
+	if !fileStat.Mode().IsRegular() {
+		return fmt.Errorf("not a regular file")
+	}
+
+	inputFile, err := os.Open(path)
+	if err != nil {
+		return fmt.Errorf("unable to open file: %w", err)
+	}
+	defer inputFile.Close()
+	logger.V(3).Info("scanning file")
+
+	reReader, err := diskbufferreader.New(inputFile)
+	if err != nil {
+		return fmt.Errorf("could not create re-readable reader: %w", err)
+	}
+	defer reReader.Close()
+
+	chunkSkel := &sources.Chunk{
+		SourceType: s.Type(),
+		SourceName: s.name,
+		SourceID:   s.SourceID(),
+		SourceMetadata: &source_metadatapb.MetaData{
+			Data: &source_metadatapb.MetaData_Filesystem{
+				Filesystem: &source_metadatapb.Filesystem{
+					File: sanitizer.UTF8(path),
+				},
+			},
+		},
+		Verify: s.verify,
+	}
+	if handlers.HandleFile(ctx, reReader, chunkSkel, chunksChan) {
+		return nil
+	}
+
+	if err := reReader.Reset(); err != nil {
+		return err
+	}
+	reReader.Stop()
+	data, err := io.ReadAll(reReader)
+	if err != nil {
+		return fmt.Errorf("unable to read file: %w", err)
+	}
+	chunksChan <- &sources.Chunk{
+		SourceType: s.Type(),
+		SourceName: s.name,
+		SourceID:   s.SourceID(),
+		Data:       data,
+		SourceMetadata: &source_metadatapb.MetaData{
+			Data: &source_metadatapb.MetaData_Filesystem{
+				Filesystem: &source_metadatapb.Filesystem{
+					File: sanitizer.UTF8(path),
+				},
+			},
+		},
+		Verify: s.verify,
 	}
 	return nil
 }

pkg/sources/filesystem/filesystem_test.go

@@ -33,7 +33,7 @@ func TestSource_Scan(t *testing.T) {
 			init: init{
 				name: "this repo",
 				connection: &sourcespb.Filesystem{
-					Directories: []string{"."},
+					Paths: []string{"."},
 				},
 				verify: true,
 			},

pkg/sources/gcs/gcs.go

@@ -0,0 +1,452 @@
+package gcs
+
+import (
+	"encoding/json"
+	"fmt"
+	"io"
+	"os"
+	"sort"
+	"sync"
+
+	diskbufferreader "github.com/bill-rich/disk-buffer-reader"
+	"github.com/go-errors/errors"
+	"github.com/go-logr/logr"
+	"google.golang.org/protobuf/proto"
+	"google.golang.org/protobuf/types/known/anypb"
+
+	"github.com/trufflesecurity/trufflehog/v3/pkg/context"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/handlers"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/source_metadatapb"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
+)
+
+// Source represents a GCS source.
+type Source struct {
+	name        string
+	jobId       int64
+	sourceId    int64
+	concurrency int
+	verify      bool
+
+	gcsManager objectManager
+	stats      *attributes
+	log        logr.Logger
+	chunksCh   chan *sources.Chunk
+
+	mu               sync.Mutex
+	sources.Progress // not safe for concurrent access.
+}
+
+// Ensure the Source satisfies the interface at compile time.
+var _ sources.Source = (*Source)(nil)
+
+// Type returns the type of source.
+// It is used for matching source types in configuration and job input.
+func (s *Source) Type() sourcespb.SourceType {
+	return sourcespb.SourceType_SOURCE_TYPE_GCS
+}
+
+// SourceID number for GCS Source.
+func (s *Source) SourceID() int64 {
+	return s.sourceId
+}
+
+// JobID number for GCS Source.
+func (s *Source) JobID() int64 {
+	return s.jobId
+}
+
+// Init returns an initialized GCS source.
+func (s *Source) Init(aCtx context.Context, name string, id int64, sourceID int64, verify bool, connection *anypb.Any, concurrency int) error {
+	s.log = aCtx.Logger()
+
+	s.name = name
+	s.verify = verify
+	s.sourceId = sourceID
+	s.jobId = id
+	s.concurrency = concurrency
+
+	var conn sourcespb.GCS
+	err := anypb.UnmarshalTo(connection, &conn, proto.UnmarshalOptions{})
+	if err != nil {
+		return errors.WrapPrefix(err, "error unmarshalling connection", 0)
+	}
+
+	var gcsManagerAuthOption gcsManagerOption
+
+	switch conn.Credential.(type) {
+	case *sourcespb.GCS_ApiKey:
+		gcsManagerAuthOption = withAPIKey(aCtx, conn.GetApiKey())
+	case *sourcespb.GCS_ServiceAccountFile:
+		b, err := os.ReadFile(conn.GetServiceAccountFile())
+		if err != nil {
+			return fmt.Errorf("error reading GCS JSON Service Account file: %w", err)
+		}
+		gcsManagerAuthOption = withJSONServiceAccount(aCtx, b)
+	case *sourcespb.GCS_JsonServiceAccount:
+		gcsManagerAuthOption = withJSONServiceAccount(aCtx, []byte(conn.GetJsonServiceAccount()))
+	case *sourcespb.GCS_Adc:
+		gcsManagerAuthOption = withDefaultADC(aCtx)
+	case *sourcespb.GCS_Unauthenticated:
+		gcsManagerAuthOption = withoutAuthentication()
+	default:
+		return fmt.Errorf("unknown GCS authentication type: %T", conn.Credential)
+
+	}
+
+	resume, err := setResumeBucketOffset(s.Progress.EncodedResumeInfo)
+	if err != nil {
+		return fmt.Errorf("error setting resume info: %w", err)
+	}
+
+	gcsManagerOpts := []gcsManagerOption{
+		withConcurrency(concurrency),
+		withBucketOffsets(resume),
+		withMaxObjectSize(conn.MaxObjectSize),
+		gcsManagerAuthOption,
+	}
+	if setGCSManagerBucketOptions(&conn) != nil {
+		gcsManagerOpts = append(gcsManagerOpts, setGCSManagerBucketOptions(&conn))
+	}
+	if setGCSManagerObjectOptions(&conn) != nil {
+		gcsManagerOpts = append(gcsManagerOpts, setGCSManagerObjectOptions(&conn))
+	}
+
+	if s.gcsManager, err = newGCSManager(conn.ProjectId, gcsManagerOpts...); err != nil {
+		return fmt.Errorf("error creating GCS manager: %w", err)
+	}
+
+	if len(resume) > 0 {
+		s.log.V(2).Info("resuming from previous run")
+		return nil
+	}
+
+	s.log.V(2).Info("new run, enumerating buckets and objects")
+	return s.enumerate(aCtx)
+}
+
+func setGCSManagerBucketOptions(conn *sourcespb.GCS) gcsManagerOption {
+	return setGCSManagerOptions(conn.GetIncludeBuckets(), conn.GetExcludeBuckets(), withIncludeBuckets, withExcludeBuckets)
+}
+
+func setGCSManagerObjectOptions(conn *sourcespb.GCS) gcsManagerOption {
+	return setGCSManagerOptions(conn.GetIncludeObjects(), conn.GetExcludeObjects(), withIncludeObjects, withExcludeObjects)
+}
+
+func setGCSManagerOptions(include, exclude []string, includeFn, excludeFn func([]string) gcsManagerOption) gcsManagerOption {
+	// Only allow one of include/exclude to be set.
+	// If both are set, include takes precedence.
+	if len(include) > 0 {
+		return includeFn(include)
+	}
+	if len(exclude) > 0 {
+		return excludeFn(exclude)
+	}
+
+	return nil
+}
+
+type progressInfo struct {
+	processedBucketCnt,
+	totalBucketsCnt,
+	processedObjectsCnt,
+	totalObjectsCnt int
+
+	// bucketObjects tracks processing and processed objects for each bucket.
+	bucketObjects map[string]*objectsProgress
+}
+
+// objectsProgress keeps track of the progress of processing objects in a bucket.
+// It is marshalled/unmarshalled to/from a string as part of the Source's Progress.
+type objectsProgress struct {
+	// IsBuckedProcessed is true if all objects in the bucket have been processed.
+	IsBucketProcessed bool
+	// ProcessedCnt is the number of objects that have been processed.
+	ProcessedCnt,
+	// TotalCnt is the total number of objects in the bucket.
+	TotalCnt int
+	// Processed is the last object that was fully processed.
+	Processed string
+	// Processing are the objects that were not fully processed.
+	Processing map[string]struct{}
+}
+
+func newObjectsProgress(cnt int) *objectsProgress {
+	return &objectsProgress{TotalCnt: cnt}
+}
+
+// newProgressInfo constructs a progressInfo from the Source's Progress information.
+func newProgressInfo(ctx context.Context, s *sources.Progress) (*progressInfo, error) {
+	totalObjCnt := s.SectionsRemaining
+	processedObjCnt := s.SectionsCompleted
+	info := &progressInfo{
+		totalObjectsCnt:     int(totalObjCnt),
+		processedObjectsCnt: int(processedObjCnt),
+	}
+
+	encodeResume := s.EncodedResumeInfo
+	if encodeResume == "" {
+		info.bucketObjects = make(map[string]*objectsProgress)
+		return info, nil
+	}
+
+	var m map[string]*objectsProgress
+	if err := json.Unmarshal([]byte(encodeResume), &m); err != nil {
+		ctx.Logger().V(1).Info("error unmarshalling progress info", "error", err)
+		return info, nil
+	}
+	info.bucketObjects = m
+
+	return info, nil
+}
+
+type progressStateFn func(string, string, *progressInfo)
+
+func (p *progressInfo) setProcessStatus(obj object, fn progressStateFn) {
+	fn(obj.bucket, obj.name, p)
+}
+
+func setProcessingBucketObject(bucket, obj string, progress *progressInfo) {
+	if progress.bucketObjects[bucket].Processing == nil {
+		progress.bucketObjects[bucket].Processing = make(map[string]struct{})
+	}
+	progress.processing(bucket, obj)
+}
+
+func (p *progressInfo) processing(bkt, obj string) {
+	p.bucketObjects[bkt].Processing[obj] = struct{}{}
+}
+
+func setProcessedBucketObject(bucket, obj string, progress *progressInfo) {
+	progress.processed(bucket, obj)
+}
+
+func (p *progressInfo) processed(bkt, obj string) {
+	// Remove the object from the processing list and increment the processed count.
+	delete(p.bucketObjects[bkt].Processing, obj)
+
+	p.bucketObjects[bkt].ProcessedCnt++
+	// If all objects in the bucket have been processed, mark the bucket as processed.
+	if p.bucketObjects[bkt].ProcessedCnt == p.bucketObjects[bkt].TotalCnt {
+		p.bucketObjects[bkt].IsBucketProcessed = true
+		p.processedBucketCnt++
+	}
+	p.processedObjectsCnt++
+
+	// If the object is not greater (lexicographically) than the last processed object, we can skip it.
+	// This ensures we keep the greatest object name as the last processed object.
+	if obj < p.bucketObjects[bkt].Processed {
+		return
+	}
+	p.bucketObjects[bkt].Processed = obj
+}
+
+func setResumeBucketOffset(s string) (map[string]offsetInfo, error) {
+	resumeInfo := progressInfo{}
+	if s != "" {
+		if err := json.Unmarshal([]byte(s), &resumeInfo.bucketObjects); err != nil {
+			return nil, fmt.Errorf("error unmarshalling resume info: %w", err)
+		}
+	}
+
+	return calcBktOffset(resumeInfo.bucketObjects)
+}
+
+// In order to calculate the bucket offset, we need to know the last object
+// that was processed for each bucket, as well as any objects that were processing.
+// If the bucket was fully processed, we don't need to set an offset.
+// If there were no objects processing, we can just use the last object that was processed.
+// If there were objects processing, we need to find the first object (lexicographically)
+// that was processing, as that is the next object that needs to be processed.
+func calcBktOffset(resumeInfo map[string]*objectsProgress) (map[string]offsetInfo, error) {
+	bucketOffset := make(map[string]offsetInfo, len(resumeInfo))
+	for objName, progress := range resumeInfo {
+		info := offsetInfo{}
+		if progress.IsBucketProcessed {
+			info.isBucketProcessed = true
+			bucketOffset[objName] = info
+			continue
+		}
+		if len(progress.Processing) == 0 {
+			info.lastProcessedObject = progress.Processed
+			bucketOffset[objName] = info
+			continue
+		}
+
+		processing := make([]string, 0, len(progress.Processing))
+		for k := range progress.Processing {
+			processing = append(processing, k)
+		}
+		sort.Strings(processing)
+		info.lastProcessedObject = processing[0]
+		bucketOffset[objName] = info
+	}
+
+	return bucketOffset, nil
+}
+
+// enumerate all the objects and buckets in the source and use the results to
+// set the progress information. This will be used track progression of the scan,
+// and to resume the scan if it is interrupted.
+func (s *Source) enumerate(ctx context.Context) error {
+	stats, err := s.gcsManager.attributes(ctx)
+	if err != nil {
+		return fmt.Errorf("error getting attributes during enumeration: %w", err)
+	}
+	s.stats = stats
+
+	info := progressInfo{
+		totalBucketsCnt: int(s.stats.numBuckets),
+		totalObjectsCnt: int(s.stats.numObjects),
+	}
+
+	progress := make(map[string]*objectsProgress, len(s.stats.bucketObjects))
+	for k, v := range s.stats.bucketObjects {
+		progress[k] = newObjectsProgress(int(v))
+	}
+	info.bucketObjects = progress
+
+	return s.setProgress(ctx, &info, fmt.Sprintf("enumerated %d buckets, %d objects", s.stats.numBuckets, s.stats.numObjects))
+}
+
+// Chunks emits chunks of bytes over a channel.
+func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) error {
+	objectCh, err := s.gcsManager.listObjects(ctx)
+	if err != nil {
+		return fmt.Errorf("error listing objects: %w", err)
+	}
+	s.chunksCh = chunksChan
+
+	progress, err := newProgressInfo(ctx, &s.Progress)
+	if err != nil {
+		return fmt.Errorf("error constructing progress info: %w", err)
+	}
+
+	var wg sync.WaitGroup
+	for obj := range objectCh {
+		obj := obj
+		o, ok := obj.(object)
+		if !ok {
+			ctx.Logger().Error(fmt.Errorf("unexpected object type: %T", obj), "GCS source unexpected object type", "name", s.name)
+			continue
+		}
+
+		if err := s.startProcessing(ctx, progress, o); err != nil {
+			ctx.Logger().V(1).Info("GCS source error starting to process objects", "error", err)
+			continue
+		}
+		wg.Add(1)
+		go func(obj object, resume *progressInfo) {
+			defer wg.Done()
+
+			if err := s.processObject(ctx, o); err != nil {
+				ctx.Logger().V(1).Info("error setting start progress progress", "name", o.name, "error", err)
+				return
+			}
+			if err := s.endProcessing(ctx, resume, o); err != nil {
+				ctx.Logger().V(1).Info("error setting end progress progress", "name", o.name, "error", err)
+			}
+		}(o, progress)
+	}
+	wg.Wait()
+
+	ctx.Logger().Info("GCS source finished processing", "name", s.name)
+	return nil
+}
+
+func (s *Source) startProcessing(ctx context.Context, progress *progressInfo, o object) error {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+
+	progress.setProcessStatus(o, setProcessingBucketObject)
+	return s.setProgress(ctx, progress, fmt.Sprintf("GCS source beginning to process object %s in bucket %s", o.name, o.bucket))
+}
+
+func (s *Source) endProcessing(ctx context.Context, progress *progressInfo, o object) error {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+
+	progress.setProcessStatus(o, setProcessedBucketObject)
+	return s.setProgress(ctx, progress, fmt.Sprintf("GCS source finished processing object %s in bucket %s", o.name, o.bucket))
+}
+
+func (s *Source) setProgress(ctx context.Context, progress *progressInfo, msg string) error {
+	ctx.Logger().V(5).Info(msg)
+
+	b, err := json.Marshal(progress.bucketObjects)
+	if err != nil {
+		return fmt.Errorf("error marshalling resume info: %w", err)
+	}
+	s.SetProgressComplete(progress.processedObjectsCnt, progress.totalObjectsCnt, msg, string(b))
+
+	return nil
+}
+
+func (s *Source) processObject(ctx context.Context, o object) error {
+	chunkSkel := &sources.Chunk{
+		SourceName: s.name,
+		SourceType: s.Type(),
+		SourceID:   s.sourceId,
+		Verify:     s.verify,
+		SourceMetadata: &source_metadatapb.MetaData{
+			Data: &source_metadatapb.MetaData_Gcs{
+				Gcs: &source_metadatapb.GCS{
+					Bucket:      o.bucket,
+					Filename:    o.name,
+					Link:        o.link,
+					Email:       o.owner,
+					ContentType: o.contentType,
+					Acls:        o.acl,
+					CreatedAt:   o.createdAt.String(),
+					UpdatedAt:   o.updatedAt.String(),
+				},
+			},
+		},
+	}
+
+	data, err := s.readObjectData(ctx, o, chunkSkel)
+	if err != nil {
+		return fmt.Errorf("error reading object data: %w", err)
+	}
+
+	// If data is nil, it means that the file was handled by a handler.
+	if data == nil {
+		return nil
+	}
+
+	chunkSkel.Data = data
+
+	select {
+	case <-ctx.Done():
+		return ctx.Err()
+	case s.chunksCh <- chunkSkel:
+	}
+
+	return nil
+}
+
+func (s *Source) readObjectData(ctx context.Context, o object, chunk *sources.Chunk) ([]byte, error) {
+	reader, err := diskbufferreader.New(o)
+	if err != nil {
+		return nil, fmt.Errorf("error creating disk buffer reader: %w", err)
+	}
+	defer reader.Close()
+
+	if handlers.HandleFile(ctx, reader, chunk, s.chunksCh) {
+		ctx.Logger().V(3).Info("File was handled", "name", s.name, "bucket", o.bucket, "object", o.name)
+		return nil, nil
+	}
+
+	if err := reader.Reset(); err != nil {
+		return nil, fmt.Errorf("error resetting reader: %w", err)
+	}
+
+	reader.Stop()
+	data, err := io.ReadAll(reader)
+	if err != nil {
+		return nil, fmt.Errorf("error reading object: %w", err)
+	}
+
+	return data, nil
+}

pkg/sources/gcs/gcs_integration_test.go

@@ -0,0 +1,190 @@
+//go:build integration
+// +build integration
+
+package gcs
+
+import (
+	"sort"
+	"testing"
+
+	"github.com/google/go-cmp/cmp"
+	"github.com/google/go-cmp/cmp/cmpopts"
+	"github.com/stretchr/testify/assert"
+
+	"github.com/trufflesecurity/trufflehog/v3/pkg/context"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/source_metadatapb"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
+)
+
+func TestChunks(t *testing.T) {
+	ctx := context.Background()
+
+	source, conn := createTestSource(&sourcespb.GCS{
+		ProjectId:      testProjectID,
+		Credential:     &sourcespb.GCS_Adc{},
+		ExcludeBuckets: []string{perfTestBucketGlob, publicBucket},
+	})
+
+	err := source.Init(ctx, "test", 1, 1, true, conn, 8)
+	assert.Nil(t, err)
+
+	chunksCh := make(chan *sources.Chunk, 1)
+
+	go func() {
+		defer close(chunksCh)
+		err := source.Chunks(ctx, chunksCh)
+		assert.Nil(t, err)
+	}()
+
+	want := createTestChunks()
+
+	got := make([]*sources.Chunk, 0, len(want))
+	for chunk := range chunksCh {
+		got = append(got, chunk)
+	}
+	sort.Slice(got, func(i, j int) bool {
+		return got[i].SourceMetadata.GetGcs().Filename < got[j].SourceMetadata.GetGcs().Filename
+	})
+
+	assert.Equal(t, len(want), len(got))
+
+	for i, chunk := range got {
+		if diff := cmp.Diff(want[i].SourceMetadata.GetGcs(), chunk.SourceMetadata.GetGcs(),
+			cmpopts.IgnoreFields(source_metadatapb.GCS{}, "state", "sizeCache", "unknownFields", "CreatedAt", "UpdatedAt"),
+		); diff != "" {
+			t.Errorf("chunk mismatch (-want +got):\n%s", diff)
+		}
+	}
+}
+
+func TestChunks_PublicBucket(t *testing.T) {
+	ctx := context.Background()
+
+	source, conn := createTestSource(&sourcespb.GCS{
+		Credential:     &sourcespb.GCS_Unauthenticated{},
+		IncludeBuckets: []string{publicBucket},
+	})
+
+	err := source.Init(ctx, "test", 1, 1, true, conn, 8)
+	assert.Nil(t, err)
+
+	chunksCh := make(chan *sources.Chunk, 1)
+
+	go func() {
+		defer close(chunksCh)
+		err := source.Chunks(ctx, chunksCh)
+		assert.Nil(t, err)
+	}()
+
+	want := []*sources.Chunk{
+		{
+			SourceName: "test",
+			SourceType: sourcespb.SourceType_SOURCE_TYPE_GCS,
+			SourceID:   0,
+			Verify:     true,
+			SourceMetadata: &source_metadatapb.MetaData{
+				Data: &source_metadatapb.MetaData_Gcs{
+					Gcs: &source_metadatapb.GCS{
+						Filename:    "aws1.txt",
+						Bucket:      publicBucket,
+						ContentType: "text/plain",
+						Email:       "",
+						Link:        "https://storage.googleapis.com/download/storage/v1/b/public-trufflehog-test-bucket/o/aws1.txt?generation=1678334408999764&alt=media",
+						Acls:        []string{},
+					},
+				},
+			},
+		},
+	}
+
+	got := make([]*sources.Chunk, 0, len(want))
+	for chunk := range chunksCh {
+		got = append(got, chunk)
+	}
+	sort.Slice(got, func(i, j int) bool {
+		return got[i].SourceMetadata.GetGcs().Filename < got[j].SourceMetadata.GetGcs().Filename
+	})
+
+	assert.Equal(t, len(want), len(got))
+
+	for i, chunk := range got {
+		if diff := cmp.Diff(want[i].SourceMetadata.GetGcs(), chunk.SourceMetadata.GetGcs(),
+			cmpopts.IgnoreFields(source_metadatapb.GCS{}, "state", "sizeCache", "unknownFields", "CreatedAt", "UpdatedAt"),
+		); diff != "" {
+			t.Errorf("chunk mismatch (-want +got):\n%s", diff)
+		}
+	}
+}
+
+func createTestChunks() []*sources.Chunk {
+	objects := []object{
+		{
+			name:        "aws1.txt",
+			bucket:      testBucket,
+			contentType: "text/plain",
+			size:        150,
+			link:        "https://storage.googleapis.com/download/storage/v1/b/test-bkt-th/o/aws1.txt?generation=1677870994890594&alt=media",
+			acl:         []string{},
+		},
+		{
+			name:        "moar2.txt",
+			bucket:      testBucket,
+			contentType: "text/plain",
+			size:        12,
+			link:        "https://storage.googleapis.com/download/storage/v1/b/test-bkt-th/o/moar2.txt?generation=1677871000378542&alt=media",
+			acl:         []string{},
+		},
+		{
+			name:        "aws3.txt",
+			bucket:      testBucket2,
+			contentType: "text/plain",
+			size:        150,
+			link:        "https://storage.googleapis.com/download/storage/v1/b/test-bkt-th2/o/aws3.txt?generation=1677871022489611&alt=media",
+			acl:         []string{},
+		},
+		{
+			name:        "moar.txt",
+			bucket:      testBucket3,
+			contentType: "text/plain",
+			size:        6,
+			link:        "https://storage.googleapis.com/download/storage/v1/b/test-bkt-th3/o/moar.txt?generation=1677871042896804&alt=media",
+			acl:         []string{},
+		},
+		{
+			name:        "AMAZON_FASHION_5.json",
+			bucket:      testBucket4,
+			contentType: "application/json",
+			size:        1413469,
+			link:        "https://storage.googleapis.com/download/storage/v1/b/test-bkt-th4/o/AMAZON_FASHION_5.json?generation=1677871063457469&alt=media",
+			acl:         []string{},
+		},
+	}
+
+	chunks := make([]*sources.Chunk, 0, len(objects))
+	for _, o := range objects {
+		chunks = append(chunks, &sources.Chunk{
+			SourceName: "test",
+			SourceType: sourcespb.SourceType_SOURCE_TYPE_GCS,
+			SourceID:   0,
+			Verify:     true,
+			SourceMetadata: &source_metadatapb.MetaData{
+				Data: &source_metadatapb.MetaData_Gcs{
+					Gcs: &source_metadatapb.GCS{
+						Filename:    o.name,
+						Bucket:      o.bucket,
+						ContentType: o.contentType,
+						Email:       o.owner,
+						Link:        o.link,
+						Acls:        o.acl,
+					},
+				},
+			},
+		})
+	}
+	sort.Slice(chunks, func(i, j int) bool {
+		return chunks[i].SourceMetadata.GetGcs().Filename < chunks[j].SourceMetadata.GetGcs().Filename
+	})
+
+	return chunks
+}

pkg/sources/gcs/gcs_manager.go

@@ -0,0 +1,716 @@
+package gcs
+
+import (
+	aCtx "context"
+	"fmt"
+	"io"
+	"runtime"
+	"sync"
+	"sync/atomic"
+	"time"
+
+	"cloud.google.com/go/storage"
+	"github.com/gobwas/glob"
+	"github.com/googleapis/gax-go/v2"
+	"github.com/pkg/errors"
+	"golang.org/x/sync/errgroup"
+	"google.golang.org/api/iterator"
+	"google.golang.org/api/option"
+
+	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/context"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
+)
+
+const (
+	defaultMaxObjectSize = 10 * 1024 * 1024 // 10MB
+	maxObjectSizeLimit   = 50 * 1024 * 1024 // 50MB
+)
+
+var (
+	defaultConcurrency = runtime.NumCPU()
+)
+
+type objectManager interface {
+	listObjects(context.Context) (chan io.Reader, error)
+	attributes(ctx context.Context) (*attributes, error)
+}
+
+// bucketManager is a simplified *storage.Client wrapper.
+// It provides only a subset of methods that are needed by the gcsManager.
+type bucketManager interface {
+	// Bucket returns a BucketHandle for the given bucket name.
+	Bucket(name string) *storage.BucketHandle
+	// Buckets returns an iterator over the buckets in the project.
+	Buckets(ctx aCtx.Context, projectID string) *storage.BucketIterator
+}
+
+// gcsManager serves as simple facade for interacting with GCS.
+// It's main purpose is to retrieve objects from GCS.
+type gcsManager struct {
+	projectID string
+	withoutAuth,
+	hasEnumerated bool
+
+	concurrency int
+	workerPool  *errgroup.Group
+
+	maxObjectSize int64
+	numBuckets    uint32
+	numObjects    uint64
+
+	includeBuckets,
+	excludeBuckets,
+	includeObjects,
+	excludeObjects map[string]struct{}
+
+	buckets map[string]bucket
+	attr    *attributes
+
+	client bucketManager
+}
+
+// bucket is a simplified *storage.BucketHandle wrapper.
+// It also provides a mechanism for resuming an interrupted iteration.
+// This reduces the number of objects that need to be re-processed.
+type bucket struct {
+	shouldInclude bool
+	name          string
+	// startOffset is the name of the object to resume from. (inclusive)
+	// This works because GCS objects are sorted lexicographically.
+	// If this is empty, the iteration will start from the beginning.
+	startOffset string
+	*storage.BucketHandle
+}
+
+// offsetInfo is used to resume an interrupted iteration.
+type offsetInfo struct {
+	isBucketProcessed   bool
+	lastProcessedObject string
+}
+
+// attributes contains metadata about the GCS source.
+// This will be collected during the initial scan.
+type attributes struct {
+	numBuckets    uint32
+	numObjects    uint64
+	mu            sync.RWMutex
+	bucketObjects map[string]uint64
+}
+
+func newStats(numBkts int) *attributes {
+	return &attributes{
+		numBuckets:    uint32(numBkts),
+		bucketObjects: make(map[string]uint64, numBkts),
+	}
+}
+
+func (s *attributes) incObjects() {
+	s.mu.Lock()
+	s.numObjects++
+	s.mu.Unlock()
+}
+
+func (s *attributes) setBucketCnt(bkt string, cnt uint64) {
+	s.mu.Lock()
+	s.bucketObjects[bkt] = cnt
+	s.mu.Unlock()
+}
+
+type gcsManagerOption func(*gcsManager) error
+
+// withAPIKey uses the provided API key when creating a new GCS client.
+// This can ONLY be used for public buckets.
+func withAPIKey(ctx context.Context, apiKey string) gcsManagerOption {
+	client, err := storage.NewClient(ctx, option.WithAPIKey(apiKey), option.WithScopes(storage.ScopeReadOnly))
+	return func(m *gcsManager) error {
+		if err != nil {
+			return err
+		}
+
+		m.client = client
+		return nil
+	}
+}
+
+// withJSONServiceAccount uses the provided JSON service account when creating a new GCS client.
+func withJSONServiceAccount(ctx context.Context, jsonServiceAccount []byte) gcsManagerOption {
+	client, err := storage.NewClient(ctx, option.WithCredentialsJSON(jsonServiceAccount), option.WithScopes(storage.ScopeReadOnly))
+	return func(m *gcsManager) error {
+		if err != nil {
+			return err
+		}
+
+		m.client = client
+		return nil
+	}
+}
+
+// withDefaultADC uses the default application credentials when creating a new GCS client.
+func withDefaultADC(ctx context.Context) gcsManagerOption {
+	client, err := defaultADC(ctx)
+	return func(m *gcsManager) error {
+		if err != nil {
+			return err
+		}
+
+		m.client = client
+		return nil
+	}
+}
+
+// withoutAuthentication uses an unauthenticated client when creating a new GCS client.
+// This can ONLY be used for public buckets.
+func withoutAuthentication() gcsManagerOption {
+	client, err := storage.NewClient(aCtx.Background(), option.WithoutAuthentication(), option.WithScopes(storage.ScopeReadOnly))
+	return func(m *gcsManager) error {
+		if err != nil {
+			return err
+		}
+		m.client = client
+		m.withoutAuth = true
+		return nil
+	}
+}
+
+func defaultADC(ctx context.Context) (*storage.Client, error) {
+	client, err := storage.NewClient(ctx, option.WithScopes(storage.ScopeReadOnly))
+	if err != nil {
+		return nil, err
+	}
+	return client, nil
+}
+
+// withIncludeBuckets sets the buckets that should be included in the scan.
+// If used in conjunction with withExcludeBuckets, the include buckets will
+// take precedence.
+func withIncludeBuckets(buckets []string) gcsManagerOption {
+	return func(m *gcsManager) error {
+		if len(buckets) == 0 {
+			return nil
+		}
+		if m.excludeBuckets != nil {
+			m.excludeBuckets = nil
+		}
+
+		m.includeBuckets = make(map[string]struct{}, len(buckets))
+		for _, bucket := range buckets {
+			m.includeBuckets[bucket] = struct{}{}
+		}
+		return nil
+	}
+}
+
+// withExcludeBuckets sets the buckets that should be excluded from the scan.
+// If used in conjunction with withIncludeBuckets, the include buckets will
+// take precedence.
+func withExcludeBuckets(buckets []string) gcsManagerOption {
+	return func(m *gcsManager) error {
+		if len(buckets) == 0 {
+			return nil
+		}
+		if m.includeBuckets != nil {
+			return nil
+		}
+
+		m.excludeBuckets = make(map[string]struct{}, len(buckets))
+		for _, bucket := range buckets {
+			m.excludeBuckets[bucket] = struct{}{}
+		}
+		return nil
+	}
+}
+
+// withIncludeObjects sets the objects that should be included in the scan.
+// Using this option in conjuection with withIncludeBuckets will result in
+// only specific buckets and objects being scanned.
+// If the bucket(s) are known, it is recommended to also use withIncludeBuckets
+// to increase the performance of the scan.
+// If used in conjunction with any of the withExclude* or withIncludeBuckets options, this
+// option will take precedence.
+func withIncludeObjects(objects []string) gcsManagerOption {
+	return func(m *gcsManager) error {
+		if len(objects) == 0 {
+			return nil
+		}
+		if m.excludeObjects != nil {
+			m.excludeObjects = nil
+		}
+
+		m.includeObjects = make(map[string]struct{}, len(objects))
+		for _, object := range objects {
+			m.includeObjects[object] = struct{}{}
+		}
+		return nil
+	}
+}
+
+// withExcludeObjects sets the objects that should be excluded from the scan.
+// If used in conjunction with withIncludeObjects, the include objects will
+// take precedence.
+func withExcludeObjects(objects []string) gcsManagerOption {
+	return func(m *gcsManager) error {
+		if len(objects) == 0 {
+			return nil
+		}
+		if m.includeObjects != nil {
+			return nil
+		}
+
+		m.excludeObjects = make(map[string]struct{}, len(objects))
+		for _, object := range objects {
+			m.excludeObjects[object] = struct{}{}
+		}
+		return nil
+	}
+}
+
+// withConcurrency sets the number of concurrent workers that will be used
+// to process objects.
+// If not set, or set to a negative number the default value is runtime.NumCPU().
+func withConcurrency(concurrency int) gcsManagerOption {
+	return func(m *gcsManager) error {
+		if concurrency <= 0 {
+			m.concurrency = defaultConcurrency
+		} else {
+			m.concurrency = concurrency
+		}
+		return nil
+	}
+}
+
+// withMaxObjectSize sets the maximum size of objects that will be scanned.
+// If not set, set to a negative number, or set larger than 1GB,
+// the default value of 500MB will be used.
+func withMaxObjectSize(maxObjectSize int64) gcsManagerOption {
+	return func(m *gcsManager) error {
+		if maxObjectSize <= 0 || maxObjectSize > maxObjectSizeLimit {
+			m.maxObjectSize = defaultMaxObjectSize
+		} else {
+			m.maxObjectSize = maxObjectSize
+		}
+
+		return nil
+	}
+}
+
+// withBucketOffsets sets the offset for each bucket.
+// This is used to resume listing objects for a bucket.
+func withBucketOffsets(offsets map[string]offsetInfo) gcsManagerOption {
+	bkts := make(map[string]bucket, len(offsets))
+	for bkt, offst := range offsets {
+		bkts[bkt] = bucket{
+			shouldInclude: !offst.isBucketProcessed,
+			name:          bkt,
+			startOffset:   offst.lastProcessedObject,
+		}
+	}
+	return func(m *gcsManager) error {
+		m.hasEnumerated = len(offsets) > 0 // offset means we are resuming.
+		m.buckets = bkts
+		return nil
+	}
+}
+
+func newGCSManager(projectID string, opts ...gcsManagerOption) (*gcsManager, error) {
+	// Default values for the manager.
+	gcs := &gcsManager{
+		projectID:     projectID,
+		concurrency:   defaultConcurrency,
+		maxObjectSize: defaultMaxObjectSize,
+		buckets:       make(map[string]bucket),
+	}
+
+	for _, opt := range opts {
+		if err := opt(gcs); err != nil {
+			return nil, fmt.Errorf("failed to apply option: %w", err)
+		}
+	}
+
+	if projectID == "" && !gcs.withoutAuth {
+		return nil, fmt.Errorf("project ID is required, when using authentication")
+	}
+
+	// If no client was provided, use the default application credentials.
+	// A client is required to perform any operations.
+	if gcs.client == nil {
+		c, err := defaultADC(context.Background())
+		if err != nil {
+			return nil, err
+		}
+		gcs.client = c
+	}
+	configureWorkers(gcs)
+
+	return gcs, nil
+}
+
+func configureWorkers(gcs *gcsManager) {
+	gcs.workerPool = new(errgroup.Group)
+	gcs.workerPool.SetLimit(gcs.concurrency)
+}
+
+// object is a representation of a GCS object.
+type object struct {
+	name        string
+	bucket      string
+	contentType string
+	owner       string
+	link        string
+	// acl represents an ACLEntities.
+	// https://pkg.go.dev/cloud.google.com/go/storage#ACLEntity
+	acl       []string
+	size      int64
+	createdAt time.Time
+	updatedAt time.Time
+
+	io.Reader
+}
+
+func (g *gcsManager) attributes(ctx context.Context) (*attributes, error) {
+	// Get all the buckets in the project.
+	buckets, err := g.listBuckets(ctx)
+	if err != nil {
+		return nil, fmt.Errorf("failed to list buckets: %w", err)
+	}
+
+	return g.enumerate(ctx, buckets)
+}
+
+func (g *gcsManager) enumerate(ctx context.Context, bkts []bucket) (*attributes, error) {
+	logger := ctx.Logger().WithValues("phase", "enumeration")
+
+	logger.V(5).Info("enumerating buckets", "numBuckets", len(bkts))
+	stats := newStats(len(bkts))
+
+	for _, bkt := range bkts {
+		bkt := bkt
+		g.workerPool.Go(func() error {
+			// List all the objects in the bucket and calculate a attributes.
+			g.setupBktHandle(&bkt)
+
+			q, err := setObjectQuery(&bkt)
+			if err != nil {
+				logger.Error(err, "failed to set object query", "bucket", bkt.name)
+				return nil
+			}
+
+			var count uint64
+			objs := bkt.Objects(ctx, q)
+			for {
+				obj, err := objs.Next()
+				if errors.Is(err, iterator.Done) {
+					logger.V(5).Info("finished listing objects in bucket")
+					break
+				}
+				if err != nil {
+					logger.V(1).Info("failed to list objects", "bucket", bkt.name, "error", err)
+					return nil
+				}
+				if obj == nil {
+					logger.V(5).Info("object is nil")
+					continue
+				}
+
+				if !g.shouldIncludeObject(ctx, obj.Name) || g.shouldExcludeObject(ctx, obj.Name) {
+					continue
+				}
+				count++
+				stats.incObjects()
+			}
+
+			stats.setBucketCnt(bkt.name, count)
+			return nil
+		})
+	}
+
+	_ = g.workerPool.Wait()
+	g.attr = stats
+	logger.V(5).Info("finished enumerating buckets", "num-objects", stats.numObjects, "num-buckets", stats.numBuckets)
+
+	return stats, nil
+}
+
+func (g *gcsManager) listObjects(ctx context.Context) (chan io.Reader, error) {
+	ch := make(chan io.Reader, 100) // TODO (ahrav): Determine optimal buffer size.
+
+	// Get all the buckets in the project.
+	buckets, err := g.listBuckets(ctx)
+	if err != nil {
+		return nil, fmt.Errorf("failed to list buckets: %w", err)
+	}
+
+	// TODO (ahrav): This can be optimized if we set the buckets from within
+	// the listBuckets method. This way seems a little more clear as to the intent.
+
+	// Update the gcsManager with all the buckets to list objects from.
+	// This is in addition to any buckets that were provided via the
+	// withBucketOffsets option.
+	for _, b := range buckets {
+		g.buckets[b.name] = b
+	}
+
+	go func() {
+		gcsErrs := sources.NewScanErrors()
+		for _, bucket := range g.buckets {
+			g.numBuckets++
+			bucket := bucket
+			g.workerPool.Go(func() error {
+				objCh, errCh := g.listBucketObjects(ctx, &bucket)
+				for {
+					select {
+					case obj, ok := <-objCh:
+						if !ok {
+							return nil
+						}
+						ch <- obj
+					case err := <-errCh:
+						gcsErrs.Add(err)
+						return nil
+					}
+				}
+			})
+		}
+
+		_ = g.workerPool.Wait()
+		if gcsErrs.Count() > 0 {
+			ctx.Logger().V(2).Info("encountered gcsErrs while scanning GCS buckets", "error-count", gcsErrs.Count(), "gcsErrs", gcsErrs.String())
+		}
+		close(ch)
+	}()
+
+	return ch, nil
+}
+
+func (g *gcsManager) listBuckets(ctx context.Context) ([]bucket, error) {
+	var buckets []bucket
+	if g.withoutAuth {
+		for name := range g.includeBuckets {
+			buckets = append(buckets, bucket{name: name})
+		}
+		return buckets, nil
+	}
+
+	bkts := g.client.Buckets(ctx, g.projectID)
+	for {
+		bkt, err := bkts.Next()
+		if errors.Is(err, iterator.Done) {
+			ctx.Logger().V(5).Info("finished listing buckets", "num_buckets", len(buckets))
+			break
+		}
+		if err != nil {
+			return nil, fmt.Errorf("failed to retrieve bucket: %w", err)
+		}
+
+		// If the bucket is already in the map, skip it, it's already accounted for.
+		// This is used to resume listing objects for a bucket.
+		if _, ok := g.buckets[bkt.Name]; ok {
+			continue
+		}
+		if !g.shouldIncludeBucket(ctx, bkt.Name) || g.shouldExcludeBucket(ctx, bkt.Name) {
+			continue
+		}
+		buckets = append(buckets, bucket{name: bkt.Name})
+
+	}
+	return buckets, nil
+}
+
+func (g *gcsManager) listBucketObjects(ctx context.Context, bkt *bucket) (chan io.Reader, chan error) {
+	ch := make(chan io.Reader, 100)
+	errCh := make(chan error, 1)
+
+	go func() {
+		defer close(ch)
+
+		logger := ctx.Logger().WithValues("bucket", bkt.name)
+		logger.V(5).Info("listing object(s) in bucket")
+
+		g.setupBktHandle(bkt)
+
+		// TODO (ahrav): Look to extend gcsManager to allow for exact buckets/objects
+		// include filters. This will increase performance substantially
+
+		if err := g.bucketObjects(ctx, bkt, ch); err != nil {
+			errCh <- fmt.Errorf("failed to list bucket objects: %w", err)
+		}
+	}()
+	return ch, errCh
+}
+
+func (g *gcsManager) setupBktHandle(bkt *bucket) {
+	b := g.client.Bucket(bkt.name).Retryer(
+		storage.WithBackoff(gax.Backoff{
+			Initial:    2 * time.Second,
+			Max:        30 * time.Second,
+			Multiplier: 1.5,
+		}),
+		storage.WithPolicy(storage.RetryAlways),
+	)
+	bkt.BucketHandle = b
+}
+
+func (g *gcsManager) bucketObjects(ctx context.Context, bkt *bucket, ch chan<- io.Reader) error {
+	q, err := setObjectQuery(bkt)
+	if err != nil {
+		return fmt.Errorf("failed to set object query: %w", err)
+	}
+
+	objs := bkt.Objects(ctx, q)
+	for {
+		obj, err := objs.Next()
+		if errors.Is(err, iterator.Done) {
+			ctx.Logger().V(5).Info("finished listing objects in bucket")
+			break
+		}
+		if err != nil {
+			return fmt.Errorf("failed to retrieve object iterator: %w", err)
+		}
+		if obj == nil {
+			ctx.Logger().V(5).Info("object is nil")
+			continue
+		}
+
+		if !g.shouldIncludeObject(ctx, obj.Name) || g.shouldExcludeObject(ctx, obj.Name) {
+			continue
+		}
+
+		o, err := g.constructObject(ctx, bkt.Object(obj.Name))
+		if err != nil {
+			ctx.Logger().V(1).Info("failed to create object", "object-name", obj.Name, "error", err)
+			continue
+		}
+		ch <- o
+	}
+	return nil
+}
+
+func setObjectQuery(bkt *bucket) (*storage.Query, error) {
+	// Setting the attribute selection is a performance optimization.
+	// https://pkg.go.dev/cloud.google.com/go/storage#Query.SetAttrSelection
+	q := new(storage.Query)
+	err := q.SetAttrSelection([]string{
+		"Name",
+		"ContentType",
+		"Owner",
+		"Size",
+		"Updated",
+		"Created",
+		"ACL",
+	})
+	if err != nil {
+		return nil, fmt.Errorf("failed to set attribute selection: %w", err)
+	}
+
+	// If a start offset is provided, then we need to set it on the query.
+	// This will begin listing objects from the start offset.
+	if bkt.startOffset != "" {
+		q.StartOffset = bkt.startOffset
+	}
+	return q, nil
+}
+
+func (g *gcsManager) constructObject(ctx context.Context, obj *storage.ObjectHandle) (object, error) {
+	o := object{}
+	attrs, err := obj.Attrs(ctx)
+	if err != nil {
+		return o, fmt.Errorf("failed to retrieve object attributes: %w", err)
+	}
+
+	if !isObjectTypeValid(ctx, attrs.Name) || !g.isObjectSizeValid(ctx, attrs.Size) {
+		return o, fmt.Errorf("object is not valid")
+	}
+
+	rc, err := obj.NewReader(ctx)
+	if err != nil {
+		return o, fmt.Errorf("failed to retrieve object reader: %w", err)
+	}
+
+	o.name = attrs.Name
+	o.bucket = attrs.Bucket
+	o.contentType = attrs.ContentType
+	o.owner = attrs.Owner
+	o.link = attrs.MediaLink
+	o.createdAt = attrs.Created
+	o.updatedAt = attrs.Updated
+	o.acl = objectACLs(attrs.ACL)
+	o.size = attrs.Size
+	o.Reader = rc
+
+	atomic.AddUint64(&g.numObjects, 1)
+
+	return o, nil
+}
+
+func objectACLs(acl []storage.ACLRule) []string {
+	acls := make([]string, 0, len(acl))
+	for _, rule := range acl {
+		acls = append(acls, string(rule.Entity))
+	}
+	return acls
+}
+
+func isObjectTypeValid(ctx context.Context, name string) bool {
+	isValid := !common.SkipFile(name)
+	if !isValid {
+		ctx.Logger().V(2).Info("object type is invalid", "object-name", name)
+		return false
+	}
+	return true
+}
+
+func (g *gcsManager) isObjectSizeValid(ctx context.Context, size int64) bool {
+	isValid := size > 0 && size <= g.maxObjectSize
+	if !isValid {
+		ctx.Logger().V(2).Info("object size is invalid", "object-size", size)
+		return false
+	}
+	return true
+}
+
+func (g *gcsManager) shouldIncludeBucket(ctx context.Context, bkt string) bool {
+	if len(g.includeBuckets) == 0 {
+		return true
+	}
+	return shouldProcess(ctx, bkt, g.includeBuckets, globMatches)
+}
+
+func (g *gcsManager) shouldExcludeBucket(ctx context.Context, bkt string) bool {
+	return shouldProcess(ctx, bkt, g.excludeBuckets, globMatches)
+}
+
+func (g *gcsManager) shouldIncludeObject(ctx context.Context, obj string) bool {
+	if len(g.includeObjects) == 0 {
+		return true
+	}
+	return shouldProcess(ctx, obj, g.includeObjects, globMatches)
+}
+
+func (g *gcsManager) shouldExcludeObject(ctx context.Context, obj string) bool {
+	return shouldProcess(ctx, obj, g.excludeObjects, globMatches)
+}
+
+type globMatcherFn func(string, glob.Glob) bool
+
+func shouldProcess(ctx context.Context, s string, matchers map[string]struct{}, matcherFn globMatcherFn) bool {
+	if len(matchers) == 0 {
+		return false
+	}
+
+	for m := range matchers {
+		g, err := glob.Compile(m)
+		if err != nil {
+			ctx.Logger().V(1).Info("failed to compile glob", "glob", m, "error", err)
+			continue
+		}
+
+		if matcherFn(s, g) {
+			return true
+		}
+	}
+	return false
+}
+
+func globMatches(s string, g glob.Glob) bool {
+	return g.Match(s)
+}

pkg/sources/gcs/gcs_manager_test.go

@@ -0,0 +1,890 @@
+package gcs
+
+import (
+	"fmt"
+	"sort"
+	"testing"
+	"time"
+
+	"github.com/google/go-cmp/cmp"
+	"github.com/google/go-cmp/cmp/cmpopts"
+	"github.com/stretchr/testify/assert"
+
+	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/context"
+)
+
+const (
+	publicBucket       = "public-trufflehog-test-bucket"
+	testProjectID      = "trufflehog-testing"
+	testAPIKey         = "somekeys"
+	testBucket         = "test-bkt-th"
+	testBucket2        = "test-bkt-th2"
+	testBucket3        = "test-bkt-th3"
+	testBucket4        = "test-bkt-th4"
+	perfTestBucketGlob = "perf-test-bkt-th*"
+	bucket1            = "bucket1"
+	bucket2            = "bucket2"
+	object1            = "object1"
+	object2            = "object2"
+)
+
+func TestNewGcsManager(t *testing.T) {
+	ctx := context.Background()
+
+	testCases := []struct {
+		name    string
+		projID  string
+		opts    []gcsManagerOption
+		want    *gcsManager
+		wantErr bool
+	}{
+		{
+			name:   "new gcs manager, no options",
+			projID: testProjectID,
+			want:   &gcsManager{projectID: testProjectID, concurrency: defaultConcurrency, maxObjectSize: defaultMaxObjectSize},
+		},
+		{
+			name:    "new gcs manager, no project id",
+			projID:  "",
+			wantErr: true,
+		},
+		{
+			name:   "new gcs manager, without auth",
+			projID: "",
+			opts:   []gcsManagerOption{withoutAuthentication()},
+			want: &gcsManager{
+				concurrency:   defaultConcurrency,
+				maxObjectSize: defaultMaxObjectSize,
+				withoutAuth:   true,
+			},
+		},
+		{
+			name:   "new gcs manager, with api key",
+			projID: testProjectID,
+			opts:   []gcsManagerOption{withAPIKey(ctx, testAPIKey)},
+			want:   &gcsManager{projectID: testProjectID, concurrency: defaultConcurrency, maxObjectSize: defaultMaxObjectSize},
+		},
+		{
+			name:   "new gcs manager, with json service account",
+			projID: testProjectID,
+			opts: []gcsManagerOption{withJSONServiceAccount(ctx, []byte(`{
+				"type": "service_account",
+				"project_id": "test-project"}`,
+			))},
+			want: &gcsManager{projectID: testProjectID, concurrency: defaultConcurrency, maxObjectSize: defaultMaxObjectSize},
+		},
+		{
+			name:   "new gcs manager, with default ADC account",
+			projID: testProjectID,
+			opts:   []gcsManagerOption{withDefaultADC(ctx)},
+			want:   &gcsManager{projectID: testProjectID, concurrency: defaultConcurrency, maxObjectSize: defaultMaxObjectSize},
+		},
+		{
+			name:   "new gcs manager, with bucket offsets",
+			projID: testProjectID,
+			opts: []gcsManagerOption{
+				withDefaultADC(ctx),
+				withBucketOffsets(map[string]offsetInfo{
+					testBucket:  {lastProcessedObject: "moar1.txt", isBucketProcessed: false},
+					testBucket2: {lastProcessedObject: "", isBucketProcessed: true}}),
+			},
+			want: &gcsManager{
+				projectID:     testProjectID,
+				hasEnumerated: true,
+				concurrency:   defaultConcurrency,
+				maxObjectSize: defaultMaxObjectSize,
+				buckets: map[string]bucket{
+					testBucket:  {name: testBucket, startOffset: "moar.txt", shouldInclude: true},
+					testBucket2: {name: testBucket2, startOffset: "", shouldInclude: false},
+				},
+			},
+		},
+		{
+			name:   "new gcs manager, with include buckets",
+			projID: testProjectID,
+			opts: []gcsManagerOption{
+				withDefaultADC(ctx),
+				withIncludeBuckets([]string{bucket1, bucket2}),
+			},
+			want: &gcsManager{
+				projectID:      testProjectID,
+				includeBuckets: map[string]struct{}{bucket1: {}, bucket2: {}},
+				concurrency:    defaultConcurrency,
+				maxObjectSize:  defaultMaxObjectSize,
+			},
+		},
+		{
+			name:   "new gcs manager, with include buckets and api key",
+			projID: testProjectID,
+			opts:   []gcsManagerOption{withIncludeBuckets([]string{bucket1, bucket2}), withAPIKey(ctx, testAPIKey)},
+			want: &gcsManager{
+				projectID:      testProjectID,
+				includeBuckets: map[string]struct{}{bucket1: {}, bucket2: {}},
+				concurrency:    defaultConcurrency,
+				maxObjectSize:  defaultMaxObjectSize,
+			},
+		},
+		{
+			name:   "new gcs manager, with exclude buckets",
+			projID: testProjectID,
+			opts:   []gcsManagerOption{withExcludeBuckets([]string{bucket1, bucket2}), withAPIKey(ctx, testAPIKey)},
+			want: &gcsManager{
+				projectID:      testProjectID,
+				excludeBuckets: map[string]struct{}{bucket1: {}, bucket2: {}},
+				concurrency:    defaultConcurrency,
+				maxObjectSize:  defaultMaxObjectSize,
+			},
+		},
+		{
+			name:   "new gcs manager, with exclude buckets and api key",
+			projID: testProjectID,
+			opts:   []gcsManagerOption{withExcludeBuckets([]string{bucket1, bucket2}), withAPIKey(ctx, testAPIKey)},
+			want: &gcsManager{
+				projectID:      testProjectID,
+				excludeBuckets: map[string]struct{}{bucket1: {}, bucket2: {}},
+				concurrency:    defaultConcurrency,
+				maxObjectSize:  defaultMaxObjectSize,
+			},
+		},
+		{
+			name:   "new gcs manager, with include and exclude buckets",
+			projID: testProjectID,
+			opts: []gcsManagerOption{
+				withDefaultADC(ctx),
+				withIncludeBuckets([]string{bucket1, bucket2}),
+				withExcludeBuckets([]string{"bucket3", "bucket4"}),
+			},
+			want: &gcsManager{
+				projectID:      testProjectID,
+				includeBuckets: map[string]struct{}{bucket1: {}, bucket2: {}},
+				concurrency:    defaultConcurrency,
+				maxObjectSize:  defaultMaxObjectSize,
+			},
+		},
+		{
+			name:   "new gcs manager, with include and exclude buckets and api key",
+			projID: testProjectID,
+			opts: []gcsManagerOption{
+				withDefaultADC(ctx),
+				withIncludeBuckets([]string{bucket1, bucket2}),
+				withExcludeBuckets([]string{"bucket3", "bucket4"}),
+				withAPIKey(ctx, testAPIKey),
+			},
+			want: &gcsManager{
+				projectID:      testProjectID,
+				includeBuckets: map[string]struct{}{bucket1: {}, bucket2: {}},
+				concurrency:    defaultConcurrency,
+				maxObjectSize:  defaultMaxObjectSize,
+			},
+		},
+		{
+			name:   "new gcs manager, with include objects, no bucket",
+			projID: testProjectID,
+			opts:   []gcsManagerOption{withDefaultADC(ctx), withIncludeObjects([]string{"object1", "object2"})},
+			want: &gcsManager{
+				projectID:      testProjectID,
+				includeObjects: map[string]struct{}{object1: {}, object2: {}},
+				concurrency:    defaultConcurrency,
+				maxObjectSize:  defaultMaxObjectSize,
+			},
+		},
+		{
+			name:   "new gcs manager, with include objects, include buckets",
+			projID: testProjectID,
+			opts: []gcsManagerOption{
+				withDefaultADC(ctx),
+				withIncludeObjects([]string{"object1", "object2"}),
+				withIncludeBuckets([]string{bucket1, bucket2}),
+			},
+			want: &gcsManager{
+				projectID:      testProjectID,
+				includeObjects: map[string]struct{}{object1: {}, object2: {}},
+				includeBuckets: map[string]struct{}{bucket1: {}, bucket2: {}},
+				concurrency:    defaultConcurrency,
+				maxObjectSize:  defaultMaxObjectSize,
+			},
+		},
+		{
+			name:   "new gcs manager, with include objects and api key, include buckets",
+			projID: testProjectID,
+			opts: []gcsManagerOption{
+				withIncludeObjects([]string{"object1", "object2"}),
+				withIncludeBuckets([]string{bucket1, bucket2}),
+				withAPIKey(ctx, testAPIKey),
+			},
+			want: &gcsManager{
+				projectID:      testProjectID,
+				includeObjects: map[string]struct{}{object1: {}, object2: {}},
+				includeBuckets: map[string]struct{}{bucket1: {}, bucket2: {}},
+				concurrency:    defaultConcurrency,
+				maxObjectSize:  defaultMaxObjectSize,
+			},
+		},
+		{
+			name:   "new gcs manager, with exclude objects",
+			projID: testProjectID,
+			opts:   []gcsManagerOption{withDefaultADC(ctx), withExcludeObjects([]string{"object1", "object2"})},
+			want: &gcsManager{
+				projectID:      testProjectID,
+				excludeObjects: map[string]struct{}{object1: {}, object2: {}},
+				concurrency:    defaultConcurrency,
+				maxObjectSize:  defaultMaxObjectSize,
+			},
+		},
+		{
+			name:   "new gcs manager, with exclude objects and api key",
+			projID: testProjectID,
+			opts:   []gcsManagerOption{withExcludeObjects([]string{"object1", "object2"}), withAPIKey(ctx, testAPIKey)},
+			want: &gcsManager{
+				projectID:      testProjectID,
+				excludeObjects: map[string]struct{}{object1: {}, object2: {}},
+				concurrency:    defaultConcurrency,
+				maxObjectSize:  defaultMaxObjectSize,
+			},
+		},
+		{
+			name:   "new gcs manager, with include and exclude objects",
+			projID: testProjectID,
+			opts: []gcsManagerOption{
+				withDefaultADC(ctx),
+				withIncludeObjects([]string{"object1", "object2"}),
+				withExcludeObjects([]string{"object3", "object4"}),
+			},
+			want: &gcsManager{
+				projectID:      testProjectID,
+				includeObjects: map[string]struct{}{object1: {}, object2: {}},
+				concurrency:    defaultConcurrency,
+				maxObjectSize:  defaultMaxObjectSize,
+			},
+		},
+		{
+			name:   "new gcs manager, with concurrency",
+			projID: testProjectID,
+			opts:   []gcsManagerOption{withDefaultADC(ctx), withConcurrency(10)},
+			want: &gcsManager{
+				projectID:     testProjectID,
+				concurrency:   10,
+				maxObjectSize: defaultMaxObjectSize,
+			},
+		},
+		{
+			name:   "new gcs manager, default concurrency",
+			projID: testProjectID,
+			opts:   []gcsManagerOption{withDefaultADC(ctx)},
+			want: &gcsManager{
+				projectID:     testProjectID,
+				concurrency:   defaultConcurrency,
+				maxObjectSize: defaultMaxObjectSize,
+			},
+		},
+		{
+			name:   "new gcs manager, with negative concurrency",
+			projID: testProjectID,
+			opts:   []gcsManagerOption{withDefaultADC(ctx), withConcurrency(-1)},
+			want: &gcsManager{
+				projectID:     testProjectID,
+				concurrency:   defaultConcurrency,
+				maxObjectSize: defaultMaxObjectSize,
+			},
+		},
+		{
+			name:   "new gcs manager, with valid max object size",
+			projID: testProjectID,
+			opts:   []gcsManagerOption{withDefaultADC(ctx), withMaxObjectSize(10)},
+			want: &gcsManager{
+				projectID:     testProjectID,
+				concurrency:   defaultConcurrency,
+				maxObjectSize: 10,
+			},
+		},
+		{
+			name:   "new gcs manager, with negative max object size",
+			projID: testProjectID,
+			opts:   []gcsManagerOption{withDefaultADC(ctx), withMaxObjectSize(-1)},
+			want: &gcsManager{
+				projectID:     testProjectID,
+				concurrency:   defaultConcurrency,
+				maxObjectSize: defaultMaxObjectSize,
+			},
+		},
+		{
+			name:   "new gcs manager, max object size above limit",
+			projID: testProjectID,
+			opts:   []gcsManagerOption{withDefaultADC(ctx), withMaxObjectSize(int64(2 * common.GB))},
+			want: &gcsManager{
+				projectID:     testProjectID,
+				concurrency:   defaultConcurrency,
+				maxObjectSize: defaultMaxObjectSize,
+			},
+		},
+		{
+			name:   "new gcs manager, with max object size 0",
+			projID: testProjectID,
+			opts:   []gcsManagerOption{withDefaultADC(ctx), withMaxObjectSize(0)},
+			want: &gcsManager{
+				projectID:     testProjectID,
+				concurrency:   defaultConcurrency,
+				maxObjectSize: defaultMaxObjectSize,
+			},
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			got, err := newGCSManager(tc.projID, tc.opts...)
+			if (err != nil) != tc.wantErr {
+				t.Errorf("newGCSManager() error = %v, wantErr %v", err, tc.wantErr)
+				return
+			}
+
+			if err != nil {
+				return
+			}
+			// The client should never be nil.
+			if got.client == nil {
+				t.Errorf("newGCSManager() client should not be nil")
+			}
+
+			diff := cmp.Diff(got, tc.want,
+				cmp.AllowUnexported(gcsManager{}, bucket{}),
+				cmpopts.IgnoreFields(gcsManager{}, "client", "workerPool", "buckets"),
+			)
+			if diff != "" {
+				t.Errorf("newGCSManager(%v, %v) got: %v, want: %v, diff: %v", tc.projID, tc.opts, got, tc.want, diff)
+			}
+		})
+	}
+}
+
+func TestGCSManagerStats(t *testing.T) {
+	ctx := context.Background()
+
+	testCases := []struct {
+		name      string
+		projectID string
+		opts      []gcsManagerOption
+		wantStats *attributes
+	}{
+		{
+			name:      "enumerate, all buckets, with objects",
+			projectID: testProjectID,
+			opts:      []gcsManagerOption{withDefaultADC(ctx), withExcludeBuckets([]string{perfTestBucketGlob, publicBucket})},
+			wantStats: &attributes{
+				numBuckets:    4,
+				numObjects:    5,
+				bucketObjects: map[string]uint64{testBucket: 2, testBucket2: 1, testBucket3: 1, testBucket4: 1},
+			},
+		},
+		{
+			name:      "list objects, with exclude objects",
+			projectID: testProjectID,
+			opts: []gcsManagerOption{
+				withDefaultADC(ctx),
+				withExcludeObjects([]string{"aws1.txt", "moar2.txt"}),
+				withExcludeBuckets([]string{perfTestBucketGlob, publicBucket}),
+			},
+			wantStats: &attributes{
+				numBuckets:    4,
+				numObjects:    3,
+				bucketObjects: map[string]uint64{testBucket: 0, testBucket2: 1, testBucket3: 1, testBucket4: 1},
+			},
+		},
+		{
+			name:      "list objects, with include objects",
+			projectID: testProjectID,
+			opts: []gcsManagerOption{
+				withDefaultADC(ctx),
+				withIncludeObjects([]string{"aws1.txt"}),
+				withExcludeBuckets([]string{perfTestBucketGlob, publicBucket}),
+			},
+			wantStats: &attributes{
+				numBuckets:    4,
+				numObjects:    1,
+				bucketObjects: map[string]uint64{testBucket: 1, testBucket2: 0, testBucket3: 0, testBucket4: 0},
+			},
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			gm, err := newGCSManager(tc.projectID, tc.opts...)
+			if err != nil {
+				t.Fatalf("newGCSManager() error = %v", err)
+			}
+
+			got, err := gm.attributes(ctx)
+			if err != nil {
+				t.Errorf("attributes() error = %v", err)
+			}
+
+			if diff := cmp.Diff(got, tc.wantStats, cmp.AllowUnexported(attributes{}), cmpopts.IgnoreFields(attributes{}, "mu")); diff != "" {
+				t.Errorf("attributes() got: %v, want: %v, diff: %v", got, tc.wantStats, diff)
+			}
+		})
+	}
+}
+
+func TestGCSManagerStats_Time(t *testing.T) {
+	ctx := context.Background()
+
+	opts := []gcsManagerOption{withDefaultADC(ctx)}
+	gm, err := newGCSManager(testProjectID, opts...)
+	if err != nil {
+		t.Fatalf("newGCSManager() error = %v", err)
+	}
+
+	start := time.Now()
+	var stats *attributes
+	stats, _ = gm.attributes(ctx)
+	end := time.Since(start).Seconds()
+
+	fmt.Printf("Time taken to get %d objects: %f seconds\n", stats.numObjects, end)
+}
+
+func TestGCSManagerListObjects(t *testing.T) {
+	ctx := context.Background()
+
+	testCases := []struct {
+		name       string
+		projectID  string
+		opts       []gcsManagerOption
+		want       []object
+		wantNumBkt uint32
+		wantNumObj uint64
+		wantErr    bool
+	}{
+		{
+			name:      "list objects, all buckets, no objects",
+			projectID: "other",
+			opts:      []gcsManagerOption{withDefaultADC(ctx)},
+			want:      []object{},
+			wantErr:   true,
+		},
+		{
+			name:      "list objects, all buckets, with objects",
+			projectID: testProjectID,
+			opts:      []gcsManagerOption{withDefaultADC(ctx), withExcludeBuckets([]string{perfTestBucketGlob, publicBucket})},
+			want: []object{
+				{
+					name:        "aws1.txt",
+					bucket:      testBucket,
+					contentType: "text/plain",
+					size:        150,
+					link:        "https://storage.googleapis.com/download/storage/v1/b/test-bkt-th/o/aws1.txt?generation=1677870994890594&alt=media",
+					acl:         []string{},
+				},
+				{
+					name:        "moar2.txt",
+					bucket:      testBucket,
+					contentType: "text/plain",
+					size:        12,
+					link:        "https://storage.googleapis.com/download/storage/v1/b/test-bkt-th/o/moar2.txt?generation=1677871000378542&alt=media",
+					acl:         []string{},
+				},
+				{
+					name:        "aws3.txt",
+					bucket:      testBucket2,
+					contentType: "text/plain",
+					size:        150,
+					link:        "https://storage.googleapis.com/download/storage/v1/b/test-bkt-th2/o/aws3.txt?generation=1677871022489611&alt=media",
+					acl:         []string{},
+				},
+				{
+					name:        "moar.txt",
+					bucket:      testBucket3,
+					contentType: "text/plain",
+					size:        6,
+					link:        "https://storage.googleapis.com/download/storage/v1/b/test-bkt-th3/o/moar.txt?generation=1677871042896804&alt=media",
+					acl:         []string{},
+				},
+				{
+					name:        "AMAZON_FASHION_5.json",
+					bucket:      testBucket4,
+					contentType: "application/json",
+					size:        1413469,
+					link:        "https://storage.googleapis.com/download/storage/v1/b/test-bkt-th4/o/AMAZON_FASHION_5.json?generation=1677871063457469&alt=media",
+					acl:         []string{},
+				},
+			},
+			wantNumBkt: 4,
+			wantNumObj: 5,
+		},
+		{
+			name:      "list objects, include buckets, with objects",
+			projectID: testProjectID,
+			opts:      []gcsManagerOption{withDefaultADC(ctx), withIncludeBuckets([]string{testBucket})},
+			want: []object{
+				{
+					name:        "aws1.txt",
+					bucket:      testBucket,
+					contentType: "text/plain",
+					size:        150,
+					link:        "https://storage.googleapis.com/download/storage/v1/b/test-bkt-th/o/aws1.txt?generation=1677870994890594&alt=media",
+					acl:         []string{},
+				},
+				{
+					name:        "moar2.txt",
+					bucket:      testBucket,
+					contentType: "text/plain",
+					size:        12,
+					link:        "https://storage.googleapis.com/download/storage/v1/b/test-bkt-th/o/moar2.txt?generation=1677871000378542&alt=media",
+					acl:         []string{},
+				},
+			},
+			wantNumBkt: 1,
+			wantNumObj: 2,
+		},
+		{
+			name:      "list objects, exclude buckets, with objects",
+			projectID: testProjectID,
+			opts:      []gcsManagerOption{withDefaultADC(ctx), withExcludeBuckets([]string{testBucket, testBucket2, perfTestBucketGlob, publicBucket})},
+			want: []object{
+				{
+					name:        "moar.txt",
+					bucket:      testBucket3,
+					contentType: "text/plain",
+					size:        6,
+					link:        "https://storage.googleapis.com/download/storage/v1/b/test-bkt-th3/o/moar.txt?generation=1677871042896804&alt=media",
+					acl:         []string{},
+				},
+				{
+					name:        "AMAZON_FASHION_5.json",
+					bucket:      testBucket4,
+					contentType: "application/json",
+					size:        1413469,
+					link:        "https://storage.googleapis.com/download/storage/v1/b/test-bkt-th4/o/AMAZON_FASHION_5.json?generation=1677871063457469&alt=media",
+					acl:         []string{},
+				},
+			},
+			wantNumBkt: 2,
+			wantNumObj: 2,
+		},
+		{
+			name:      "list objects, with exclude objects",
+			projectID: testProjectID,
+			opts: []gcsManagerOption{
+				withDefaultADC(ctx),
+				withExcludeObjects([]string{"aws1.txt", "moar2.txt"}),
+				withExcludeBuckets([]string{perfTestBucketGlob, publicBucket}),
+			},
+			want: []object{
+				{
+					name:        "aws3.txt",
+					bucket:      testBucket2,
+					contentType: "text/plain",
+					size:        150,
+					link:        "https://storage.googleapis.com/download/storage/v1/b/test-bkt-th2/o/aws3.txt?generation=1677871022489611&alt=media",
+					acl:         []string{},
+				},
+				{
+					name:        "moar.txt",
+					bucket:      testBucket3,
+					contentType: "text/plain",
+					size:        6,
+					link:        "https://storage.googleapis.com/download/storage/v1/b/test-bkt-th3/o/moar.txt?generation=1677871042896804&alt=media",
+					acl:         []string{},
+				},
+				{
+					name:        "AMAZON_FASHION_5.json",
+					bucket:      testBucket4,
+					contentType: "application/json",
+					size:        1413469,
+					link:        "https://storage.googleapis.com/download/storage/v1/b/test-bkt-th4/o/AMAZON_FASHION_5.json?generation=1677871063457469&alt=media",
+					acl:         []string{},
+				},
+			},
+			wantNumBkt: 4, // We still list objects in all buckets.
+			wantNumObj: 3,
+		},
+		{
+			name:      "list objects, with include objects, include bucket",
+			projectID: testProjectID,
+			opts: []gcsManagerOption{
+				withDefaultADC(ctx),
+				withIncludeObjects([]string{"aws1.txt"}),
+				withIncludeBuckets([]string{testBucket}),
+			},
+			want: []object{
+				{
+					name:        "aws1.txt",
+					bucket:      testBucket,
+					contentType: "text/plain",
+					size:        150,
+					link:        "https://storage.googleapis.com/download/storage/v1/b/test-bkt-th/o/aws1.txt?generation=1677870994890594&alt=media",
+					acl:         []string{},
+				},
+			},
+			wantNumBkt: 1,
+			wantNumObj: 1,
+		},
+		{
+			name:      "list objects, with include objects",
+			projectID: testProjectID,
+			opts: []gcsManagerOption{
+				withDefaultADC(ctx),
+				withIncludeObjects([]string{"aws1.txt"}),
+				withExcludeBuckets([]string{perfTestBucketGlob, publicBucket}),
+			},
+			want: []object{
+				{
+					name:        "aws1.txt",
+					bucket:      testBucket,
+					contentType: "text/plain",
+					size:        150,
+					link:        "https://storage.googleapis.com/download/storage/v1/b/test-bkt-th/o/aws1.txt?generation=1677870994890594&alt=media",
+					acl:         []string{},
+				},
+			},
+			wantNumBkt: 4,
+			wantNumObj: 1,
+		},
+		{
+			name:      "list objects, with include bucket, exclude object",
+			projectID: testProjectID,
+			opts: []gcsManagerOption{
+				withDefaultADC(ctx),
+				withIncludeBuckets([]string{testBucket}),
+				withExcludeObjects([]string{"aws1.txt"}),
+				withExcludeBuckets([]string{perfTestBucketGlob, publicBucket}),
+			},
+			want: []object{
+				{
+					name:        "moar2.txt",
+					bucket:      testBucket,
+					contentType: "text/plain",
+					size:        12,
+					link:        "https://storage.googleapis.com/download/storage/v1/b/test-bkt-th/o/moar2.txt?generation=1677871000378542&alt=media",
+					acl:         []string{},
+				},
+			},
+			wantNumBkt: 1,
+			wantNumObj: 1,
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			mgr, err := newGCSManager(tc.projectID, tc.opts...)
+			assert.Nil(t, err)
+
+			got, err := mgr.listObjects(ctx)
+			if (err != nil) != tc.wantErr {
+				t.Errorf("GCSManager.ListObjects() error = %v, wantErr %v", err, tc.wantErr)
+				return
+			}
+
+			// If we expect an error, we're done.
+			// Short-circuit the rest of the test.
+			if err != nil {
+				return
+			}
+
+			doneCh := make(chan struct{})
+			defer close(doneCh)
+			go func() {
+				defer func() {
+					doneCh <- struct{}{}
+				}()
+				res := make([]object, 0, len(tc.want))
+				for obj := range got {
+					res = append(res, obj.(object))
+				}
+
+				if len(res) != len(tc.want) {
+					t.Errorf("gcsManager.listObjects() got: %v, want: %v", res, tc.want)
+				}
+
+				// Test the bucket and object counts.
+				assert.Equal(t, tc.wantNumBkt, mgr.numBuckets)
+				assert.Equal(t, tc.wantNumObj, mgr.numObjects)
+
+				// Sort the objects by name to make the test deterministic.
+				// This is necessary because we list bucket objects concurrently.
+				sort.Slice(res, func(i, j int) bool { return res[i].name < res[j].name })
+				sort.Slice(tc.want, func(i, j int) bool { return tc.want[i].name < tc.want[j].name })
+
+				// Test the objects are equal.
+				if diff := cmp.Diff(res, tc.want, cmp.AllowUnexported(object{}), cmpopts.IgnoreFields(object{}, "Reader", "createdAt", "updatedAt")); diff != "" {
+					t.Errorf("gcsManager.listObjects() mismatch (-want +got):\n%s", diff)
+				}
+			}()
+
+			<-doneCh
+		})
+	}
+}
+
+func TestGCSManagerListObjects_Resuming(t *testing.T) {
+	ctx := context.Background()
+
+	testCases := []struct {
+		name       string
+		projectID  string
+		opts       []gcsManagerOption
+		want       []object
+		wantNumBkt uint32
+		wantNumObj uint64
+		wantErr    bool
+	}{
+		{
+			name:      "list objects, testBucket, resume from moar2.txt",
+			projectID: testProjectID,
+			opts: []gcsManagerOption{
+				withDefaultADC(ctx),
+				withIncludeBuckets([]string{testBucket}),
+				withBucketOffsets(map[string]offsetInfo{testBucket: {lastProcessedObject: "moar2.txt"}}),
+			},
+			want: []object{
+				{
+					name:        "moar2.txt",
+					bucket:      testBucket,
+					contentType: "text/plain",
+					size:        12,
+					link:        "https://storage.googleapis.com/download/storage/v1/b/test-bkt-th/o/moar2.txt?generation=1677871000378542&alt=media",
+					acl:         []string{},
+				},
+			},
+			wantNumBkt: 1,
+			wantNumObj: 1,
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			mgr, err := newGCSManager(tc.projectID, tc.opts...)
+			assert.Nil(t, err)
+
+			got, err := mgr.listObjects(ctx)
+			if (err != nil) != tc.wantErr {
+				t.Errorf("GCSManager.ListObjects() error = %v, wantErr %v", err, tc.wantErr)
+				return
+			}
+
+			// If we expect an error, we're done.
+			// Short-circuit the rest of the test.
+			if err != nil {
+				return
+			}
+
+			doneCh := make(chan struct{})
+			defer close(doneCh)
+			go func() {
+				defer func() {
+					doneCh <- struct{}{}
+				}()
+				res := make([]object, 0, len(tc.want))
+				for obj := range got {
+					res = append(res, obj.(object))
+				}
+
+				if len(res) != len(tc.want) {
+					t.Errorf("gcsManager.listObjects() got: %v, want: %v", res, tc.want)
+				}
+
+				// Test the bucket and object counts.
+				assert.Equal(t, tc.wantNumBkt, mgr.numBuckets)
+				assert.Equal(t, tc.wantNumObj, mgr.numObjects)
+
+				// Sort the objects by name to make the test deterministic.
+				// This is necessary because we list bucket objects concurrently.
+				sort.Slice(res, func(i, j int) bool { return res[i].name < res[j].name })
+				sort.Slice(tc.want, func(i, j int) bool { return tc.want[i].name < tc.want[j].name })
+
+				// Test the objects are equal.
+				if diff := cmp.Diff(res, tc.want, cmp.AllowUnexported(object{}), cmpopts.IgnoreFields(object{}, "Reader", "createdAt", "updatedAt")); diff != "" {
+					t.Errorf("gcsManager.listObjects() mismatch (-want +got):\n%s", diff)
+				}
+			}()
+
+			<-doneCh
+		})
+	}
+}
+
+func Test_isObjectTypeValid(t *testing.T) {
+	testCases := []struct {
+		name    string
+		objName string
+		want    bool
+	}{
+		{
+			name:    "valid object type, text",
+			objName: "aws1.txt",
+			want:    true,
+		},
+		{
+			name:    "valid object type, zip",
+			objName: "aws1.zip",
+			want:    true,
+		},
+		{
+			name:    "invalid object type",
+			objName: "video.mp4",
+			want:    false,
+		},
+	}
+
+	ctx := context.Background()
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			got := isObjectTypeValid(ctx, tc.objName)
+			assert.Equal(t, tc.want, got)
+		})
+	}
+}
+
+func Test_isObjectSizeValid(t *testing.T) {
+	testCases := []struct {
+		name    string
+		objSize int64
+		want    bool
+	}{
+		{
+			name:    "valid object size, 150 bytes",
+			objSize: 150,
+			want:    true,
+		},
+		{
+			name:    "valid object size, 1 MB",
+			objSize: int64(1 * common.MB),
+			want:    true,
+		},
+		{
+			name:    "valid object size, 49 MB",
+			objSize: int64(49 * common.MB),
+			want:    true,
+		},
+		{
+			name:    "invalid object size, 1 GB",
+			objSize: int64(1 * common.GB),
+			want:    false,
+		},
+		{
+			name:    "invalid object size, 2 GB",
+			objSize: int64(2 * common.GB),
+			want:    false,
+		},
+		{
+			name:    "invalid object size, empty object",
+			objSize: 0,
+			want:    false,
+		},
+		{
+			name:    "invalid object size, negative object size",
+			objSize: -1,
+			want:    false,
+		},
+	}
+
+	ctx := context.Background()
+	for _, tc := range testCases {
+		g := &gcsManager{
+			maxObjectSize: maxObjectSizeLimit,
+		}
+		t.Run(tc.name, func(t *testing.T) {
+			got := g.isObjectSizeValid(ctx, tc.objSize)
+			assert.Equal(t, tc.want, got)
+		})
+	}
+}

pkg/sources/gcs/gcs_test.go

@@ -0,0 +1,422 @@
+package gcs
+
+import (
+	"encoding/json"
+	"fmt"
+	"io"
+	"sort"
+	"testing"
+
+	"github.com/google/go-cmp/cmp"
+	"github.com/google/go-cmp/cmp/cmpopts"
+	"github.com/stretchr/testify/assert"
+	"google.golang.org/protobuf/types/known/anypb"
+
+	"github.com/trufflesecurity/trufflehog/v3/pkg/context"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/source_metadatapb"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
+)
+
+func createTestSource(src *sourcespb.GCS) (*Source, *anypb.Any) {
+	s := &Source{}
+	conn, err := anypb.New(src)
+	if err != nil {
+		panic(err)
+	}
+	return s, conn
+}
+
+func TestSourceInit(t *testing.T) {
+	source, conn := createTestSource(&sourcespb.GCS{
+		ProjectId: testProjectID,
+		IncludeBuckets: []string{
+			"bucket1",
+		},
+		ExcludeBuckets: []string{
+			perfTestBucketGlob,
+		},
+		ExcludeObjects: []string{
+			"object1",
+		},
+		Credential: &sourcespb.GCS_ApiKey{
+			ApiKey: testAPIKey,
+		},
+	})
+
+	// Set the EncodeResumeInfo to prevent enumeration.
+	b, err := json.Marshal(map[string]objectsProgress{"other": {}})
+	assert.Nil(t, err)
+	source.Progress.EncodedResumeInfo = string(b)
+
+	err = source.Init(context.Background(), "test", 1, 1, true, conn, 8)
+	assert.Nil(t, err)
+	assert.NotNil(t, source.gcsManager)
+}
+
+func TestSourceInit_Conn(t *testing.T) {
+	testCases := []struct {
+		name    string
+		conn    *sourcespb.GCS
+		want    *gcsManager
+		wantErr bool
+	}{
+		{
+			name:    "nil conn",
+			wantErr: true,
+		},
+		{
+			name: "valid conn, bare config",
+			conn: &sourcespb.GCS{
+				ProjectId:  testProjectID,
+				Credential: &sourcespb.GCS_Adc{},
+				ExcludeBuckets: []string{
+					perfTestBucketGlob,
+				},
+			},
+			want: &gcsManager{
+				projectID:      testProjectID,
+				excludeBuckets: map[string]struct{}{perfTestBucketGlob: {}},
+				hasEnumerated:  true,
+			},
+		},
+		{
+			name: "valid conn, include and exclude buckets",
+			conn: &sourcespb.GCS{
+				ProjectId:  testProjectID,
+				Credential: &sourcespb.GCS_Adc{},
+				IncludeBuckets: []string{
+					"bucket1",
+				},
+				ExcludeBuckets: []string{
+					perfTestBucketGlob,
+				},
+			},
+			want: &gcsManager{
+				projectID:      testProjectID,
+				includeBuckets: map[string]struct{}{"bucket1": {}},
+				hasEnumerated:  true,
+			},
+		},
+		{
+			name: "valid conn, include and exclude objects",
+			conn: &sourcespb.GCS{
+				ProjectId:  testProjectID,
+				Credential: &sourcespb.GCS_Adc{},
+				IncludeObjects: []string{
+					"object1",
+				},
+				ExcludeObjects: []string{
+					"object2",
+				},
+				ExcludeBuckets: []string{
+					perfTestBucketGlob,
+				},
+			},
+			want: &gcsManager{
+				projectID:      testProjectID,
+				includeObjects: map[string]struct{}{"object1": {}},
+				excludeBuckets: map[string]struct{}{perfTestBucketGlob: {}},
+				hasEnumerated:  true,
+			},
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			source, conn := createTestSource(tc.conn)
+			// Set the EncodeResumeInfo to prevent enumeration.
+			b, err := json.Marshal(map[string]objectsProgress{"other": {}})
+			assert.Nil(t, err)
+			source.Progress.EncodedResumeInfo = string(b)
+
+			err = source.Init(context.Background(), "test", 1, 1, true, conn, 8)
+			if err != nil && !tc.wantErr {
+				t.Errorf("source.Init() got: %v, want: %v", err, nil)
+				return
+			}
+
+			if !tc.wantErr {
+				if diff := cmp.Diff(tc.want, source.gcsManager,
+					cmp.AllowUnexported(gcsManager{}),
+					cmpopts.IgnoreFields(gcsManager{}, "client", "workerPool", "concurrency", "buckets", "maxObjectSize", "attr"),
+				); diff != "" {
+					t.Errorf("source.Init() diff: (-want +got)\n%s", diff)
+				}
+			}
+		})
+	}
+}
+
+func TestSourceInit_Progress(t *testing.T) {
+	complete := map[string]objectsProgress{testBucket: {
+		Processed:         object2,
+		IsBucketProcessed: true,
+		ProcessedCnt:      1,
+		TotalCnt:          1,
+	}}
+	completeEncodeResumeInfo, err := json.Marshal(complete)
+	assert.Nil(t, err)
+
+	processing := map[string]objectsProgress{
+		testBucket: {
+			IsBucketProcessed: false,
+			ProcessedCnt:      1,
+			Processed:         object2,
+			Processing:        map[string]struct{}{object1: {}},
+			TotalCnt:          2,
+		},
+	}
+	processingEncodeResumeInfo, err := json.Marshal(processing)
+	assert.Nil(t, err)
+
+	testCases := []struct {
+		name              string
+		conn              *sourcespb.GCS
+		encodeResumeInfo  string
+		gcsManagerBuckets map[string]bucket
+	}{
+		{
+			name:              "no progress",
+			conn:              &sourcespb.GCS{ProjectId: testProjectID, Credential: &sourcespb.GCS_Adc{}, ExcludeBuckets: []string{perfTestBucketGlob}},
+			gcsManagerBuckets: map[string]bucket{},
+		},
+		{
+			name:              "progress set, no processing objects in any buckets",
+			conn:              &sourcespb.GCS{ProjectId: testProjectID, Credential: &sourcespb.GCS_Adc{}},
+			encodeResumeInfo:  string(completeEncodeResumeInfo),
+			gcsManagerBuckets: map[string]bucket{testBucket: {name: testBucket, startOffset: "", shouldInclude: false}},
+		},
+		{
+			name:              "progress set, processing objects in some buckets",
+			conn:              &sourcespb.GCS{ProjectId: testProjectID, Credential: &sourcespb.GCS_Adc{}},
+			encodeResumeInfo:  string(processingEncodeResumeInfo),
+			gcsManagerBuckets: map[string]bucket{testBucket: {name: testBucket, startOffset: object1, shouldInclude: true}},
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			source, conn := createTestSource(tc.conn)
+			source.Progress.EncodedResumeInfo = tc.encodeResumeInfo
+
+			err := source.Init(context.Background(), "test", 1, 1, true, conn, 8)
+			assert.Nil(t, err)
+			assert.Equal(t, tc.gcsManagerBuckets, source.gcsManager.(*gcsManager).buckets)
+		})
+	}
+}
+
+type mockObjectManager struct {
+	wantErr bool
+}
+
+func (m *mockObjectManager) attributes(_ context.Context) (*attributes, error) {
+	if m.wantErr {
+		return nil, fmt.Errorf("some error")
+	}
+
+	return &attributes{
+		numObjects:    5,
+		numBuckets:    1,
+		bucketObjects: map[string]uint64{testBucket: 5},
+	}, nil
+}
+
+type mockReader struct {
+	offset int
+	data   []byte
+}
+
+func (m *mockReader) Read(p []byte) (n int, err error) {
+	if m.offset >= len(m.data) {
+		return 0, io.EOF
+	}
+
+	n = copy(p, m.data[m.offset:])
+	m.offset += n
+	return
+}
+
+func (m *mockObjectManager) listObjects(context.Context) (chan io.Reader, error) {
+	if m.wantErr {
+		return nil, fmt.Errorf("some error")
+	}
+
+	ch := make(chan io.Reader)
+	go func() {
+		defer close(ch)
+		// Add 5 objects to the channel.
+		for i := 0; i < 5; i++ {
+			ch <- createTestObject(i)
+		}
+	}()
+
+	return ch, nil
+}
+
+func createTestObject(id int) object {
+	return object{
+		name:        fmt.Sprintf("object%d", id),
+		bucket:      testBucket,
+		contentType: "plain/text",
+		owner:       "testman@test.com",
+		link:        fmt.Sprintf("https://storage.googleapis.com/%s/%s", testBucket, fmt.Sprintf("object%d", id)),
+		acl:         []string{"authenticatedUsers"},
+		size:        42,
+		Reader:      &mockReader{data: []byte(fmt.Sprintf("hello world %d", id))},
+	}
+}
+
+func createTestSourceChunk(id int) *sources.Chunk {
+	return &sources.Chunk{
+		SourceName: "test",
+		SourceType: sourcespb.SourceType_SOURCE_TYPE_GCS,
+		SourceID:   0,
+		Verify:     true,
+		Data:       []byte(fmt.Sprintf("hello world %d", id)),
+		SourceMetadata: &source_metadatapb.MetaData{
+			Data: &source_metadatapb.MetaData_Gcs{
+				Gcs: &source_metadatapb.GCS{
+					Filename:    fmt.Sprintf("object%d", id),
+					Bucket:      testBucket,
+					ContentType: "plain/text",
+					Email:       "testman@test.com",
+					Link:        fmt.Sprintf("https://storage.googleapis.com/%s/%s", testBucket, fmt.Sprintf("object%d", id)),
+					Acls:        []string{"authenticatedUsers"},
+				},
+			},
+		},
+	}
+}
+
+func TestSourceChunks_ListObjects(t *testing.T) {
+	ctx := context.Background()
+	chunksCh := make(chan *sources.Chunk, 1)
+
+	source := &Source{
+		gcsManager: &mockObjectManager{},
+		chunksCh:   chunksCh,
+	}
+
+	err := source.enumerate(ctx)
+	assert.Nil(t, err)
+
+	go func() {
+		defer close(chunksCh)
+		err := source.Chunks(ctx, chunksCh)
+		assert.Nil(t, err)
+	}()
+
+	want := make([]*sources.Chunk, 0, 5)
+	for i := 0; i < 5; i++ {
+		want = append(want, createTestSourceChunk(i))
+	}
+
+	count := 0
+	got := make([]*sources.Chunk, 0, 5)
+	for ch := range chunksCh {
+		got = append(got, ch)
+		count++
+	}
+
+	// Ensure we get 5 objects back.
+	assert.Equal(t, 5, count)
+
+	// Sort the results to ensure deterministic ordering.
+	sort.Slice(want, func(i, j int) bool {
+		return want[i].SourceMetadata.GetGcs().Filename < want[j].SourceMetadata.GetGcs().Filename
+	})
+	sort.Slice(got, func(i, j int) bool {
+		return got[i].SourceMetadata.GetGcs().Filename < got[j].SourceMetadata.GetGcs().Filename
+	})
+
+	for _, c := range got {
+		assert.Equal(t, c.SourceMetadata.GetGcs().Filename, c.SourceMetadata.GetGcs().Filename)
+		assert.Equal(t, c.Data, c.Data)
+	}
+
+}
+
+func TestSourceInit_Enumerate(t *testing.T) {
+	ctx := context.Background()
+	source := &Source{gcsManager: &mockObjectManager{}}
+
+	err := source.enumerate(ctx)
+	assert.Nil(t, err)
+
+	// Ensure the attributes are set.
+	assert.Equal(t, uint64(5), source.stats.numObjects)
+	assert.Equal(t, uint32(1), source.stats.numBuckets)
+	assert.Equal(t, uint64(5), source.stats.bucketObjects[testBucket])
+	// Ensure progress is initialized correctly.
+	p := map[string]*objectsProgress{testBucket: {IsBucketProcessed: false, ProcessedCnt: 0, TotalCnt: 5}}
+	b, err := json.Marshal(p)
+	assert.Nil(t, err)
+	assert.Equal(t, string(b), source.Progress.EncodedResumeInfo)
+}
+
+func TestSourceChunks_ListObjects_Error(t *testing.T) {
+	ctx := context.Background()
+	source := &Source{gcsManager: &mockObjectManager{wantErr: true}}
+
+	chunksCh := make(chan *sources.Chunk, 1)
+
+	defer close(chunksCh)
+	err := source.Chunks(ctx, chunksCh)
+	assert.True(t, err != nil)
+}
+
+func TestSourceChunks_ProgressSet(t *testing.T) {
+	ctx := context.Background()
+	chunksCh := make(chan *sources.Chunk, 1)
+
+	source := &Source{
+		gcsManager: &mockObjectManager{},
+		chunksCh:   chunksCh,
+	}
+	err := source.enumerate(ctx)
+	assert.Nil(t, err)
+
+	go func() {
+		defer close(chunksCh)
+		err := source.Chunks(ctx, chunksCh)
+		assert.Nil(t, err)
+	}()
+
+	want := make([]*sources.Chunk, 0, 5)
+	for i := 0; i < 5; i++ {
+		want = append(want, createTestSourceChunk(i))
+	}
+
+	got := make([]*sources.Chunk, 0, 5)
+	for ch := range chunksCh {
+		got = append(got, ch)
+	}
+
+	// Ensure we get 5 objects back.
+	assert.Equal(t, len(want), len(got))
+
+	processing := map[string]struct{}{"object0": {}, "object1": {}, "object2": {}, "object3": {}, "object4": {}}
+	for i := 0; i < 5; i++ {
+		delete(processing, fmt.Sprintf("object%d", i))
+	}
+
+	// Test that the resume progress is set.
+	// The processed values should be the greatest object name lexicographically.
+	resume := map[string]objectsProgress{
+		testBucket: {
+			Processed:         "object4",
+			Processing:        processing,
+			IsBucketProcessed: true,
+			ProcessedCnt:      5,
+			TotalCnt:          5,
+		},
+	}
+	b, err := json.Marshal(resume)
+	assert.Nil(t, err)
+
+	assert.Equal(t, string(b), source.Progress.EncodedResumeInfo)
+	assert.Equal(t, int32(5), source.Progress.SectionsCompleted)
+	assert.Equal(t, int64(100), source.Progress.PercentComplete)
+}

pkg/sources/git/git.go

@@ -5,7 +5,6 @@ import (
 	"bytes"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"net/url"
 	"os"
 	"os/exec"
@@ -257,7 +256,7 @@ func CloneRepo(ctx context.Context, userInfo *url.Userinfo, gitUrl string, args
 	if err := gitCmdCheck(); err != nil {
 		return "", nil, err
 	}
-	clonePath, err := ioutil.TempDir(os.TempDir(), "trufflehog")
+	clonePath, err := os.MkdirTemp(os.TempDir(), "trufflehog")
 	if err != nil {
 		return "", nil, err
 	}
@@ -296,7 +295,7 @@ func CloneRepo(ctx context.Context, userInfo *url.Userinfo, gitUrl string, args
 		return "", nil, errors.New("clone command exited with no output")
 	}
 	if cloneCmd.ProcessState != nil && cloneCmd.ProcessState.ExitCode() != 0 {
-		logger.V(1).Info("git clone failed", "error", err)
+		logger.V(1).Info("git clone failed", "output", output, "error", err)
 		return "", nil, fmt.Errorf("could not clone repo: %s, %w", safeUrl, err)
 	}
 
@@ -379,7 +378,7 @@ func (s *Git) ScanCommits(ctx context.Context, repo *git.Repository, path string
 			var email, hash, when string
 			email = commit.Author
 			hash = commit.Hash
-			when = commit.Date.String()
+			when = commit.Date.Format("2006-01-02 15:04:05 -0700")
 
 			// Handle binary files by reading the entire file rather than using the diff.
 			if diff.IsBinary {
@@ -473,9 +472,9 @@ func (s *Git) gitChunk(ctx context.Context, diff gitparse.Diff, fileName, email,
 	}
 }
 
-// ScanUnstaged chunks unstaged changes.
-func (s *Git) ScanUnstaged(ctx context.Context, repo *git.Repository, path string, scanOptions *ScanOptions, chunksChan chan *sources.Chunk) error {
-	// get the URL metadata for reporting (may be empty)
+// ScanStaged chunks staged changes.
+func (s *Git) ScanStaged(ctx context.Context, repo *git.Repository, path string, scanOptions *ScanOptions, chunksChan chan *sources.Chunk) error {
+	// Get the URL metadata for reporting (may be empty).
 	urlMetadata := getSafeRemoteURL(repo, "origin")
 
 	commitChan, err := gitparse.NewParser().Unstaged(ctx, path)
@@ -489,11 +488,11 @@ func (s *Git) ScanUnstaged(ctx context.Context, repo *git.Repository, path strin
 	var depth int64
 	var reachedBase = false
 
-	ctx.Logger().V(1).Info("scanning unstaged changes", "path", path)
+	ctx.Logger().V(1).Info("scanning staged changes", "path", path)
 	for commit := range commitChan {
 		for _, diff := range commit.Diffs {
 			logger := ctx.Logger().WithValues("filename", diff.PathB, "commit", commit.Hash, "file", diff.PathB)
-			logger.V(2).Info("scanning unstaged changes from git")
+			logger.V(2).Info("scanning staged changes from git")
 
 			if scanOptions.MaxDepth > 0 && depth >= scanOptions.MaxDepth {
 				logger.V(1).Info("reached max depth")
@@ -521,12 +520,12 @@ func (s *Git) ScanUnstaged(ctx context.Context, repo *git.Repository, path strin
 			var email, hash, when string
 			email = commit.Author
 			hash = commit.Hash
-			when = commit.Date.String()
+			when = commit.Date.Format("2006-01-02 15:04:05 -0700")
 
 			// Handle binary files by reading the entire file rather than using the diff.
 			if diff.IsBinary {
 				commitHash := plumbing.NewHash(hash)
-				metadata := s.sourceMetadataFunc(fileName, email, "Unstaged", when, urlMetadata, 0)
+				metadata := s.sourceMetadataFunc(fileName, email, "Staged", when, urlMetadata, 0)
 				chunkSkel := &sources.Chunk{
 					SourceName:     s.sourceName,
 					SourceID:       s.sourceID,
@@ -540,7 +539,7 @@ func (s *Git) ScanUnstaged(ctx context.Context, repo *git.Repository, path strin
 				continue
 			}
 
-			metadata := s.sourceMetadataFunc(fileName, email, "Unstaged", when, urlMetadata, int64(diff.LineStart))
+			metadata := s.sourceMetadataFunc(fileName, email, "Staged", when, urlMetadata, int64(diff.LineStart))
 			chunksChan <- &sources.Chunk{
 				SourceName:     s.sourceName,
 				SourceID:       s.sourceID,
@@ -565,7 +564,7 @@ func (s *Git) ScanRepo(ctx context.Context, repo *git.Repository, repoPath strin
 	if err := s.ScanCommits(ctx, repo, repoPath, scanOptions, chunksChan); err != nil {
 		return err
 	}
-	if err := s.ScanUnstaged(ctx, repo, repoPath, scanOptions, chunksChan); err != nil {
+	if err := s.ScanStaged(ctx, repo, repoPath, scanOptions, chunksChan); err != nil {
 		ctx.Logger().V(1).Info("error scanning unstaged changes", "error", err)
 	}
 

pkg/sources/git/git_test.go

@@ -137,7 +137,7 @@ func TestSource_Scan(t *testing.T) {
 			}
 			chunksCh := make(chan *sources.Chunk, 1)
 			go func() {
-				s.Chunks(ctx, chunksCh)
+				assert.NoError(t, s.Chunks(ctx, chunksCh))
 			}()
 			gotChunk := <-chunksCh
 			gotChunk.Data = nil