Add method to check for unverified.

Co-authored-by: Zachary Rice <zachary.rice@trufflesec.com>

go.mod

@@ -6,7 +6,7 @@ replace github.com/jpillora/overseer => github.com/trufflesecurity/overseer v1.1
 
 require (
 	cloud.google.com/go/secretmanager v1.11.0
-	cloud.google.com/go/storage v1.29.0
+	cloud.google.com/go/storage v1.30.1
 	github.com/Azure/go-autorest/autorest/azure/auth v0.5.11
 	github.com/TheZeroSlave/zapsentry v1.15.0
 	github.com/aws/aws-sdk-go v1.44.83
@@ -14,6 +14,7 @@ require (
 	github.com/bill-rich/go-syslog v0.0.0-20220413021637-49edb52a574c
 	github.com/bitfinexcom/bitfinex-api-go v0.0.0-20210608095005-9e0b26f200fb
 	github.com/bradleyfalzon/ghinstallation/v2 v2.4.0
+	github.com/couchbase/gocb/v2 v2.6.3
 	github.com/crewjam/rfc5424 v0.1.0
 	github.com/denisenkom/go-mssqldb v0.12.3
 	github.com/envoyproxy/protoc-gen-validate v1.0.1
@@ -22,7 +23,7 @@ require (
 	github.com/getsentry/sentry-go v0.22.0
 	github.com/go-errors/errors v1.4.2
 	github.com/go-git/go-git/v5 v5.7.0
-	github.com/go-ldap/ldap/v3 v3.4.4
+	github.com/go-ldap/ldap/v3 v3.4.5
 	github.com/go-logr/logr v1.2.4
 	github.com/go-logr/zapr v1.2.4
 	github.com/go-redis/redis v6.15.9+incompatible
@@ -50,13 +51,13 @@ require (
 	github.com/sergi/go-diff v1.3.1
 	github.com/stretchr/testify v1.8.4
 	github.com/tailscale/depaware v0.0.0-20210622194025-720c4b409502
-	github.com/xanzy/go-gitlab v0.85.0
-	go.mongodb.org/mongo-driver v1.11.6
+	github.com/xanzy/go-gitlab v0.86.0
+	go.mongodb.org/mongo-driver v1.12.0
 	go.uber.org/zap v1.24.0
 	golang.org/x/crypto v0.10.0
 	golang.org/x/exp v0.0.0-20221018205818-5c77f4b2bbd7
 	golang.org/x/oauth2 v0.9.0
-	golang.org/x/sync v0.2.0
+	golang.org/x/sync v0.3.0
 	golang.org/x/text v0.10.0
 	google.golang.org/api v0.128.0
 	google.golang.org/protobuf v1.30.0
@@ -77,7 +78,7 @@ require (
 	github.com/Azure/go-autorest/autorest/date v0.3.0 // indirect
 	github.com/Azure/go-autorest/logger v0.2.1 // indirect
 	github.com/Azure/go-autorest/tracing v0.6.0 // indirect
-	github.com/Azure/go-ntlmssp v0.0.0-20220621081337-cb9428e4ac1e // indirect
+	github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 // indirect
 	github.com/Microsoft/go-winio v0.6.1 // indirect
 	github.com/ProtonMail/go-crypto v0.0.0-20230518184743-7afd39499903 // indirect
 	github.com/acomagu/bufpipe v1.0.4 // indirect
@@ -93,6 +94,7 @@ require (
 	github.com/cloudflare/circl v1.3.3 // indirect
 	github.com/connesc/cipherio v0.2.1 // indirect
 	github.com/containerd/stargz-snapshotter/estargz v0.14.3 // indirect
+  github.com/couchbase/gocbcore/v10 v10.2.3 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/dimchansky/utfbom v1.1.1 // indirect
 	github.com/docker/cli v23.0.5+incompatible // indirect
@@ -153,8 +155,8 @@ require (
 	github.com/vbatts/tar-split v0.11.3 // indirect
 	github.com/xanzy/ssh-agent v0.3.3 // indirect
 	github.com/xdg-go/pbkdf2 v1.0.0 // indirect
-	github.com/xdg-go/scram v1.1.1 // indirect
-	github.com/xdg-go/stringprep v1.0.3 // indirect
+	github.com/xdg-go/scram v1.1.2 // indirect
+	github.com/xdg-go/stringprep v1.0.4 // indirect
 	github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d // indirect
 	github.com/yusufpapurcu/wmi v1.2.2 // indirect
 	go.opencensus.io v0.24.0 // indirect

go.sum

@@ -25,8 +25,8 @@ cloud.google.com/go/secretmanager v1.11.0 h1:tNzxvs3fP57iTrhaS1XdEPwg4LSPKiDNgp+
 cloud.google.com/go/secretmanager v1.11.0/go.mod h1:qeQq0/jyJqrGeULu0GkRsVSPKTvf98AEqJnuEIQiJwA=
 cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw=
 cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos=
-cloud.google.com/go/storage v1.29.0 h1:6weCgzRvMg7lzuUurI4697AqIRPU1SvzHhynwpW31jI=
-cloud.google.com/go/storage v1.29.0/go.mod h1:4puEjyTKnku6gfKoTfNOU/W+a9JyuVNxjpS5GBrB8h4=
+cloud.google.com/go/storage v1.30.1 h1:uOdMxAs8HExqBlnLtnQyP0YkvbiDpdGShGKtx6U/oNM=
+cloud.google.com/go/storage v1.30.1/go.mod h1:NfxhC0UJE1aXSx7CIIbCf7y9HKT7BiccwkR7+P7gN8E=
 dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
 github.com/Azure/azure-sdk-for-go/sdk/azcore v0.19.0/go.mod h1:h6H6c8enJmmocHUbLiiGY6sx7f9i+X3m1CHdd5c6Rdw=
 github.com/Azure/azure-sdk-for-go/sdk/azidentity v0.11.0/go.mod h1:HcM1YX14R7CJcghJGOYCgdezslRSVzqwLf/q+4Y2r/0=
@@ -49,8 +49,8 @@ github.com/Azure/go-autorest/logger v0.2.1 h1:IG7i4p/mDa2Ce4TRyAO8IHnVhAVF3RFU+Z
 github.com/Azure/go-autorest/logger v0.2.1/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZmbF5NWuPV8+WeEW8=
 github.com/Azure/go-autorest/tracing v0.6.0 h1:TYi4+3m5t6K48TGI9AUdb+IzbnSxvnvUMfuitfgcfuo=
 github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU=
-github.com/Azure/go-ntlmssp v0.0.0-20220621081337-cb9428e4ac1e h1:NeAW1fUYUEWhft7pkxDf6WoUvEZJ/uOKsvtpjLnn8MU=
-github.com/Azure/go-ntlmssp v0.0.0-20220621081337-cb9428e4ac1e/go.mod h1:chxPXzSsl7ZWRAuOIE23GDNzjWuZquvFlgA8xmpunjU=
+github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 h1:mFRzDkZVAjdal+s7s0MwaRv9igoPqLRdzOLzw/8Xvq8=
+github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358/go.mod h1:chxPXzSsl7ZWRAuOIE23GDNzjWuZquvFlgA8xmpunjU=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/BurntSushi/toml v1.2.1/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ=
 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
@@ -68,6 +68,8 @@ github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 h1:JYp7IbQjafo
 github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
 github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 h1:s6gZFSlWYmbqAuRjVTiNNhvNRfY2Wxp9nhfyel4rklc=
 github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137/go.mod h1:OMCwj8VM1Kc9e19TLln2VL61YJF0x1XFtfdL4JdbSyE=
+github.com/alexbrainman/sspi v0.0.0-20210105120005-909beea2cc74 h1:Kk6a4nehpJ3UuJRqlA3JxYxBZEqCeOmATOvrbT4p9RA=
+github.com/alexbrainman/sspi v0.0.0-20210105120005-909beea2cc74/go.mod h1:cEWa1LVoE5KvSD9ONXsZrj0z6KqySlCCNKHlLzbqAt4=
 github.com/andybalholm/brotli v1.0.5 h1:8uQZIdzKmjc/iuPu7O2ioW48L81FgatrcpfFmiq/cCs=
 github.com/andybalholm/brotli v1.0.5/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig=
 github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be h1:9AeTilPcZAjCFIImctFaOjnTIavg87rW78vTPkQqLI8=
@@ -113,6 +115,13 @@ github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWH
 github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
 github.com/connesc/cipherio v0.2.1 h1:FGtpTPMbKNNWByNrr9aEBtaJtXjqOzkIXNYJp6OEycw=
 github.com/connesc/cipherio v0.2.1/go.mod h1:ukY0MWJDFnJEbXMQtOcn2VmTpRfzcTz4OoVrWGGJZcA=
+github.com/couchbase/gocb/v2 v2.6.3 h1:5RsMo+RRfK0mVxHLAfpBz3/tHlgXZb1WBNItLk9Ab+c=
+github.com/couchbase/gocb/v2 v2.6.3/go.mod h1:yF5F6BHTZ/ZowhEuZbySbXrlI4rHd1TIhm5azOaMbJU=
+github.com/couchbase/gocbcore/v10 v10.2.3 h1:PEkRSNSkKjUBXx82Ucr094+anoiCG5GleOOQZOHo6D4=
+github.com/couchbase/gocbcore/v10 v10.2.3/go.mod h1:lYQIIk+tzoMcwtwU5GzPbDdqEkwkH3isI2rkSpfL0oM=
+github.com/couchbaselabs/gocaves/client v0.0.0-20230307083111-cc3960c624b1/go.mod h1:AVekAZwIY2stsJOMWLAS/0uA/+qdp7pjO8EHnl61QkY=
+github.com/couchbaselabs/gocaves/client v0.0.0-20230404095311-05e3ba4f0259 h1:2TXy68EGEzIMHOx9UvczR5ApVecwCfQZ0LjkmwMI6g4=
+github.com/couchbaselabs/gocaves/client v0.0.0-20230404095311-05e3ba4f0259/go.mod h1:AVekAZwIY2stsJOMWLAS/0uA/+qdp7pjO8EHnl61QkY=
 github.com/containerd/stargz-snapshotter/estargz v0.14.3 h1:OqlDCK3ZVUO6C3B/5FSkDwbkEETK84kQgEeFwDC+62k=
 github.com/containerd/stargz-snapshotter/estargz v0.14.3/go.mod h1:KY//uOCIkSuNAHhJogcZtrNHdKrA99/FCCRjE3HD36o=
 github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
@@ -173,8 +182,8 @@ github.com/go-git/go-git/v5 v5.7.0 h1:t9AudWVLmqzlo+4bqdf7GY+46SUuRsx59SboFxkq2a
 github.com/go-git/go-git/v5 v5.7.0/go.mod h1:coJHKEOk5kUClpsNlXrUvPrDxY3w3gjHvhcZd8Fodw8=
 github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
 github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
-github.com/go-ldap/ldap/v3 v3.4.4 h1:qPjipEpt+qDa6SI/h1fzuGWoRUY+qqQ9sOZq67/PYUs=
-github.com/go-ldap/ldap/v3 v3.4.4/go.mod h1:fe1MsuN5eJJ1FeLT/LEBVdWfNWKh459R7aXgXtJC+aI=
+github.com/go-ldap/ldap/v3 v3.4.5 h1:ekEKmaDrpvR2yf5Nc/DClsGG9lAmdDixe44mLzlW5r8=
+github.com/go-ldap/ldap/v3 v3.4.5/go.mod h1:bMGIq3AGbytbaMwf8wdv5Phdxz0FWHTIYMSzyrYgnQs=
 github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ=
 github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
 github.com/go-logr/zapr v1.2.4 h1:QHVo+6stLbfJmYGkQ7uGHUCu5hnAFAj6mDe6Ea0SeOo=
@@ -419,6 +428,7 @@ github.com/smartystreets/gunit v1.1.3 h1:32x+htJCu3aMswhPw3teoJ+PnWPONqdNgaGs6Qt
 github.com/smartystreets/gunit v1.1.3/go.mod h1:EH5qMBab2UclzXUcpR8b93eHsIlp9u+pDQIRp5DZNzQ=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
+github.com/stretchr/objx v0.5.0 h1:1zr/of2m5FGMsad5YfcqgdqdWrIhu+EBEJRhR1U7z/c=
 github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
@@ -427,17 +437,15 @@ github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5
 github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
-github.com/stretchr/testify v1.7.2/go.mod h1:R6va5+xMeoiuVRoj+gSkQ7d3FALtqAAGI1FQKckRals=
 github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
 github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
+github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
 github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
 github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
 github.com/tailscale/depaware v0.0.0-20210622194025-720c4b409502 h1:34icjjmqJ2HPjrSuJYEkdZ+0ItmGQAQ75cRHIiftIyE=
 github.com/tailscale/depaware v0.0.0-20210622194025-720c4b409502/go.mod h1:p9lPsd+cx33L3H9nNoecRRxPssFKUwwI50I3pZ0yT+8=
 github.com/therootcompany/xz v1.0.1 h1:CmOtsn1CbtmyYiusbfmhmkpAAETj0wBIH6kCYaX+xzw=
 github.com/therootcompany/xz v1.0.1/go.mod h1:3K3UH1yCKgBneZYhuQUvJ9HPD19UEXEI0BWbMn8qNMY=
-github.com/tidwall/pretty v1.0.0 h1:HsD+QiTn7sK6flMKIvNmpqz1qrpP3Ps6jOKIKMooyg4=
-github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk=
 github.com/trufflesecurity/overseer v1.1.7-custom5 h1:xu+Fg6fkSRifUPzUCl7N8HmobJ6WGOkIApGnM7mJS6w=
 github.com/trufflesecurity/overseer v1.1.7-custom5/go.mod h1:nT9w37AiO1Nop2VhVhNfzAFaPjthvxgpDV3XKsxYkcI=
 github.com/ulikunitz/xz v0.5.6/go.mod h1:2bypXElzHzzJZwzH67Y6wb67pO62Rzfn7BSiF4ABRW8=
@@ -446,16 +454,16 @@ github.com/ulikunitz/xz v0.5.10/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0o
 github.com/urfave/cli v1.22.12/go.mod h1:sSBEIC79qR6OvcmsD4U3KABeOTxDqQtdDnaFuUN30b8=
 github.com/vbatts/tar-split v0.11.3 h1:hLFqsOLQ1SsppQNTMpkpPXClLDfC2A3Zgy9OUU+RVck=
 github.com/vbatts/tar-split v0.11.3/go.mod h1:9QlHN18E+fEH7RdG+QAJJcuya3rqT7eXSTY7wGrAokY=
-github.com/xanzy/go-gitlab v0.85.0 h1:E/wjnsd/mM5kV6O9y5+i6zxjx+wfAwa97sgcT1ETNwk=
-github.com/xanzy/go-gitlab v0.85.0/go.mod h1:5ryv+MnpZStBH8I/77HuQBsMbBGANtVpLWC15qOjWAw=
+github.com/xanzy/go-gitlab v0.86.0 h1:jR8V9cK9jXRQDb46KOB20NCF3ksY09luaG0IfXE6p7w=
+github.com/xanzy/go-gitlab v0.86.0/go.mod h1:5ryv+MnpZStBH8I/77HuQBsMbBGANtVpLWC15qOjWAw=
 github.com/xanzy/ssh-agent v0.3.3 h1:+/15pJfg/RsTxqYcX6fHqOXZwwMP+2VyYWJeWM2qQFM=
 github.com/xanzy/ssh-agent v0.3.3/go.mod h1:6dzNDKs0J9rVPHPhaGCukekBHKqfl+L3KghI1Bc68Uw=
 github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c=
 github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI=
-github.com/xdg-go/scram v1.1.1 h1:VOMT+81stJgXW3CpHyqHN3AXDYIMsx56mEFrB37Mb/E=
-github.com/xdg-go/scram v1.1.1/go.mod h1:RaEWvsqvNKKvBPvcKeFjrG2cJqOkHTiyTpzz23ni57g=
-github.com/xdg-go/stringprep v1.0.3 h1:kdwGpVNwPFtjs98xCGkHjQtGKh86rDcRZN17QEMCOIs=
-github.com/xdg-go/stringprep v1.0.3/go.mod h1:W3f5j4i+9rC0kuIEJL0ky1VpHXQU3ocBgklLGvcBnW8=
+github.com/xdg-go/scram v1.1.2 h1:FHX5I5B4i4hKRVRBCFRxq1iQRej7WO3hhBuJf+UUySY=
+github.com/xdg-go/scram v1.1.2/go.mod h1:RT/sEzTbU5y00aCK8UOx6R7YryM0iF1N2MOmC3kKLN4=
+github.com/xdg-go/stringprep v1.0.4 h1:XLI/Ng3O1Atzq0oBs3TWm+5ZVgkq2aqdlvP9JtoZ6c8=
+github.com/xdg-go/stringprep v1.0.4/go.mod h1:mPGuuIYwz7CmR2bT9j4GbQqutWS1zV24gijq1dTyGkM=
 github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d h1:splanxYIlg+5LfHAM6xpdFEAYOk8iySO56hMFq6uLyA=
 github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA=
 github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
@@ -463,8 +471,8 @@ github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1
 github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
 github.com/yusufpapurcu/wmi v1.2.2 h1:KBNDSne4vP5mbSWnJbO+51IMOXJB67QiYCSBrubbPRg=
 github.com/yusufpapurcu/wmi v1.2.2/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0=
-go.mongodb.org/mongo-driver v1.11.6 h1:XM7G6PjiGAO5betLF13BIa5TlLUUE3uJ/2Ox3Lz1K+o=
-go.mongodb.org/mongo-driver v1.11.6/go.mod h1:G9TgswdsWjX4tmDA5zfs2+6AEPpYJwqblyjsfuh8oXY=
+go.mongodb.org/mongo-driver v1.12.0 h1:aPx33jmn/rQuJXPQLZQ8NtfPQG8CaqgLThFtqRb0PiE=
+go.mongodb.org/mongo-driver v1.12.0/go.mod h1:AZkxhPnFJUoH7kZlFkVKucV20K387miPfm7oimrSmK0=
 go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
 go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
 go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
@@ -578,8 +586,8 @@ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJ
 golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.2.0 h1:PUR+T4wwASmuSTYdKjYHI5TD22Wy5ogLU5qZCOLxBrI=
-golang.org/x/sync v0.2.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.3.0 h1:ftCYgMx6zT/asHUrPw8BLLscYtGznsLAnjq5RH9P66E=
+golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y=
 golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=

main.go

@@ -476,7 +476,7 @@ func run(state overseer.State) {
 		}
 	}
 	// asynchronously wait for scanning to finish and cleanup
-	go e.Finish(ctx)
+	go e.Finish(ctx, logFatal)
 
 	if !*jsonLegacy && !*jsonOut {
 		fmt.Fprintf(os.Stderr, "🐷🔑🐷  TruffleHog. Unearth your secrets. 🐷🔑🐷\n\n")

pkg/common/patterns.go

@@ -2,6 +2,7 @@ package common
 
 import (
 	"fmt"
+	"regexp"
 	"strconv"
 	"strings"
 )
@@ -15,7 +16,11 @@ const RegexPattern = "0-9a-z"
 const AlphaNumPattern = "0-9a-zA-Z"
 const HexPattern = "0-9a-f"
 
-//Custom Regex functions
+type RegexState struct {
+	compiledRegex *regexp.Regexp
+}
+
+// Custom Regex functions
 func BuildRegex(pattern string, specialChar string, length int) string {
 	return fmt.Sprintf(`\b([%s%s]{%s})\b`, pattern, specialChar, strconv.Itoa(length))
 }
@@ -37,3 +42,30 @@ func RangeValidation(rangeInput string) bool {
 func ToUpperCase(input string) string {
 	return strings.ToUpper(input)
 }
+
+func (r RegexState) Matches(data []byte) []string {
+	matches := r.compiledRegex.FindAllStringSubmatch(string(data), -1)
+
+	res := make([]string, 0, len(matches))
+
+	// trim off spaces and different quote types ('").
+	for i := range matches {
+		res = append(res, strings.Trim(matches[i][1], `"' )`))
+	}
+
+	return res
+}
+
+// UsernameRegexCheck constructs an username usernameRegex pattern from a given pattern of excluded characters.
+func UsernameRegexCheck(pattern string) RegexState {
+	raw := fmt.Sprintf(`(?im)(?:user|usr)\S{0,40}?[:=\s]{1,3}[ '"=]{0,1}([^:%+v]{4,40})\b`, pattern)
+
+	return RegexState{regexp.MustCompile(raw)}
+}
+
+// PasswordRegexCheck constructs an username usernameRegex pattern from a given pattern of excluded characters.
+func PasswordRegexCheck(pattern string) RegexState {
+	raw := fmt.Sprintf(`(?im)(?:pass|password)\S{0,40}?[:=\s]{1,3}[ '"=]{0,1}([^:%+v]{4,40})`, pattern)
+
+	return RegexState{regexp.MustCompile(raw)}
+}

pkg/common/patterns_test.go

@@ -0,0 +1,59 @@
+package common
+
+import (
+	"github.com/stretchr/testify/assert"
+	"regexp"
+	"testing"
+)
+
+const (
+	usernamePattern = `?()/\+=\s\n`
+	passwordPattern = `^<>;.*&|£\n\s`
+	usernameRegex   = `(?im)(?:user|usr)\S{0,40}?[:=\s]{1,3}[ '"=]{0,1}([^:?()/\+=\s\n]{4,40})\b`
+	passwordRegex   = `(?im)(?:pass|password)\S{0,40}?[:=\s]{1,3}[ '"=]{0,1}([^:^<>;.*&|£\n\s]{4,40})`
+)
+
+func TestUsernameRegexCheck(t *testing.T) {
+	usernameRegexPat := UsernameRegexCheck(usernamePattern)
+
+	expectedRegexPattern := regexp.MustCompile(usernameRegex)
+
+	if usernameRegexPat.compiledRegex.String() != expectedRegexPattern.String() {
+		t.Errorf("\n got %v \n want %v", usernameRegexPat.compiledRegex, expectedRegexPattern)
+	}
+
+	testString := `username = "johnsmith123"
+                   username='johnsmith123'
+				   username:="johnsmith123"
+                   username = johnsmith123
+                   username=johnsmith123`
+
+	expectedStr := []string{"johnsmith123", "johnsmith123", "johnsmith123", "johnsmith123", "johnsmith123"}
+
+	usernameRegexMatches := usernameRegexPat.Matches([]byte(testString))
+
+	assert.Exactly(t, usernameRegexMatches, expectedStr)
+
+}
+
+func TestPasswordRegexCheck(t *testing.T) {
+	passwordRegexPat := PasswordRegexCheck(passwordPattern)
+
+	expectedRegexPattern := regexp.MustCompile(passwordRegex)
+	assert.Equal(t, passwordRegexPat.compiledRegex, expectedRegexPattern)
+
+	testString := `password = "johnsmith123$!"
+                   password='johnsmith123$!'
+				   password:="johnsmith123$!"
+                   password = johnsmith123$!
+                   password=johnsmith123$!
+				   PasswordAuthenticator(username, "johnsmith123$!")`
+
+	expectedStr := []string{"johnsmith123$!", "johnsmith123$!", "johnsmith123$!", "johnsmith123$!", "johnsmith123$!",
+		"johnsmith123$!"}
+
+	passwordRegexMatches := passwordRegexPat.Matches([]byte(testString))
+
+	assert.Exactly(t, passwordRegexMatches, expectedStr)
+
+}

pkg/detectors/couchbase/couchbase.go

@@ -0,0 +1,152 @@
+package couchbase
+
+import (
+	"context"
+	"fmt"
+	"log"
+	"regexp"
+	"strings"
+	"time"
+	"unicode"
+
+	"github.com/couchbase/gocb/v2"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+type Scanner struct{}
+
+// Ensure the Scanner satisfies the interface at compile time.
+var _ detectors.Detector = (*Scanner)(nil)
+
+var (
+
+	// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
+	connectionStringPat = regexp.MustCompile(`\bcb\.[a-z0-9]+\.cloud\.couchbase\.com\b`)
+	usernamePat         = `?()/\+=\s\n`
+	passwordPat         = `^<>;.*&|£\n\s`
+	//passwordPat         = regexp.MustCompile(`(?i)(?:pass|pwd)(?:.|[\n\r]){0,15}(\b[^<>;.*&|£\n\s]{8,100}$)`)
+	// passwordPat = regexp.MustCompile(`(?im)(?:pass|pwd)\S{0,40}?[:=\s]{1,3}[ '"=]{0,1}([^:?()/\+=\s\n]{4,40})\b`)
+)
+
+func meetsCouchbasePasswordRequirements(password string) (string, bool) {
+	var hasLower, hasUpper, hasNumber, hasSpecialChar bool
+	for _, char := range password {
+		switch {
+		case unicode.IsLower(char):
+			hasLower = true
+		case unicode.IsUpper(char):
+			hasUpper = true
+		case unicode.IsNumber(char):
+			hasNumber = true
+		case unicode.IsPunct(char) || unicode.IsSymbol(char):
+			hasSpecialChar = true
+		}
+
+		if hasLower && hasUpper && hasNumber && hasSpecialChar {
+			return password, true
+		}
+	}
+
+	return "", false
+}
+
+// Keywords are used for efficiently pre-filtering chunks.
+// Use identifiers in the secret preferably, or the provider name.
+func (s Scanner) Keywords() []string {
+	return []string{"couchbase://", "couchbases://"}
+}
+
+// FromData will find and optionally verify Couchbase secrets in a given set of bytes.
+func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
+	dataStr := string(data)
+
+	connectionStringMatches := connectionStringPat.FindAllStringSubmatch(dataStr, -1)
+
+	// prepend 'couchbases://' to the connection string as the connection
+	// string format is couchbases://cb.stuff.cloud.couchbase.com but the
+	// cb.stuff.cloud.couchbase.com may be separated from the couchbases:// in codebases.
+	for i, connectionStringMatch := range connectionStringMatches {
+		connectionStringMatches[i][0] = "couchbases://" + connectionStringMatch[0]
+	}
+
+	usernameRegexState := common.UsernameRegexCheck(usernamePat)
+	usernameMatches := usernameRegexState.Matches(data)
+
+	passwordRegexState := common.PasswordRegexCheck(passwordPat)
+	passwordMatches := passwordRegexState.Matches(data)
+
+	for _, connectionStringMatch := range connectionStringMatches {
+		resConnectionStringMatch := strings.TrimSpace(connectionStringMatch[0])
+
+		for _, resUsernameMatch := range usernameMatches {
+
+			for _, resPasswordMatch := range passwordMatches {
+				_, metPasswordRequirements := meetsCouchbasePasswordRequirements(resPasswordMatch)
+
+				if !metPasswordRequirements {
+					continue
+				}
+
+				s1 := detectors.Result{
+					DetectorType: detectorspb.DetectorType_Couchbase,
+					Raw:          []byte(fmt.Sprintf("%s:%s@%s", resUsernameMatch, resPasswordMatch, resConnectionStringMatch)),
+				}
+
+				if verify {
+
+					options := gocb.ClusterOptions{
+						Authenticator: gocb.PasswordAuthenticator{
+							Username: resUsernameMatch,
+							Password: resPasswordMatch,
+						},
+					}
+
+					// Sets a pre-configured profile called "wan-development" to help avoid latency issues
+					// when accessing Capella from a different Wide Area Network
+					// or Availability Zone (e.g. your laptop).
+					if err := options.ApplyProfile(gocb.ClusterConfigProfileWanDevelopment); err != nil {
+						log.Fatal("apply profile err", err)
+					}
+
+					// Initialize the Connection
+					cluster, err := gocb.Connect(resConnectionStringMatch, options)
+					if err != nil {
+						continue
+					}
+
+					// We'll ping the KV nodes in our cluster.
+					pings, err := cluster.Ping(&gocb.PingOptions{
+						Timeout: time.Second * 5,
+					})
+
+					if err != nil {
+						continue
+					}
+
+					for _, ping := range pings.Services {
+						for _, pingEndpoint := range ping {
+							if pingEndpoint.State == gocb.PingStateOk {
+								s1.Verified = true
+								break
+							} else {
+								// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
+								if detectors.IsKnownFalsePositive(resPasswordMatch, detectors.DefaultFalsePositives, true) {
+									continue
+								}
+							}
+						}
+					}
+				}
+
+				results = append(results, s1)
+			}
+		}
+	}
+	return results, nil
+}
+
+func (s Scanner) Type() detectorspb.DetectorType {
+	return detectorspb.DetectorType_Couchbase
+}

pkg/detectors/couchbase/couchbase_test.go

@@ -0,0 +1,122 @@
+//go:build detectors
+// +build detectors
+
+package couchbase
+
+import (
+	"context"
+	"fmt"
+	"testing"
+	"time"
+
+	"github.com/kylelemons/godebug/pretty"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+
+	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestCouchbase_FromChunk(t *testing.T) {
+	ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+	defer cancel()
+	testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")
+	if err != nil {
+		t.Fatalf("could not get test secrets from GCP: %s", err)
+	}
+	endpoint := testSecrets.MustGetField("COUCHBASE_ENDPOINT")
+	username := testSecrets.MustGetField("COUCHBASE_USERNAME")
+	password := testSecrets.MustGetField("COUCHBASE_PASSWORD")
+	inactiveSecret := testSecrets.MustGetField("COUCHBASE_INACTIVE_PASSWORD")
+
+	type args struct {
+		ctx    context.Context
+		data   []byte
+		verify bool
+	}
+	tests := []struct {
+		name    string
+		s       Scanner
+		args    args
+		want    []detectors.Result
+		wantErr bool
+	}{
+		{
+			name: "found, verified",
+			s:    Scanner{},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte(fmt.Sprintf("db uri: %s \n username = %s \n password = %s", endpoint, username, password)),
+				verify: true,
+			},
+			want: []detectors.Result{
+				{
+					DetectorType: detectorspb.DetectorType_Couchbase,
+					Verified:     true,
+				},
+			},
+			wantErr: false,
+		},
+		{
+			name: "found, unverified",
+			s:    Scanner{},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte(fmt.Sprintf("db uri: %s \n username = %s \n password = %s", endpoint, username, inactiveSecret)), // the secret would satisfy the regex but not pass validation
+				verify: true,
+			},
+			want: []detectors.Result{
+				{
+					DetectorType: detectorspb.DetectorType_Couchbase,
+					Verified:     false,
+				},
+			},
+			wantErr: false,
+		},
+		{
+			name: "not found",
+			s:    Scanner{},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte("You cannot find the secret within"),
+				verify: true,
+			},
+			want:    nil,
+			wantErr: false,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			s := Scanner{}
+
+			got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("Couchbase.FromData() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			for i := range got {
+				if len(got[i].Raw) == 0 {
+					t.Fatalf("no raw secret present: \n %+v", got[i])
+				}
+				got[i].Raw = nil
+			}
+			if diff := pretty.Compare(got, tt.want); diff != "" {
+				t.Errorf("Couchbase.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+			}
+		})
+	}
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+	ctx := context.Background()
+	s := Scanner{}
+	for name, data := range detectors.MustGetBenchmarkData() {
+		benchmark.Run(name, func(b *testing.B) {
+			for n := 0; n < b.N; n++ {
+				_, err := s.FromData(ctx, false, data)
+				if err != nil {
+					b.Fatal(err)
+				}
+			}
+		})
+	}
+}

pkg/detectors/detectors.go

@@ -39,6 +39,8 @@ type EndpointCustomizer interface {
 	DefaultEndpoint() string
 }
 
+// Result is a secret that has been found by a Detector and optionally verified.
+// It contains the raw secret identification data, redacted data, and extra data.
 type Result struct {
 	// DetectorType is the type of Detector.
 	DetectorType detectorspb.DetectorType
@@ -46,7 +48,15 @@ type Result struct {
 	DetectorName string
 	// DecoderType is the type of Decoder.
 	DecoderType detectorspb.DecoderType
-	Verified    bool
+	// Verified is true if the secret has been confirmed to be a valid secret.
+	Verified bool
+	// HasVerificationIndeterminate is true if the secret could NOT be verified.
+	// This is used to differentiate between secrets that are NOT verified and secrets that COULD NOT be verified.
+	// Ex: A 429 response from a service, or a timeout would set this to true.
+	HasVerificationIndeterminate bool
+	// VerificationIndeterminateError holds the error if the secret could NOT be verified.
+	// This is used specifically for cases that resulted in indeterminate resolution, such as a 429 or timeout.
+	VerificationIndeterminateError error
 	// Raw contains the raw secret identifier data. Prefer IDs over secrets since it is used for deduping after hashing.
 	Raw []byte
 	// RawV2 contains the raw secret identifier that is a combination of both the ID and the secret.
@@ -59,6 +69,17 @@ type Result struct {
 	StructuredData *detectorspb.StructuredData
 }
 
+// IsVerified checks if a result is guaranteed to be verified.
+func (r Result) IsVerified() bool {
+	return r.Verified && !r.HasVerificationIndeterminate
+}
+
+// IsUnverified checks if a result is guaranteed to be unverified.
+// This is used to differentiate between secrets that are NOT verified and secrets that COULD NOT be verified.
+func (r Result) IsUnverified() bool {
+	return !r.Verified && !r.HasVerificationIndeterminate
+}
+
 type ResultWithMetadata struct {
 	// SourceMetadata contains source-specific contextual information.
 	SourceMetadata *source_metadatapb.MetaData

pkg/detectors/detectors_test.go

@@ -3,7 +3,9 @@
 
 package detectors
 
-import "testing"
+import (
+	"testing"
+)
 
 func TestPrefixRegex(t *testing.T) {
 	tests := []struct {
@@ -37,3 +39,101 @@ func BenchmarkPrefixRegex(b *testing.B) {
 		PrefixRegex(kws)
 	}
 }
+
+func TestIsVerified(t *testing.T) {
+	tests := []struct {
+		name   string
+		result Result
+		want   bool
+	}{
+		{
+			name: "Verified true, HasVerificationIndeterminate false",
+			result: Result{
+				Verified:                     true,
+				HasVerificationIndeterminate: false,
+			},
+			want: true,
+		},
+		{
+			name: "Verified false, HasVerificationIndeterminate false",
+			result: Result{
+				Verified:                     false,
+				HasVerificationIndeterminate: false,
+			},
+			want: false,
+		},
+		{
+			name: "Verified true, HasVerificationIndeterminate true",
+			result: Result{
+				Verified:                     true,
+				HasVerificationIndeterminate: true,
+			},
+			want: false,
+		},
+		{
+			name: "Verified false, HasVerificationIndeterminate true",
+			result: Result{
+				Verified:                     false,
+				HasVerificationIndeterminate: true,
+			},
+			want: false,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := tt.result.IsVerified(); got != tt.want {
+				t.Errorf("IsVerified() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}
+
+func TestIsUnverified(t *testing.T) {
+	tests := []struct {
+		name   string
+		result Result
+		want   bool
+	}{
+		{
+			name: "Verified true, HasVerificationIndeterminate false",
+			result: Result{
+				Verified:                     true,
+				HasVerificationIndeterminate: false,
+			},
+			want: false,
+		},
+		{
+			name: "Verified false, HasVerificationIndeterminate false",
+			result: Result{
+				Verified:                     false,
+				HasVerificationIndeterminate: false,
+			},
+			want: true,
+		},
+		{
+			name: "Verified true, HasVerificationIndeterminate true",
+			result: Result{
+				Verified:                     true,
+				HasVerificationIndeterminate: true,
+			},
+			want: false,
+		},
+		{
+			name: "Verified false, HasVerificationIndeterminate true",
+			result: Result{
+				Verified:                     false,
+				HasVerificationIndeterminate: true,
+			},
+			want: false,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := tt.result.IsUnverified(); got != tt.want {
+				t.Errorf("IsUnverified() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}

pkg/detectors/falsepositives.go

@@ -6,7 +6,7 @@ import (
 	"unicode"
 )
 
-var DefaultFalsePositives = []FalsePositive{"example", "xxxxxx", "aaaaaa", "abcde", "00000", "sample"}
+var DefaultFalsePositives = []FalsePositive{"example", "xxxxxx", "aaaaaa", "abcde", "00000", "sample", "www"}
 
 type FalsePositive string
 
@@ -31,9 +31,9 @@ var FalsePositiveWordlists = Wordlists{
 	programmingBookWords: bytesToCleanWordList(programmingBookWords),
 }
 
-//IsKnownFalsePositives will not return a valid secret finding if any of the disqualifying conditions are met
-//Currently that includes: No number, english word in key, or matches common example pattens.
-//Only the secret key material should be passed into this function
+// IsKnownFalsePositives will not return a valid secret finding if any of the disqualifying conditions are met
+// Currently that includes: No number, english word in key, or matches common example pattens.
+// Only the secret key material should be passed into this function
 func IsKnownFalsePositive(match string, falsePositives []FalsePositive, wordCheck bool) bool {
 
 	for _, fp := range falsePositives {

pkg/detectors/razorpay/razorpay.go

@@ -38,19 +38,18 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 	keyMatches := keyPat.FindAllString(dataStr, -1)
 
 	for _, key := range keyMatches {
+		secMatches := secretPat.FindAllString(dataStr, -1)
 
-		if verify {
-			secMatches := secretPat.FindAllString(dataStr, -1)
+		for _, secret := range secMatches {
 
-			for _, secret := range secMatches {
-
-				s1 := detectors.Result{
-					DetectorType: detectorspb.DetectorType_RazorPay,
-					Raw:          []byte(key),
-					RawV2:        []byte(key + secret),
-					Redacted:     key,
-				}
+			s1 := detectors.Result{
+				DetectorType: detectorspb.DetectorType_RazorPay,
+				Raw:          []byte(key),
+				RawV2:        []byte(key + secret),
+				Redacted:     key,
+			}
 
+			if verify {
 				req, err := http.NewRequest("GET", "https://api.razorpay.com/v1/items?count=1", nil)
 				if err != nil {
 					continue
@@ -66,25 +65,21 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 					if res.StatusCode >= 200 && res.StatusCode < 300 {
 						if json.Valid(bodyBytes) {
 							s1.Verified = true
-						} else {
-							s1.Verified = false
-						}
-					} else {
-						// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
-						if detectors.IsKnownFalsePositive(key, detectors.DefaultFalsePositives, true) {
-							continue
 						}
 					}
 				}
-
-				results = append(results, s1)
 			}
+
+			if !s1.Verified && detectors.IsKnownFalsePositive(key, detectors.DefaultFalsePositives, true) {
+				continue
+			}
+
+			results = append(results, s1)
 		}
 
 	}
 
-	results = detectors.CleanResults(results)
-	return
+	return detectors.CleanResults(results), nil
 }
 
 func (s Scanner) Type() detectorspb.DetectorType {

pkg/detectors/slackwebhook/slackwebhook.go

@@ -62,7 +62,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 					continue
 				}
 				body := string(bodyBytes)
-				if (res.StatusCode >= 200 && res.StatusCode < 300) || (res.StatusCode == 400 && strings.Contains(body, "no_text")) {
+				if (res.StatusCode >= 200 && res.StatusCode < 300) || (res.StatusCode == 400 && (strings.Contains(body, "no_text") || strings.Contains(body, "missing_text"))) {
 					s1.Verified = true
 				}
 			}

pkg/detectors/twilio/twilio.go

@@ -21,6 +21,7 @@ var (
 	identifierPat = regexp.MustCompile(`(?i)sid.{0,20}AC[0-9a-f]{32}`) // Should we have this? Seems restrictive.
 	sidPat        = regexp.MustCompile(`\bAC[0-9a-f]{32}\b`)
 	keyPat        = regexp.MustCompile(`\b[0-9a-f]{32}\b`)
+	client        = common.SaneHttpClient()
 )
 
 // Keywords are used for efficiently pre-filtering chunks.
@@ -43,15 +44,15 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 	sidMatches := sidPat.FindAllString(dataStr, -1)
 
 	for _, sid := range sidMatches {
-		s := detectors.Result{
-			DetectorType: detectorspb.DetectorType_Twilio,
-			Raw:          []byte(sid),
-			Redacted:     sid,
-		}
+		for _, key := range keyMatches {
+			s1 := detectors.Result{
+				DetectorType: detectorspb.DetectorType_Twilio,
+				Raw:          []byte(sid),
+				RawV2:        []byte(sid + key),
+				Redacted:     sid,
+			}
 
-		if verify {
-			client := common.SaneHttpClient()
-			for _, key := range keyMatches {
+			if verify {
 
 				form := url.Values{}
 				form.Add("FriendlyName", "MyServiceName")
@@ -71,22 +72,22 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 					res.Body.Close() // The request body is unused.
 
 					if res.StatusCode >= 200 && res.StatusCode < 300 {
-						s.Verified = true
+						s1.Verified = true
 					}
 				}
 			}
-		}
 
-		if !s.Verified && detectors.IsKnownFalsePositive(string(s.Raw), detectors.DefaultFalsePositives, true) {
-			continue
-		}
+			if !s1.Verified && detectors.IsKnownFalsePositive(string(s1.Raw), detectors.DefaultFalsePositives, true) {
+				continue
+			}
 
-		if len(keyMatches) > 0 {
-			results = append(results, s)
+			if len(keyMatches) > 0 {
+				results = append(results, s1)
+			}
 		}
 	}
 
-	return
+	return detectors.CleanResults(results), nil
 }
 
 func (s Scanner) Type() detectorspb.DetectorType {

pkg/detectors/twilio/twilio_test.go

@@ -98,6 +98,7 @@ func TestTwilio_FromChunk(t *testing.T) {
 					t.Fatal("no raw secret present")
 				}
 				got[i].Raw = nil
+				got[i].RawV2 = nil
 			}
 			if diff := pretty.Compare(got, tt.want); diff != "" {
 				t.Errorf("Twilio.FromData() %s diff: (-got +want)\n%s", tt.name, diff)

pkg/engine/circleci.go

@@ -1,6 +1,7 @@
 package engine
 
 import (
+	"fmt"
 	"runtime"
 
 	"github.com/go-errors/errors"
@@ -38,14 +39,13 @@ func (e *Engine) ScanCircleCI(ctx context.Context, token string) error {
 		return errors.WrapPrefix(err, "failed to init Circle CI source", 0)
 	}
 
-	e.sourcesWg.Add(1)
-	go func() {
+	e.sourcesWg.Go(func() error {
 		defer common.RecoverWithExit(ctx)
-		defer e.sourcesWg.Done()
 		err := circleSource.Chunks(ctx, e.ChunksChan())
 		if err != nil {
-			ctx.Logger().Error(err, "error scanning Circle CI")
+			return fmt.Errorf("error scanning CircleCI: %w", err)
 		}
-	}()
+		return nil
+	})
 	return nil
 }

pkg/engine/docker.go

@@ -1,6 +1,7 @@
 package engine
 
 import (
+	"fmt"
 	"runtime"
 
 	"github.com/go-errors/errors"
@@ -25,14 +26,13 @@ func (e *Engine) ScanDocker(ctx context.Context, conn *anypb.Any) error {
 		return errors.WrapPrefix(err, "could not init docker source", 0)
 	}
 
-	e.sourcesWg.Add(1)
-	go func() {
+	e.sourcesWg.Go(func() error {
 		defer common.RecoverWithExit(ctx)
-		defer e.sourcesWg.Done()
 		err := dockerSource.Chunks(ctx, e.ChunksChan())
 		if err != nil {
-			ctx.Logger().Error(err, "error scanning docker image")
+			return fmt.Errorf("error scanning docker image: %w", err)
 		}
-	}()
+		return nil
+	})
 	return nil
 }

pkg/engine/engine.go

@@ -10,6 +10,7 @@ import (
 	"time"
 
 	ahocorasick "github.com/petar-dambovaliev/aho-corasick"
+	"golang.org/x/sync/errgroup"
 	"google.golang.org/protobuf/proto"
 
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
@@ -32,7 +33,7 @@ type Engine struct {
 	chunksScanned   uint64
 	bytesScanned    uint64
 	detectorAvgTime sync.Map
-	sourcesWg       sync.WaitGroup
+	sourcesWg       *errgroup.Group
 	workersWg       sync.WaitGroup
 	// filterUnverified is used to reduce the number of unverified results.
 	// If there are multiple unverified results for the same chunk for the same detector,
@@ -123,6 +124,11 @@ func Start(ctx context.Context, options ...EngineOption) *Engine {
 	}
 	ctx.Logger().V(2).Info("engine started", "workers", e.concurrency)
 
+	sourcesWg, egCtx := errgroup.WithContext(ctx)
+	sourcesWg.SetLimit(e.concurrency)
+	e.sourcesWg = sourcesWg
+	ctx.SetParent(egCtx)
+
 	if len(e.decoders) == 0 {
 		e.decoders = decoders.DefaultDecoders()
 	}
@@ -188,10 +194,14 @@ func Start(ctx context.Context, options ...EngineOption) *Engine {
 // Finish waits for running sources to complete and workers to finish scanning
 // chunks before closing their respective channels. Once Finish is called, no
 // more sources may be scanned by the engine.
-func (e *Engine) Finish(ctx context.Context) {
+func (e *Engine) Finish(ctx context.Context, logFunc func(error, string, ...any)) {
 	defer common.RecoverWithExit(ctx)
 	// wait for the sources to finish putting chunks onto the chunks channel
-	e.sourcesWg.Wait()
+	sourceErr := e.sourcesWg.Wait()
+	if sourceErr != nil {
+		logFunc(sourceErr, "error occurred while collecting chunks")
+	}
+
 	close(e.chunks)
 	// wait for the workers to finish processing all of the chunks and putting
 	// results onto the results channel

pkg/engine/filesystem.go

@@ -1,6 +1,7 @@
 package engine
 
 import (
+	"fmt"
 	"runtime"
 
 	"github.com/go-errors/errors"
@@ -37,14 +38,13 @@ func (e *Engine) ScanFileSystem(ctx context.Context, c sources.FilesystemConfig)
 		return errors.WrapPrefix(err, "could not init filesystem source", 0)
 	}
 	fileSystemSource.WithFilter(c.Filter)
-	e.sourcesWg.Add(1)
-	go func() {
+	e.sourcesWg.Go(func() error {
 		defer common.RecoverWithExit(ctx)
-		defer e.sourcesWg.Done()
 		err := fileSystemSource.Chunks(ctx, e.ChunksChan())
 		if err != nil {
-			ctx.Logger().Error(err, "error scanning filesystem")
+			return fmt.Errorf("error scanning filesystem: %w", err)
 		}
-	}()
+		return nil
+	})
 	return nil
 }

pkg/engine/gcs.go

@@ -54,14 +54,13 @@ func (e *Engine) ScanGCS(ctx context.Context, c sources.GCSConfig) error {
 		return fmt.Errorf("failed to initialize GCS source: %w", err)
 	}
 
-	e.sourcesWg.Add(1)
-	go func() {
+	e.sourcesWg.Go(func() error {
 		defer common.RecoverWithExit(ctx)
-		defer e.sourcesWg.Done()
 		if err := source.Chunks(ctx, e.ChunksChan()); err != nil {
-			ctx.Logger().Error(err, "could not scan GCS")
+			return fmt.Errorf("could not scan GCS: %w", err)
 		}
-	}()
+		return nil
+	})
 	return nil
 }
 

pkg/engine/gcs_test.go

@@ -5,6 +5,7 @@ import (
 	"testing"
 
 	"github.com/trufflesecurity/trufflehog/v3/pkg/context"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/decoders"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
 )
 
@@ -55,12 +56,30 @@ func TestScanGCS(t *testing.T) {
 
 	for _, test := range tests {
 		t.Run(test.name, func(t *testing.T) {
-			e := &Engine{}
-			err := e.ScanGCS(context.Background(), test.gcsConfig)
+			ctx, cancel := context.WithCancel(context.TODO())
+			defer cancel()
+
+			e := Start(ctx,
+				WithConcurrency(1),
+				WithDecoders(decoders.DefaultDecoders()...),
+				WithDetectors(false, DefaultDetectors()...),
+			)
+			go func() {
+				resultCount := 0
+				for range e.ResultsChan() {
+					resultCount++
+				}
+			}()
+
+			err := e.ScanGCS(ctx, test.gcsConfig)
 			if err != nil && !test.wantErr && !strings.Contains(err.Error(), "googleapi: Error 400: Bad Request") {
 				t.Errorf("ScanGCS() got: %v, want: %v", err, nil)
 				return
 			}
+			logFatalFunc := func(_ error, _ string, _ ...any) {
+				t.Fatalf("error logging function should not have been called")
+			}
+			e.Finish(ctx, logFatalFunc)
 
 			if err == nil && test.wantErr {
 				t.Errorf("ScanGCS() got: %v, want: %v", err, "error")

pkg/engine/git.go

@@ -66,14 +66,13 @@ func (e *Engine) ScanGit(ctx context.Context, c sources.GitConfig) error {
 		"source_type", sourcespb.SourceType_SOURCE_TYPE_GIT.String(),
 		"source_name", "git",
 	)
-	e.sourcesWg.Add(1)
-	go func() {
+	e.sourcesWg.Go(func() error {
 		defer common.RecoverWithExit(ctx)
-		defer e.sourcesWg.Done()
 		err := gitSource.ScanRepo(ctx, repo, c.RepoPath, scanOptions, e.ChunksChan())
 		if err != nil {
-			ctx.Logger().Error(err, "could not scan repo")
+			return fmt.Errorf("could not scan repo: %w", err)
 		}
-	}()
+		return nil
+	})
 	return nil
 }

pkg/engine/git_test.go

@@ -53,39 +53,45 @@ func TestGitEngine(t *testing.T) {
 			base:   "2f251b8c1e72135a375b659951097ec7749d4af9",
 		},
 	} {
-		e := Start(ctx,
-			WithConcurrency(1),
-			WithDecoders(decoders.DefaultDecoders()...),
-			WithDetectors(false, DefaultDetectors()...),
-		)
-		cfg := sources.GitConfig{
-			RepoPath: path,
-			HeadRef:  tTest.branch,
-			BaseRef:  tTest.base,
-			MaxDepth: tTest.maxDepth,
-			Filter:   tTest.filter,
-		}
-		if err := e.ScanGit(ctx, cfg); err != nil {
-			return
-		}
-		go e.Finish(ctx)
-		resultCount := 0
-		for result := range e.ResultsChan() {
-			switch meta := result.SourceMetadata.GetData().(type) {
-			case *source_metadatapb.MetaData_Git:
-				if tTest.expected[meta.Git.Commit].B != string(result.Raw) {
-					t.Errorf("%s: unexpected result. Got: %s, Expected: %s", tName, string(result.Raw), tTest.expected[meta.Git.Commit].B)
-				}
-				if tTest.expected[meta.Git.Commit].LineNumber != result.SourceMetadata.GetGit().Line {
-					t.Errorf("%s: unexpected line number. Got: %d, Expected: %d", tName, result.SourceMetadata.GetGit().Line, tTest.expected[meta.Git.Commit].LineNumber)
-				}
+		t.Run(tName, func(t *testing.T) {
+			e := Start(ctx,
+				WithConcurrency(1),
+				WithDecoders(decoders.DefaultDecoders()...),
+				WithDetectors(false, DefaultDetectors()...),
+			)
+			cfg := sources.GitConfig{
+				RepoPath: path,
+				HeadRef:  tTest.branch,
+				BaseRef:  tTest.base,
+				MaxDepth: tTest.maxDepth,
+				Filter:   tTest.filter,
+			}
+			if err := e.ScanGit(ctx, cfg); err != nil {
+				return
 			}
-			resultCount++
 
-		}
-		if resultCount != len(tTest.expected) {
-			t.Errorf("%s: unexpected number of results. Got: %d, Expected: %d", tName, resultCount, len(tTest.expected))
-		}
+			logFatalFunc := func(_ error, _ string, _ ...any) {
+				t.Fatalf("error logging function should not have been called")
+			}
+			go e.Finish(ctx, logFatalFunc)
+			resultCount := 0
+			for result := range e.ResultsChan() {
+				switch meta := result.SourceMetadata.GetData().(type) {
+				case *source_metadatapb.MetaData_Git:
+					if tTest.expected[meta.Git.Commit].B != string(result.Raw) {
+						t.Errorf("%s: unexpected result. Got: %s, Expected: %s", tName, string(result.Raw), tTest.expected[meta.Git.Commit].B)
+					}
+					if tTest.expected[meta.Git.Commit].LineNumber != result.SourceMetadata.GetGit().Line {
+						t.Errorf("%s: unexpected line number. Got: %d, Expected: %d", tName, result.SourceMetadata.GetGit().Line, tTest.expected[meta.Git.Commit].LineNumber)
+					}
+				}
+				resultCount++
+
+			}
+			if resultCount != len(tTest.expected) {
+				t.Errorf("%s: unexpected number of results. Got: %d, Expected: %d", tName, resultCount, len(tTest.expected))
+			}
+		})
 	}
 }
 
@@ -124,5 +130,8 @@ func BenchmarkGitEngine(b *testing.B) {
 			return
 		}
 	}
-	e.Finish(ctx)
+	logFatalFunc := func(_ error, _ string, _ ...any) {
+		b.Fatalf("error logging function should not have been called")
+	}
+	e.Finish(ctx, logFatalFunc)
 }

pkg/engine/github.go

@@ -1,6 +1,8 @@
 package engine
 
 import (
+	"fmt"
+
 	gogit "github.com/go-git/go-git/v5"
 	"google.golang.org/protobuf/proto"
 	"google.golang.org/protobuf/types/known/anypb"
@@ -58,14 +60,13 @@ func (e *Engine) ScanGitHub(ctx context.Context, c sources.GithubConfig) error {
 	scanOptions := git.NewScanOptions(opts...)
 	source.WithScanOptions(scanOptions)
 
-	e.sourcesWg.Add(1)
-	go func() {
+	e.sourcesWg.Go(func() error {
 		defer common.RecoverWithExit(ctx)
-		defer e.sourcesWg.Done()
 		err := source.Chunks(ctx, e.ChunksChan())
 		if err != nil {
-			ctx.Logger().Error(err, "could not scan github")
+			return fmt.Errorf("could not scan github: %w", err)
 		}
-	}()
+		return nil
+	})
 	return nil
 }

pkg/engine/gitlab.go

@@ -63,14 +63,13 @@ func (e *Engine) ScanGitLab(ctx context.Context, c sources.GitlabConfig) error {
 	}
 	gitlabSource.WithScanOptions(scanOptions)
 
-	e.sourcesWg.Add(1)
-	go func() {
+	e.sourcesWg.Go(func() error {
 		defer common.RecoverWithExit(ctx)
-		defer e.sourcesWg.Done()
 		err := gitlabSource.Chunks(ctx, e.ChunksChan())
 		if err != nil {
-			ctx.Logger().Error(err, "error scanning GitLab")
+			return fmt.Errorf("error scanning GitLab: %w", err)
 		}
-	}()
+		return nil
+	})
 	return nil
 }

pkg/engine/s3.go

@@ -65,14 +65,13 @@ func (e *Engine) ScanS3(ctx context.Context, c sources.S3Config) error {
 		return errors.WrapPrefix(err, "failed to init S3 source", 0)
 	}
 
-	e.sourcesWg.Add(1)
-	go func() {
+	e.sourcesWg.Go(func() error {
 		defer common.RecoverWithExit(ctx)
-		defer e.sourcesWg.Done()
 		err := s3Source.Chunks(ctx, e.ChunksChan())
 		if err != nil {
-			ctx.Logger().Error(err, "error scanning S3")
+			return fmt.Errorf("error scanning S3: %w", err)
 		}
-	}()
+		return nil
+	})
 	return nil
 }

pkg/engine/syslog.go

@@ -1,6 +1,7 @@
 package engine
 
 import (
+	"fmt"
 	"os"
 
 	"github.com/go-errors/errors"
@@ -53,14 +54,13 @@ func (e *Engine) ScanSyslog(ctx context.Context, c sources.SyslogConfig) error {
 		return err
 	}
 
-	e.sourcesWg.Add(1)
-	go func() {
+	e.sourcesWg.Go(func() error {
 		defer common.RecoverWithExit(ctx)
-		defer e.sourcesWg.Done()
 		err := source.Chunks(ctx, e.ChunksChan())
 		if err != nil {
-			ctx.Logger().Error(err, "could not scan syslog")
+			return fmt.Errorf("could not scan syslog: %w", err)
 		}
-	}()
+		return nil
+	})
 	return nil
 }

pkg/pb/detectorspb/detectors.pb.go

@@ -991,6 +991,7 @@ const (
 	DetectorType_Aiven                         DetectorType = 917
 	DetectorType_Prefect                       DetectorType = 918
 	DetectorType_Docusign                      DetectorType = 919
+	DetectorType_Couchbase                     DetectorType = 920
 )
 
 // Enum value maps for DetectorType.
@@ -1912,6 +1913,7 @@ var (
 		917: "Aiven",
 		918: "Prefect",
 		919: "Docusign",
+		920: "Couchbase",
 	}
 	DetectorType_value = map[string]int32{
 		"Alibaba":                       0,
@@ -2830,6 +2832,7 @@ var (
 		"Aiven":                         917,
 		"Prefect":                       918,
 		"Docusign":                      919,
+		"Couchbase":                     920,
 	}
 )
 
@@ -3194,7 +3197,7 @@ var file_detectors_proto_rawDesc = []byte{
 	0x0a, 0x0b, 0x44, 0x65, 0x63, 0x6f, 0x64, 0x65, 0x72, 0x54, 0x79, 0x70, 0x65, 0x12, 0x0b, 0x0a,
 	0x07, 0x55, 0x4e, 0x4b, 0x4e, 0x4f, 0x57, 0x4e, 0x10, 0x00, 0x12, 0x09, 0x0a, 0x05, 0x50, 0x4c,
 	0x41, 0x49, 0x4e, 0x10, 0x01, 0x12, 0x0a, 0x0a, 0x06, 0x42, 0x41, 0x53, 0x45, 0x36, 0x34, 0x10,
-	0x02, 0x12, 0x09, 0x0a, 0x05, 0x55, 0x54, 0x46, 0x31, 0x36, 0x10, 0x03, 0x2a, 0xf9, 0x72, 0x0a,
+	0x02, 0x12, 0x09, 0x0a, 0x05, 0x55, 0x54, 0x46, 0x31, 0x36, 0x10, 0x03, 0x2a, 0x89, 0x73, 0x0a,
 	0x0c, 0x44, 0x65, 0x74, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x54, 0x79, 0x70, 0x65, 0x12, 0x0b, 0x0a,
 	0x07, 0x41, 0x6c, 0x69, 0x62, 0x61, 0x62, 0x61, 0x10, 0x00, 0x12, 0x08, 0x0a, 0x04, 0x41, 0x4d,
 	0x51, 0x50, 0x10, 0x01, 0x12, 0x07, 0x0a, 0x03, 0x41, 0x57, 0x53, 0x10, 0x02, 0x12, 0x09, 0x0a,
@@ -4114,7 +4117,8 @@ var file_detectors_proto_rawDesc = []byte{
 	0x0b, 0x4e, 0x75, 0x47, 0x65, 0x74, 0x41, 0x70, 0x69, 0x4b, 0x65, 0x79, 0x10, 0x94, 0x07, 0x12,
 	0x0a, 0x0a, 0x05, 0x41, 0x69, 0x76, 0x65, 0x6e, 0x10, 0x95, 0x07, 0x12, 0x0c, 0x0a, 0x07, 0x50,
 	0x72, 0x65, 0x66, 0x65, 0x63, 0x74, 0x10, 0x96, 0x07, 0x12, 0x0d, 0x0a, 0x08, 0x44, 0x6f, 0x63,
-	0x75, 0x73, 0x69, 0x67, 0x6e, 0x10, 0x97, 0x07, 0x42, 0x3d, 0x5a, 0x3b, 0x67, 0x69, 0x74, 0x68,
+	0x75, 0x73, 0x69, 0x67, 0x6e, 0x10, 0x97, 0x07, 0x12, 0x0e, 0x0a, 0x09, 0x43, 0x6f, 0x75, 0x63,
+	0x68, 0x62, 0x61, 0x73, 0x65, 0x10, 0x98, 0x07, 0x42, 0x3d, 0x5a, 0x3b, 0x67, 0x69, 0x74, 0x68,
 	0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x74, 0x72, 0x75, 0x66, 0x66, 0x6c, 0x65, 0x73, 0x65,
 	0x63, 0x75, 0x72, 0x69, 0x74, 0x79, 0x2f, 0x74, 0x72, 0x75, 0x66, 0x66, 0x6c, 0x65, 0x68, 0x6f,
 	0x67, 0x2f, 0x76, 0x33, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x70, 0x62, 0x2f, 0x64, 0x65, 0x74, 0x65,

pkg/sources/circleci/circleci.go

@@ -31,10 +31,12 @@ type Source struct {
 	jobPool  *errgroup.Group
 	sources.Progress
 	client *http.Client
+	sources.CommonSourceUnitUnmarshaller
 }
 
-// Ensure the Source satisfies the interface at compile time.
+// Ensure the Source satisfies the interfaces at compile time.
 var _ sources.Source = (*Source)(nil)
+var _ sources.SourceUnitUnmarshaller = (*Source)(nil)
 
 // Type returns the type of source.
 // It is used for matching source types in configuration and job input.

pkg/sources/docker/docker.go

@@ -30,12 +30,14 @@ type Source struct {
 	verify   bool
 	conn     sourcespb.Docker
 	sources.Progress
+	sources.CommonSourceUnitUnmarshaller
 }
 
 var FilesizeLimitBytes int64 = 50 * 1024 * 1024 // 50MB
 
-// Ensure the Source satisfies the interface at compile time.
+// Ensure the Source satisfies the interfaces at compile time.
 var _ sources.Source = (*Source)(nil)
+var _ sources.SourceUnitUnmarshaller = (*Source)(nil)
 
 // Type returns the type of source.
 // It is used for matching source types in configuration and job input.

pkg/sources/filesystem/filesystem.go

@@ -39,10 +39,12 @@ type Source struct {
 	log      logr.Logger
 	filter   *common.Filter
 	sources.Progress
+	sources.CommonSourceUnitUnmarshaller
 }
 
-// Ensure the Source satisfies the interface at compile time
+// Ensure the Source satisfies the interfaces at compile time
 var _ sources.Source = (*Source)(nil)
+var _ sources.SourceUnitUnmarshaller = (*Source)(nil)
 
 // Type returns the type of source.
 // It is used for matching source types in configuration and job input.

pkg/sources/gcs/gcs.go

@@ -30,8 +30,9 @@ import (
 
 const defaultCachePersistIncrement = 2500
 
-// Ensure the Source satisfies the interface at compile time.
+// Ensure the Source satisfies the interfaces at compile time.
 var _ sources.Source = (*Source)(nil)
+var _ sources.SourceUnitUnmarshaller = (*Source)(nil)
 
 // Type returns the type of source.
 // It is used for matching source types in configuration and job input.
@@ -69,6 +70,7 @@ type Source struct {
 
 	mu               sync.Mutex
 	sources.Progress // progress is not thread safe
+	sources.CommonSourceUnitUnmarshaller
 }
 
 // persistableCache is a wrapper around cache.Cache that allows

pkg/sources/git/git.go

@@ -67,8 +67,9 @@ func NewGit(sourceType sourcespb.SourceType, jobID, sourceID int64, sourceName s
 	}
 }
 
-// Ensure the Source satisfies the interface at compile time.
+// Ensure the Source satisfies the interfaces at compile time.
 var _ sources.Source = (*Source)(nil)
+var _ sources.SourceUnitUnmarshaller = (*Source)(nil)
 
 // Type returns the type of source.
 // It is used for matching source types in configuration and job input.
@@ -738,7 +739,7 @@ func PrepareRepoSinceCommit(ctx context.Context, uriString, commitHash string) (
 	var path string
 	switch {
 	case uri.User != nil:
-		ctx.Logger().V(1).Info("cloning repo with authentication", "uri", uri)
+		ctx.Logger().V(1).Info("cloning repo with authentication", "uri", uri.Redacted())
 		password, ok := uri.User.Password()
 		if !ok {
 			return "", true, fmt.Errorf("password must be included in Git repo URL when username is provided")
@@ -776,7 +777,7 @@ func PrepareRepo(ctx context.Context, uriString string) (string, bool, error) {
 		remote = true
 		switch {
 		case uri.User != nil:
-			ctx.Logger().V(1).Info("cloning repo with authentication", "uri", uri)
+			ctx.Logger().V(1).Info("cloning repo with authentication", "uri", uri.Redacted())
 			password, ok := uri.User.Password()
 			if !ok {
 				return "", remote, fmt.Errorf("password must be included in Git repo URL when username is provided")
@@ -875,3 +876,7 @@ func handleBinary(ctx context.Context, repo *git.Repository, chunksChan chan *so
 
 	return nil
 }
+
+func (s *Source) UnmarshalSourceUnit(data []byte) (sources.SourceUnit, error) {
+	return UnmarshalUnit(data)
+}

pkg/sources/git/unit.go

@@ -0,0 +1,40 @@
+package git
+
+import (
+	"encoding/json"
+	"fmt"
+
+	"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
+)
+
+const (
+	UnitRepo string = "repo"
+	UnitDir  string = "dir"
+)
+
+// Ensure SourceUnit implements the interface at compile time.
+var _ sources.SourceUnit = SourceUnit{}
+
+// A git source unit can be two kinds of units: either a local directory path
+// or a remote repository.
+type SourceUnit struct {
+	Kind string `json:"kind"`
+	ID   string `json:"id"`
+}
+
+// Implement sources.SourceUnit interface.
+func (u SourceUnit) SourceUnitID() string {
+	return u.ID
+}
+
+// Helper function to unmarshal raw bytes into our SourceUnit struct.
+func UnmarshalUnit(data []byte) (sources.SourceUnit, error) {
+	var unit SourceUnit
+	if err := json.Unmarshal(data, &unit); err != nil {
+		return nil, err
+	}
+	if unit.ID == "" || (unit.Kind != UnitRepo && unit.Kind != UnitDir) {
+		return nil, fmt.Errorf("not a git.SourceUnit")
+	}
+	return unit, nil
+}

pkg/sources/github/github.go

@@ -69,14 +69,17 @@ type Source struct {
 	mu                sync.Mutex
 	publicMap         map[string]source_metadatapb.Visibility
 	sources.Progress
+	sources.CommonSourceUnitUnmarshaller
 }
 
 func (s *Source) WithScanOptions(scanOptions *git.ScanOptions) {
 	s.scanOptions = scanOptions
 }
 
-// Ensure the Source satisfies the interface at compile time
+// Ensure the Source satisfies the interfaces at compile time
 var _ sources.Source = (*Source)(nil)
+var _ sources.SourceUnitUnmarshaller = (*Source)(nil)
+
 var endsWithGithub = regexp.MustCompile(`github\.com/?$`)
 
 // Type returns the type of source.
@@ -663,17 +666,11 @@ func (s *Source) scan(ctx context.Context, installationClient *github.Client, ch
 			defer func(start time.Time) {
 				logger.V(2).Info(fmt.Sprintf("scanned %d/%d repos", scanned, len(s.repos)), "repo_size", repoSize, "duration_seconds", time.Since(start).Seconds())
 			}(now)
-
-			fmt.Printf("Scanning repoURL %s, path %+v, repo %+v \n ", repoURL, path, repo)
 			if err = s.git.ScanRepo(ctx, repo, path, s.scanOptions, chunksChan); err != nil {
 				scanErrs.Add(fmt.Errorf("error scanning repo %s: %w", repoURL, err))
 				return nil
 			}
 			atomic.AddUint64(&scanned, 1)
-			fmt.Println("SCANNNNN COMMENTS")
-			if err = s.scanComments(ctx, repoURL, chunksChan); err != nil {
-				fmt.Println("test")
-			}
 
 			return nil
 		})
@@ -922,82 +919,3 @@ func (s *Source) setProgressCompleteWithRepo(index int, offset int, repoURL stri
 
 	s.SetProgressComplete(index+offset, len(s.repos)+offset, fmt.Sprintf("Repo: %s", repoURL), encodedResumeInfo)
 }
-
-func (s *Source) scanComments(ctx context.Context, repoPath string, chunksChan chan *sources.Chunk) error {
-	s.log.Info("scanning comments")
-	fmt.Print("repoPath: ", repoPath, "\n")
-
-	trimmedURL := removeURLAndSplit(repoPath)
-	owner := trimmedURL[1]
-	repo := trimmedURL[2]
-
-	//for _, part := range trimmedURL {
-	//	fmt.Println(part)
-	//}
-
-	sortType := "created"
-	directionType := "desc"
-
-	issueOpts := &github.IssueListCommentsOptions{
-		Sort:      &sortType,
-		Direction: &directionType,
-		ListOptions: github.ListOptions{
-			PerPage: defaultPagination,
-			Page:    1,
-		},
-	}
-	issueComments, resp, err := s.apiClient.Issues.ListComments(ctx, owner, repo, 0, issueOpts)
-
-	prOpts := &github.PullRequestListCommentsOptions{
-		Sort:      sortType,
-		Direction: directionType,
-		ListOptions: github.ListOptions{
-			PerPage: defaultPagination,
-			Page:    1,
-		},
-	}
-
-	prComments, resp, err := s.apiClient.PullRequests.ListComments(ctx, owner, repo, 0, prOpts)
-
-	fmt.Println("issue comments: ", issueComments)
-	fmt.Println("pr comments: ", prComments)
-	fmt.Println("resp: ", resp)
-	fmt.Println("err: ", err)
-
-	//for _, comment := range comments {
-	//	// Create chunk and send it to the channel.
-	//	chunk := &sources.Chunk{
-	//		SourceName: s.name,
-	//		SourceID:   s.SourceID(),
-	//		SourceType: s.Type(),
-	//		SourceMetadata: &source_metadatapb.MetaData{
-	//			Data: &source_metadatapb.MetaData_Github{
-	//				Github: &source_metadatapb.Github{
-	//					Link:       sanitizer.UTF8(comment.GetHTMLURL()),
-	//					Username:   sanitizer.UTF8(comment.GetUser().GetLogin()),
-	//					Email:      sanitizer.UTF8(comment.GetUser().GetEmail()),
-	//					Repository: sanitizer.UTF8(repo),
-	//					Timestamp:  sanitizer.UTF8(comment.GetCreatedAt().String()),
-	//				},
-	//			},
-	//		},
-	//		Data:   []byte(sanitizer.UTF8(comment.GetBody())),
-	//		Verify: s.verify,
-	//	}
-	//
-	//	select {
-	//	case <-ctx.Done():
-	//		return ctx.Err()
-	//	case chunksChan <- chunk:
-	//	}
-	//}
-	return nil
-}
-
-func removeURLAndSplit(url string) []string {
-	trimmedURL := strings.TrimPrefix(url, "https://")
-	trimmedURL = strings.TrimSuffix(trimmedURL, ".git")
-	splitURL := strings.Split(trimmedURL, "/")
-
-	return splitURL
-}

pkg/sources/gitlab/gitlab.go

@@ -47,10 +47,12 @@ type Source struct {
 	resumeInfoMutex sync.Mutex
 	sources.Progress
 	jobPool *errgroup.Group
+	sources.CommonSourceUnitUnmarshaller
 }
 
-// Ensure the Source satisfies the interface at compile time.
+// Ensure the Source satisfies the interfaces at compile time.
 var _ sources.Source = (*Source)(nil)
+var _ sources.SourceUnitUnmarshaller = (*Source)(nil)
 
 // Type returns the type of source.
 // It is used for matching source types in configuration and job input.

pkg/sources/s3/s3.go

@@ -46,10 +46,12 @@ type Source struct {
 	conn          *sourcespb.S3
 	jobPool       *errgroup.Group
 	maxObjectSize int64
+	sources.CommonSourceUnitUnmarshaller
 }
 
-// Ensure the Source satisfies the interface at compile time
+// Ensure the Source satisfies the interfaces at compile time
 var _ sources.Source = (*Source)(nil)
+var _ sources.SourceUnitUnmarshaller = (*Source)(nil)
 
 // Type returns the type of source
 func (s *Source) Type() sourcespb.SourceType {
@@ -195,7 +197,10 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) err
 			})
 
 		if err != nil {
-			s.log.Error(err, "could not list objects in s3 bucket", "bucket", bucket)
+			return fmt.Errorf(
+				"could not list objects in s3 bucket: bucket %s: %w",
+				bucket,
+				err)
 		}
 	}
 	s.SetProgressComplete(len(bucketsToScan), len(bucketsToScan), fmt.Sprintf("Completed scanning source %s. %d objects scanned.", s.name, objectCount), "")

pkg/sources/source_unit.go

@@ -1,12 +1,37 @@
 package sources
 
+import (
+	"encoding/json"
+	"fmt"
+)
+
+// Ensure CommonSourceUnit implements SourceUnit at compile time.
+var _ SourceUnit = CommonSourceUnit{}
+
 // CommonSourceUnit is a common implementation of SourceUnit that Sources can
 // use instead of implementing their own types.
 type CommonSourceUnit struct {
 	ID string `json:"source_unit_id"`
 }
 
-// Implement the SourceUnit interface.
+// SourceUnitID implements the SourceUnit interface.
 func (c CommonSourceUnit) SourceUnitID() string {
 	return c.ID
 }
+
+// CommonSourceUnitUnmarshaller is an implementation of SourceUnitUnmarshaller
+// for the CommonSourceUnit. A source can embed this struct to gain the
+// functionality of converting []byte to a CommonSourceUnit.
+type CommonSourceUnitUnmarshaller struct{}
+
+// UnmarshalSourceUnit implements the SourceUnitUnmarshaller interface.
+func (c CommonSourceUnitUnmarshaller) UnmarshalSourceUnit(data []byte) (SourceUnit, error) {
+	var unit CommonSourceUnit
+	if err := json.Unmarshal(data, &unit); err != nil {
+		return nil, err
+	}
+	if unit.ID == "" {
+		return nil, fmt.Errorf("not a CommonSourceUnit")
+	}
+	return unit, nil
+}

proto/detectors.proto

@@ -928,6 +928,7 @@ enum DetectorType {
   Aiven = 917;
   Prefect = 918;
   Docusign = 919;
+  Couchbase = 920;
 }
 
 message Result {