updating secret to s1 to match convention

update gitlabv2 to tri-state
add tri state verification to slack (not slack webhook) (#1731 )
2023-09-11 16:07:19 -04:00 · 2023-09-11 14:36:48 -04:00 · 2023-09-11 08:38:43 -07:00 · 2023-09-11 08:18:48 -07:00 · 2023-09-11 08:18:34 -07:00 · 2023-09-11 10:42:14 -04:00
62 changed files with 2595 additions and 692 deletions
@@ -32,7 +32,7 @@ Have questions? Feedback? Jump in slack or discord and hang out with us

 Join our [Slack Community](https://join.slack.com/t/trufflehog-community/shared_invite/zt-pw2qbi43-Aa86hkiimstfdKH9UCpPzQ)

-Join the [Secret Scanning Discord](https://discord.gg/sydS6AHTUP)
+Join the [Secret Scanning Discord](https://discord.gg/8Hzbrnkr7E)

 # :tv: Demo

@@ -1,6 +1,6 @@
 module github.com/trufflesecurity/trufflehog/v3

-go 1.20
+go 1.21

 replace github.com/jpillora/overseer => github.com/trufflesecurity/overseer v1.1.7-custom5

@@ -50,6 +50,7 @@ require (
 	github.com/joho/godotenv v1.5.1
 	github.com/jpillora/overseer v1.1.6
 	github.com/kylelemons/godebug v1.1.0
+	github.com/launchdarkly/go-server-sdk/v6 v6.1.0
 	github.com/lib/pq v1.10.9
 	github.com/lrstanley/bubblezone v0.0.0-20221222153816-e95291e2243e
 	github.com/mattn/go-isatty v0.0.18
@@ -170,8 +171,8 @@ require (
 	github.com/google/uuid v1.3.0 // indirect
 	github.com/googleapis/enterprise-certificate-proxy v0.2.5 // indirect
 	github.com/gorilla/css v1.0.0 // indirect
-	github.com/gsterjov/go-libsecret v0.0.0-20161001094733-a6f4afe4910c // indirect
 	github.com/gregjones/httpcache v0.0.0-20171119193500-2bcd89a1743f // indirect
+	github.com/gsterjov/go-libsecret v0.0.0-20161001094733-a6f4afe4910c // indirect
 	github.com/h2non/parth v0.0.0-20190131123155-b4df798d6542 // indirect
 	github.com/hashicorp/errwrap v1.0.0 // indirect
 	github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
@@ -192,7 +193,6 @@ require (
 	github.com/launchdarkly/go-sdk-events/v2 v2.0.1 // indirect
 	github.com/launchdarkly/go-semver v1.0.2 // indirect
 	github.com/launchdarkly/go-server-sdk-evaluation/v2 v2.0.2 // indirect
-	github.com/launchdarkly/go-server-sdk/v6 v6.1.0 // indirect
 	github.com/lucasb-eyer/go-colorful v1.2.0 // indirect
 	github.com/mailru/easyjson v0.7.7 // indirect
 	github.com/mattn/go-colorable v0.1.13 // indirect
@@ -39,6 +39,7 @@ github.com/Azure/azure-sdk-for-go/sdk/azcore v1.4.0 h1:rTnT/Jrcm+figWlYz4Ixzt0SJ
 github.com/Azure/azure-sdk-for-go/sdk/azcore v1.4.0/go.mod h1:ON4tFdPTwRcgWEaVDrN3584Ef+b7GgSJaXxe5fW9t4M=
 github.com/Azure/azure-sdk-for-go/sdk/azidentity v0.11.0/go.mod h1:HcM1YX14R7CJcghJGOYCgdezslRSVzqwLf/q+4Y2r/0=
 github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.1.0 h1:QkAcEIAKbNL4KoFr4SathZPhDhF4mVwpBMFlYjyAqy8=
+github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.1.0/go.mod h1:bhXu1AjYL+wutSL/kpSq6s7733q2Rb0yuot9Zgfqa/0=
 github.com/Azure/azure-sdk-for-go/sdk/internal v0.7.0/go.mod h1:yqy467j36fJxcRV2TzfVZ1pCb5vxm4BtZPUdYWe/Xo8=
 github.com/Azure/azure-sdk-for-go/sdk/internal v1.1.2 h1:+5VZ72z0Qan5Bog5C+ZkgSqUbeVUd9wgtHOrIKuc5b8=
 github.com/Azure/azure-sdk-for-go/sdk/internal v1.1.2/go.mod h1:eWRD7oawr1Mu1sLCawqVc0CUiF43ia3qQMxLscsKQ9w=
@@ -65,6 +66,7 @@ github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBp
 github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 h1:mFRzDkZVAjdal+s7s0MwaRv9igoPqLRdzOLzw/8Xvq8=
 github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358/go.mod h1:chxPXzSsl7ZWRAuOIE23GDNzjWuZquvFlgA8xmpunjU=
 github.com/AzureAD/microsoft-authentication-library-for-go v0.5.1 h1:BWe8a+f/t+7KY7zH2mqygeUD0t8hNFXe08p1Pb3/jKE=
+github.com/AzureAD/microsoft-authentication-library-for-go v0.5.1/go.mod h1:Vt9sXTKwMyGcOxSmLDMnGPgqsUg7m8pe215qMLrDXw4=
 github.com/BobuSumisu/aho-corasick v1.0.3 h1:uuf+JHwU9CHP2Vx+wAy6jcksJThhJS9ehR8a+4nPE9g=
 github.com/BobuSumisu/aho-corasick v1.0.3/go.mod h1:hm4jLcvZKI2vRF2WDU1N4p/jpWtpOzp3nLmi9AzX/XE=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
@@ -93,12 +95,14 @@ github.com/alexbrainman/sspi v0.0.0-20210105120005-909beea2cc74/go.mod h1:cEWa1L
 github.com/andybalholm/brotli v1.0.5 h1:8uQZIdzKmjc/iuPu7O2ioW48L81FgatrcpfFmiq/cCs=
 github.com/andybalholm/brotli v1.0.5/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig=
 github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be h1:9AeTilPcZAjCFIImctFaOjnTIavg87rW78vTPkQqLI8=
+github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be/go.mod h1:ySMOLuWl6zY27l47sB3qLNK6tF2fkHG55UZxx8oIVo4=
 github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY=
 github.com/apache/arrow/go/v12 v12.0.1 h1:JsR2+hzYYjgSUkBSaahpqCetqZMr76djX80fF/DiJbg=
 github.com/apache/arrow/go/v12 v12.0.1/go.mod h1:weuTY7JvTG/HDPtMQxEUp7pU73vkLWMLpY67QwZ/WWw=
 github.com/apache/thrift v0.16.0 h1:qEy6UW60iVOlUy+b9ZR0d5WzUWYGOo4HfopoyBaNmoY=
 github.com/apache/thrift v0.16.0/go.mod h1:PHK3hniurgQaNMZYaCLEqXKsYK8upmhPbmdP2FXSqgU=
 github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio=
+github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs=
 github.com/atotto/clipboard v0.1.4 h1:EH0zSVneZPSuFR11BlR9YppQTVDbh5+16AmcJi4g1z4=
 github.com/atotto/clipboard v0.1.4/go.mod h1:ZY9tmq7sm5xIbd9bOK4onWV4S6X0u6GY7Vn0Yu86PYI=
 github.com/aws/aws-sdk-go v1.44.83 h1:7+Rtc2Eio6EKUNoZeMV/IVxzVrY5oBQcNPtCcgIHYJA=
@@ -237,6 +241,7 @@ github.com/dsnet/golib v0.0.0-20171103203638-1ea166775780/go.mod h1:Lj+Z9rebOhdf
 github.com/dvsekhvalnov/jose2go v1.5.0 h1:3j8ya4Z4kMCwT5nXIKFSV84YS+HdqSSO0VsTQxaLAeM=
 github.com/dvsekhvalnov/jose2go v1.5.0/go.mod h1:QsHjhyTlD/lAVqn/NSbVZmSCGeDehTB/mPZadG+mhXU=
 github.com/elazarl/goproxy v0.0.0-20221015165544-a0805db90819 h1:RIB4cRk+lBqKK3Oy0r2gRX4ui7tuhiZq2SuTtTCi0/0=
+github.com/elazarl/goproxy v0.0.0-20221015165544-a0805db90819/go.mod h1:Ro8st/ElPeALwNFlcTpWmkr6IoMFfkjXAvTHpevnDsM=
 github.com/emirpasic/gods v1.18.1 h1:FXtiHYKDGKCW2KzwZKx0iC0PQmdlorYgdFG9jPXJ1Bc=
 github.com/emirpasic/gods v1.18.1/go.mod h1:8tpGGwCnJ5H4r6BWwaV6OrWmMoPhUl5jm/FMNAnJvWQ=
 github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
@@ -262,6 +267,7 @@ github.com/getsentry/sentry-go v0.22.0 h1:XNX9zKbv7baSEI65l+H1GEJgSeIC1c7EN5kluW
 github.com/getsentry/sentry-go v0.22.0/go.mod h1:lc76E2QywIyW8WuBnwl8Lc4bkmQH4+w1gwTf25trprY=
 github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
 github.com/gliderlabs/ssh v0.3.5 h1:OcaySEmAQJgyYcArR+gGGTHCyE7nvhEMTlYY+Dp8CpY=
+github.com/gliderlabs/ssh v0.3.5/go.mod h1:8XB4KraRrX39qHhT6yxPsHedjA08I/uBVwj4xC+/+z4=
 github.com/go-asn1-ber/asn1-ber v1.5.4 h1:vXT6d/FNDiELJnLb6hGNa309LMsrCoYFvpwHDF0+Y1A=
 github.com/go-asn1-ber/asn1-ber v1.5.4/go.mod h1:hEBeB/ic+5LoWskz+yKT7vGhhPYkProFKoKdwZRWMe0=
 github.com/go-errors/errors v1.4.2 h1:J6MZopCL4uSllY1OfXM374weqZFFItUbrImctkmUxIA=
@@ -271,6 +277,7 @@ github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376/go.mod h1:an3vInlBmS
 github.com/go-git/go-billy/v5 v5.4.1 h1:Uwp5tDRkPr+l/TnbHOQzp+tmJfLceOlbVucgpTz8ix4=
 github.com/go-git/go-billy/v5 v5.4.1/go.mod h1:vjbugF6Fz7JIflbVpl1hJsGjSHNltrSw45YK/ukIvQg=
 github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20230305113008-0c11038e723f h1:Pz0DHeFij3XFhoBRGUDPzSJ+w2UcK5/0JvF8DRI58r8=
+github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20230305113008-0c11038e723f/go.mod h1:8LHG1a3SRW71ettAD/jW13h8c6AqjVSeL11RAdgaqpo=
 github.com/go-git/go-git/v5 v5.8.1 h1:Zo79E4p7TRk0xoRgMq0RShiTHGKcKI4+DI6BfJc/Q+A=
 github.com/go-git/go-git/v5 v5.8.1/go.mod h1:FHFuoD6yGz5OSKEBK+aWN9Oah0q54Jxl0abmj6GnqAo=
 github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
@@ -366,6 +373,7 @@ github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17
 github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPgecwXBIDzw5no=
 github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
 github.com/google/martian/v3 v3.3.2 h1:IqNFLAmvJOgVlpdEBiQbDc2EwKW77amAycfTuWKdfvw=
+github.com/google/martian/v3 v3.3.2/go.mod h1:oBOf6HBosgwRXnUGWUB05QECsc6uvmMiJ3+6W4l/CUk=
 github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
 github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
 github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
@@ -427,6 +435,7 @@ github.com/jpillora/s3 v1.1.4 h1:YCCKDWzb/Ye9EBNd83ATRF/8wPEy0xd43Rezb6u6fzc=
 github.com/jpillora/s3 v1.1.4/go.mod h1:yedE603V+crlFi1Kl/5vZJaBu9pUzE9wvKegU/lF2zs=
 github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
 github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk=
+github.com/karlseguin/expect v1.0.2-0.20190806010014-778a5f0c6003 h1:vJ0Snvo+SLMY72r5J4sEfkuE7AFbixEP2qRbEcum/wA=
 github.com/karlseguin/expect v1.0.2-0.20190806010014-778a5f0c6003/go.mod h1:zNBxMY8P21owkeogJELCLeHIt+voOSduHYTFUbwRAV8=
 github.com/kevinburke/ssh_config v1.2.0 h1:x584FjTGwHzMwvHx18PXxbBVzfnxogHaAReU4gf13a4=
 github.com/kevinburke/ssh_config v1.2.0/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM=
@@ -445,6 +454,7 @@ github.com/klauspost/pgzip v1.2.5/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQ
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
 github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
 github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
+github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
@@ -467,7 +477,10 @@ github.com/launchdarkly/go-server-sdk-evaluation/v2 v2.0.2 h1:PAM0GvE0nIUBeOkjdi
 github.com/launchdarkly/go-server-sdk-evaluation/v2 v2.0.2/go.mod h1:Mztipcz+7ZMatXVun3k/IfPa8IOgUnAqiZawtFh2MRg=
 github.com/launchdarkly/go-server-sdk/v6 v6.1.0 h1:pp/VBIon5JNbtSw7ih7I5MXN9mXHfo6T5xikKVy52dI=
 github.com/launchdarkly/go-server-sdk/v6 v6.1.0/go.mod h1:3TfS6vNsNksPT4LGeGuGKtT3zTjTbbOsCUK3nrj1neA=
+github.com/launchdarkly/go-test-helpers/v2 v2.2.0 h1:L3kGILP/6ewikhzhdNkHy1b5y4zs50LueWenVF0sBbs=
 github.com/launchdarkly/go-test-helpers/v2 v2.2.0/go.mod h1:L7+th5govYp5oKU9iN7To5PgznBuIjBPn+ejqKR0avw=
+github.com/launchdarkly/go-test-helpers/v3 v3.0.2 h1:rh0085g1rVJM5qIukdaQ8z1XTWZztbJ49vRZuveqiuU=
+github.com/launchdarkly/go-test-helpers/v3 v3.0.2/go.mod h1:u2ZvJlc/DDJTFrshWW50tWMZHLVYXofuSHUfTU/eIwM=
 github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw=
 github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
 github.com/lrstanley/bubblezone v0.0.0-20221222153816-e95291e2243e h1:XoxHx8K6ZKoMtjzWOMDuM69LCdjDDsTOtTfWGrT/fns=
@@ -548,6 +561,7 @@ github.com/paulbellamy/ratecounter v0.2.0/go.mod h1:Hfx1hDpSGoqxkVVpBi/IlYD7kChl
 github.com/pierrec/lz4/v4 v4.1.17 h1:kV4Ip+/hUBC+8T6+2EgburRtkE9ef4nbY3f4dFhGjMc=
 github.com/pierrec/lz4/v4 v4.1.17/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
 github.com/pingcap/errors v0.11.4 h1:lFuQV/oaUMGcD2tqt+01ROSmJs75VG1ToEOkZIZ4nE4=
+github.com/pingcap/errors v0.11.4/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8=
 github.com/pjbgf/sha1cd v0.3.0 h1:4D5XXmUUBUl/xQ6IjCkEAbqXskkq/4O7LmGn0AqMDs4=
 github.com/pjbgf/sha1cd v0.3.0/go.mod h1:nZ1rrWOcGJ5uZgEEVL1VUM9iRQiZvWdbZjkKyFzPPsI=
 github.com/pkg/browser v0.0.0-20180916011732-0a3d74bf9ce4/go.mod h1:4OwLy04Bl9Ef3GJJCoec+30X3LQs/0/m4HFRt/2LUSA=
@@ -626,6 +640,7 @@ github.com/ulikunitz/xz v0.5.10/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0o
 github.com/urfave/cli v1.22.12/go.mod h1:sSBEIC79qR6OvcmsD4U3KABeOTxDqQtdDnaFuUN30b8=
 github.com/vbatts/tar-split v0.11.3 h1:hLFqsOLQ1SsppQNTMpkpPXClLDfC2A3Zgy9OUU+RVck=
 github.com/vbatts/tar-split v0.11.3/go.mod h1:9QlHN18E+fEH7RdG+QAJJcuya3rqT7eXSTY7wGrAokY=
+github.com/wsxiaoys/terminal v0.0.0-20160513160801-0940f3fc43a0 h1:3UeQBvD0TFrlVjOeLOBz+CPAI8dnbqNSVwUwRrkp7vQ=
 github.com/wsxiaoys/terminal v0.0.0-20160513160801-0940f3fc43a0/go.mod h1:IXCdmsXIht47RaVFLEdVnh1t+pgYtTAhQGj73kz+2DM=
 github.com/xanzy/go-gitlab v0.88.0 h1:9GHBrxyCUNZZNuAsbJ1NbEH6XAYsKyTn6NfE0wYO5SY=
 github.com/xanzy/go-gitlab v0.88.0/go.mod h1:5ryv+MnpZStBH8I/77HuQBsMbBGANtVpLWC15qOjWAw=
@@ -649,6 +664,7 @@ github.com/yuin/goldmark-emoji v1.0.1/go.mod h1:2w1E6FEWLcDQkoTE+7HU6QF1F6SLlNGj
 github.com/yusufpapurcu/wmi v1.2.2 h1:KBNDSne4vP5mbSWnJbO+51IMOXJB67QiYCSBrubbPRg=
 github.com/yusufpapurcu/wmi v1.2.2/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0=
 github.com/zeebo/assert v1.3.0 h1:g7C04CbJuIDKNPFHmsk4hwZDO5O+kntRxzaUoNXj+IQ=
+github.com/zeebo/assert v1.3.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0=
 github.com/zeebo/xxh3 v1.0.2 h1:xZmwmqxHZA8AI603jOQ0tMqmBr9lPeFwGg6d+xy9DC0=
 github.com/zeebo/xxh3 v1.0.2/go.mod h1:5NWz9Sef7zIDm2JHfFlcQvNekmcEl9ekUZQQKCYaDcA=
 go.mongodb.org/mongo-driver v1.12.0 h1:aPx33jmn/rQuJXPQLZQ8NtfPQG8CaqgLThFtqRb0PiE=
@@ -885,6 +901,7 @@ golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8T
 golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 h1:H2TDz8ibqkAF6YGhCdN3jS9O0/s90v0rJh3X/OLHEUk=
 golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8=
 gonum.org/v1/gonum v0.11.0 h1:f1IJhK4Km5tBJmaiJXtk/PkL4cdVX6J+tGiM187uT5E=
+gonum.org/v1/gonum v0.11.0/go.mod h1:fSG4YDCxxUZQJ7rKsQrj0gMOg00Il0Z96/qMA4bVQhA=
 google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE=
 google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M=
 google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg=
@@ -980,6 +997,7 @@ gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gotest.tools/v3 v3.0.3 h1:4AuOwCGf4lLR9u3YOe2awrHygurzhO/HeQ6laiA6Sx0=
+gotest.tools/v3 v3.0.3/go.mod h1:Z7Lb0S5l+klDB31fvDQX8ss/FlKDxtlFlw3Oa8Ymbl8=
 honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
@@ -44,6 +44,18 @@ If you think that something should be included outside of these guidelines, plea
 - Go 1.17+
 - Make

+### Adding New Token Formats to an Existing Scanner
+
+In some instances, services will update their token format, requiring a new regex to properly detect secrets in addition to supporting the previous token format. Accomodating this can be done without adding a net-new detector. [We provide a `Versioner` interface](https://github.com/trufflesecurity/trufflehog/blob/e18cfd5e0af1469a9f05b8d5732bcc94c39da49c/pkg/detectors/detectors.go#L30) that can be implemented.
+
+1. Create a copy of the package and append `_v2` to the package and file names. Ex: `<packagename>/` -> `<packagename>_v2`, `<packagename>.go` -> `<packagename>_v2.go`
+
+Note: Be sure to update the tests to reference the new secret values in GSM, or the tests will fail.
+
+2. Implement the `Versioner` interface. [GitHub example implementation.](/pkg/detectors/github_old/github_old.go#L22)
+
+3. Proceed from step 3 of [Creating a new Secret Scanner](#creating-a-new-secret-scanner)
+
 ### Creating a new Secret Scanner

 1. Identify the Secret Detector name from the [/proto/detectors.proto](/proto/detectors.proto) `DetectorType` enum.
@@ -96,7 +108,7 @@ Do not embed test credentials in the test code. Instead, use GCP Secrets Manager
   Note: `/tmp/s` is a valid path on Linux. You will need to change that for Windows or OSX, otherwise you will see an error. On Windows you will also need to install [WSL](https://docs.microsoft.com/en-us/windows/wsl/install).

   ```bash
-   gcloud secrets versions access --project trufflehog-testing --secret detectors3 latest > /tmp/s
+   gcloud secrets versions access --project trufflehog-testing --secret detectors5 latest > /tmp/s
   ```

 2. Add the secret that you need for testing.
@@ -113,8 +125,9 @@ Do not embed test credentials in the test code. Instead, use GCP Secrets Manager
 3. Update the secret version with your modification.

   ```bash
-   gcloud secrets versions add --project trufflehog-testing detectors3 --data-file /tmp/s
+   gcloud secrets versions add --project trufflehog-testing detectors5 --data-file /tmp/s
   ```
+   Note: We increment the detectors file name `detectors(n+1)` once the previous one exceeds the max size allowed by GSM (65kb).

 4. Access the secret value as shown in the [example code](pkg/detectors/heroku/heroku_test.go).

@@ -40,6 +40,18 @@ If you think that something should be included outside of these guidelines, plea
 - Go 1.17+
 - Make

+### Adding New Token Formats to an Existing Scanner
+
+In some instances, services will update their token format, requiring a new regex to properly detect secrets in addition to supporting the previous token format. Accomodating this can be done without adding a net-new detector. [We provide a `Versioner` interface](https://github.com/trufflesecurity/trufflehog/blob/e18cfd5e0af1469a9f05b8d5732bcc94c39da49c/pkg/detectors/detectors.go#L30) that can be implemented.
+
+1. Create a copy of the package and append `_v2` to the package and file names. Ex: `<packagename>/` -> `<packagename>_v2`, `<packagename>.go` -> `<packagename>_v2.go`
+
+Note: Be sure to update the tests to reference the new secret values in GSM, or the tests will fail.
+
+2. Implement the `Versioner` interface. [GitHub example implementation.](/pkg/detectors/github_old/github_old.go#L22)
+
+3. Proceed from step 3 of [Creating a new Secret Scanner](#creating-a-new-secret-scanner)
+
 ### Creating a new Secret Scanner

 1. Identify the Secret Detector name from the [/proto/detectors.proto](/proto/detectors.proto) `DetectorType` enum. If necessary, run `make protos` when adding new ones.
@@ -105,11 +105,11 @@ var (
 	filesystemScanIncludePaths = filesystemScan.Flag("include-paths", "Path to file with newline separated regexes for files to include in scan.").Short('i').String()
 	filesystemScanExcludePaths = filesystemScan.Flag("exclude-paths", "Path to file with newline separated regexes for files to exclude in scan.").Short('x').String()

-	s3Scan    = cli.Command("s3", "Find credentials in S3 buckets.")
-	s3ScanKey = s3Scan.Flag("key", "S3 key used to authenticate. Can be provided with environment variable AWS_ACCESS_KEY_ID.").Envar("AWS_ACCESS_KEY_ID").String()
-	s3ScanRoleArns     = s3Scan.Flag("role-arn", "Specify the ARN of an IAM role to assume for scanning. You can repeat this flag.").Strings()
-	s3ScanSecret       = s3Scan.Flag("secret", "S3 secret used to authenticate. Can be provided with environment variable AWS_SECRET_ACCESS_KEY.").Envar("AWS_SECRET_ACCESS_KEY").String()
-	s3ScanSessionToken = s3Scan.Flag("session-token", "S3 session token used to authenticate temporary credentials. Can be provided with environment variable AWS_SESSION_TOKEN.").Envar("AWS_SESSION_TOKEN").String()
+	s3Scan              = cli.Command("s3", "Find credentials in S3 buckets.")
+	s3ScanKey           = s3Scan.Flag("key", "S3 key used to authenticate. Can be provided with environment variable AWS_ACCESS_KEY_ID.").Envar("AWS_ACCESS_KEY_ID").String()
+	s3ScanRoleArns      = s3Scan.Flag("role-arn", "Specify the ARN of an IAM role to assume for scanning. You can repeat this flag.").Strings()
+	s3ScanSecret        = s3Scan.Flag("secret", "S3 secret used to authenticate. Can be provided with environment variable AWS_SECRET_ACCESS_KEY.").Envar("AWS_SECRET_ACCESS_KEY").String()
+	s3ScanSessionToken  = s3Scan.Flag("session-token", "S3 session token used to authenticate temporary credentials. Can be provided with environment variable AWS_SESSION_TOKEN.").Envar("AWS_SESSION_TOKEN").String()
 	s3ScanCloudEnv      = s3Scan.Flag("cloud-environment", "Use IAM credentials in cloud environment.").Bool()
 	s3ScanBuckets       = s3Scan.Flag("bucket", "Name of S3 bucket to scan. You can repeat this flag.").Strings()
 	s3ScanMaxObjectSize = s3Scan.Flag("max-object-size", "Maximum size of objects to scan. Objects larger than this will be skipped. (Byte units eg. 512B, 2KB, 4MB)").Default("250MB").Bytes()
@@ -354,6 +354,10 @@ func run(state overseer.State) {
 		printer = new(output.PlainPrinter)
 	}

+	if !*jsonLegacy && !*jsonOut {
+		fmt.Fprintf(os.Stderr, "🐷🔑🐷  TruffleHog. Unearth your secrets. 🐷🔑🐷\n\n")
+	}
+
 	e, err := engine.Start(ctx,
 		engine.WithConcurrency(uint8(*concurrency)),
 		engine.WithDecoders(decoders.DefaultDecoders()...),
@@ -469,7 +473,7 @@ func run(state overseer.State) {
 			Secret:        *s3ScanSecret,
 			SessionToken:  *s3ScanSessionToken,
 			Buckets:       *s3ScanBuckets,
-			Roles:	       *s3ScanRoleArns,
+			Roles:         *s3ScanRoleArns,
 			CloudCred:     *s3ScanCloudEnv,
 			MaxObjectSize: int64(*s3ScanMaxObjectSize),
 		}
@@ -525,10 +529,6 @@ func run(state overseer.State) {
 		}
 	}

-	if !*jsonLegacy && !*jsonOut {
-		fmt.Fprintf(os.Stderr, "🐷🔑🐷  TruffleHog. Unearth your secrets. 🐷🔑🐷\n\n")
-	}
-
 	// Wait for all workers to finish.
 	if err = e.Finish(ctx); err != nil {
 		logFatal(err, "engine failed to finish execution")
@@ -26,9 +26,10 @@ type Context interface {
 	Logger() logr.Logger
 }

-// CancelFunc is a type alias to context.CancelFunc to allow use as if they are
-// the same types.
+// CancelFunc and CancelCauseFunc are type aliases to allow use as if they are
+// the same types as the standard library variants.
 type CancelFunc = context.CancelFunc
+type CancelCauseFunc = context.CancelCauseFunc

 // logCtx implements Context.
 type logCtx struct {
@@ -68,6 +69,16 @@ func WithCancel(parent Context) (Context, context.CancelFunc) {
 	return lCtx, cancel
 }

+// WithCancelCause returns context.WithCancelCause with the log object propagated.
+func WithCancelCause(parent Context) (Context, context.CancelCauseFunc) {
+	ctx, cancel := context.WithCancelCause(parent)
+	lCtx := logCtx{
+		log:     parent.Logger(),
+		Context: ctx,
+	}
+	return lCtx, cancel
+}
+
 // WithDeadline returns context.WithDeadline with the log object propagated and
 // the deadline added to the structured log values.
 func WithDeadline(parent Context, d time.Time) (Context, context.CancelFunc) {
@@ -79,6 +90,17 @@ func WithDeadline(parent Context, d time.Time) (Context, context.CancelFunc) {
 	return lCtx, cancel
 }

+// WithDeadlineCause returns context.WithDeadlineCause with the log object
+// propagated and the deadline added to the structured log values.
+func WithDeadlineCause(parent Context, d time.Time, cause error) (Context, context.CancelFunc) {
+	ctx, cancel := context.WithDeadlineCause(parent, d, cause)
+	lCtx := logCtx{
+		log:     parent.Logger().WithValues("deadline", d),
+		Context: ctx,
+	}
+	return lCtx, cancel
+}
+
 // WithTimeout returns context.WithTimeout with the log object propagated and
 // the timeout added to the structured log values.
 func WithTimeout(parent Context, timeout time.Duration) (Context, context.CancelFunc) {
@@ -90,6 +112,22 @@ func WithTimeout(parent Context, timeout time.Duration) (Context, context.Cancel
 	return lCtx, cancel
 }

+// WithTimeoutCause returns context.WithTimeoutCause with the log object
+// propagated and the timeout added to the structured log values.
+func WithTimeoutCause(parent Context, timeout time.Duration, cause error) (Context, context.CancelFunc) {
+	ctx, cancel := context.WithTimeoutCause(parent, timeout, cause)
+	lCtx := logCtx{
+		log:     parent.Logger().WithValues("timeout", timeout),
+		Context: ctx,
+	}
+	return lCtx, cancel
+}
+
+// Cause returns the context.Cause of the context.
+func Cause(ctx context.Context) error {
+	return context.Cause(ctx)
+}
+
 // WithValue returns context.WithValue with the log object propagated and
 // the value added to the structured log values (if the key is a string).
 func WithValue(parent Context, key, val any) Context {
@@ -3,6 +3,7 @@ package context
 import (
 	"bytes"
 	"context"
+	"fmt"
 	"strings"
 	"testing"
 	"time"
@@ -177,3 +178,10 @@ func TestRace(t *testing.T) {
 	cancel()
 	_ = ctx.Err()
 }
+
+func TestCause(t *testing.T) {
+	ctx, cancel := WithCancelCause(Background())
+	err := fmt.Errorf("oh no")
+	cancel(err)
+	assert.Equal(t, err, Cause(ctx))
+}
@@ -118,118 +118,21 @@ func (s scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 			}

 			if verify {
-				// REQUEST VALUES.
-				method := "GET"
-				service := "sts"
-				host := "sts.amazonaws.com"
-				region := "us-east-1"
-				endpoint := "https://sts.amazonaws.com"
-				now := time.Now().UTC()
-				datestamp := now.Format("20060102")
-				amzDate := now.Format("20060102T150405Z0700")
-
-				req, err := http.NewRequestWithContext(ctx, method, endpoint, nil)
-				if err != nil {
-					continue
-				}
-				req.Header.Set("Accept", "application/json")
-
-				// TASK 1: CREATE A CANONICAL REQUEST.
-				// http://docs.aws.amazon.com/general/latest/gr/sigv4-create-canonical-request.html
-				canonicalURI := "/"
-				canonicalHeaders := "host:" + host + "\n"
-				signedHeaders := "host"
-				algorithm := "AWS4-HMAC-SHA256"
-				credentialScope := fmt.Sprintf("%s/%s/%s/aws4_request", datestamp, region, service)
-
-				params := req.URL.Query()
-				params.Add("Action", "GetCallerIdentity")
-				params.Add("Version", "2011-06-15")
-				params.Add("X-Amz-Algorithm", algorithm)
-				params.Add("X-Amz-Credential", resIDMatch+"/"+credentialScope)
-				params.Add("X-Amz-Date", amzDate)
-				params.Add("X-Amz-Expires", "30")
-				params.Add("X-Amz-SignedHeaders", signedHeaders)
-
-				canonicalQuerystring := params.Encode()
-				payloadHash := GetHash("") // empty payload
-				canonicalRequest := method + "\n" + canonicalURI + "\n" + canonicalQuerystring + "\n" + canonicalHeaders + "\n" + signedHeaders + "\n" + payloadHash
-
-				// TASK 2: CREATE THE STRING TO SIGN.
-				stringToSign := algorithm + "\n" + amzDate + "\n" + credentialScope + "\n" + GetHash(canonicalRequest)
-
-				// TASK 3: CALCULATE THE SIGNATURE.
-				// https://docs.aws.amazon.com/general/latest/gr/sigv4-calculate-signature.html
-				hash := GetHMAC([]byte(fmt.Sprintf("AWS4%s", resSecretMatch)), []byte(datestamp))
-				hash = GetHMAC(hash, []byte(region))
-				hash = GetHMAC(hash, []byte(service))
-				hash = GetHMAC(hash, []byte("aws4_request"))
-
-				signature2 := GetHMAC(hash, []byte(stringToSign)) // Get Signature HMAC SHA256
-				signature := hex.EncodeToString(signature2)
-
-				// TASK 4: ADD SIGNING INFORMATION TO THE REQUEST.
-				params.Add("X-Amz-Signature", signature)
-				req.Header.Add("Content-type", "application/x-www-form-urlencoded; charset=utf-8")
-				req.URL.RawQuery = params.Encode()
-
-				client := s.verificationClient
-				if client == nil {
-					client = defaultVerificationClient
-				}
-				res, err := client.Do(req)
-				if err == nil {
-
-					if res.StatusCode >= 200 && res.StatusCode < 300 {
-						identityInfo := identityRes{}
-						err := json.NewDecoder(res.Body).Decode(&identityInfo)
-						if err == nil {
-							s1.Verified = true
-							s1.ExtraData = map[string]string{
-								"account": identityInfo.GetCallerIdentityResponse.GetCallerIdentityResult.Account,
-								"user_id": identityInfo.GetCallerIdentityResponse.GetCallerIdentityResult.UserID,
-								"arn":     identityInfo.GetCallerIdentityResponse.GetCallerIdentityResult.Arn,
-							}
-						} else {
-							s1.VerificationError = err
-						}
-						res.Body.Close()
-					} else {
-						// This function will check false positives for common test words, but also it will make sure the key appears "random" enough to be a real key.
-						if detectors.IsKnownFalsePositive(resSecretMatch, detectors.DefaultFalsePositives, true) {
-							continue
-						}
-
-						if res.StatusCode == 403 {
-							var body awsErrorResponseBody
-							err = json.NewDecoder(res.Body).Decode(&body)
-							res.Body.Close()
-							if err == nil {
-								// All instances of the code I've seen in the wild are PascalCased but this check is
-								// case-insensitive out of an abundance of caution
-								if strings.EqualFold(body.Error.Code, "InvalidClientTokenId") {
-									// determinate failure - nothing to do
-								} else {
-									// We see a surprising number of false-negative signature mismatch errors here
-									// (The official SDK somehow elicits even more than just making the request
-									// ourselves)
-									s1.VerificationError = fmt.Errorf("request to %v returned status %d with an unexpected reason (%s: %s)", res.Request.URL, res.StatusCode, body.Error.Code, body.Error.Message)
-								}
-							} else {
-								s1.VerificationError = fmt.Errorf("couldn't parse the sts response body (%v)", err)
-							}
-						} else {
-							s1.VerificationError = fmt.Errorf("request to %v returned unexpected status %d", res.Request.URL, res.StatusCode)
-						}
-					}
-				} else {
-					s1.VerificationError = err
-				}
+				verified, extraData, verificationErr := s.verifyMatch(ctx, resIDMatch, resSecretMatch, true)
+				s1.Verified = verified
+				s1.ExtraData = extraData
+				s1.VerificationError = verificationErr
 			}

-			// If the result is unverified and matches something like a git hash, don't include it in the results.
-			if !s1.Verified && falsePositiveSecretCheck.MatchString(resSecretMatch) {
-				continue
+			if !s1.Verified {
+				// Unverified results that contain common test words are probably not secrets
+				if detectors.IsKnownFalsePositive(resSecretMatch, detectors.DefaultFalsePositives, true) {
+					continue
+				}
+				// Unverified results that look like hashes are probably not secrets
+				if falsePositiveSecretCheck.MatchString(resSecretMatch) {
+					continue
+				}
 			}

 			results = append(results, s1)
@@ -242,6 +145,123 @@ func (s scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 	return awsCustomCleanResults(results), nil
 }

+func (s scanner) verifyMatch(ctx context.Context, resIDMatch, resSecretMatch string, retryOn403 bool) (bool, map[string]string, error) {
+	// REQUEST VALUES.
+	method := "GET"
+	service := "sts"
+	host := "sts.amazonaws.com"
+	region := "us-east-1"
+	endpoint := "https://sts.amazonaws.com"
+	now := time.Now().UTC()
+	datestamp := now.Format("20060102")
+	amzDate := now.Format("20060102T150405Z0700")
+
+	req, err := http.NewRequestWithContext(ctx, method, endpoint, nil)
+	if err != nil {
+		return false, nil, err
+	}
+	req.Header.Set("Accept", "application/json")
+
+	// TASK 1: CREATE A CANONICAL REQUEST.
+	// http://docs.aws.amazon.com/general/latest/gr/sigv4-create-canonical-request.html
+	canonicalURI := "/"
+	canonicalHeaders := "host:" + host + "\n"
+	signedHeaders := "host"
+	algorithm := "AWS4-HMAC-SHA256"
+	credentialScope := fmt.Sprintf("%s/%s/%s/aws4_request", datestamp, region, service)
+
+	params := req.URL.Query()
+	params.Add("Action", "GetCallerIdentity")
+	params.Add("Version", "2011-06-15")
+	params.Add("X-Amz-Algorithm", algorithm)
+	params.Add("X-Amz-Credential", resIDMatch+"/"+credentialScope)
+	params.Add("X-Amz-Date", amzDate)
+	params.Add("X-Amz-Expires", "30")
+	params.Add("X-Amz-SignedHeaders", signedHeaders)
+
+	canonicalQuerystring := params.Encode()
+	payloadHash := GetHash("") // empty payload
+	canonicalRequest := method + "\n" + canonicalURI + "\n" + canonicalQuerystring + "\n" + canonicalHeaders + "\n" + signedHeaders + "\n" + payloadHash
+
+	// TASK 2: CREATE THE STRING TO SIGN.
+	stringToSign := algorithm + "\n" + amzDate + "\n" + credentialScope + "\n" + GetHash(canonicalRequest)
+
+	// TASK 3: CALCULATE THE SIGNATURE.
+	// https://docs.aws.amazon.com/general/latest/gr/sigv4-calculate-signature.html
+	hash := GetHMAC([]byte(fmt.Sprintf("AWS4%s", resSecretMatch)), []byte(datestamp))
+	hash = GetHMAC(hash, []byte(region))
+	hash = GetHMAC(hash, []byte(service))
+	hash = GetHMAC(hash, []byte("aws4_request"))
+
+	signature2 := GetHMAC(hash, []byte(stringToSign)) // Get Signature HMAC SHA256
+	signature := hex.EncodeToString(signature2)
+
+	// TASK 4: ADD SIGNING INFORMATION TO THE REQUEST.
+	params.Add("X-Amz-Signature", signature)
+	req.Header.Add("Content-type", "application/x-www-form-urlencoded; charset=utf-8")
+	req.URL.RawQuery = params.Encode()
+
+	client := s.verificationClient
+	if client == nil {
+		client = defaultVerificationClient
+	}
+
+	res, err := client.Do(req)
+	if err == nil {
+		defer res.Body.Close()
+		if res.StatusCode >= 200 && res.StatusCode < 300 {
+			identityInfo := identityRes{}
+			err := json.NewDecoder(res.Body).Decode(&identityInfo)
+			if err == nil {
+				extraData := map[string]string{
+					"account": identityInfo.GetCallerIdentityResponse.GetCallerIdentityResult.Account,
+					"user_id": identityInfo.GetCallerIdentityResponse.GetCallerIdentityResult.UserID,
+					"arn":     identityInfo.GetCallerIdentityResponse.GetCallerIdentityResult.Arn,
+				}
+				return true, extraData, nil
+			} else {
+				return false, nil, err
+			}
+		} else if res.StatusCode == 403 {
+			// Experimentation has indicated that if you make two GetCallerIdentity requests within five seconds that
+			// share a key ID but are signed with different secrets the second one will be rejected with a 403 that
+			// carries a SignatureDoesNotMatch code in its body. This happens even if the second ID-secret pair is
+			// valid. Since this is exactly our access pattern, we need to work around it.
+			//
+			// Fortunately, experimentation has also revealed a workaround: simply resubmit the second request. The
+			// response to the resubmission will be as expected. But there's a caveat: You can't have closed the body of
+			// the response to the original second request, or read to its end, or the resubmission will also yield a
+			// SignatureDoesNotMatch. For this reason, we have to re-request all 403s. We can't re-request only
+			// SignatureDoesNotMatch responses, because we can only tell whether a given 403 is a SignatureDoesNotMatch
+			// after decoding its response body, which requires reading the entire response body, which disables the
+			// workaround.
+			//
+			// We are clearly deep in the guts of AWS implementation details here, so this all might change with no
+			// notice. If you're here because something in this detector broke, you have my condolences.
+			if retryOn403 {
+				return s.verifyMatch(ctx, resIDMatch, resSecretMatch, false)
+			}
+			var body awsErrorResponseBody
+			err = json.NewDecoder(res.Body).Decode(&body)
+			if err == nil {
+				// All instances of the code I've seen in the wild are PascalCased but this check is
+				// case-insensitive out of an abundance of caution
+				if strings.EqualFold(body.Error.Code, "InvalidClientTokenId") {
+					return false, nil, nil
+				} else {
+					return false, nil, fmt.Errorf("request to %v returned status %d with an unexpected reason (%s: %s)", res.Request.URL, res.StatusCode, body.Error.Code, body.Error.Message)
+				}
+			} else {
+				return false, nil, fmt.Errorf("couldn't parse the sts response body (%v)", err)
+			}
+		} else {
+			return false, nil, fmt.Errorf("request to %v returned unexpected status %d", res.Request.URL, res.StatusCode)
+		}
+	} else {
+		return false, nil, err
+	}
+}
+
 func awsCustomCleanResults(results []detectors.Result) []detectors.Result {
 	if len(results) == 0 {
 		return results
@@ -248,6 +248,28 @@ func TestAWS_FromChunk(t *testing.T) {
 			wantErr:               false,
 			wantVerificationError: true,
 		},
+		{
+			name: "verified secret checked directly after unverified secret with same key id",
+			s:    scanner{},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte(fmt.Sprintf("%s\n%s\n%s", inactiveSecret, id, secret)),
+				verify: true,
+			},
+			want: []detectors.Result{
+				{
+					DetectorType: detectorspb.DetectorType_AWS,
+					Verified:     true,
+					Redacted:     "AKIASP2TPHJSQH3FJRUX",
+					ExtraData: map[string]string{
+						"account": "171436882533",
+						"arn":     "arn:aws:iam::171436882533:user/canarytokens.com@@4dxkh0pdeop3bzu9zx5wob793",
+						"user_id": "AIDASP2TPHJSUFRSTTZX4",
+					},
+				},
+			},
+			wantErr: false,
+		},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
@@ -53,7 +53,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 			resMatch = keyString[len(keyString)-1]
 		}

-		secret := detectors.Result{
+		s1 := detectors.Result{
 			DetectorType: detectorspb.DetectorType_Gitlab,
 			Raw:          []byte(match[1]),
 		}
@@ -84,26 +84,26 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 					// 401 is bad key
 					switch res.StatusCode {
 					case http.StatusOK:
-						secret.Verified = true
+						s1.Verified = true
 					case http.StatusForbidden:
 						// Good key but not the right scope
-						secret.Verified = true
+						s1.Verified = true
 					case http.StatusUnauthorized:
 						// Nothing to do; zero values are the ones we want
 					default:
-						secret.VerificationError = fmt.Errorf("unexpected HTTP response status %d", res.StatusCode)
+						s1.VerificationError = fmt.Errorf("unexpected HTTP response status %d", res.StatusCode)
 					}
 				} else {
-					secret.VerificationError = err
+					s1.VerificationError = err
 				}
 			}
 		}

-		if !secret.Verified && detectors.IsKnownFalsePositive(string(secret.Raw), detectors.DefaultFalsePositives, true) {
+		if !s1.Verified && detectors.IsKnownFalsePositive(string(s1.Raw), detectors.DefaultFalsePositives, true) {
 			continue
 		}

-		results = append(results, secret)
+		results = append(results, s1)
 	}

 	return results, nil
@@ -6,11 +6,12 @@ package gitlab
 import (
 	"context"
 	"fmt"
-	"github.com/google/go-cmp/cmp"
-	"github.com/google/go-cmp/cmp/cmpopts"
 	"testing"
 	"time"

+	"github.com/google/go-cmp/cmp"
+	"github.com/google/go-cmp/cmp/cmpopts"
+
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
@@ -11,7 +11,10 @@ import (
 	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
 )

-type Scanner struct{ detectors.EndpointSetter }
+type Scanner struct {
+	client *http.Client
+	detectors.EndpointSetter
+}

 // Ensure the Scanner satisfies the interfaces at compile time.
 var _ detectors.Detector = (*Scanner)(nil)
@@ -22,7 +25,8 @@ func (Scanner) Version() int            { return 2 }
 func (Scanner) DefaultEndpoint() string { return "https://gitlab.com" }

 var (
-	keyPat = regexp.MustCompile(`\b(glpat-[a-zA-Z0-9\-=_]{20,22})\b`)
+	defaultClient = common.SaneHttpClient()
+	keyPat        = regexp.MustCompile(`\b(glpat-[a-zA-Z0-9\-=_]{20,22})\b`)
 )

 // Keywords are used for efficiently pre-filtering chunks.
@@ -42,7 +46,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 			continue
 		}

-		secret := detectors.Result{
+		s1 := detectors.Result{
 			DetectorType: detectorspb.DetectorType_Gitlab,
 			Raw:          []byte(match[1]),
 		}
@@ -52,7 +56,10 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 			// they all grant access to different parts of the API. I couldn't find an endpoint that every
 			// one of these scopes has access to, so we just check an example endpoint for each scope. If any
 			// of them contain data, we know we have a valid key, but if they all fail, we don't
-			client := common.SaneHttpClient()
+			client := s.client
+			if client == nil {
+				client = defaultClient
+			}
 			for _, baseURL := range s.Endpoints(s.DefaultEndpoint()) {
 				// test `read_user` scope
 				req, err := http.NewRequestWithContext(ctx, "GET", baseURL+"/api/v4/user", nil)
@@ -67,19 +74,29 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 					// 200 means good key and has `read_user` scope
 					// 403 means good key but not the right scope
 					// 401 is bad key
-					if res.StatusCode == http.StatusOK || res.StatusCode == http.StatusForbidden {
-						secret.Verified = true
+					switch res.StatusCode {
+					case http.StatusOK:
+						s1.Verified = true
+					case http.StatusForbidden:
+						// Good key but not the right scope
+						s1.Verified = true
+					case http.StatusUnauthorized:
+						// Nothing to do; zero values are the ones we want
+					default:
+						s1.VerificationError = fmt.Errorf("unexpected HTTP response status %d", res.StatusCode)
 					}
+				} else {
+					s1.VerificationError = err
 				}
 			}

 		}

-		if !secret.Verified && detectors.IsKnownFalsePositive(string(secret.Raw), detectors.DefaultFalsePositives, true) {
+		if !s1.Verified && detectors.IsKnownFalsePositive(string(s1.Raw), detectors.DefaultFalsePositives, true) {
 			continue
 		}

-		results = append(results, secret)
+		results = append(results, s1)
 	}

 	return
@@ -9,7 +9,8 @@ import (
 	"testing"
 	"time"

-	"github.com/kylelemons/godebug/pretty"
+	"github.com/google/go-cmp/cmp"
+	"github.com/google/go-cmp/cmp/cmpopts"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
@@ -18,7 +19,7 @@ import (
 func TestGitlab_FromChunk(t *testing.T) {
 	ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
 	defer cancel()
-	testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")
+	testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors4")
 	if err != nil {
 		t.Fatalf("could not get test secrets from GCP: %s", err)
 	}
@@ -31,18 +32,19 @@ func TestGitlab_FromChunk(t *testing.T) {
 		verify bool
 	}
 	tests := []struct {
-		name    string
-		s       Scanner
-		args    args
-		want    []detectors.Result
-		wantErr bool
+		name                string
+		s                   Scanner
+		args                args
+		want                []detectors.Result
+		wantErr             bool
+		wantVerificationErr bool
 	}{
 		{
 			name: "found, verified",
 			s:    Scanner{},
 			args: args{
 				ctx:    context.Background(),
-				data:   []byte(fmt.Sprintf("You can find a aws secret %s within", secret)),
+				data:   []byte(fmt.Sprintf("You can find a gitlab secret %s within", secret)),
 				verify: true,
 			},
 			want: []detectors.Result{
@@ -58,10 +60,65 @@ func TestGitlab_FromChunk(t *testing.T) {
 			s:    Scanner{},
 			args: args{
 				ctx:    context.Background(),
-				data:   []byte(fmt.Sprintf("You can find a aws secret %s within", secretInactive)),
+				data:   []byte(fmt.Sprintf("You can find a gitlab secret %s within", secretInactive)),
 				verify: true,
 			},
-			want:    nil,
+			want: []detectors.Result{
+				{
+					DetectorType: detectorspb.DetectorType_Gitlab,
+					Verified:     false,
+				},
+			},
+			wantErr: false,
+		},
+		{
+			name: "found, would be verified but for timeout",
+			s:    Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte(fmt.Sprintf("You can find a gitlab super secret %s within", secret)),
+				verify: true,
+			},
+			want: []detectors.Result{
+				{
+					DetectorType: detectorspb.DetectorType_Gitlab,
+					Verified:     false,
+				},
+			},
+			wantErr:             false,
+			wantVerificationErr: true,
+		},
+		{
+			name: "found and valid but unexpected api response",
+			s:    Scanner{client: common.ConstantResponseHttpClient(500, "")},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte(fmt.Sprintf("You can find a gitlab super secret %s within", secret)),
+				verify: true,
+			},
+			want: []detectors.Result{
+				{
+					DetectorType: detectorspb.DetectorType_Gitlab,
+					Verified:     false,
+				},
+			},
+			wantErr:             false,
+			wantVerificationErr: true,
+		},
+		{
+			name: "found, good key but wrong scope",
+			s:    Scanner{client: common.ConstantResponseHttpClient(403, "")},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte(fmt.Sprintf("You can find a gitlab super secret %s within", secret)),
+				verify: true,
+			},
+			want: []detectors.Result{
+				{
+					DetectorType: detectorspb.DetectorType_Gitlab,
+					Verified:     true,
+				},
+			},
 			wantErr: false,
 		},
 		{
@@ -78,8 +135,7 @@ func TestGitlab_FromChunk(t *testing.T) {
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			s := Scanner{}
-			got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+			got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
 			if (err != nil) != tt.wantErr {
 				t.Errorf("Gitlab.FromData() error = %v, wantErr %v", err, tt.wantErr)
 				return
@@ -88,9 +144,12 @@ func TestGitlab_FromChunk(t *testing.T) {
 				if len(got[i].Raw) == 0 {
 					t.Fatal("no raw secret present")
 				}
-				got[i].Raw = nil
+				if (got[i].VerificationError != nil) != tt.wantVerificationErr {
+					t.Fatalf(" wantVerificationError = %v, verification error = %v,", tt.wantVerificationErr, got[i].VerificationError)
+				}
 			}
-			if diff := pretty.Compare(got, tt.want); diff != "" {
+			opts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "VerificationError")
+			if diff := cmp.Diff(got, tt.want, opts); diff != "" {
 				t.Errorf("Gitlab.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
 			}
 		})
@@ -13,13 +13,18 @@ import (
 	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
 )

-type Scanner struct{}
+type Scanner struct {
+	client *http.Client
+}

 // Ensure the Scanner satisfies the interface at compile time.
 var _ detectors.Detector = (*Scanner)(nil)
+var _ detectors.Versioner = (*Scanner)(nil)
+
+func (Scanner) Version() int { return 1 }

 var (
-	client = common.SaneHttpClient()
+	defaultClient = common.SaneHttpClient()

 	// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
 	tokenPat  = regexp.MustCompile(detectors.PrefixRegex([]string{"jira"}) + `\b([a-zA-Z-0-9]{24})\b`)
@@ -70,6 +75,11 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 				}

 				if verify {
+					client := s.client
+					if client == nil {
+						client = defaultClient
+					}
+
 					data := fmt.Sprintf("%s:%s", resEmail, resToken)
 					sEnc := b64.StdEncoding.EncodeToString([]byte(data))
 					req, err := http.NewRequestWithContext(ctx, "GET", "https://"+resDomain+"/rest/api/3/dashboard", nil)
@@ -85,9 +95,15 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 						// If the request is successful and the login reason is not failed authentication, then the token is valid.
 						// This is because Jira returns a 200 status code even if the token is invalid.
 						// Jira returns a default dashboard page.
-						if (res.StatusCode >= 200 && res.StatusCode < 300) && res.Header.Get(loginReasonHeaderKey) != failedAuth {
-							s1.Verified = true
+						if res.StatusCode >= 200 && res.StatusCode < 300 {
+							if res.Header.Get(loginReasonHeaderKey) != failedAuth {
+								s1.Verified = true
+							}
+						} else {
+							s1.VerificationError = fmt.Errorf("unexpected HTTP response status %d", res.StatusCode)
 						}
+					} else {
+						s1.VerificationError = err
 					}
 				}

@@ -9,7 +9,8 @@ import (
 	"testing"
 	"time"

-	"github.com/kylelemons/godebug/pretty"
+	"github.com/google/go-cmp/cmp"
+	"github.com/google/go-cmp/cmp/cmpopts"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"

 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
@@ -19,12 +20,12 @@ import (
 func TestJiraToken_FromChunk(t *testing.T) {
 	ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
 	defer cancel()
-	testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
+	testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")
 	if err != nil {
 		t.Fatalf("could not get test secrets from GCP: %s", err)
 	}
-	token := testSecrets.MustGetField("JIRA_TOKEN")
-	inactiveToken := testSecrets.MustGetField("JIRA_INACTIVE")
+	secret := testSecrets.MustGetField("JIRA_TOKEN")
+	inactiveSecret := testSecrets.MustGetField("JIRA_INACTIVE")
 	email := testSecrets.MustGetField("JIRA_EMAIL")
 	domain := testSecrets.MustGetField("JIRA_DOMAIN")

@@ -34,18 +35,19 @@ func TestJiraToken_FromChunk(t *testing.T) {
 		verify bool
 	}
 	tests := []struct {
-		name    string
-		s       Scanner
-		args    args
-		want    []detectors.Result
-		wantErr bool
+		name                string
+		s                   Scanner
+		args                args
+		want                []detectors.Result
+		wantErr             bool
+		wantVerificationErr bool
 	}{
 		{
 			name: "found, verified",
 			s:    Scanner{},
 			args: args{
 				ctx:    context.Background(),
-				data:   []byte(fmt.Sprintf("You can find a jira secret %s within jira %s with jira %s", token, email, domain)),
+				data:   []byte(fmt.Sprintf("You can find a jira secret %s within jira %s with jira %s", secret, email, domain)),
 				verify: true,
 			},
 			want: []detectors.Result{
@@ -54,14 +56,15 @@ func TestJiraToken_FromChunk(t *testing.T) {
 					Verified:     true,
 				},
 			},
-			wantErr: false,
+			wantErr:             false,
+			wantVerificationErr: false,
 		},
 		{
 			name: "found, unverified",
 			s:    Scanner{},
 			args: args{
 				ctx:    context.Background(),
-				data:   []byte(fmt.Sprintf("You can find a jira secret %s within jira %s but not jira %s valid", inactiveToken, email, domain)), // the secret would satisfy the regex but not pass validation
+				data:   []byte(fmt.Sprintf("You can find a jira secret %s within jira %s but not jira %s valid", inactiveSecret, email, domain)), // the secret would satisfy the regex but not pass validation
 				verify: true,
 			},
 			want: []detectors.Result{
@@ -70,7 +73,8 @@ func TestJiraToken_FromChunk(t *testing.T) {
 					Verified:     false,
 				},
 			},
-			wantErr: false,
+			wantErr:             false,
+			wantVerificationErr: false,
 		},
 		{
 			name: "not found",
@@ -80,14 +84,48 @@ func TestJiraToken_FromChunk(t *testing.T) {
 				data:   []byte("You cannot find the secret within"),
 				verify: true,
 			},
-			want:    nil,
-			wantErr: false,
+			want:                nil,
+			wantErr:             false,
+			wantVerificationErr: false,
+		},
+		{
+			name: "found, would be verified if not for timeout",
+			s:    Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte(fmt.Sprintf("You can find a jira secret %s within jira %s with jira %s", secret, email, domain)),
+				verify: true,
+			},
+			want: []detectors.Result{
+				{
+					DetectorType: detectorspb.DetectorType_JiraToken,
+					Verified:     false,
+				},
+			},
+			wantErr:             false,
+			wantVerificationErr: true,
+		},
+		{
+			name: "found, verified but unexpected api surface",
+			s:    Scanner{client: common.ConstantResponseHttpClient(404, "")},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte(fmt.Sprintf("You can find a jira secret %s within jira %s with jira %s", secret, email, domain)),
+				verify: true,
+			},
+			want: []detectors.Result{
+				{
+					DetectorType: detectorspb.DetectorType_JiraToken,
+					Verified:     false,
+				},
+			},
+			wantErr:             false,
+			wantVerificationErr: true,
 		},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			s := Scanner{}
-			got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+			got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
 			if (err != nil) != tt.wantErr {
 				t.Errorf("JiraToken.FromData() error = %v, wantErr %v", err, tt.wantErr)
 				return
@@ -96,10 +134,12 @@ func TestJiraToken_FromChunk(t *testing.T) {
 				if len(got[i].Raw) == 0 {
 					t.Fatalf("no raw secret present: \n %+v", got[i])
 				}
-				got[i].Raw = nil
-				got[i].RawV2 = nil
+				if (got[i].VerificationError != nil) != tt.wantVerificationErr {
+					t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError)
+				}
 			}
-			if diff := pretty.Compare(got, tt.want); diff != "" {
+			ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "RawV2", "VerificationError")
+			if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {
 				t.Errorf("JiraToken.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
 			}
 		})
@@ -0,0 +1,129 @@
+package jiratoken_v2
+
+import (
+	"context"
+	b64 "encoding/base64"
+	"fmt"
+	"net/http"
+	"regexp"
+	"strings"
+
+	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+type Scanner struct {
+	client *http.Client
+}
+
+// Ensure the Scanner satisfies the interface at compile time.
+var _ detectors.Detector = (*Scanner)(nil)
+var _ detectors.Versioner = (*Scanner)(nil)
+
+func (Scanner) Version() int { return 2 }
+
+var (
+	defaultClient = common.SaneHttpClient()
+
+	// https://support.atlassian.com/atlassian-account/docs/manage-api-tokens-for-your-atlassian-account/
+	// Tokens created after Jan 18 2023 use a variable length
+	tokenPat  = regexp.MustCompile(detectors.PrefixRegex([]string{"jira"}) + `\b([A-Za-z0-9+/=_-]+=[A-Za-z0-9]{8})\b`)
+	domainPat = regexp.MustCompile(detectors.PrefixRegex([]string{"jira"}) + `\b([a-zA-Z-0-9]{5,24}\.[a-zA-Z-0-9]{3,16}\.[a-zA-Z-0-9]{3,16})\b`)
+	emailPat  = regexp.MustCompile(detectors.PrefixRegex([]string{"jira"}) + `\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Z|a-z]{2,}\b`)
+)
+
+const (
+	failedAuth           = "AUTHENTICATED_FAILED"
+	loginReasonHeaderKey = "X-Seraph-LoginReason"
+)
+
+// Keywords are used for efficiently pre-filtering chunks.
+// Use identifiers in the secret preferably, or the provider name.
+func (s Scanner) Keywords() []string {
+	return []string{"jira"}
+}
+
+// FromData will find and optionally verify JiraToken secrets in a given set of bytes.
+func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
+	dataStr := string(data)
+
+	tokens := tokenPat.FindAllStringSubmatch(dataStr, -1)
+	domains := domainPat.FindAllStringSubmatch(dataStr, -1)
+	emails := emailPat.FindAllStringSubmatch(dataStr, -1)
+
+	for _, email := range emails {
+		email = strings.Split(email[0], " ")
+		if len(email) != 2 {
+			continue
+		}
+		resEmail := strings.TrimSpace(email[1])
+		for _, token := range tokens {
+			if len(token) != 2 {
+				continue
+			}
+			resToken := strings.TrimSpace(token[1])
+			for _, domain := range domains {
+				if len(domain) != 2 {
+					continue
+				}
+				resDomain := strings.TrimSpace(domain[1])
+
+				s1 := detectors.Result{
+					DetectorType: detectorspb.DetectorType_JiraToken,
+					Raw:          []byte(resToken),
+					RawV2:        []byte(fmt.Sprintf("%s:%s:%s", resEmail, resToken, resDomain)),
+				}
+
+				if verify {
+					client := s.client
+					if client == nil {
+						client = defaultClient
+					}
+
+					data := fmt.Sprintf("%s:%s", resEmail, resToken)
+					sEnc := b64.StdEncoding.EncodeToString([]byte(data))
+					req, err := http.NewRequestWithContext(ctx, "GET", "https://"+resDomain+"/rest/api/3/dashboard", nil)
+					if err != nil {
+						continue
+					}
+					req.Header.Add("Accept", "application/json")
+					req.Header.Add("Authorization", fmt.Sprintf("Basic %s", sEnc))
+					res, err := client.Do(req)
+					if err == nil {
+						defer res.Body.Close()
+
+						// If the request is successful and the login reason is not failed authentication, then the token is valid.
+						// This is because Jira returns a 200 status code even if the token is invalid.
+						// Jira returns a default dashboard page.
+						if res.StatusCode >= 200 && res.StatusCode < 300 {
+							if res.Header.Get(loginReasonHeaderKey) != failedAuth {
+								s1.Verified = true
+							}
+						} else {
+							s1.VerificationError = fmt.Errorf("unexpected HTTP response status %d", res.StatusCode)
+						}
+					} else {
+						s1.VerificationError = err
+					}
+				}
+
+				if !s1.Verified {
+					if detectors.IsKnownFalsePositive(string(s1.Raw), detectors.DefaultFalsePositives, true) {
+						continue
+					}
+				}
+
+				results = append(results, s1)
+
+			}
+
+		}
+	}
+
+	return results, nil
+}
+
+func (s Scanner) Type() detectorspb.DetectorType {
+	return detectorspb.DetectorType_JiraToken
+}
@@ -0,0 +1,163 @@
+//go:build detectors
+// +build detectors
+
+package jiratoken_v2
+
+import (
+	"context"
+	"fmt"
+	"testing"
+	"time"
+
+	"github.com/google/go-cmp/cmp"
+	"github.com/google/go-cmp/cmp/cmpopts"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+
+	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestJiraToken_FromChunk(t *testing.T) {
+	ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+	defer cancel()
+	testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")
+	if err != nil {
+		t.Fatalf("could not get test secrets from GCP: %s", err)
+	}
+	secret := testSecrets.MustGetField("JIRA_V2_TOKEN")
+	inactiveSecret := testSecrets.MustGetField("JIRA_V2_INACTIVE")
+	email := testSecrets.MustGetField("JIRA_V2_EMAIL")
+	domain := testSecrets.MustGetField("JIRA_V2_DOMAIN")
+
+	type args struct {
+		ctx    context.Context
+		data   []byte
+		verify bool
+	}
+	tests := []struct {
+		name                string
+		s                   Scanner
+		args                args
+		want                []detectors.Result
+		wantErr             bool
+		wantVerificationErr bool
+	}{
+		{
+			name: "found, verified",
+			s:    Scanner{},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte(fmt.Sprintf("You can find a jira secret %s within jira %s with jira %s", secret, email, domain)),
+				verify: true,
+			},
+			want: []detectors.Result{
+				{
+					DetectorType: detectorspb.DetectorType_JiraToken,
+					Verified:     true,
+				},
+			},
+			wantErr:             false,
+			wantVerificationErr: false,
+		},
+		{
+			name: "found, unverified",
+			s:    Scanner{},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte(fmt.Sprintf("You can find a jira secret %s within jira %s but not jira %s valid", inactiveSecret, email, domain)), // the secret would satisfy the regex but not pass validation
+				verify: true,
+			},
+			want: []detectors.Result{
+				{
+					DetectorType: detectorspb.DetectorType_JiraToken,
+					Verified:     false,
+				},
+			},
+			wantErr:             false,
+			wantVerificationErr: false,
+		},
+		{
+			name: "not found",
+			s:    Scanner{},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte("You cannot find the secret within"),
+				verify: true,
+			},
+			want:                nil,
+			wantErr:             false,
+			wantVerificationErr: false,
+		},
+		{
+			name: "found, would be verified if not for timeout",
+			s:    Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte(fmt.Sprintf("You can find a jira secret %s within jira %s with jira %s", secret, email, domain)),
+				verify: true,
+			},
+			want: []detectors.Result{
+				{
+					DetectorType: detectorspb.DetectorType_JiraToken,
+					Verified:     false,
+				},
+			},
+			wantErr:             false,
+			wantVerificationErr: true,
+		},
+		{
+			name: "found, verified but unexpected api surface",
+			s:    Scanner{client: common.ConstantResponseHttpClient(404, "")},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte(fmt.Sprintf("You can find a jira secret %s within jira %s with jira %s", secret, email, domain)),
+				verify: true,
+			},
+			want: []detectors.Result{
+				{
+					DetectorType: detectorspb.DetectorType_JiraToken,
+					Verified:     false,
+				},
+			},
+			wantErr:             false,
+			wantVerificationErr: true,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("JiraToken.FromData() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			for i := range got {
+				if len(got[i].Raw) == 0 {
+					t.Fatalf("no raw secret present: \n %+v", got[i])
+				}
+				if (got[i].VerificationError != nil) != tt.wantVerificationErr {
+					t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError)
+				}
+			}
+			ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "RawV2", "VerificationError")
+			if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {
+				t.Errorf("JiraToken.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+			}
+		})
+	}
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+	ctx := context.Background()
+	s := Scanner{}
+	for name, data := range detectors.MustGetBenchmarkData() {
+		benchmark.Run(name, func(b *testing.B) {
+			b.ResetTimer()
+			for n := 0; n < b.N; n++ {
+				_, err := s.FromData(ctx, false, data)
+				if err != nil {
+					b.Fatal(err)
+				}
+			}
+		})
+	}
+}
@@ -21,13 +21,13 @@ var (
 	client = common.SaneHttpClient()

 	// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
-	keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"pagerduty"}) + `\b([a-z]{1}\+[a-zA-Z]{9}\-[a-z]{2}\-[a-z0-9]{5})\b`)
+	keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"pagerduty", "pd"}) + `\b([a-zA-Z0-9_+-]{20})\b`)
 )

 // Keywords are used for efficiently pre-filtering chunks.
 // Use identifiers in the secret preferably, or the provider name.
 func (s Scanner) Keywords() []string {
-	return []string{"pagerduty"}
+	return []string{"pagerduty", "pd"}
 }

 // FromData will find and optionally verify PagerDutyApiKey secrets in a given set of bytes.
@@ -43,8 +43,6 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 	instanceMatches := instancePat.FindAllStringSubmatch(dataStr, -1)
 	tokenMatches := accessTokenPat.FindAllStringSubmatch(dataStr, -1)

-	fmt.Printf("instanceMatches: %v\n", instanceMatches)
-
 	for _, instance := range instanceMatches {
 		if len(instance) != 1 {
 			continue
@@ -2,6 +2,7 @@ package sendbird

 import (
 	"context"
+	"encoding/json"
 	"fmt"
 	"net/http"
 	"regexp"
@@ -12,19 +13,25 @@ import (
 	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
 )

-type Scanner struct{}
+type Scanner struct {
+	client *http.Client
+}

 // Ensure the Scanner satisfies the interface at compile time.
 var _ detectors.Detector = (*Scanner)(nil)

 var (
-	client = common.SaneHttpClient()
+	defaultClient = common.SaneHttpClient()

 	// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
 	keyPat   = regexp.MustCompile(detectors.PrefixRegex([]string{"sendbird"}) + `\b([0-9a-f]{40})\b`)
 	appIdPat = regexp.MustCompile(detectors.PrefixRegex([]string{"sendbird"}) + `\b([0-9A-F]{8}-[0-9A-F]{4}-[0-9A-F]{4}-[0-9A-F]{4}-[0-9A-F]{12})\b`)
 )

+type userResp struct {
+	Code int `json:"code"`
+}
+
 // Keywords are used for efficiently pre-filtering chunks.
 // Use identifiers in the secret preferably, or the provider name.
 func (s Scanner) Keywords() []string {
@@ -61,20 +68,40 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 					continue
 				}
 				req.Header.Add("Api-Token", resMatch)
+
+				client := s.client
+				if client == nil {
+					client = defaultClient
+				}
+
 				res, err := client.Do(req)
 				if err == nil {
 					defer res.Body.Close()
 					if res.StatusCode >= 200 && res.StatusCode < 300 {
 						s1.Verified = true
-					} else {
-						// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
-						if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
-							continue
+					} else if res.StatusCode == 400 { // Sendbird returns 400 for all errors
+						var userResp userResp
+						err := json.NewDecoder(res.Body).Decode(&userResp)
+						if err != nil {
+							s1.VerificationError = fmt.Errorf("error decoding json response body: %w", err)
+						} else if userResp.Code != 400401 {
+							// https://sendbird.com/docs/chat/platform-api/v3/error-codes
+							// Sendbird always includes its own error codes with 400 responses
+							// 400401 (InvalidApiToken) is the only one that indicates a bad token
+							s1.VerificationError = fmt.Errorf("unexpected response code: %d", userResp.Code)
 						}
+					} else {
+						s1.VerificationError = fmt.Errorf("unexpected HTTP response status %d", res.StatusCode)
 					}
+				} else {
+					s1.VerificationError = err
 				}
 			}

+			// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
+			if !s1.Verified && detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
+				continue
+			}
 			results = append(results, s1)
 		}
 	}
@@ -9,8 +9,8 @@ import (
 	"testing"
 	"time"

-	"github.com/kylelemons/godebug/pretty"
-
+	"github.com/google/go-cmp/cmp"
+	"github.com/google/go-cmp/cmp/cmpopts"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
@@ -19,7 +19,7 @@ import (
 func TestSendbird_FromChunk(t *testing.T) {
 	ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
 	defer cancel()
-	testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
+	testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")
 	if err != nil {
 		t.Fatalf("could not get test secrets from GCP: %s", err)
 	}
@@ -33,11 +33,12 @@ func TestSendbird_FromChunk(t *testing.T) {
 		verify bool
 	}
 	tests := []struct {
-		name    string
-		s       Scanner
-		args    args
-		want    []detectors.Result
-		wantErr bool
+		name                string
+		s                   Scanner
+		args                args
+		want                []detectors.Result
+		wantErr             bool
+		wantVerificationErr bool
 	}{
 		{
 			name: "found, verified",
@@ -53,7 +54,8 @@ func TestSendbird_FromChunk(t *testing.T) {
 					Verified:     true,
 				},
 			},
-			wantErr: false,
+			wantErr:             false,
+			wantVerificationErr: false,
 		},
 		{
 			name: "found, unverified",
@@ -69,7 +71,8 @@ func TestSendbird_FromChunk(t *testing.T) {
 					Verified:     false,
 				},
 			},
-			wantErr: false,
+			wantErr:             false,
+			wantVerificationErr: false,
 		},
 		{
 			name: "not found",
@@ -79,14 +82,82 @@ func TestSendbird_FromChunk(t *testing.T) {
 				data:   []byte("You cannot find the secret within"),
 				verify: true,
 			},
-			want:    nil,
-			wantErr: false,
+			want:                nil,
+			wantErr:             false,
+			wantVerificationErr: false,
+		},
+		{
+			name: "found, would be verified if not for timeout",
+			s:    Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte(fmt.Sprintf("You can find a sendbird application ID %s with sendbird secret %s within", appId, secret)),
+				verify: true,
+			},
+			want: []detectors.Result{
+				{
+					DetectorType: detectorspb.DetectorType_Sendbird,
+					Verified:     false,
+				},
+			},
+			wantErr:             false,
+			wantVerificationErr: true,
+		},
+		{
+			name: "found, verified but unexpected api surface",
+			s:    Scanner{client: common.ConstantResponseHttpClient(404, "")},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte(fmt.Sprintf("You can find a sendbird application ID %s with sendbird secret %s within", appId, secret)),
+				verify: true,
+			},
+			want: []detectors.Result{
+				{
+					DetectorType: detectorspb.DetectorType_Sendbird,
+					Verified:     false,
+				},
+			},
+			wantErr:             false,
+			wantVerificationErr: true,
+		},
+		{
+			name: "found, verified but unexpected error code",
+			s:    Scanner{client: common.ConstantResponseHttpClient(400, `{"code":400402}`)},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte(fmt.Sprintf("You can find a sendbird application ID %s with sendbird secret %s within", appId, secret)),
+				verify: true,
+			},
+			want: []detectors.Result{
+				{
+					DetectorType: detectorspb.DetectorType_Sendbird,
+					Verified:     false,
+				},
+			},
+			wantErr:             false,
+			wantVerificationErr: true,
+		},
+		{
+			name: "found, verified but failed to decode json",
+			s:    Scanner{client: common.ConstantResponseHttpClient(400, `{`)},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte(fmt.Sprintf("You can find a sendbird application ID %s with sendbird secret %s within", appId, secret)),
+				verify: true,
+			},
+			want: []detectors.Result{
+				{
+					DetectorType: detectorspb.DetectorType_Sendbird,
+					Verified:     false,
+				},
+			},
+			wantErr:             false,
+			wantVerificationErr: true,
 		},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			s := Scanner{}
-			got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+			got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
 			if (err != nil) != tt.wantErr {
 				t.Errorf("Sendbird.FromData() error = %v, wantErr %v", err, tt.wantErr)
 				return
@@ -95,9 +166,12 @@ func TestSendbird_FromChunk(t *testing.T) {
 				if len(got[i].Raw) == 0 {
 					t.Fatalf("no raw secret present: \n %+v", got[i])
 				}
-				got[i].Raw = nil
+				if (got[i].VerificationError != nil) != tt.wantVerificationErr {
+					t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError)
+				}
 			}
-			if diff := pretty.Compare(got, tt.want); diff != "" {
+			ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "VerificationError")
+			if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {
 				t.Errorf("Sendbird.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
 			}
 		})
@@ -13,13 +13,15 @@ import (
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 )

-type Scanner struct{}
+type Scanner struct {
+	client *http.Client
+}

 // Ensure the Scanner satisfies the interface at compile time.
 var _ detectors.Detector = (*Scanner)(nil)

 var (
-	client = common.SaneHttpClient()
+	defaultClient = common.SaneHttpClient()

 	keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"sendgrid"}) + `(SG\.[\w\-_]{20,24}\.[\w\-_]{39,50})\b`)
 )
@@ -42,7 +44,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 		}
 		res := strings.TrimSpace(match[1])

-		s := detectors.Result{
+		s1 := detectors.Result{
 			DetectorType: detectorspb.DetectorType_SendGrid,
 			Raw:          []byte(res),
 		}
@@ -52,6 +54,11 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 			// 403 will be issued if the scope is wrong but the key is correct
 			baseURL := "https://api.sendgrid.com/v3/templates"

+			client := s.client
+			if client == nil {
+				client = defaultClient
+			}
+
 			// test `read_user` scope
 			req, err := http.NewRequestWithContext(ctx, "GET", baseURL, nil)
 			if err != nil {
@@ -67,16 +74,18 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 				// 403 means good key but not the right scope
 				// 401 is bad key
 				if res.StatusCode == http.StatusOK || res.StatusCode == http.StatusForbidden {
-					s.Verified = true
+					s1.Verified = true
+				} else if res.StatusCode != http.StatusUnauthorized {
+					s1.VerificationError = fmt.Errorf("unexpected HTTP response status %d", res.StatusCode)
 				}
 			}
 		}

-		if !s.Verified && detectors.IsKnownFalsePositive(string(s.Raw), detectors.DefaultFalsePositives, true) {
+		if !s1.Verified && detectors.IsKnownFalsePositive(string(s1.Raw), detectors.DefaultFalsePositives, true) {
 			continue
 		}

-		results = append(results, s)
+		results = append(results, s1)
 	}

 	return
@@ -9,7 +9,8 @@ import (
 	"testing"
 	"time"

-	"github.com/kylelemons/godebug/pretty"
+	"github.com/google/go-cmp/cmp"
+	"github.com/google/go-cmp/cmp/cmpopts"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
@@ -18,7 +19,7 @@ import (
 func TestSendgrid_FromChunk(t *testing.T) {
 	ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
 	defer cancel()
-	testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")
+	testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")
 	if err != nil {
 		t.Fatalf("could not get test secrets from GCP: %s", err)
 	}
@@ -30,11 +31,12 @@ func TestSendgrid_FromChunk(t *testing.T) {
 		verify bool
 	}
 	tests := []struct {
-		name    string
-		s       Scanner
-		args    args
-		want    []detectors.Result
-		wantErr bool
+		name                string
+		s                   Scanner
+		args                args
+		want                []detectors.Result
+		wantErr             bool
+		wantVerificationErr bool
 	}{
 		{
 			name: "found, verified",
@@ -50,7 +52,8 @@ func TestSendgrid_FromChunk(t *testing.T) {
 					Verified:     true,
 				},
 			},
-			wantErr: false,
+			wantErr:             false,
+			wantVerificationErr: false,
 		},
 		{
 			name: "found, unverified",
@@ -66,7 +69,8 @@ func TestSendgrid_FromChunk(t *testing.T) {
 					Verified:     false,
 				},
 			},
-			wantErr: false,
+			wantErr:             false,
+			wantVerificationErr: false,
 		},
 		{
 			name: "not found",
@@ -76,22 +80,54 @@ func TestSendgrid_FromChunk(t *testing.T) {
 				data:   []byte("You cannot find the secret within"),
 				verify: true,
 			},
-			want:    nil,
-			wantErr: false,
+			want:                nil,
+			wantErr:             false,
+			wantVerificationErr: false,
+		},
+		{
+			name: "found, would be verified if not for timeout",
+			s:    Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte(fmt.Sprintf("You can find a sendgrid secret %s within", secret)),
+				verify: true,
+			},
+			want: []detectors.Result{
+				{
+					DetectorType: detectorspb.DetectorType_SendGrid,
+					Verified:     false,
+				},
+			},
+			wantErr:             false,
+			wantVerificationErr: true,
+		},
+		{
+			name: "found, verified but unexpected api surface",
+			s:    Scanner{client: common.ConstantResponseHttpClient(404, "")},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte(fmt.Sprintf("You can find a sendgrid secret %s within", secret)),
+				verify: true,
+			},
+			want: []detectors.Result{
+				{
+					DetectorType: detectorspb.DetectorType_SendGrid,
+					Verified:     false,
+				},
+			},
+			wantErr:             false,
+			wantVerificationErr: true,
 		},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			s := Scanner{}
-			got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+			got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
 			if (err != nil) != tt.wantErr {
 				t.Errorf("Sendgrid.FromData() error = %v, wantErr %v", err, tt.wantErr)
 				return
 			}
-			for i := range got {
-				got[i].Raw = nil
-			}
-			if diff := pretty.Compare(got, tt.want); diff != "" {
+			ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "VerificationError")
+			if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {
 				t.Errorf("Sendgrid.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
 			}
 		})
@@ -2,7 +2,10 @@ package sentrytoken

 import (
 	"context"
+	"encoding/json"
+	"errors"
 	"fmt"
+	"io"
 	"net/http"
 	"regexp"
 	"strings"
@@ -12,16 +15,20 @@ import (
 	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
 )

-type Scanner struct{}
+type Scanner struct {
+	client *http.Client
+}

 // Ensure the Scanner satisfies the interface at compile time.
 var _ detectors.Detector = (*Scanner)(nil)

 var (
-	client = common.SaneHttpClient()
+	defaultClient = common.SaneHttpClient()

 	// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
 	keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"sentry"}) + `\b([a-f0-9]{64})\b`)
+
+	errUnauthorized = fmt.Errorf("token unauthorized")
 )

 // Keywords are used for efficiently pre-filtering chunks.
@@ -47,32 +54,81 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 		}

 		if verify {
-			req, err := http.NewRequestWithContext(ctx, "GET", "https://sentry.io/api/0/projects/", nil)
-			if err != nil {
-				continue
+			client := s.client
+			if client == nil {
+				client = defaultClient
 			}
-			req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", resMatch))
-			// req.Header.Add("Cookie", "sc=g2ceDPcsBwPLGXugHUv0coGBi09TQYKqmtp0u4JYdIvOTFd28yVvn1yiuXgmJJa8; session=.eJwNyE0OwiAQQGGsPwtP4kqoGQa4gUdgZ2CGiaZJCUqjGxOPbnfve7_h29QpHpVSvbw61To9StvEwzre9TkVbkPcr7jN5dPbNl7X1jUt_a4vLpEHgUycQSgFZ0YBRkGxnthxwIyBAgdngcF4YxBssiMjekGv2245_wHyySMG:1m0J5S:FjpRlBVuFx577UQp6GcSGWTq0uk")
-			res, err := client.Do(req)
-			if err == nil {
-				defer res.Body.Close()
-				if res.StatusCode >= 200 && res.StatusCode < 300 {
-					s1.Verified = true
-				} else {
-					// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
-					if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
-						continue
-					}
-				}
+			isVerified, verificationErr := verifyToken(ctx, client, resMatch)
+
+			switch {
+			case errors.Is(verificationErr, errUnauthorized):
+				s1.Verified = false
+			case isVerified:
+				s1.Verified = true
+			default:
+				s1.VerificationError = verificationErr
 			}
 		}

+		if !s1.Verified && detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
+			continue
+		}
 		results = append(results, s1)
 	}

 	return results, nil
 }

+type Response []Project
+
+type Project struct {
+	Organization Organization `json:"organization"`
+}
+
+type Organization struct {
+	ID   string `json:"id"`
+	Name string `json:"name"`
+}
+
+func verifyToken(ctx context.Context, client *http.Client, token string) (bool, error) {
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, "https://sentry.io/api/0/projects/", nil)
+	if err != nil {
+		return false, err
+	}
+	req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", token))
+
+	res, err := client.Do(req)
+	if err != nil {
+		return false, err
+	}
+	defer res.Body.Close()
+
+	var isVerified bool
+	switch res.StatusCode {
+	case http.StatusOK, http.StatusForbidden:
+		isVerified = true
+	case http.StatusUnauthorized:
+		return false, errUnauthorized
+	default:
+		return false, fmt.Errorf("unexpected HTTP response status %d", res.StatusCode)
+	}
+
+	bytes, readErr := io.ReadAll(res.Body)
+	if readErr != nil {
+		return false, readErr
+	}
+
+	var resp Response
+	if err = json.Unmarshal(bytes, &resp); err != nil {
+		return false, err
+	}
+	if len(resp) == 0 {
+		return false, fmt.Errorf("unexpected response body: %s", string(bytes))
+	}
+
+	return isVerified, err
+}
+
 func (s Scanner) Type() detectorspb.DetectorType {
 	return detectorspb.DetectorType_SentryToken
 }
@@ -9,7 +9,9 @@ import (
 	"testing"
 	"time"

-	"github.com/kylelemons/godebug/pretty"
+	"github.com/google/go-cmp/cmp"
+	"github.com/google/go-cmp/cmp/cmpopts"
+
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"

 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
@@ -32,11 +34,12 @@ func TestSentryToken_FromChunk(t *testing.T) {
 		verify bool
 	}
 	tests := []struct {
-		name    string
-		s       Scanner
-		args    args
-		want    []detectors.Result
-		wantErr bool
+		name                string
+		s                   Scanner
+		args                args
+		want                []detectors.Result
+		wantErr             bool
+		wantVerificationErr bool
 	}{
 		{
 			name: "found, verified",
@@ -70,6 +73,90 @@ func TestSentryToken_FromChunk(t *testing.T) {
 			},
 			wantErr: false,
 		},
+		{
+			name: "found, would be verified but for timeout",
+			s:    Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte(fmt.Sprintf("You can find a sentry super secret %s within", secret)),
+				verify: true,
+			},
+			want: []detectors.Result{
+				{
+					DetectorType: detectorspb.DetectorType_SentryToken,
+					Verified:     false,
+				},
+			},
+			wantErr:             false,
+			wantVerificationErr: true,
+		},
+		{
+			name: "found and valid but unexpected api response",
+			s:    Scanner{client: common.ConstantResponseHttpClient(500, "")},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte(fmt.Sprintf("You can find a sentry super secret %s within", secret)),
+				verify: true,
+			},
+			want: []detectors.Result{
+				{
+					DetectorType: detectorspb.DetectorType_SentryToken,
+					Verified:     false,
+				},
+			},
+			wantErr:             false,
+			wantVerificationErr: true,
+		},
+		{
+			name: "found, good key but wrong scope",
+			s:    Scanner{client: common.ConstantResponseHttpClient(403, responseBody403)},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte(fmt.Sprintf("You can find a sentry super secret %s within", secret)),
+				verify: true,
+			},
+			want: []detectors.Result{
+				{
+					DetectorType: detectorspb.DetectorType_SentryToken,
+					Verified:     true,
+				},
+			},
+			wantErr: false,
+		},
+		{
+			name: "found, account deactivated",
+			s:    Scanner{client: common.ConstantResponseHttpClient(200, reponseAccountDeactivated)},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte(fmt.Sprintf("You can find a sentry super secret %s within", secret)),
+				verify: true,
+			},
+			want: []detectors.Result{
+				{
+					DetectorType: detectorspb.DetectorType_SentryToken,
+					Verified:     false,
+				},
+			},
+			wantErr:             false,
+			wantVerificationErr: true,
+		},
+		{
+			name: "found, account deactivated",
+			s:    Scanner{client: common.ConstantResponseHttpClient(200, responseEnmpty)},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte(fmt.Sprintf("You can find a sentry super secret %s within", secret)),
+				verify: true,
+			},
+			want: []detectors.Result{
+				{
+					DetectorType: detectorspb.DetectorType_SentryToken,
+					Verified:     false,
+				},
+			},
+			wantErr:             false,
+			wantVerificationErr: true,
+		},
 		{
 			name: "not found",
 			s:    Scanner{},
@@ -82,27 +169,50 @@ func TestSentryToken_FromChunk(t *testing.T) {
 			wantErr: false,
 		},
 	}
+
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			s := Scanner{}
-			got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+			got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
 			if (err != nil) != tt.wantErr {
-				t.Errorf("SentryToken.FromData() error = %v, wantErr %v", err, tt.wantErr)
+				t.Errorf("Gitlab.FromData() error = %v, wantErr %v", err, tt.wantErr)
 				return
 			}
 			for i := range got {
 				if len(got[i].Raw) == 0 {
-					t.Fatalf("no raw secret present: \n %+v", got[i])
+					t.Fatal("no raw secret present")
+				}
+				if (got[i].VerificationError != nil) != tt.wantVerificationErr {
+					t.Fatalf("wantVerificationError = %v, verification error = %v,", tt.wantVerificationErr, got[i].VerificationError)
 				}
-				got[i].Raw = nil
 			}
-			if diff := pretty.Compare(got, tt.want); diff != "" {
-				t.Errorf("SentryToken.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+			opts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "VerificationError")
+			if diff := cmp.Diff(got, tt.want, opts); diff != "" {
+				t.Errorf("Gitlab.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
 			}
 		})
 	}
 }

+const (
+	responseBody403 = `
+[
+  {
+    "organization": {
+      "id": "911964",
+      "slug": "wigslap",
+      "status": {
+        "id": "active",
+        "name": "active"
+      },
+      "name": "wigslap"
+    }
+  }
+]
+`
+	reponseAccountDeactivated = `{"detail": "Authentication credentials were not provided"}`
+	responseEnmpty            = `[]`
+)
+
 func BenchmarkFromData(benchmark *testing.B) {
 	ctx := context.Background()
 	s := Scanner{}
@@ -13,13 +13,16 @@ import (
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 )

-type Scanner struct{}
+type Scanner struct {
+	client *http.Client
+}

 // Check that the Slack scanner implements the SecretScanner interface at compile time.
 var _ detectors.Detector = Scanner{}

 var (
-	tokenPats = map[string]*regexp.Regexp{
+	defaultClient = common.SaneHttpClient()
+	tokenPats     = map[string]*regexp.Regexp{
 		"Slack Bot Token":               regexp.MustCompile(`xoxb\-[0-9]{10,13}\-[0-9]{10,13}[a-zA-Z0-9\-]*`),
 		"Slack User Token":              regexp.MustCompile(`xoxp\-[0-9]{10,13}\-[0-9]{10,13}[a-zA-Z0-9\-]*`),
 		"Slack Workspace Access Token":  regexp.MustCompile(`xoxa\-[0-9]{10,13}\-[0-9]{10,13}[a-zA-Z0-9\-]*`),
@@ -53,34 +56,54 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 		tokens := tokenPat.FindAllString(dataStr, -1)

 		for _, token := range tokens {
-			s := detectors.Result{
+			s1 := detectors.Result{
 				DetectorType: detectorspb.DetectorType_Slack,
 				Raw:          []byte(token),
 			}
 			if verify {
-				client := common.SaneHttpClient()
+				client := s.client
+				if s.client == nil {
+					client = defaultClient
+				}
+
 				req, err := http.NewRequestWithContext(ctx, "POST", verifyURL, nil)
 				if err != nil {
 					continue
 				}
+
 				req.Header.Add("Content-Type", "application/json; charset=utf-8")
 				req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", token))
+
 				res, err := client.Do(req)
 				if err == nil {
 					defer res.Body.Close()
 					var authResponse authRes
 					if err := json.NewDecoder(res.Body).Decode(&authResponse); err != nil {
-						continue
+						s1.VerificationError = fmt.Errorf("failed to decode auth response: %w", err)
 					}
-					s.Verified = authResponse.Ok
+
+					if authResponse.Ok {
+						s1.Verified = true
+						// Slack API returns 200 even if the token is invalid. We need to check the error field.
+					} else if authResponse.Error == "invalid_auth" {
+						// The secret is determinately not verified (nothing to do)
+					} else {
+						s1.VerificationError = fmt.Errorf("unexpected error auth response %+v", authResponse.Error)
+					}
+				} else {
+					s1.VerificationError = err
 				}
 			}

-			results = append(results, s)
+			if !s1.Verified && detectors.IsKnownFalsePositive(string(s1.Raw), detectors.DefaultFalsePositives, true) {
+				continue
+			}
+
+			results = append(results, s1)
 		}
 	}

-	return
+	return results, nil
 }

 func (s Scanner) Type() detectorspb.DetectorType {
@@ -9,13 +9,15 @@ import (
 	"testing"
 	"time"

-	"github.com/kylelemons/godebug/pretty"
+	"github.com/google/go-cmp/cmp"
+	"github.com/google/go-cmp/cmp/cmpopts"
+
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
 )

-func TestScanner_FromChunk(t *testing.T) {
+func TestSlack_FromChunk(t *testing.T) {
 	ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
 	defer cancel()
 	testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")
@@ -24,17 +26,28 @@ func TestScanner_FromChunk(t *testing.T) {
 	}
 	secret := testSecrets.MustGetField("SLACK")
 	secretInactive := testSecrets.MustGetField("SLACK_INACTIVE")
+
+	type args struct {
+		ctx    context.Context
+		data   []byte
+		verify bool
+	}
+
 	tests := []struct {
-		name        string
-		data        []byte
-		verify      bool
-		wantResults []detectors.Result
-		wantErr     bool
+		name                string
+		s                   Scanner
+		args                args
+		wantResults         []detectors.Result
+		wantErr             bool
+		wantVerificationErr bool
 	}{
 		{
-			name:   "found, verified",
-			data:   []byte(fmt.Sprintf("You can find a slack secret %s within", secret)),
-			verify: true,
+			name: "found, verified",
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte(fmt.Sprintf("You can find a slack secret %s within", secret)),
+				verify: true,
+			},
 			wantResults: []detectors.Result{
 				{
 					DetectorType: detectorspb.DetectorType_Slack,
@@ -44,9 +57,12 @@ func TestScanner_FromChunk(t *testing.T) {
 			wantErr: false,
 		},
 		{
-			name:   "found but unverified",
-			data:   []byte(fmt.Sprintf("You can find a slack secret %s within", secretInactive)),
-			verify: true,
+			name: "found but unverified",
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte(fmt.Sprintf("You can find a slack secret %s within", secretInactive)),
+				verify: true,
+			},
 			wantResults: []detectors.Result{
 				{
 					DetectorType: detectorspb.DetectorType_Slack,
@@ -55,23 +71,62 @@ func TestScanner_FromChunk(t *testing.T) {
 			},
 			wantErr: false,
 		},
+		{
+			name: "found, would be verified if not for timeout",
+			s:    Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte(fmt.Sprintf("You can find a slack secret %s within", secret)),
+				verify: true,
+			},
+			wantResults: []detectors.Result{
+				{
+					DetectorType: detectorspb.DetectorType_Slack,
+					Verified:     false,
+				},
+			},
+			wantErr:             false,
+			wantVerificationErr: true,
+		},
+		{
+			name: "unexpected auth response",
+			s:    Scanner{client: common.ConstantResponseHttpClient(200, `{"ok": false, "error": "unexpected_error"}`)},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte(fmt.Sprintf("You can find a slack secret %s within", secret)),
+				verify: true,
+			},
+			wantResults: []detectors.Result{
+				{
+					DetectorType: detectorspb.DetectorType_Slack,
+					Verified:     false,
+				},
+			},
+			wantErr:             false,
+			wantVerificationErr: true,
+		},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			s := Scanner{}
-			got, err := s.FromData(context.Background(), tt.verify, tt.data)
+			got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
 			if (err != nil) != tt.wantErr {
-				t.Errorf("Scanner.FromData) error = %v, wantErr %v", err, tt.wantErr)
+				t.Errorf("Slack.FromData() error = %v, wantErr %v", err, tt.wantErr)
 				return
 			}
+
 			for i := range got {
 				if len(got[i].Raw) == 0 {
 					t.Fatal("no raw secret present")
 				}
 				got[i].Raw = nil
+
+				if (got[i].VerificationError != nil) != tt.wantVerificationErr {
+					t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError)
+				}
 			}
-			if diff := pretty.Compare(got, tt.wantResults); diff != "" {
-				t.Errorf("Scanner.FromData) %s diff: (-got +want)\n%s", tt.name, diff)
+			ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "VerificationError")
+			if diff := cmp.Diff(got, tt.wantResults, ignoreOpts); diff != "" {
+				t.Errorf("Slack.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
 			}
 		})
 	}
@@ -2,6 +2,7 @@ package slackwebhook

 import (
 	"context"
+	"fmt"
 	"io"
 	"net/http"
 	"regexp"
@@ -12,14 +13,15 @@ import (
 	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
 )

-type Scanner struct{}
+type Scanner struct {
+	client *http.Client
+}

 // Ensure the Scanner satisfies the interface at compile time.
 var _ detectors.Detector = (*Scanner)(nil)

 var (
-	client = common.SaneHttpClient()
-
+	defaultClient = common.SaneHttpClient()
 	// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
 	keyPat = regexp.MustCompile(`(https://hooks\.slack\.com/services/T[A-Z0-9]+/B[A-Z0-9]+/[A-Za-z0-9]{23,25})`)
 )
@@ -48,6 +50,12 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 		}

 		if verify {
+
+			client := s.client
+			if client == nil {
+				client = defaultClient
+			}
+
 			payload := strings.NewReader(`{"text": ""}`)
 			req, err := http.NewRequestWithContext(ctx, "POST", resMatch, payload)
 			if err != nil {
@@ -62,9 +70,18 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 					continue
 				}
 				body := string(bodyBytes)
-				if (res.StatusCode >= 200 && res.StatusCode < 300) || (res.StatusCode == 400 && (strings.Contains(body, "no_text") || strings.Contains(body, "missing_text"))) {
+
+				defer res.Body.Close()
+
+				if res.StatusCode >= 200 && res.StatusCode < 300 || (res.StatusCode == 400 && (strings.Contains(body, "no_text") || strings.Contains(body, "missing_text"))) {
 					s1.Verified = true
+				} else if res.StatusCode == 401 || res.StatusCode == 403 {
+					// The secret is determinately not verified (nothing to do)
+				} else {
+					s1.VerificationError = fmt.Errorf("unexpected HTTP response status %d", res.StatusCode)
 				}
+			} else {
+				s1.VerificationError = err
 			}
 		}

@@ -6,10 +6,11 @@ package slackwebhook
 import (
 	"context"
 	"fmt"
+	"github.com/google/go-cmp/cmp"
+	"github.com/google/go-cmp/cmp/cmpopts"
 	"testing"
 	"time"

-	"github.com/kylelemons/godebug/pretty"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"

 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
@@ -32,11 +33,12 @@ func TestSlackWebhook_FromChunk(t *testing.T) {
 		verify bool
 	}
 	tests := []struct {
-		name    string
-		s       Scanner
-		args    args
-		want    []detectors.Result
-		wantErr bool
+		name                string
+		s                   Scanner
+		args                args
+		want                []detectors.Result
+		wantErr             bool
+		wantVerificationErr bool
 	}{
 		{
 			name: "found, verified",
@@ -52,7 +54,8 @@ func TestSlackWebhook_FromChunk(t *testing.T) {
 					Verified:     true,
 				},
 			},
-			wantErr: false,
+			wantErr:             false,
+			wantVerificationErr: false,
 		},
 		{
 			name: "found, unverified",
@@ -68,7 +71,8 @@ func TestSlackWebhook_FromChunk(t *testing.T) {
 					Verified:     false,
 				},
 			},
-			wantErr: false,
+			wantErr:             false,
+			wantVerificationErr: false,
 		},
 		{
 			name: "not found",
@@ -78,14 +82,48 @@ func TestSlackWebhook_FromChunk(t *testing.T) {
 				data:   []byte("You cannot find the secret within"),
 				verify: true,
 			},
-			want:    nil,
-			wantErr: false,
+			want:                nil,
+			wantErr:             false,
+			wantVerificationErr: false,
+		},
+		{
+			name: "found, would be verified if not for timeout",
+			s:    Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte(fmt.Sprintf("You can find a slackwebhook secret %s within", secret)),
+				verify: true,
+			},
+			want: []detectors.Result{
+				{
+					DetectorType: detectorspb.DetectorType_SlackWebhook,
+					Verified:     false,
+				},
+			},
+			wantErr:             false,
+			wantVerificationErr: true,
+		},
+		{
+			name: "found, verified but unexpected api surface",
+			s:    Scanner{client: common.ConstantResponseHttpClient(404, "")},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte(fmt.Sprintf("You can find a slackwebhook secret %s within", secret)),
+				verify: true,
+			},
+			want: []detectors.Result{
+				{
+					DetectorType: detectorspb.DetectorType_SlackWebhook,
+					Verified:     false,
+				},
+			},
+			wantErr:             false,
+			wantVerificationErr: true,
 		},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			s := Scanner{}
-			got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+			got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
 			if (err != nil) != tt.wantErr {
 				t.Errorf("SlackWebhook.FromData() error = %v, wantErr %v", err, tt.wantErr)
 				return
@@ -94,9 +132,12 @@ func TestSlackWebhook_FromChunk(t *testing.T) {
 				if len(got[i].Raw) == 0 {
 					t.Fatalf("no raw secret present: \n %+v", got[i])
 				}
-				got[i].Raw = nil
+				if (got[i].VerificationError != nil) != tt.wantVerificationErr {
+					t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError)
+				}
 			}
-			if diff := pretty.Compare(got, tt.want); diff != "" {
+			ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "VerificationError")
+			if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {
 				t.Errorf("SlackWebhook.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
 			}
 		})
@@ -0,0 +1,94 @@
+package sourcegraph
+
+import (
+	"context"
+	"fmt"
+	"net/http"
+	"regexp"
+	"strings"
+
+	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+type Scanner struct {
+	client *http.Client
+}
+
+// Ensure the Scanner satisfies the interface at compile time.
+var _ detectors.Detector = (*Scanner)(nil)
+
+var (
+	defaultClient = common.SaneHttpClient()
+	// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
+	keyPat = regexp.MustCompile(`\b(sgp_[a-f0-9]{40})\b`)
+)
+
+// Keywords are used for efficiently pre-filtering chunks.
+// Use identifiers in the secret preferably, or the provider name.
+func (s Scanner) Keywords() []string {
+	return []string{"sgp_"}
+}
+
+// FromData will find and optionally verify Sourcegraph secrets in a given set of bytes.
+func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
+	dataStr := string(data)
+
+	matches := keyPat.FindAllStringSubmatch(dataStr, -1)
+
+	for _, match := range matches {
+		if len(match) != 2 {
+			continue
+		}
+		resMatch := strings.TrimSpace(match[1])
+
+		s1 := detectors.Result{
+			DetectorType: detectorspb.DetectorType_Sourcegraph,
+			Raw:          []byte(resMatch),
+		}
+
+		if verify {
+			client := s.client
+			if client == nil {
+				client = defaultClient
+			}
+			payload := strings.NewReader("{\"query\": \"query { currentUser { username } }\"}")
+
+			req, err := http.NewRequestWithContext(ctx, "POST", "https://sourcegraph.com/.api/graphql", payload)
+			req.Header.Add("Authorization", "token "+resMatch)
+			req.Header.Add("Content-Type", "application/json")
+
+			if err != nil {
+				continue
+			}
+			res, err := client.Do(req)
+			if err == nil {
+				defer res.Body.Close()
+
+				if res.StatusCode >= 200 && res.StatusCode < 300 {
+					s1.Verified = true
+				} else if res.StatusCode == 401 {
+					// The secret is determinately not verified (nothing to do)
+				} else {
+					s1.VerificationError = fmt.Errorf("unexpected HTTP response status %d", res.StatusCode)
+				}
+			} else {
+				s1.VerificationError = err
+			}
+		}
+
+		// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
+		if !s1.Verified && detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
+			continue
+		}
+
+		results = append(results, s1)
+	}
+
+	return results, nil
+}
+
+func (s Scanner) Type() detectorspb.DetectorType {
+	return detectorspb.DetectorType_Sourcegraph
+}
@@ -0,0 +1,161 @@
+//go:build detectors
+// +build detectors
+
+package sourcegraph
+
+import (
+	"context"
+	"fmt"
+	"github.com/google/go-cmp/cmp"
+	"github.com/google/go-cmp/cmp/cmpopts"
+	"testing"
+	"time"
+
+	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+
+	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestSourcegraph_FromChunk(t *testing.T) {
+	ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+	defer cancel()
+	testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors4")
+	if err != nil {
+		t.Fatalf("could not get test secrets from GCP: %s", err)
+	}
+	secret := testSecrets.MustGetField("SOURCEGRAPH")
+	inactiveSecret := testSecrets.MustGetField("SOURCEGRAPH_INACTIVE")
+
+	type args struct {
+		ctx    context.Context
+		data   []byte
+		verify bool
+	}
+	tests := []struct {
+		name                string
+		s                   Scanner
+		args                args
+		want                []detectors.Result
+		wantErr             bool
+		wantVerificationErr bool
+	}{
+		{
+			name: "found, verified",
+			s:    Scanner{},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte(fmt.Sprintf("You can find a sourcegraph secret %s within", secret)),
+				verify: true,
+			},
+			want: []detectors.Result{
+				{
+					DetectorType: detectorspb.DetectorType_Sourcegraph,
+					Verified:     true,
+				},
+			},
+			wantErr:             false,
+			wantVerificationErr: false,
+		},
+		{
+			name: "found, unverified",
+			s:    Scanner{},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte(fmt.Sprintf("You can find a sourcegraph secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+				verify: true,
+			},
+			want: []detectors.Result{
+				{
+					DetectorType: detectorspb.DetectorType_Sourcegraph,
+					Verified:     false,
+				},
+			},
+			wantErr:             false,
+			wantVerificationErr: false,
+		},
+		{
+			name: "not found",
+			s:    Scanner{},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte("You cannot find the secret within"),
+				verify: true,
+			},
+			want:                nil,
+			wantErr:             false,
+			wantVerificationErr: false,
+		},
+		{
+			name: "found, would be verified if not for timeout",
+			s:    Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte(fmt.Sprintf("You can find a sourcegraph secret %s within", secret)),
+				verify: true,
+			},
+			want: []detectors.Result{
+				{
+					DetectorType: detectorspb.DetectorType_Sourcegraph,
+					Verified:     false,
+				},
+			},
+			wantErr:             false,
+			wantVerificationErr: true,
+		},
+		{
+			name: "found, verified but unexpected api surface",
+			s:    Scanner{client: common.ConstantResponseHttpClient(404, "")},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte(fmt.Sprintf("You can find a sourcegraph secret %s within", secret)),
+				verify: true,
+			},
+			want: []detectors.Result{
+				{
+					DetectorType: detectorspb.DetectorType_Sourcegraph,
+					Verified:     false,
+				},
+			},
+			wantErr:             false,
+			wantVerificationErr: true,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("Sourcegraph.FromData() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			for i := range got {
+				if len(got[i].Raw) == 0 {
+					t.Fatalf("no raw secret present: \n %+v", got[i])
+				}
+				if (got[i].VerificationError != nil) != tt.wantVerificationErr {
+					t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError)
+				}
+			}
+			ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "VerificationError")
+			if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {
+				t.Errorf("Sourcegraph.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+			}
+		})
+	}
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+	ctx := context.Background()
+	s := Scanner{}
+	for name, data := range detectors.MustGetBenchmarkData() {
+		benchmark.Run(name, func(b *testing.B) {
+			b.ResetTimer()
+			for n := 0; n < b.N; n++ {
+				_, err := s.FromData(ctx, false, data)
+				if err != nil {
+					b.Fatal(err)
+				}
+			}
+		})
+	}
+}
@@ -0,0 +1,85 @@
+package tailscale
+
+import (
+	"context"
+	"fmt"
+	"net/http"
+	"net/url"
+	"regexp"
+	"strings"
+
+	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+type Scanner struct{}
+
+// Ensure the Scanner satisfies the interface at compile time.
+var _ detectors.Detector = (*Scanner)(nil)
+
+var (
+	client = common.SaneHttpClient()
+
+	// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
+	keyPat = regexp.MustCompile(`\btskey-[a-z]+-[0-9A-Za-z_]+-[0-9A-Za-z_]+\b`)
+)
+
+// Keywords are used for efficiently pre-filtering chunks.
+// Use identifiers in the secret preferably, or the provider name.
+func (s Scanner) Keywords() []string {
+	return []string{"tskey-", "tskey-api-", "tskey-oauth-"}
+}
+
+// FromData will find and optionally verify Tailscaleapi secrets in a given set of bytes.
+func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
+	dataStr := string(data)
+
+	matches := keyPat.FindAllStringSubmatch(dataStr, -1)
+
+	for _, match := range matches {
+		resMatch := strings.TrimSpace(match[0])
+
+		s1 := detectors.Result{
+			DetectorType: detectorspb.DetectorType_Tailscale,
+			Raw:          []byte(resMatch),
+		}
+
+		if verify {
+			const u = "https://api.tailscale.com/api/v2/secret-scanning/verify"
+			vals := url.Values{"key": []string{resMatch}}
+			req, err := http.NewRequestWithContext(ctx, http.MethodPost, u, strings.NewReader(vals.Encode()))
+			if err != nil {
+				continue
+			}
+			req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
+			res, err := client.Do(req)
+			if err == nil {
+				defer res.Body.Close()
+				switch res.StatusCode {
+				case http.StatusNoContent:
+					s1.Verified = true
+				case http.StatusUnauthorized:
+					// The secret is determinately not verified (nothing to do)
+				default:
+					s1.VerificationError = fmt.Errorf("unexpected HTTP response status %d", res.StatusCode)
+				}
+			} else {
+				s1.VerificationError = err
+			}
+		}
+
+		// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
+		if !s1.Verified && detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
+			continue
+		}
+
+		results = append(results, s1)
+	}
+
+	return results, nil
+}
+
+func (s Scanner) Type() detectorspb.DetectorType {
+	return detectorspb.DetectorType_Tailscale
+}
@@ -1,4 +1,7 @@
-package gitlab
+//go:build detectors
+// +build detectors
+
+package tailscale

 import (
 	"context"
@@ -7,20 +10,23 @@ import (
 	"time"

 	"github.com/kylelemons/godebug/pretty"
-	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
+
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+
+	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
 )

-func TestGitlabv2_FromChunk(t *testing.T) {
+func TestTailscaleapi_FromChunk(t *testing.T) {
 	ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
 	defer cancel()
 	testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors4")
-	if err != nil {
+	if err != nil && false {
 		t.Fatalf("could not get test secrets from GCP: %s", err)
 	}
-	secret := testSecrets.MustGetField("GITLABV2")
-	secretInactive := testSecrets.MustGetField("GITLABV2_INACTIVE")
+	secret := testSecrets.MustGetField("TAILSCALEAPI")
+	inactiveSecret := testSecrets.MustGetField("TAILSCALEAPI_INACTIVE")
+
 	type args struct {
 		ctx    context.Context
 		data   []byte
@@ -34,16 +40,16 @@ func TestGitlabv2_FromChunk(t *testing.T) {
 		wantErr bool
 	}{
 		{
-			name: "found",
+			name: "found, verified",
 			s:    Scanner{},
 			args: args{
 				ctx:    context.Background(),
-				data:   []byte(fmt.Sprintf("You can find a gitlab secret %s within", secret)),
+				data:   []byte(fmt.Sprintf("You can find a tailscaleapi secret %s within", secret)),
 				verify: true,
 			},
 			want: []detectors.Result{
 				{
-					DetectorType: detectorspb.DetectorType_Gitlab,
+					DetectorType: detectorspb.DetectorType_Tailscale,
 					Verified:     true,
 				},
 			},
@@ -54,12 +60,12 @@ func TestGitlabv2_FromChunk(t *testing.T) {
 			s:    Scanner{},
 			args: args{
 				ctx:    context.Background(),
-				data:   []byte(fmt.Sprintf("You can find a gitlab secret %s within", secretInactive)),
+				data:   []byte(fmt.Sprintf("You can find a tailscaleapi secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
 				verify: true,
 			},
 			want: []detectors.Result{
 				{
-					DetectorType: detectorspb.DetectorType_Gitlab,
+					DetectorType: detectorspb.DetectorType_Tailscale,
 					Verified:     false,
 				},
 			},
@@ -82,28 +88,27 @@ func TestGitlabv2_FromChunk(t *testing.T) {
 			s := Scanner{}
 			got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
 			if (err != nil) != tt.wantErr {
-				t.Errorf("Gitlab.FromData() error = %v, wantErr %v", err, tt.wantErr)
+				t.Errorf("Tailscaleapi.FromData() error = %v, wantErr %v", err, tt.wantErr)
 				return
 			}
 			for i := range got {
 				if len(got[i].Raw) == 0 {
-					t.Fatal("no raw secret present")
+					t.Fatalf("no raw secret present: \n %+v", got[i])
 				}
 				got[i].Raw = nil
 			}
 			if diff := pretty.Compare(got, tt.want); diff != "" {
-				t.Errorf("Gitlab.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+				t.Errorf("Tailscaleapi.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
 			}
 		})
 	}
 }

-func BenchmarkFromData2(benchmark *testing.B) {
+func BenchmarkFromData(benchmark *testing.B) {
 	ctx := context.Background()
 	s := Scanner{}
 	for name, data := range detectors.MustGetBenchmarkData() {
 		benchmark.Run(name, func(b *testing.B) {
-			b.ResetTimer()
 			for n := 0; n < b.N; n++ {
 				_, err := s.FromData(ctx, false, data)
 				if err != nil {
@@ -2,6 +2,7 @@ package twilio

 import (
 	"context"
+	"fmt"
 	"net/http"
 	"regexp"

@@ -10,12 +11,15 @@ import (
 	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
 )

-type Scanner struct{}
+type Scanner struct {
+	client *http.Client
+}

 // Ensure the Scanner satisfies the interface at compile time.
 var _ detectors.Detector = (*Scanner)(nil)

 var (
+	defaultClient = common.SaneHttpClient()
 	identifierPat = regexp.MustCompile(`(?i)sid.{0,20}AC[0-9a-f]{32}`) // Should we have this? Seems restrictive.
 	sidPat        = regexp.MustCompile(`\bAC[0-9a-f]{32}\b`)
 	keyPat        = regexp.MustCompile(`\b[0-9a-f]{32}\b`)
@@ -51,6 +55,11 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 			}

 			if verify {
+				client = s.client
+				if client == nil {
+					client = defaultClient
+				}
+
 				req, err := http.NewRequestWithContext(
 					ctx, "GET", "https://verify.twilio.com/v2/Services", nil)
 				if err != nil {
@@ -65,7 +74,13 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result

 					if res.StatusCode >= 200 && res.StatusCode < 300 {
 						s1.Verified = true
+					} else if res.StatusCode == 401 || res.StatusCode == 403 {
+						// The secret is determinately not verified (nothing to do)
+					} else {
+						s1.VerificationError = fmt.Errorf("unexpected HTTP response status %d", res.StatusCode)
 					}
+				} else {
+					s1.VerificationError = err
 				}
 			}

@@ -6,11 +6,11 @@ package twilio
 import (
 	"context"
 	"fmt"
+	"github.com/google/go-cmp/cmp"
+	"github.com/google/go-cmp/cmp/cmpopts"
 	"testing"
 	"time"

-	"github.com/kylelemons/godebug/pretty"
-
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
@@ -33,11 +33,12 @@ func TestTwilio_FromChunk(t *testing.T) {
 		verify bool
 	}
 	tests := []struct {
-		name    string
-		s       Scanner
-		args    args
-		want    []detectors.Result
-		wantErr bool
+		name                string
+		s                   Scanner
+		args                args
+		want                []detectors.Result
+		wantErr             bool
+		wantVerificationErr bool
 	}{
 		{
 			name: "found, verified",
@@ -52,6 +53,7 @@ func TestTwilio_FromChunk(t *testing.T) {
 					DetectorType: detectorspb.DetectorType_Twilio,
 					Verified:     true,
 					Redacted:     id,
+					RawV2:        []byte(id + secret),
 				},
 			},
 			wantErr: false,
@@ -69,6 +71,7 @@ func TestTwilio_FromChunk(t *testing.T) {
 					DetectorType: detectorspb.DetectorType_Twilio,
 					Verified:     false,
 					Redacted:     id,
+					RawV2:        []byte(id + secretInactive),
 				},
 			},
 			wantErr: false,
@@ -84,23 +87,62 @@ func TestTwilio_FromChunk(t *testing.T) {
 			want:    nil,
 			wantErr: false,
 		},
+		{
+			name: "found, would be verified if not for timeout",
+			s:    Scanner{client: common.SaneHttpClientTimeOut(100 * time.Microsecond)},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte(fmt.Sprintf("You can find a twillio secret %s within awsId %s", secret, id)),
+				verify: true,
+			},
+			want: []detectors.Result{
+				{
+					DetectorType: detectorspb.DetectorType_Twilio,
+					Verified:     false,
+					Redacted:     id,
+					RawV2:        []byte(id + secret),
+				},
+			},
+			wantErr:             false,
+			wantVerificationErr: true,
+		},
+		{
+			name: "found, verified but unexpected api surface",
+			s:    Scanner{client: common.ConstantResponseHttpClient(404, "")},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte(fmt.Sprintf("You can find a twillio secret %s within awsId %s", secret, id)),
+				verify: true,
+			},
+			want: []detectors.Result{
+				{
+					DetectorType: detectorspb.DetectorType_Twilio,
+					Verified:     false,
+					Redacted:     id,
+					RawV2:        []byte(id + secret),
+				},
+			},
+			wantErr:             false,
+			wantVerificationErr: true,
+		},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			s := Scanner{}
-			got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+			got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
 			if (err != nil) != tt.wantErr {
 				t.Errorf("Twilio.FromData() error = %v, wantErr %v", err, tt.wantErr)
 				return
 			}
 			for i := range got {
 				if len(got[i].Raw) == 0 {
-					t.Fatal("no raw secret present")
+					t.Fatalf("no raw secret present: \n %+v", got[i])
+				}
+				if (got[i].VerificationError != nil) != tt.wantVerificationErr {
+					t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError)
 				}
-				got[i].Raw = nil
-				got[i].RawV2 = nil
 			}
-			if diff := pretty.Compare(got, tt.want); diff != "" {
+			ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "VerificationError")
+			if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {
 				t.Errorf("Twilio.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
 			}
 		})
@@ -2,6 +2,12 @@ package engine

 import (
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/envoyapikey"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/huggingface"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/salesforce"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/snowflake"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/sourcegraph"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/tailscale"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/trufflehogenterprise"

 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/abbysale"
@@ -287,6 +293,7 @@ import (
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/getresponse"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/getsandbox"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/github"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/github_oauth2"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/github_old"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/githubapp"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/gitlab"
@@ -556,7 +563,6 @@ import (
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/satismeterwritekey"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/saucelabs"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/scalewaykey"
-	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/github_oauth2"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/scalr"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/scrapeowl"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/scraperapi"
@@ -1532,6 +1538,12 @@ func DefaultDetectors() []detectors.Detector {
 		couchbase.Scanner{},
 		envoyapikey.Scanner{},
 		github_oauth2.Scanner{},
+		snowflake.Scanner{},
+		huggingface.Scanner{},
+		trufflehogenterprise.Scanner{},
+		salesforce.Scanner{},
+		sourcegraph.Scanner{},
+		tailscale.Scanner{},
 	}

 }
@@ -565,11 +565,18 @@ func (e *Engine) notifyResults(ctx context.Context) {
 		}
 		atomic.AddUint32(&e.numFoundResults, 1)

+		// Dedupe results by comparing the detector type, raw result, and source metadata.
+		// We want to avoid duplicate results with different decoder types, but we also
+		// want to include duplicate results with the same decoder type.
+		// Duplicate results with the same decoder type SHOULD have their own entry in the
+		// results list, this would happen if the same secret is found multiple times.
 		key := fmt.Sprintf("%s%s%s%+v", r.DetectorType.String(), r.Raw, r.RawV2, r.SourceMetadata)
-		if _, ok := e.dedupeCache.Get(key); ok {
-			continue
+		if val, ok := e.dedupeCache.Get(key); ok {
+			if res, ok := val.(detectorspb.DecoderType); ok && res != r.DecoderType {
+				continue
+			}
 		}
-		e.dedupeCache.Add(key, struct{}{})
+		e.dedupeCache.Add(key, r.DecoderType)

 		if r.Verified {
 			atomic.AddUint64(&e.metrics.VerifiedSecretsFound, 1)
@@ -2,8 +2,13 @@ package engine

 import (
 	"fmt"
+	"path/filepath"
 	"testing"
+	"time"

+	"github.com/stretchr/testify/assert"
+
+	"github.com/trufflesecurity/trufflehog/v3/pkg/context"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/decoders"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
@@ -171,3 +176,32 @@ func BenchmarkSupportsLineNumbersLoop(b *testing.B) {
 		_ = SupportsLineNumbers(sourceType)
 	}
 }
+
+// TestEngine_DuplicatSecrets is a test that detects ALL duplicate secrets with the same decoder.
+func TestEngine_DuplicatSecrets(t *testing.T) {
+	ctx := context.Background()
+
+	absPath, err := filepath.Abs("./testdata")
+	assert.Nil(t, err)
+
+	ctx, cancel := context.WithTimeout(ctx, 10*time.Second)
+	defer cancel()
+
+	e, err := Start(ctx,
+		WithConcurrency(1),
+		WithDecoders(decoders.DefaultDecoders()...),
+		WithDetectors(true, DefaultDetectors()...),
+		WithPrinter(new(discardPrinter)),
+	)
+	assert.Nil(t, err)
+
+	cfg := sources.FilesystemConfig{Paths: []string{absPath}}
+	if err := e.ScanFileSystem(ctx, cfg); err != nil {
+		return
+	}
+
+	// Wait for all the chunks to be processed.
+	assert.Nil(t, e.Finish(ctx))
+	want := uint64(5)
+	assert.Equal(t, want, e.GetMetrics().UnverifiedSecretsFound)
+}
@@ -0,0 +1,6 @@
+  1 aws AKIAWARWQKZNHMZBLY4I
+  2 secret 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
+ sentry 27ac84f4bcdb4fca9701f4d6f6f58cd7d96b69c9d9754d40800645a51d668f90
+ sentry 27ac84f4bcdb4fca9701f4d6f6f58cd7d96b69c9d9754d40800645a51d668f90
+ sentry 27ac84f4bcdb4fca9701f4d6f6f58cd7d96b69c9d9754d40800645a51d668f90
+ sentry 27ac84f4bcdb4fca9701f4d6f6f58cd7d96b69c9d9754d40800645a51d668f90
@@ -119,7 +119,18 @@ func GenerateLink(repo, commit, file string, line int64) string {
 	case providerBitbucket:
 		return repo[:len(repo)-4] + "/commits/" + commit

+	case providerAzure:
+		baseLink := repo + "/commit/" + commit + "/" + file
+		if line > 0 {
+			baseLink += "?line=" + strconv.FormatInt(line, 10)
+		}
+		return baseLink
+
 	case providerGithub, providerGitlab:
+		// If the provider name isn't one of the cloud defaults, it is probably an on-prem github or gitlab.
+		// So do the same thing.
+		fallthrough
+	default:
 		var baseLink string
 		if file == "" {
 			baseLink = repo[:len(repo)-4] + "/commit/" + commit
@@ -130,15 +141,5 @@ func GenerateLink(repo, commit, file string, line int64) string {
 			}
 		}
 		return baseLink
-
-	case providerAzure:
-		baseLink := repo + "?path=" + file + "&version=GB" + commit
-		if line > 0 {
-			baseLink += "&line=" + strconv.FormatInt(line, 10)
-		}
-		return baseLink
-
-	default:
-		return ""
 	}
 }
@@ -148,7 +148,7 @@ func TestGenerateLink(t *testing.T) {
 				commit: "abcdef",
 				file:   "main.go",
 			},
-			want: "https://dev.azure.com/org/project/_git/repo?path=main.go&version=GBabcdef",
+			want: "https://dev.azure.com/org/project/_git/repo/commit/abcdef/main.go",
 		},
 		{
 			name: "Azure link gen with line",
@@ -158,7 +158,25 @@ func TestGenerateLink(t *testing.T) {
 				file:   "main.go",
 				line:   int64(20),
 			},
-			want: "https://dev.azure.com/org/project/_git/repo?path=main.go&version=GBabcdef&line=20",
+			want: "https://dev.azure.com/org/project/_git/repo/commit/abcdef/main.go?line=20",
+		},
+		{
+			name: "Unknown provider on-prem instance",
+			args: args{
+				repo:   "https://onprem.customdomain.com/org/repo.git",
+				commit: "xyz123",
+				file:   "main.go",
+				line:   int64(30),
+			},
+			want: "https://onprem.customdomain.com/org/repo/blob/xyz123/main.go#L30",
+		},
+		{
+			name: "Unknown provider on-prem instance - no file",
+			args: args{
+				repo:   "https://onprem.customdomain.com/org/repo.git",
+				commit: "xyz123",
+			},
+			want: "https://onprem.customdomain.com/org/repo/commit/xyz123",
 		},
 	}
 	for _, tt := range tests {
@@ -17,18 +17,23 @@ import (

 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	logContext "github.com/trufflesecurity/trufflehog/v3/pkg/context"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
 )

 type ctxKey int

 const (
 	depthKey ctxKey = iota
+
+	errMaxArchiveDepthReached = "max archive depth reached"
 )

 var (
 	maxDepth   = 5
 	maxSize    = 250 * 1024 * 1024 // 20MB
 	maxTimeout = time.Duration(30) * time.Second
+
+	defaultBufferSize = 512
 )

 // Ensure the Archive satisfies the interfaces at compile time.
@@ -62,7 +67,7 @@ func SetArchiveMaxTimeout(timeout time.Duration) {

 // FromFile extracts the files from an archive.
 func (a *Archive) FromFile(originalCtx context.Context, data io.Reader) chan []byte {
-	archiveChan := make(chan []byte, 512)
+	archiveChan := make(chan []byte, defaultBufferSize)
 	go func() {
 		ctx, cancel := context.WithTimeout(originalCtx, maxTimeout)
 		logger := logContext.AddLogger(ctx).Logger()
@@ -79,51 +84,66 @@ func (a *Archive) FromFile(originalCtx context.Context, data io.Reader) chan []b
 	return archiveChan
 }

+type decompressorInfo struct {
+	depth       int
+	reader      io.Reader
+	archiveChan chan []byte
+	archiver    archiver.Decompressor
+}
+
 // openArchive takes a reader and extracts the contents up to the maximum depth.
 func (a *Archive) openArchive(ctx context.Context, depth int, reader io.Reader, archiveChan chan []byte) error {
 	if depth >= maxDepth {
-		return fmt.Errorf("max archive depth reached")
+		return fmt.Errorf(errMaxArchiveDepthReached)
 	}
-	format, reader, err := archiver.Identify("", reader)
-	if err != nil {
-		if errors.Is(err, archiver.ErrNoMatch) && depth > 0 {
-			chunkSize := 10 * 1024
-			for {
-				chunk := make([]byte, chunkSize)
-				n, _ := reader.Read(chunk)
-				archiveChan <- chunk
-				if n < chunkSize {
-					break
-				}
-			}
-			return nil
-		}
-		return err
+
+	format, arReader, err := archiver.Identify("", reader)
+	if errors.Is(err, archiver.ErrNoMatch) && depth > 0 {
+		return a.handleNonArchiveContent(ctx, arReader, archiveChan)
 	}
+
 	switch archive := format.(type) {
 	case archiver.Decompressor:
-		compReader, err := archive.OpenReader(reader)
-		if err != nil {
-			return err
-		}
-		fileBytes, err := a.ReadToMax(ctx, compReader)
-		if err != nil {
-			return err
-		}
-		newReader := bytes.NewReader(fileBytes)
-		return a.openArchive(ctx, depth+1, newReader, archiveChan)
+		info := decompressorInfo{depth: depth, reader: arReader, archiveChan: archiveChan, archiver: archive}
+		return a.handleDecompressor(ctx, info)
 	case archiver.Extractor:
-		err := archive.Extract(context.WithValue(ctx, depthKey, depth+1), reader, nil, a.extractorHandler(archiveChan))
-		if err != nil {
+		return archive.Extract(context.WithValue(ctx, depthKey, depth+1), reader, nil, a.extractorHandler(archiveChan))
+	default:
+		return fmt.Errorf("unknown archive type: %s", format.Name())
+	}
+}
+
+func (a *Archive) handleNonArchiveContent(ctx context.Context, reader io.Reader, archiveChan chan []byte) error {
+	aCtx := logContext.AddLogger(ctx)
+	chunkReader := sources.NewChunkReader()
+	chunkResChan := chunkReader(aCtx, reader)
+	for data := range chunkResChan {
+		if err := data.Error(); err != nil {
+			aCtx.Logger().Error(err, "error reading chunk")
+			continue
+		}
+		if err := common.CancellableWrite(ctx, archiveChan, data.Bytes()); err != nil {
 			return err
 		}
-		return nil
 	}
-	return fmt.Errorf("Unknown archive type: %s", format.Name())
+	return nil
+}
+
+func (a *Archive) handleDecompressor(ctx context.Context, info decompressorInfo) error {
+	compReader, err := info.archiver.OpenReader(info.reader)
+	if err != nil {
+		return err
+	}
+	fileBytes, err := a.ReadToMax(ctx, compReader)
+	if err != nil {
+		return err
+	}
+	newReader := bytes.NewReader(fileBytes)
+	return a.openArchive(ctx, info.depth+1, newReader, info.archiveChan)
 }

 // IsFiletype returns true if the provided reader is an archive.
-func (a *Archive) IsFiletype(ctx context.Context, reader io.Reader) (io.Reader, bool) {
+func (a *Archive) IsFiletype(_ context.Context, reader io.Reader) (io.Reader, bool) {
 	format, readerB, err := archiver.Identify("", reader)
 	if err != nil {
 		return readerB, false
@@ -133,8 +153,9 @@ func (a *Archive) IsFiletype(ctx context.Context, reader io.Reader) (io.Reader,
 		return readerB, true
 	case archiver.Decompressor:
 		return readerB, true
+	default:
+		return readerB, false
 	}
-	return readerB, false
 }

 // extractorHandler is applied to each file in an archiver.Extractor file.
@@ -178,34 +199,33 @@ func (a *Archive) ReadToMax(ctx context.Context, reader io.Reader) (data []byte,
 			if e, ok := r.(error); ok {
 				err = e
 			} else {
-				err = fmt.Errorf("Panic occurred: %v", r)
+				err = fmt.Errorf("panic occurred: %v", r)
 			}
 			logger.Error(err, "Panic occurred when reading archive")
 		}
 	}()
-	fileContent := bytes.Buffer{}
-	logger.V(5).Info("Remaining buffer capacity", "bytes", maxSize-a.size)
-	for i := 0; i <= maxSize/512; i++ {
-		if common.IsDone(ctx) {
-			return nil, ctx.Err()
-		}
-		fileChunk := make([]byte, 512)
-		bRead, err := reader.Read(fileChunk)
-		if err != nil && !errors.Is(err, io.EOF) {
-			return []byte{}, err
-		}
-		a.size += bRead
-		if len(fileChunk) > 0 {
-			fileContent.Write(fileChunk[0:bRead])
-		}
-		if bRead < 512 {
-			return fileContent.Bytes(), nil
-		}
-		if a.size >= maxSize && bRead == 512 {
-			logger.V(2).Info("Max archive size reached.")
-			return fileContent.Bytes(), nil
-		}
+
+	if common.IsDone(ctx) {
+		return nil, ctx.Err()
 	}
+
+	var fileContent bytes.Buffer
+	// Create a limited reader to ensure we don't read more than the max size.
+	lr := io.LimitReader(reader, int64(maxSize))
+
+	// Using io.CopyBuffer for performance advantages. Though buf is mandatory
+	// for the method, due to the internal implementation of io.CopyBuffer, when
+	// *bytes.Buffer implements io.WriterTo or io.ReaderFrom, the provided buf
+	// is simply ignored. Thus, we can pass nil for the buf parameter.
+	_, err = io.CopyBuffer(&fileContent, lr, nil)
+	if err != nil && !errors.Is(err, io.EOF) {
+		return nil, err
+	}
+
+	if fileContent.Len() == maxSize {
+		logger.V(2).Info("Max archive size reached.")
+	}
+
 	return fileContent.Bytes(), nil
 }

@@ -282,7 +302,7 @@ func (a *Archive) HandleSpecialized(ctx logContext.Context, reader io.Reader) (i
 // The caller is responsible for closing the returned reader.
 func (a *Archive) extractDebContent(ctx logContext.Context, file io.Reader) (io.ReadCloser, error) {
 	if a.currentDepth >= maxDepth {
-		return nil, fmt.Errorf("max archive depth reached")
+		return nil, fmt.Errorf(errMaxArchiveDepthReached)
 	}

 	tmpEnv, err := a.createTempEnv(ctx, file)
@@ -1,6 +1,7 @@
 package handlers

 import (
+	"bytes"
 	"context"
 	"io"
 	"net/http"
@@ -126,6 +127,58 @@ func TestHandleFile(t *testing.T) {
 	assert.Equal(t, 1, len(ch))
 }

+func TestReadToMax(t *testing.T) {
+	tests := []struct {
+		name     string
+		input    []byte
+		expected []byte
+	}{
+		{
+			name:     "read full content within maxSize",
+			input:    []byte("abcdefg"),
+			expected: []byte("abcdefg"),
+		},
+		{
+			name:     "read content larger than maxSize",
+			input:    make([]byte, maxSize+10), // this creates a byte slice 10 bytes larger than maxSize
+			expected: make([]byte, maxSize),
+		},
+		{
+			name:     "empty input",
+			input:    []byte(""),
+			expected: []byte(""),
+		},
+	}
+
+	a := &Archive{}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			reader := bytes.NewReader(tt.input)
+			output, err := a.ReadToMax(context.Background(), reader)
+			assert.Nil(t, err)
+
+			assert.Equal(t, tt.expected, output)
+		})
+	}
+}
+
+func BenchmarkReadToMax(b *testing.B) {
+	data := bytes.Repeat([]byte("a"), 1024*1000) // 1MB of data.
+	reader := bytes.NewReader(data)
+	a := &Archive{}
+
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		b.StartTimer()
+		_, _ = a.ReadToMax(context.Background(), reader)
+		b.StopTimer()
+
+		_, _ = reader.Seek(0, 0) // Reset the reader position.
+		a.size = 0               // Reset archive size.
+	}
+}
+
 func TestExtractDebContent(t *testing.T) {
 	// Open the sample .deb file from the testdata folder.
 	file, err := os.Open("testdata/test.deb")
@@ -133,7 +133,7 @@ func BranchHeads(repo *gogit.Repository) (map[string]*object.Commit, error) {
 		}
 		headCommit, err := repo.CommitObject(*headHash)
 		if err != nil {
-			logger.Error(err, "unable to get commit", "commit", headCommit.String())
+			logger.Error(err, "unable to get commit", "head_hash", headHash.String())
 			return nil
 		}
 		branches[branchName] = headCommit
@@ -999,6 +999,8 @@ const (
 	DetectorType_Salesforce                    DetectorType = 925
 	DetectorType_HuggingFace                   DetectorType = 926
 	DetectorType_Snowflake                     DetectorType = 927
+	DetectorType_Sourcegraph                   DetectorType = 928
+	DetectorType_Tailscale                     DetectorType = 929
 )

 // Enum value maps for DetectorType.
@@ -1928,6 +1930,8 @@ var (
 		925: "Salesforce",
 		926: "HuggingFace",
 		927: "Snowflake",
+		928: "Sourcegraph",
+		929: "Tailscale",
 	}
 	DetectorType_value = map[string]int32{
 		"Alibaba":                       0,
@@ -2854,6 +2858,8 @@ var (
 		"Salesforce":                    925,
 		"HuggingFace":                   926,
 		"Snowflake":                     927,
+		"Sourcegraph":                   928,
+		"Tailscale":                     929,
 	}
 )

@@ -3232,7 +3238,7 @@ var file_detectors_proto_rawDesc = []byte{
 	0x44, 0x65, 0x63, 0x6f, 0x64, 0x65, 0x72, 0x54, 0x79, 0x70, 0x65, 0x12, 0x0b, 0x0a, 0x07, 0x55,
 	0x4e, 0x4b, 0x4e, 0x4f, 0x57, 0x4e, 0x10, 0x00, 0x12, 0x09, 0x0a, 0x05, 0x50, 0x4c, 0x41, 0x49,
 	0x4e, 0x10, 0x01, 0x12, 0x0a, 0x0a, 0x06, 0x42, 0x41, 0x53, 0x45, 0x36, 0x34, 0x10, 0x02, 0x12,
-	0x09, 0x0a, 0x05, 0x55, 0x54, 0x46, 0x31, 0x36, 0x10, 0x03, 0x2a, 0x8c, 0x74, 0x0a, 0x0c, 0x44,
+	0x09, 0x0a, 0x05, 0x55, 0x54, 0x46, 0x31, 0x36, 0x10, 0x03, 0x2a, 0xae, 0x74, 0x0a, 0x0c, 0x44,
 	0x65, 0x74, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x54, 0x79, 0x70, 0x65, 0x12, 0x0b, 0x0a, 0x07, 0x41,
 	0x6c, 0x69, 0x62, 0x61, 0x62, 0x61, 0x10, 0x00, 0x12, 0x08, 0x0a, 0x04, 0x41, 0x4d, 0x51, 0x50,
 	0x10, 0x01, 0x12, 0x07, 0x0a, 0x03, 0x41, 0x57, 0x53, 0x10, 0x02, 0x12, 0x09, 0x0a, 0x05, 0x41,
@@ -4161,11 +4167,14 @@ var file_detectors_proto_rawDesc = []byte{
 	0x74, 0x68, 0x32, 0x10, 0x9c, 0x07, 0x12, 0x0f, 0x0a, 0x0a, 0x53, 0x61, 0x6c, 0x65, 0x73, 0x66,
 	0x6f, 0x72, 0x63, 0x65, 0x10, 0x9d, 0x07, 0x12, 0x10, 0x0a, 0x0b, 0x48, 0x75, 0x67, 0x67, 0x69,
 	0x6e, 0x67, 0x46, 0x61, 0x63, 0x65, 0x10, 0x9e, 0x07, 0x12, 0x0e, 0x0a, 0x09, 0x53, 0x6e, 0x6f,
-	0x77, 0x66, 0x6c, 0x61, 0x6b, 0x65, 0x10, 0x9f, 0x07, 0x42, 0x3d, 0x5a, 0x3b, 0x67, 0x69, 0x74,
-	0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x74, 0x72, 0x75, 0x66, 0x66, 0x6c, 0x65, 0x73,
-	0x65, 0x63, 0x75, 0x72, 0x69, 0x74, 0x79, 0x2f, 0x74, 0x72, 0x75, 0x66, 0x66, 0x6c, 0x65, 0x68,
-	0x6f, 0x67, 0x2f, 0x76, 0x33, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x70, 0x62, 0x2f, 0x64, 0x65, 0x74,
-	0x65, 0x63, 0x74, 0x6f, 0x72, 0x73, 0x70, 0x62, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
+	0x77, 0x66, 0x6c, 0x61, 0x6b, 0x65, 0x10, 0x9f, 0x07, 0x12, 0x10, 0x0a, 0x0b, 0x53, 0x6f, 0x75,
+	0x72, 0x63, 0x65, 0x67, 0x72, 0x61, 0x70, 0x68, 0x10, 0xa0, 0x07, 0x12, 0x0e, 0x0a, 0x09, 0x54,
+	0x61, 0x69, 0x6c, 0x73, 0x63, 0x61, 0x6c, 0x65, 0x10, 0xa1, 0x07, 0x42, 0x3d, 0x5a, 0x3b, 0x67,
+	0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x74, 0x72, 0x75, 0x66, 0x66, 0x6c,
+	0x65, 0x73, 0x65, 0x63, 0x75, 0x72, 0x69, 0x74, 0x79, 0x2f, 0x74, 0x72, 0x75, 0x66, 0x66, 0x6c,
+	0x65, 0x68, 0x6f, 0x67, 0x2f, 0x76, 0x33, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x70, 0x62, 0x2f, 0x64,
+	0x65, 0x74, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x73, 0x70, 0x62, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74,
+	0x6f, 0x33,
 }

 var (
@@ -20,7 +20,7 @@ const (

 // Chunker takes a chunk and splits it into chunks of ChunkSize.
 func Chunker(originalChunk *Chunk) chan *Chunk {
-	chunkChan := make(chan *Chunk)
+	chunkChan := make(chan *Chunk, 1)
 	go func() {
 		defer close(chunkChan)
 		if len(originalChunk.Data) <= TotalChunkSize {
@@ -75,7 +75,7 @@ func (s *Source) Init(_ context.Context, name string, jobId, sourceId int64, ver
 }

 // Chunks emits chunks of bytes over a channel.
-func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) error {
+func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk, _ ...sources.ChunkingTarget) error {
 	projects, err := s.projects(ctx)
 	if err != nil {
 		return fmt.Errorf("error getting projects: %w", err)
@@ -238,6 +238,7 @@ func (s *Source) chunkAction(ctx context.Context, proj project, bld build, act a
 			SourceType: s.Type(),
 			SourceName: s.name,
 			SourceID:   s.SourceID(),
+			JobID:      s.JobID(),
 			Data:       removeCircleSha1Line(data.Bytes()),
 			SourceMetadata: &source_metadatapb.MetaData{
 				Data: &source_metadatapb.MetaData_Circleci{
@@ -2,7 +2,6 @@ package docker

 import (
 	"archive/tar"
-	"bytes"
 	"compress/gzip"
 	"errors"
 	"fmt"
@@ -14,9 +13,11 @@ import (
 	v1 "github.com/google/go-containerregistry/pkg/v1"
 	"github.com/google/go-containerregistry/pkg/v1/remote"
 	"github.com/google/go-containerregistry/pkg/v1/tarball"
+	"golang.org/x/sync/errgroup"
 	"google.golang.org/protobuf/proto"
 	"google.golang.org/protobuf/types/known/anypb"

+	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/context"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/source_metadatapb"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
@@ -24,17 +25,16 @@ import (
 )

 type Source struct {
-	name     string
-	sourceId int64
-	jobId    int64
-	verify   bool
-	conn     sourcespb.Docker
+	name        string
+	sourceId    int64
+	jobId       int64
+	verify      bool
+	concurrency int
+	conn        sourcespb.Docker
 	sources.Progress
 	sources.CommonSourceUnitUnmarshaller
 }

-var FilesizeLimitBytes int64 = 50 * 1024 * 1024 // 50MB
-
 // Ensure the Source satisfies the interfaces at compile time.
 var _ sources.Source = (*Source)(nil)
 var _ sources.SourceUnitUnmarshaller = (*Source)(nil)
@@ -59,6 +59,7 @@ func (s *Source) Init(_ context.Context, name string, jobId, sourceId int64, ver
 	s.sourceId = sourceId
 	s.jobId = jobId
 	s.verify = verify
+	s.concurrency = concurrency

 	// Reset metrics for this source at initialization time.
 	dockerImagesScanned.WithLabelValues(s.name).Set(0)
@@ -71,163 +72,207 @@ func (s *Source) Init(_ context.Context, name string, jobId, sourceId int64, ver
 	return nil
 }

+type imageInfo struct {
+	image v1.Image
+	base  string
+	tag   string
+}
+
+type layerInfo struct {
+	digest v1.Hash
+	base   string
+	tag    string
+}
+
 // Chunks emits data over a channel that is decoded and scanned for secrets.
-func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) error {
-	remoteOpts, err := s.remoteOpts()
-	if err != nil {
-		return err
-	}
+func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk, _ ...sources.ChunkingTarget) error {
+	ctx = context.WithValues(ctx, "source_type", s.Type(), "source_name", s.name)

-	logger := ctx.Logger().WithValues("source_type", s.Type(), "source_name", s.name)
+	workers := new(errgroup.Group)
+	workers.SetLimit(s.concurrency)

+	scanErrs := sources.NewScanErrors()
 	for _, image := range s.conn.GetImages() {
-		var img v1.Image
-		var err error
-		var base, tag string
-		var hasDigest bool
-		var imageName name.Reference
+		image := image
+		workers.Go(func() error {
+			if common.IsDone(ctx) {
+				return nil
+			}

-		if strings.HasPrefix(image, "file://") {
-			image = strings.TrimPrefix(image, "file://")
-			base = image
-			img, err = tarball.ImageFromPath(image, nil)
+			imgInfo, err := s.processImage(ctx, image)
 			if err != nil {
-				return err
-			}
-		} else {
-			base, tag, hasDigest = baseAndTagFromImage(image)
-
-			if hasDigest {
-				imageName, err = name.NewDigest(image)
-				if err != nil {
-					return err
-				}
-			} else {
-				imageName, err = name.NewTag(image)
-				if err != nil {
-					return err
-				}
+				scanErrs.Add(err)
+				return nil
 			}

-			img, err = remote.Image(imageName, remoteOpts...)
+			ctx = context.WithValues(ctx, "image", imgInfo.base, "tag", imgInfo.tag)
+			ctx.Logger().V(2).Info("scanning image")
+
+			layers, err := imgInfo.image.Layers()
 			if err != nil {
-				return err
-			}
-		}
-
-		logger = logger.WithValues("image", base, "tag", tag)
-		logger.V(2).Info("scanning image")
-
-		layers, err := img.Layers()
-		if err != nil {
-			return err
-		}
-
-		for _, layer := range layers {
-
-			digest, err := layer.Digest()
-			if err != nil {
-				return err
+				scanErrs.Add(err)
+				return nil
 			}

-			logger = logger.WithValues("layer", digest.String())
-			logger.V(2).Info("scanning layer")
-
-			rc, err := layer.Compressed()
-			if err != nil {
-				return err
+			for _, layer := range layers {
+				if err := s.processLayer(ctx, layer, imgInfo, chunksChan); err != nil {
+					scanErrs.Add(err)
+					return nil
+				}
+				dockerLayersScanned.WithLabelValues(s.name).Inc()
 			}

-			defer rc.Close()
+			dockerImagesScanned.WithLabelValues(s.name).Inc()

-			gzipReader, err := gzip.NewReader(rc)
-			if err != nil {
-				return err
-			}
-
-			defer gzipReader.Close()
-
-			tarReader := tar.NewReader(gzipReader)
-
-			for {
-				header, err := tarReader.Next()
-				if errors.Is(err, io.EOF) {
-					break // End of archive
-				}
-				if err != nil {
-					return err
-				}
-
-				// Skip files larger than FilesizeLimitBytes
-				if header.Size > FilesizeLimitBytes {
-					continue
-				}
-
-				file := bytes.NewBuffer(make([]byte, 0, header.Size))
-
-				_, err = io.Copy(file, tarReader)
-				if err != nil {
-					return err
-				}
-
-				chunk := &sources.Chunk{
-					SourceType: s.Type(),
-					SourceName: s.name,
-					SourceID:   s.SourceID(),
-					Data:       file.Bytes(),
-					SourceMetadata: &source_metadatapb.MetaData{
-						Data: &source_metadatapb.MetaData_Docker{
-							Docker: &source_metadatapb.Docker{
-								File:  "/" + header.Name, // Prepend /, because the file is relative to the root of the image
-								Image: base,
-								Tag:   tag,
-								Layer: digest.String(),
-							},
-						},
-					},
-					Verify: s.verify,
-				}
-
-				select {
-				case chunksChan <- chunk:
-				case <-ctx.Done():
-					return ctx.Err()
-				}
-			}
-
-			dockerLayersScanned.WithLabelValues(s.name).Inc()
-		}
-
-		dockerImagesScanned.WithLabelValues(s.name).Inc()
+			return nil
+		})
+	}
+	_ = workers.Wait()
+	if scanErrs.Count() > 0 {
+		ctx.Logger().V(2).Info("scan errors", "errors", scanErrs.String())
 	}

 	return nil
 }

-func baseAndTagFromImage(image string) (base, tag string, hasDigest bool) {
-	regRepoDelimiter := "/"
-	tagDelim := ":"
-	digestDelim := "@"
+// processImage processes an individual image and prepares it for further processing.
+func (s *Source) processImage(ctx context.Context, image string) (imageInfo, error) {
+	var (
+		imgInfo   imageInfo
+		hasDigest bool
+		imageName name.Reference
+	)

-	// Split on digest first, if present.
-	parts := strings.SplitN(image, digestDelim, 2)
-	if len(parts) > 1 {
-		base = parts[0]
-		tag = parts[1]
-		hasDigest = true
-		return
+	remoteOpts, err := s.remoteOpts()
+	if err != nil {
+		return imgInfo, err
 	}
-	parts = strings.Split(image, tagDelim)
-	// Verify that we aren't confusing a tag for a hostname w/ port for the purposes of weak validation.
-	if len(parts) > 1 && !strings.Contains(parts[len(parts)-1], regRepoDelimiter) {
-		base = strings.Join(parts[:len(parts)-1], tagDelim)
-		tag = parts[len(parts)-1]
+
+	const filePrefix = "file://"
+	if strings.HasPrefix(image, filePrefix) {
+		image = strings.TrimPrefix(image, filePrefix)
+		imgInfo.base = image
+		imgInfo.image, err = tarball.ImageFromPath(image, nil)
+		if err != nil {
+			return imgInfo, err
+		}
 	} else {
-		base = image
-		tag = "latest"
+		imgInfo.base, imgInfo.tag, hasDigest = baseAndTagFromImage(image)
+
+		if hasDigest {
+			imageName, err = name.NewDigest(image)
+		} else {
+			imageName, err = name.NewTag(image)
+		}
+		if err != nil {
+			return imgInfo, err
+		}
+
+		imgInfo.image, err = remote.Image(imageName, remoteOpts...)
+		if err != nil {
+			return imgInfo, err
+		}
 	}

-	return
+	ctx.Logger().WithValues("image", imgInfo.base, "tag", imgInfo.tag).V(2).Info("scanning image")
+
+	return imgInfo, nil
+}
+
+// processLayer processes an individual layer of an image.
+func (s *Source) processLayer(ctx context.Context, layer v1.Layer, imgInfo imageInfo, chunksChan chan *sources.Chunk) error {
+	layerInfo := layerInfo{
+		base: imgInfo.base,
+		tag:  imgInfo.tag,
+	}
+
+	var err error
+	layerInfo.digest, err = layer.Digest()
+	if err != nil {
+		return err
+	}
+
+	ctx.Logger().WithValues("layer", layerInfo.digest.String()).V(2).Info("scanning layer")
+
+	rc, err := layer.Compressed()
+	if err != nil {
+		return err
+	}
+	defer rc.Close()
+
+	gzipReader, err := gzip.NewReader(rc)
+	if err != nil {
+		return err
+	}
+	defer gzipReader.Close()
+
+	tarReader := tar.NewReader(gzipReader)
+	for {
+		header, err := tarReader.Next()
+		if errors.Is(err, io.EOF) {
+			break
+		}
+		if err != nil {
+			return err
+		}
+
+		info := chunkProcessingInfo{size: header.Size, name: header.Name, reader: tarReader, layer: layerInfo}
+		if err := s.processChunk(ctx, info, chunksChan); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+type chunkProcessingInfo struct {
+	size   int64
+	name   string
+	reader io.Reader
+	layer  layerInfo
+}
+
+// processChunk processes an individual chunk of a layer.
+func (s *Source) processChunk(ctx context.Context, info chunkProcessingInfo, chunksChan chan *sources.Chunk) error {
+	const filesizeLimitBytes int64 = 50 * 1024 * 1024 // 50MB
+	if info.size > filesizeLimitBytes {
+		ctx.Logger().V(4).Info("skipping large file", "file", info.name, "size", info.size)
+		return nil
+	}
+
+	chunkReader := sources.NewChunkReader()
+	chunkResChan := chunkReader(ctx, info.reader)
+
+	for data := range chunkResChan {
+		if err := data.Error(); err != nil {
+			ctx.Logger().Error(err, "error reading chunk.")
+			continue
+		}
+
+		chunk := &sources.Chunk{
+			SourceType: s.Type(),
+			SourceName: s.name,
+			SourceID:   s.SourceID(),
+			SourceMetadata: &source_metadatapb.MetaData{
+				Data: &source_metadatapb.MetaData_Docker{
+					Docker: &source_metadatapb.Docker{
+						File:  "/" + info.name,
+						Image: info.layer.base,
+						Tag:   info.layer.tag,
+						Layer: info.layer.digest.String(),
+					},
+				},
+			},
+			Verify: s.verify,
+		}
+		chunk.Data = data.Bytes()
+
+		if err := common.CancellableWrite(ctx, chunksChan, chunk); err != nil {
+			return err
+		}
+	}
+
+	return nil
 }

 func (s *Source) remoteOpts() ([]remote.Option, error) {
@@ -255,3 +300,40 @@ func (s *Source) remoteOpts() ([]remote.Option, error) {
 		return nil, fmt.Errorf("unknown credential type: %T", s.conn.Credential)
 	}
 }
+
+func baseAndTagFromImage(image string) (base, tag string, hasDigest bool) {
+	if base, tag, hasDigest = extractDigest(image); hasDigest {
+		return base, tag, true
+	}
+
+	base, tag = extractTagOrUseDefault(image)
+	return base, tag, false
+}
+
+// extractDigest tries to split the image string on the digest delimiter.
+// If successful, it means the image has a digest.
+func extractDigest(image string) (base, tag string, hasDigest bool) {
+	const digestDelim = "@"
+
+	if parts := strings.SplitN(image, digestDelim, 2); len(parts) > 1 {
+		return parts[0], parts[1], true
+	}
+	return "", "", false
+}
+
+// extractTagOrUseDefault extracts the tag from the image string.
+// If no tag is found, it defaults to "latest".
+func extractTagOrUseDefault(image string) (base, tag string) {
+	const (
+		tagDelim     = ":"
+		regRepoDelim = "/"
+	)
+
+	parts := strings.Split(image, tagDelim)
+
+	// Check if the last part is not a hostname with a port (for weak validation)
+	if len(parts) > 1 && !strings.Contains(parts[len(parts)-1], regRepoDelim) {
+		return strings.Join(parts[:len(parts)-1], tagDelim), parts[len(parts)-1]
+	}
+	return image, "latest"
+}
@@ -77,7 +77,7 @@ func (s *Source) WithFilter(filter *common.Filter) {
 }

 // Chunks emits chunks of bytes over a channel.
-func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) error {
+func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk, _ ...sources.ChunkingTarget) error {
 	for i, path := range s.paths {
 		logger := ctx.Logger().WithValues("path", path)
 		if common.IsDone(ctx) {
@@ -161,6 +161,7 @@ func (s *Source) scanFile(ctx context.Context, path string, chunksChan chan *sou
 		SourceType: s.Type(),
 		SourceName: s.name,
 		SourceID:   s.SourceID(),
+		JobID:      s.JobID(),
 		SourceMetadata: &source_metadatapb.MetaData{
 			Data: &source_metadatapb.MetaData_Filesystem{
 				Filesystem: &source_metadatapb.Filesystem{
@@ -191,6 +192,7 @@ func (s *Source) scanFile(ctx context.Context, path string, chunksChan chan *sou
 			SourceType: s.Type(),
 			SourceName: s.name,
 			SourceID:   s.SourceID(),
+			JobID:      s.JobID(),
 			Data:       data.Bytes(),
 			SourceMetadata: &source_metadatapb.MetaData{
 				Data: &source_metadatapb.MetaData_Filesystem{
@@ -248,7 +248,7 @@ func (s *Source) enumerate(ctx context.Context) error {
 }

 // Chunks emits chunks of bytes over a channel.
-func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) error {
+func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk, _ ...sources.ChunkingTarget) error {
 	persistableCache := s.setupCache(ctx)

 	objectCh, err := s.gcsManager.ListObjects(ctx)
@@ -324,6 +324,7 @@ func (s *Source) processObject(ctx context.Context, o object) error {
 	chunkSkel := &sources.Chunk{
 		SourceName: s.name,
 		SourceType: s.Type(),
+		JobID:      s.JobID(),
 		SourceID:   s.sourceId,
 		Verify:     s.verify,
 		SourceMetadata: &source_metadatapb.MetaData{
@@ -153,7 +153,7 @@ func (s *Source) Init(aCtx context.Context, name string, jobId, sourceId int64,
 }

 // Chunks emits chunks of bytes over a channel.
-func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) error {
+func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk, _ ...sources.ChunkingTarget) error {
 	if err := s.scanRepos(ctx, chunksChan); err != nil {
 		return err
 	}
@@ -446,6 +446,7 @@ func (s *Git) ScanCommits(ctx context.Context, repo *git.Repository, path string
 				chunkSkel := &sources.Chunk{
 					SourceName:     s.sourceName,
 					SourceID:       s.sourceID,
+					JobID:          s.jobID,
 					SourceType:     s.sourceType,
 					SourceMetadata: metadata,
 					Verify:         s.verify,
@@ -464,6 +465,7 @@ func (s *Git) ScanCommits(ctx context.Context, repo *git.Repository, path string
 			chunksChan <- &sources.Chunk{
 				SourceName:     s.sourceName,
 				SourceID:       s.sourceID,
+				JobID:          s.jobID,
 				SourceType:     s.sourceType,
 				SourceMetadata: metadata,
 				Data:           diff.Content.Bytes(),
@@ -490,6 +492,7 @@ func (s *Git) gitChunk(ctx context.Context, diff gitparse.Diff, fileName, email,
 				chunksChan <- &sources.Chunk{
 					SourceName:     s.sourceName,
 					SourceID:       s.sourceID,
+					JobID:          s.jobID,
 					SourceType:     s.sourceType,
 					SourceMetadata: metadata,
 					Data:           append([]byte{}, newChunkBuffer.Bytes()...),
@@ -504,6 +507,7 @@ func (s *Git) gitChunk(ctx context.Context, diff gitparse.Diff, fileName, email,
 				chunksChan <- &sources.Chunk{
 					SourceName:     s.sourceName,
 					SourceID:       s.sourceID,
+					JobID:          s.jobID,
 					SourceType:     s.sourceType,
 					SourceMetadata: metadata,
 					Data:           line,
@@ -523,6 +527,7 @@ func (s *Git) gitChunk(ctx context.Context, diff gitparse.Diff, fileName, email,
 		chunksChan <- &sources.Chunk{
 			SourceName:     s.sourceName,
 			SourceID:       s.sourceID,
+			JobID:          s.jobID,
 			SourceType:     s.sourceType,
 			SourceMetadata: metadata,
 			Data:           append([]byte{}, newChunkBuffer.Bytes()...),
@@ -588,6 +593,7 @@ func (s *Git) ScanStaged(ctx context.Context, repo *git.Repository, path string,
 				chunkSkel := &sources.Chunk{
 					SourceName:     s.sourceName,
 					SourceID:       s.sourceID,
+					JobID:          s.jobID,
 					SourceType:     s.sourceType,
 					SourceMetadata: metadata,
 					Verify:         s.verify,
@@ -602,6 +608,7 @@ func (s *Git) ScanStaged(ctx context.Context, repo *git.Repository, path string,
 			chunksChan <- &sources.Chunk{
 				SourceName:     s.sourceName,
 				SourceID:       s.sourceID,
+				JobID:          s.jobID,
 				SourceType:     s.sourceType,
 				SourceMetadata: metadata,
 				Data:           diff.Content.Bytes(),
@@ -149,14 +149,16 @@ func (s *Source) newFilteredRepoCache(c cache.Cache, include, exclude []string)
 	for _, ig := range include {
 		g, err := glob.Compile(ig)
 		if err != nil {
-			s.log.V(1).Info("invalid include glob", "glob", g, "err", err)
+			s.log.V(1).Info("invalid include glob", "include_value", ig, "err", err)
+			continue
 		}
 		includeGlobs = append(includeGlobs, g)
 	}
 	for _, eg := range exclude {
 		g, err := glob.Compile(eg)
 		if err != nil {
-			s.log.V(1).Info("invalid exclude glob", "glob", g, "err", err)
+			s.log.V(1).Info("invalid exclude glob", "exclude_value", eg, "err", err)
+			continue
 		}
 		excludeGlobs = append(excludeGlobs, g)
 	}
@@ -411,7 +413,7 @@ func (s *Source) visibilityOf(ctx context.Context, repoURL string) (visibility s
 }

 // Chunks emits chunks of bytes over a channel.
-func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) error {
+func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk, _ ...sources.ChunkingTarget) error {
 	apiEndpoint := s.conn.Endpoint
 	if len(apiEndpoint) == 0 || endsWithGithub.MatchString(apiEndpoint) {
 		apiEndpoint = "https://api.github.com"
@@ -1223,6 +1225,7 @@ func (s *Source) chunkIssueComments(ctx context.Context, repo, repoPath string,
 		chunk := &sources.Chunk{
 			SourceName: s.name,
 			SourceID:   s.SourceID(),
+			JobID:      s.JobID(),
 			SourceType: s.Type(),
 			SourceMetadata: &source_metadatapb.MetaData{
 				Data: &source_metadatapb.MetaData_Github{
@@ -1256,6 +1259,7 @@ func (s *Source) chunkPullRequestComments(ctx context.Context, repo string, comm
 			SourceName: s.name,
 			SourceID:   s.SourceID(),
 			SourceType: s.Type(),
+			JobID:      s.JobID(),
 			SourceMetadata: &source_metadatapb.MetaData{
 				Data: &source_metadatapb.MetaData_Github{
 					Github: &source_metadatapb.Github{
@@ -1287,6 +1291,7 @@ func (s *Source) chunkGistComments(ctx context.Context, gistUrl string, comments
 			SourceName: s.name,
 			SourceID:   s.SourceID(),
 			SourceType: s.Type(),
+			JobID:      s.JobID(),
 			SourceMetadata: &source_metadatapb.MetaData{
 				Data: &source_metadatapb.MetaData_Github{
 					Github: &source_metadatapb.Github{
@@ -139,7 +139,7 @@ func (s *Source) Init(_ context.Context, name string, jobId, sourceId int64, ver
 }

 // Chunks emits chunks of bytes over a channel.
-func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) error {
+func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk, _ ...sources.ChunkingTarget) error {
 	// Start client.
 	apiClient, err := s.newClient()
 	if err != nil {
@@ -71,11 +71,11 @@ func (r *JobProgressRef) Done() <-chan struct{} {
 // CancelRun requests that the job this is referencing is cancelled and stops
 // running. This method will have no effect if the job does not allow
 // cancellation.
-func (r *JobProgressRef) CancelRun() {
+func (r *JobProgressRef) CancelRun(cause error) {
 	if r.jobProgress == nil || r.jobProgress.jobCancel == nil {
 		return
 	}
-	r.jobProgress.jobCancel()
+	r.jobProgress.jobCancel(cause)
 }

 // Fatal is a wrapper around error to differentiate non-fatal errors from fatal
@@ -108,7 +108,7 @@ type JobProgress struct {
 	ctx    context.Context
 	cancel context.CancelFunc
 	// Requests to cancel the job.
-	jobCancel context.CancelFunc
+	jobCancel context.CancelCauseFunc
 	// Metrics.
 	metrics     JobProgressMetrics
 	metricsLock sync.Mutex
@@ -150,7 +150,7 @@ func WithHooks(hooks ...JobProgressHook) func(*JobProgress) {
 }

 // WithCancel allows cancelling the job by the JobProgressRef.
-func WithCancel(cancel context.CancelFunc) func(*JobProgress) {
+func WithCancel(cancel context.CancelCauseFunc) func(*JobProgress) {
 	return func(jp *JobProgress) { jp.jobCancel = cancel }
 }

@@ -31,6 +31,7 @@ import (
 )

 const (
+	defaultAWSRegion     = "us-east-1"
 	defaultMaxObjectSize = 250 * 1024 * 1024 // 250 MiB
 	maxObjectSizeLimit   = 250 * 1024 * 1024 // 250 MiB
 )
@@ -53,6 +54,7 @@ type Source struct {
 // Ensure the Source satisfies the interfaces at compile time
 var _ sources.Source = (*Source)(nil)
 var _ sources.SourceUnitUnmarshaller = (*Source)(nil)
+var _ sources.Validator = (*Source)(nil)

 // Type returns the type of source
 func (s *Source) Type() sourcespb.SourceType {
@@ -93,6 +95,23 @@ func (s *Source) Init(aCtx context.Context, name string, jobId, sourceId int64,
 	return nil
 }

+func (s *Source) Validate(ctx context.Context) []error {
+	var errors []error
+	visitor := func(c context.Context, defaultRegionClient *s3.S3, roleArn string, buckets []string) {
+		roleErrs := s.validateBucketAccess(c, defaultRegionClient, roleArn, buckets)
+		if len(roleErrs) > 0 {
+			errors = append(errors, roleErrs...)
+		}
+	}
+
+	err := s.visitRoles(ctx, visitor)
+	if err != nil {
+		errors = append(errors, err)
+	}
+
+	return errors
+}
+
 // setMaxObjectSize sets the maximum size of objects that will be scanned. If
 // not set, set to a negative number, or set larger than the
 // maxObjectSizeLimit, the defaultMaxObjectSizeLimit will be used.
@@ -104,22 +123,34 @@ func (s *Source) setMaxObjectSize(maxObjectSize int64) {
 	}
 }

-func (s *Source) newClient(region string) (*s3.S3, error) {
+func (s *Source) newClient(region, roleArn string) (*s3.S3, error) {
 	cfg := aws.NewConfig()
 	cfg.CredentialsChainVerboseErrors = aws.Bool(true)
 	cfg.Region = aws.String(region)

-	switch cred := s.conn.GetCredential().(type) {
-	case *sourcespb.S3_SessionToken:
-		cfg.Credentials = credentials.NewStaticCredentials(cred.SessionToken.Key, cred.SessionToken.Secret, cred.SessionToken.SessionToken)
-	case *sourcespb.S3_AccessKey:
-		cfg.Credentials = credentials.NewStaticCredentials(cred.AccessKey.Key, cred.AccessKey.Secret, "")
-	case *sourcespb.S3_Unauthenticated:
-		cfg.Credentials = credentials.AnonymousCredentials
-	case *sourcespb.S3_CloudEnvironment:
-		// Nothing needs to be done!
-	default:
-		return nil, errors.Errorf("invalid configuration given for %s source", s.name)
+	if roleArn == "" {
+		switch cred := s.conn.GetCredential().(type) {
+		case *sourcespb.S3_SessionToken:
+			cfg.Credentials = credentials.NewStaticCredentials(cred.SessionToken.Key, cred.SessionToken.Secret, cred.SessionToken.SessionToken)
+		case *sourcespb.S3_AccessKey:
+			cfg.Credentials = credentials.NewStaticCredentials(cred.AccessKey.Key, cred.AccessKey.Secret, "")
+		case *sourcespb.S3_Unauthenticated:
+			cfg.Credentials = credentials.AnonymousCredentials
+		case *sourcespb.S3_CloudEnvironment:
+			// Nothing needs to be done!
+		default:
+			return nil, errors.Errorf("invalid configuration given for %s source", s.name)
+		}
+	} else {
+		sess, err := session.NewSession(cfg)
+		if err != nil {
+			return nil, err
+		}
+
+		stsClient := sts.New(sess)
+		cfg.Credentials = stscreds.NewCredentialsWithClient(stsClient, roleArn, func(p *stscreds.AssumeRoleProvider) {
+			p.RoleSessionName = "trufflehog"
+		})
 	}

 	sess, err := session.NewSessionWithOptions(session.Options{
@@ -133,40 +164,6 @@ func (s *Source) newClient(region string) (*s3.S3, error) {
 	return s3.New(sess), nil
 }

-// Separate role assumption functionality into a different newClient function
-func (s *Source) newRoleClient(region string, roleArn string) (*s3.S3, error) {
-
-	cfg := aws.NewConfig()
-	cfg.CredentialsChainVerboseErrors = aws.Bool(true)
-	cfg.Region = aws.String(region)
-
-	var baseCredentials *credentials.Credentials
-
-	sess, err := session.NewSession(cfg)
-	if err != nil {
-		return nil, err
-	}
-
-	stsClient := sts.New(sess)
-	baseCredentials = stscreds.NewCredentialsWithClient(stsClient, roleArn, func(p *stscreds.AssumeRoleProvider) {
-		p.RoleSessionName = "trufflehog"
-	})
-
-	cfg.Credentials = baseCredentials
-
-	sess, err = session.NewSessionWithOptions(session.Options{
-		SharedConfigState: session.SharedConfigEnable,
-		Config:            *cfg,
-	})
-
-	if err != nil {
-		return nil, err
-	}
-
-	// Create and return the S3 client with the final configuration
-	return s3.New(sess), nil
-}
-
 // IAM identity needs s3:ListBuckets permission
 func (s *Source) getBucketsToScan(client *s3.S3) ([]string, error) {
 	if len(s.conn.Buckets) > 0 {
@@ -175,7 +172,7 @@ func (s *Source) getBucketsToScan(client *s3.S3) ([]string, error) {

 	res, err := client.ListBuckets(&s3.ListBucketsInput{})
 	if err != nil {
-		return nil, fmt.Errorf("could not list s3 buckets: %w", err)
+		return nil, err
 	}

 	var bucketsToScan []string
@@ -185,35 +182,28 @@ func (s *Source) getBucketsToScan(client *s3.S3) ([]string, error) {
 	return bucketsToScan, nil
 }

-func (s *Source) scanBuckets(ctx context.Context, client *s3.S3, role string, bucketsToScan []string, chunksChan chan *sources.Chunk) error {
-	const defaultAWSRegion = "us-east-1"
+func (s *Source) scanBuckets(ctx context.Context, client *s3.S3, role string, bucketsToScan []string, chunksChan chan *sources.Chunk) {
 	objectCount := uint64(0)

+	logger := s.log
+	if role != "" {
+		logger = logger.WithValues("roleArn", role)
+	}
+
 	for i, bucket := range bucketsToScan {
+		logger := logger.WithValues("bucket", bucket)
+
 		if common.IsDone(ctx) {
-			return nil
+			return
 		}

 		s.SetProgressComplete(i, len(bucketsToScan), fmt.Sprintf("Bucket: %s", bucket), "")
-		s.log.Info("Scanning bucket", "bucket", bucket)
-		region, err := s3manager.GetBucketRegionWithClient(ctx, client, bucket)
-		if err != nil {
-			s.log.Error(err, "could not get s3 region for bucket", "bucket: ", bucket)
-			continue
-		}
+		logger.Info("Scanning bucket")

-		var regionalClient *s3.S3
-		if region != defaultAWSRegion {
-			if role != "" {
-				regionalClient, err = s.newRoleClient(region, role)
-			} else {
-				regionalClient, err = s.newClient(region)
-			}
-			if err != nil {
-				s.log.Error(err, "could not make regional s3 client")
-			}
-		} else {
-			regionalClient = client
+		regionalClient, err := s.getRegionalClientForBucket(ctx, client, role, bucket)
+		if err != nil {
+			logger.Error(err, "could not get regional client for bucket")
+			continue
 		}

 		errorCount := sync.Map{}
@@ -226,51 +216,46 @@ func (s *Source) scanBuckets(ctx context.Context, client *s3.S3, role string, bu
 			})

 		if err != nil {
-			s.log.Error(err, "could not list objects in s3 bucket", "bucket: ", bucket)
-			continue
+			if role == "" {
+				logger.Error(err, "could not list objects in bucket")
+			} else {
+				// Our documentation blesses specifying a role to assume without specifying buckets to scan, which will
+				// often cause this to happen a lot (because in that case the scanner tries to scan every bucket in the
+				// account, but the role probably doesn't have access to all of them). This makes it expected behavior
+				// and therefore not an error.
+				logger.V(3).Info("could not list objects in bucket",
+					"err", err)
+			}
 		}
 	}
 	s.SetProgressComplete(len(bucketsToScan), len(bucketsToScan), fmt.Sprintf("Completed scanning source %s. %d objects scanned.", s.name, objectCount), "")
-	return nil
 }

 // Chunks emits chunks of bytes over a channel.
-func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) error {
-	const defaultAWSRegion = "us-east-1"
-
-	if len(s.conn.Roles) > 0 {
-		for _, role := range s.conn.Roles {
-			client, err := s.newRoleClient(defaultAWSRegion, role)
-			if err != nil {
-				return errors.WrapPrefix(err, "could not create s3 client", 0)
-			}
-
-			bucketsToScan, err := s.getBucketsToScan(client)
-			if err != nil {
-				return err
-			}
-
-			if err := s.scanBuckets(ctx, client, role, bucketsToScan, chunksChan); err != nil {
-				return err
-			}
-		}
-	} else {
-		client, err := s.newClient(defaultAWSRegion)
-		if err != nil {
-			return errors.WrapPrefix(err, "could not create s3 client", 0)
-		}
-
-		bucketsToScan, err := s.getBucketsToScan(client)
-		if err != nil {
-			return err
-		}
-
-		if err := s.scanBuckets(ctx, client, "", bucketsToScan, chunksChan); err != nil {
-			return err
-		}
+func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk, _ ...sources.ChunkingTarget) error {
+	visitor := func(c context.Context, defaultRegionClient *s3.S3, roleArn string, buckets []string) {
+		s.scanBuckets(c, defaultRegionClient, roleArn, buckets, chunksChan)
 	}

-	return nil
+	return s.visitRoles(ctx, visitor)
+}
+
+func (s *Source) getRegionalClientForBucket(ctx context.Context, defaultRegionClient *s3.S3, role, bucket string) (*s3.S3, error) {
+	region, err := s3manager.GetBucketRegionWithClient(ctx, defaultRegionClient, bucket)
+	if err != nil {
+		return nil, errors.WrapPrefix(err, "could not get s3 region for bucket", 0)
+	}
+
+	if region == defaultAWSRegion {
+		return defaultRegionClient, nil
+	}
+
+	regionalClient, err := s.newClient(region, role)
+	if err != nil {
+		return nil, errors.WrapPrefix(err, "could not create regional s3 client", 0)
+	}
+
+	return regionalClient, nil
 }

 // pageChunker emits chunks onto the given channel from a page
@@ -376,6 +361,7 @@ func (s *Source) pageChunker(ctx context.Context, client *s3.S3, chunksChan chan
 				SourceType: s.Type(),
 				SourceName: s.name,
 				SourceID:   s.SourceID(),
+				JobID:      s.JobID(),
 				SourceMetadata: &source_metadatapb.MetaData{
 					Data: &source_metadatapb.MetaData_S3{
 						S3: &source_metadatapb.S3{
@@ -431,6 +417,65 @@ func (s *Source) pageChunker(ctx context.Context, client *s3.S3, chunksChan chan
 	_ = s.jobPool.Wait()
 }

+func (s *Source) validateBucketAccess(ctx context.Context, client *s3.S3, roleArn string, buckets []string) []error {
+	shouldHaveAccessToAllBuckets := roleArn == ""
+	wasAbleToListAnyBucket := false
+	var errors []error
+
+	for _, bucket := range buckets {
+		if common.IsDone(ctx) {
+			return append(errors, ctx.Err())
+		}
+
+		regionalClient, err := s.getRegionalClientForBucket(ctx, client, roleArn, bucket)
+		if err != nil {
+			errors = append(errors, fmt.Errorf("could not get regional client for bucket %q: %w", bucket, err))
+			continue
+		}
+
+		_, err = regionalClient.ListObjectsV2(&s3.ListObjectsV2Input{Bucket: &bucket})
+
+		if err == nil {
+			wasAbleToListAnyBucket = true
+		} else if shouldHaveAccessToAllBuckets {
+			errors = append(errors, fmt.Errorf("could not list objects in bucket %q: %w", bucket, err))
+		}
+	}
+
+	if !wasAbleToListAnyBucket {
+		if roleArn == "" {
+			errors = append(errors, fmt.Errorf("could not list objects in any bucket"))
+		} else {
+			errors = append(errors, fmt.Errorf("role %q could not list objects in any bucket", roleArn))
+		}
+	}
+
+	return errors
+}
+
+func (s *Source) visitRoles(ctx context.Context, f func(c context.Context, defaultRegionClient *s3.S3, roleArn string, buckets []string)) error {
+	roles := s.conn.Roles
+	if len(roles) == 0 {
+		roles = []string{""}
+	}
+
+	for _, role := range roles {
+		client, err := s.newClient(defaultAWSRegion, role)
+		if err != nil {
+			return errors.WrapPrefix(err, "could not create s3 client", 0)
+		}
+
+		bucketsToScan, err := s.getBucketsToScan(client)
+		if err != nil {
+			return fmt.Errorf("role %q could not list any s3 buckets for scanning: %w", role, err)
+		}
+
+		f(ctx, client, role, bucketsToScan)
+	}
+
+	return nil
+}
+
 // S3 links currently have the general format of:
 // https://[bucket].s3[.region unless us-east-1].amazonaws.com/[key]
 func makeS3Link(bucket, region, key string) string {
@@ -4,10 +4,14 @@
 package s3

 import (
+	"fmt"
+	"sync"
 	"testing"
 	"time"

 	"github.com/stretchr/testify/assert"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/credentialspb"
 	"google.golang.org/protobuf/types/known/anypb"

 	"github.com/trufflesecurity/trufflehog/v3/pkg/context"
@@ -45,3 +49,127 @@ func TestSource_ChunksCount(t *testing.T) {
 	}
 	assert.Greater(t, got, wantChunkCount)
 }
+
+func TestSource_Validate(t *testing.T) {
+	ctx, cancel := context.WithTimeout(context.Background(), time.Second*15)
+	defer cancel()
+
+	secret, err := common.GetTestSecret(ctx)
+	if err != nil {
+		t.Fatal(fmt.Errorf("failed to access secret: %v", err))
+	}
+
+	s3key := secret.MustGetField("AWS_S3_KEY")
+	s3secret := secret.MustGetField("AWS_S3_SECRET")
+
+	tests := []struct {
+		name         string
+		roles        []string
+		buckets      []string
+		wantErrCount int
+	}{
+		{
+			name: "buckets without roles, can access all buckets",
+			buckets: []string{
+				"truffletestbucket-s3-tests",
+			},
+			wantErrCount: 0,
+		},
+		{
+			name: "buckets without roles, one error per inaccessible bucket",
+			buckets: []string{
+				"truffletestbucket-s3-tests",
+				"truffletestbucket-s3-role-assumption",
+				"truffletestbucket-no-access",
+			},
+			wantErrCount: 2,
+		},
+		{
+			name: "roles without buckets, all can access at least one account bucket",
+			roles: []string{
+				"arn:aws:iam::619888638459:role/s3-test-assume-role",
+			},
+			wantErrCount: 0,
+		},
+		{
+			name: "roles without buckets, one error per role that cannot access any account buckets",
+			roles: []string{
+				"arn:aws:iam::619888638459:role/s3-test-assume-role",
+				"arn:aws:iam::619888638459:role/test-no-access",
+			},
+			wantErrCount: 1,
+		},
+		{
+			name: "role and buckets, can access at least one bucket",
+			roles: []string{
+				"arn:aws:iam::619888638459:role/s3-test-assume-role",
+			},
+			buckets: []string{
+				"truffletestbucket-s3-role-assumption",
+				"truffletestbucket-no-access",
+			},
+			wantErrCount: 0,
+		},
+		{
+			name: "roles and buckets, one error per role that cannot access at least one bucket",
+			roles: []string{
+				"arn:aws:iam::619888638459:role/s3-test-assume-role",
+				"arn:aws:iam::619888638459:role/test-no-access",
+			},
+			buckets: []string{
+				"truffletestbucket-s3-role-assumption",
+				"truffletestbucket-no-access",
+			},
+			wantErrCount: 1,
+		},
+		{
+			name: "role and buckets, a bucket doesn't even exist",
+			roles: []string{
+				"arn:aws:iam::619888638459:role/s3-test-assume-role",
+			},
+			buckets: []string{
+				"truffletestbucket-s3-role-assumption",
+				"not-a-real-bucket-asljdhmglasjgvklhsdaljfh", // need a bucket name that nobody is likely to ever create
+			},
+			wantErrCount: 1,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			ctx, cancel := context.WithTimeout(context.Background(), time.Second*15)
+			var cancelOnce sync.Once
+			defer cancelOnce.Do(cancel)
+
+			// These are used by the tests that assume roles
+			t.Setenv("AWS_ACCESS_KEY_ID", s3key)
+			t.Setenv("AWS_SECRET_ACCESS_KEY", s3secret)
+
+			s := &Source{}
+
+			conn, err := anypb.New(&sourcespb.S3{
+				// These are used by the tests that don't assume roles
+				Credential: &sourcespb.S3_AccessKey{
+					AccessKey: &credentialspb.KeySecret{
+						Key:    s3key,
+						Secret: s3secret,
+					},
+				},
+				Buckets: tt.buckets,
+				Roles:   tt.roles,
+			})
+			if err != nil {
+				t.Fatal(err)
+			}
+
+			err = s.Init(ctx, tt.name, 0, 0, false, conn, 0)
+			if err != nil {
+				t.Fatal(err)
+			}
+
+			errs := s.Validate(ctx)
+
+			assert.Equal(t, tt.wantErrCount, len(errs))
+		})
+	}
+}
@@ -36,7 +36,10 @@ type SourceManager struct {
 	handles     map[handle]sourceInfo
 	handlesLock sync.Mutex
 	// Pool limiting the amount of concurrent sources running.
-	pool            errgroup.Group
+	pool                errgroup.Group
+	poolLimit           int
+	currentRunningCount int32
+	// Max number of units to scan concurrently per source.
 	concurrentUnits int
 	// Run the sources using source unit enumeration / chunking if available.
 	useSourceUnits bool
@@ -68,7 +71,10 @@ func WithReportHook(hook JobProgressHook) func(*SourceManager) {

 // WithConcurrentSources limits the concurrent number of sources a manager can run.
 func WithConcurrentSources(concurrency int) func(*SourceManager) {
-	return func(mgr *SourceManager) { mgr.pool.SetLimit(concurrency) }
+	return func(mgr *SourceManager) {
+		mgr.pool.SetLimit(concurrency)
+		mgr.poolLimit = concurrency
+	}
 }

 // WithBufferedOutput sets the size of the buffer used for the Chunks() channel.
@@ -164,15 +170,17 @@ func (s *SourceManager) asyncRun(ctx context.Context, handle handle) (JobProgres
 		return JobProgressRef{SourceID: int64(handle), SourceName: sourceName}, err
 	}
 	// Create a JobProgress object for tracking progress.
-	ctx, cancel := context.WithCancel(ctx)
+	ctx, cancel := context.WithCancelCause(ctx)
 	progress := NewJobProgress(jobID, int64(handle), sourceName, WithHooks(s.hooks...), WithCancel(cancel))
 	s.pool.Go(func() error {
+		atomic.AddInt32(&s.currentRunningCount, 1)
+		defer atomic.AddInt32(&s.currentRunningCount, -1)
 		ctx := context.WithValues(ctx,
 			"job_id", jobID,
 			"source_manager_worker_id", common.RandomID(5),
 		)
 		defer common.Recover(ctx)
-		defer cancel()
+		defer cancel(nil)
 		return s.run(ctx, handle, jobID, progress)
 	})
 	return progress.Ref(), nil
@@ -207,6 +215,16 @@ func (s *SourceManager) ScanChunk(chunk *Chunk) {
 	s.outputChunks <- chunk
 }

+// AvailableCapacity returns the number of concurrent jobs the manager can
+// accommodate at this time. If there is no limit, -1 is returned.
+func (s *SourceManager) AvailableCapacity() int {
+	if s.poolLimit == 0 {
+		return -1
+	}
+	runCount := atomic.LoadInt32(&s.currentRunningCount)
+	return s.poolLimit - int(runCount)
+}
+
 // preflightChecks is a helper method to check the Manager or the context isn't
 // done and that the handle is valid.
 func (s *SourceManager) preflightChecks(ctx context.Context, handle handle) error {
@@ -229,6 +247,12 @@ func (s *SourceManager) run(ctx context.Context, handle handle, jobID int64, rep
 	report.Start(time.Now())
 	defer func() { report.End(time.Now()) }()

+	defer func() {
+		if err := context.Cause(ctx); err != nil {
+			report.ReportError(Fatal{err})
+		}
+	}()
+
 	// Initialize the source.
 	sourceInfo, ok := s.getSourceInfo(handle)
 	if !ok {
@@ -6,10 +6,11 @@ import (
 	"testing"

 	"github.com/stretchr/testify/assert"
+	"google.golang.org/protobuf/types/known/anypb"
+
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/context"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
-	"google.golang.org/protobuf/types/known/anypb"
 )

 // DummySource implements Source and is used for testing a SourceManager.
@@ -31,7 +32,7 @@ func (d *DummySource) GetProgress() *Progress { return nil }

 // Interface to easily test different chunking methods.
 type chunker interface {
-	Chunks(context.Context, chan *Chunk) error
+	Chunks(context.Context, chan *Chunk, ...ChunkingTarget) error
 	ChunkUnit(ctx context.Context, unit SourceUnit, reporter ChunkReporter) error
 	Enumerate(ctx context.Context, reporter UnitReporter) error
 }
@@ -42,7 +43,7 @@ type counterChunker struct {
 	count        int
 }

-func (c *counterChunker) Chunks(ctx context.Context, ch chan *Chunk) error {
+func (c *counterChunker) Chunks(ctx context.Context, ch chan *Chunk, _ ...ChunkingTarget) error {
 	for i := 0; i < c.count; i++ {
 		select {
 		case ch <- &Chunk{Data: []byte{c.chunkCounter}}:
@@ -75,9 +76,9 @@ func (c *counterChunker) ChunkUnit(ctx context.Context, unit SourceUnit, reporte
 // Chunk method that always returns an error.
 type errorChunker struct{ error }

-func (c errorChunker) Chunks(context.Context, chan *Chunk) error                  { return c }
-func (c errorChunker) Enumerate(context.Context, UnitReporter) error              { return c }
-func (c errorChunker) ChunkUnit(context.Context, SourceUnit, ChunkReporter) error { return c }
+func (c errorChunker) Chunks(context.Context, chan *Chunk, ...ChunkingTarget) error { return c }
+func (c errorChunker) Enumerate(context.Context, UnitReporter) error                { return c }
+func (c errorChunker) ChunkUnit(context.Context, SourceUnit, ChunkReporter) error   { return c }

 // enrollDummy is a helper function to enroll a DummySource with a SourceManager.
 func enrollDummy(mgr *SourceManager, chunkMethod chunker) (handle, error) {
@@ -176,7 +177,7 @@ type unitChunk struct {

 type unitChunker struct{ steps []unitChunk }

-func (c *unitChunker) Chunks(ctx context.Context, ch chan *Chunk) error {
+func (c *unitChunker) Chunks(ctx context.Context, ch chan *Chunk, _ ...ChunkingTarget) error {
 	for _, step := range c.steps {
 		if step.err != "" {
 			continue
@@ -294,7 +295,9 @@ type callbackChunker struct {
 	cb func(context.Context, chan *Chunk) error
 }

-func (c callbackChunker) Chunks(ctx context.Context, ch chan *Chunk) error           { return c.cb(ctx, ch) }
+func (c callbackChunker) Chunks(ctx context.Context, ch chan *Chunk, _ ...ChunkingTarget) error {
+	return c.cb(ctx, ch)
+}
 func (c callbackChunker) Enumerate(context.Context, UnitReporter) error              { return nil }
 func (c callbackChunker) ChunkUnit(context.Context, SourceUnit, ChunkReporter) error { return nil }

@@ -312,8 +315,31 @@ func TestSourceManagerCancelRun(t *testing.T) {
 	ref, err := mgr.ScheduleRun(context.Background(), handle)
 	assert.NoError(t, err)

-	ref.CancelRun()
+	cancelErr := fmt.Errorf("abort! abort!")
+	ref.CancelRun(cancelErr)
 	<-ref.Done()
 	assert.Error(t, ref.Snapshot().FatalError())
 	assert.True(t, errors.Is(ref.Snapshot().FatalError(), returnedErr))
+	assert.True(t, errors.Is(ref.Snapshot().FatalErrors(), cancelErr))
+}
+
+func TestSourceManagerAvailableCapacity(t *testing.T) {
+	mgr := NewManager(WithConcurrentSources(1337))
+	start, end := make(chan struct{}), make(chan struct{})
+	handle, err := enrollDummy(mgr, callbackChunker{func(context.Context, chan *Chunk) error {
+		start <- struct{}{} // Send start signal.
+		<-end               // Wait for end signal.
+		return nil
+	}})
+	assert.NoError(t, err)
+
+	assert.Equal(t, 1337, mgr.AvailableCapacity())
+	ref, err := mgr.ScheduleRun(context.Background(), handle)
+	assert.NoError(t, err)
+
+	<-start // Wait for start signal.
+	assert.Equal(t, 1336, mgr.AvailableCapacity())
+	end <- struct{}{} // Send end signal.
+	<-ref.Done()      // Wait for the job to finish.
+	assert.Equal(t, 1337, mgr.AvailableCapacity())
 }
@@ -30,6 +30,16 @@ type Chunk struct {
 	Verify bool
 }

+// ChunkingTarget specifies criteria for a targeted chunking process.
+// Instead of collecting data indiscriminately, this struct allows the caller
+// to specify particular subsets of data they're interested in. This becomes
+// especially useful when one needs to verify or recheck specific data points
+// without processing the entire dataset.
+type ChunkingTarget struct {
+	// QueryCriteria represents specific parameters or conditions to target the chunking process.
+	QueryCriteria source_metadatapb.MetaData
+}
+
 // Source defines the interface required to implement a source chunker.
 type Source interface {
 	// Type returns the source type, used for matching against configuration and jobs.
@@ -40,8 +50,12 @@ type Source interface {
 	JobID() int64
 	// Init initializes the source.
 	Init(aCtx context.Context, name string, jobId, sourceId int64, verify bool, connection *anypb.Any, concurrency int) error
-	// Chunks emits data over a channel that is decoded and scanned for secrets.
-	Chunks(ctx context.Context, chunksChan chan *Chunk) error
+	// Chunks emits data over a channel which is then decoded and scanned for secrets.
+	// By default, data is obtained indiscriminately. However, by providing one or more
+	// ChunkingTarget parameters, the caller can direct the function to retrieve
+	// specific chunks of data. This targeted approach allows for efficient and
+	// intentional data processing, beneficial when verifying or rechecking specific data points.
+	Chunks(ctx context.Context, chunksChan chan *Chunk, targets ...ChunkingTarget) error
 	// GetProgress is the completion progress (percentage) for Scanned Source.
 	GetProgress() *Progress
 }
@@ -182,7 +182,7 @@ func (s *Source) verifyConnectionConfig() error {
 }

 // Chunks emits chunks of bytes over a channel.
-func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) error {
+func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk, _ ...sources.ChunkingTarget) error {
 	switch {
 	case s.conn.TlsCert != nilString || s.conn.TlsKey != nilString:
 		cert, err := tls.X509KeyPair([]byte(s.conn.TlsCert), []byte(s.conn.TlsKey))
@@ -272,6 +272,7 @@ func (s *Source) monitorConnection(ctx context.Context, conn net.Conn, chunksCha
 			SourceName:     s.syslog.sourceName,
 			SourceID:       s.syslog.sourceID,
 			SourceType:     s.syslog.sourceType,
+			JobID:          s.JobID(),
 			SourceMetadata: metadata,
 			Data:           input,
 			Verify:         s.verify,
@@ -313,6 +314,7 @@ func (s *Source) acceptUDPConnections(ctx context.Context, netListener net.Packe
 		chunksChan <- &sources.Chunk{
 			SourceName:     s.syslog.sourceName,
 			SourceID:       s.syslog.sourceID,
+			JobID:          s.JobID(),
 			SourceType:     s.syslog.sourceType,
 			SourceMetadata: metadata,
 			Data:           input,
@@ -936,6 +936,8 @@ enum DetectorType {
  Salesforce = 925;
  HuggingFace = 926;
  Snowflake = 927;
+  Sourcegraph = 928;
+  Tailscale = 929;
 }

 message Result {
Author	SHA1	Message	Date
ahmed	fd840b72cb	updating secret to s1 to match convention	2023-09-11 16:07:19 -04:00
ahmed	953e7f43bd	update gitlabv2 to tri-state	2023-09-11 14:36:48 -04:00
Zubair Khan	3f84a6700e	add tri state verification to slack (not slack webhook) (#1731 ) * add client, add known false positive check * fix idiosyncracies * cleanup * cleanup comment * add unexpected error test case --------- Co-authored-by: Dustin Decker <dustin@trufflesec.com>	2023-09-11 08:38:43 -07:00
ahrav	fdeccf06a0	cache dupes w/ different decoders (#1754 ) * only cache dupes that have different decoders. * add test. * remove file. * update comment.	2023-09-11 08:18:48 -07:00
ahrav	70cdff915b	add test for custom providers. (#1759 )	2023-09-11 08:18:34 -07:00
Cody Rose	2c32b14df1	always close aws response body (#1758 )	2023-09-11 10:42:14 -04:00
ahrav	e53a72abd2	[chore] - Sentry detector update (#1746 ) * add test case for account deactivated. * Handle empty case.	2023-09-11 07:26:09 -07:00
Cody Rose	62ce9bac8b	Retry AWS verification 403s (#1757 ) This PR introduces retries on 403s in the AWS detector in attempt to work around erroneous SignatureDoesNotMatch errors. As part of the work, the detector has been refactored somewhat, resulting in two minor semantic changes: Errors crafting the verification HTTP request no longer result in the candidate secret being discarded. The known-words-based false positive check now runs (and potentially discards candidate secrets) even if verification is disabled. This unifies its behavior with the hash-based false positive check.	2023-09-08 15:32:53 -04:00
trufflesteeeve	e68ace5338	Always attempt to return a git link (#1756 )	2023-09-08 15:17:29 -04:00
Marwan Sulaiman	3aa5369608	Add Tailscale detector (#1719 ) * Add tailscale detector * PR feedback: match on first element	2023-09-07 19:11:17 -07:00
ah̳̕mͭͭͨͩ̐e̘ͬ́͋ͬ̊̓͂d	8d66fde6de	updating sendgrid detector to use tri-state verification (#1735 ) * updating sendgrid detector to use tri-state verification	2023-09-07 14:21:03 -04:00
ahrav	2a9f34962d	Add optional param to Chunks (#1747 ) * Add interface for targeted chunking. * use optional args. * update Chunks method signature. * update tests. * fix test. * update QueryCriteria type.	2023-09-07 09:03:37 -07:00
ahrav	f6512ac4ca	Use common chunker for archive handler (#1717 ) * optimize the ReadToMax. * add comment. * remove dumb comment. * update comment. * fix test. * lint. * Expired invite link fix (#1713) * Use comon chunker for archive handler. --------- Co-authored-by: Zachary Rice <zachary.rice@trufflesec.com>	2023-09-06 09:26:33 -07:00
Harmon Herring	bf581ae9f9	Fix pagerdutyapikey Detector (#1749 ) * Fix pagerdutyapikey detector by broadening regex * Add 'pd' to list of pagerdutyapikey keywords	2023-09-06 09:15:39 -07:00
ah̳̕mͭͭͨͩ̐e̘ͬ́͋ͬ̊̓͂d	d3e7c5acc2	updating jiratoken and jiratokenV2 to use tri-state verification + updating tests (#1744 )	2023-09-05 16:32:05 -04:00
ahrav	abb131e502	[chore] - update Docker source (#1708 ) * Add concurrency and common chunker. * lint. * address comments.	2023-09-05 07:40:38 -07:00
ah̳̕mͭͭͨͩ̐e̘ͬ́͋ͬ̊̓͂d	b28a2b9177	updating sendbird detector to use tri-state verification (#1737 ) * updating sendbird detector to use tri-state verification	2023-09-05 10:39:05 -04:00
Cody Rose	afe708519b	Validate S3 source (#1715 ) This PR adds S3 source validation. This is accomplished by factoring out common "bucket visiting" logic to be used by both scanning and validation.	2023-09-05 10:18:58 -04:00
ahrav	c9e6086644	Correctly create azure git links. (#1743 )	2023-09-01 10:38:51 -07:00
ahrav	000065b225	[chore] - fix sentry detector (#1738 ) * fix sentry detector to check response. * use err. * address comments.	2023-09-01 10:33:21 -07:00
s.shivasurya	0a949d7131	iterating on suggestions (#1742 )	2023-09-01 10:20:18 -07:00
Mike Vanbuskirk	2e4b17d3f4	update jira detector to match new variable tokens (#1720 ) * update jira detector to match new variable tokens * add versioned interface * use _v2 format for naming packages w. versioner - also added documentation for internal/external contrib. * migrate jira and jira_v2 secrets tests to newer version * add v2 specific domain and email * add support for tri-state verification --------- Co-authored-by: Zubair Khan <zkhan124@umd.edu>	2023-09-01 12:14:36 -04:00
Zubair Khan	78bbb89a30	add tri-state verification for twilio detector (#1729 ) * add tri state for twilio * save progress * fix twilio tristate test * resolve lint issue	2023-08-31 12:06:18 -04:00
s.shivasurya	6695cf1dce	added sourcegraph token verification detection (#1730 )	2023-08-31 08:47:13 -07:00
Miccah	4aa91337e5	Update to Go 1.21 (#1733 ) * Update to Go 1.21 * go mod tidy	2023-08-31 08:46:43 -07:00
Zubair Khan	07702ea06d	update slack webhook with tri-state verification (#1724 ) * add tristate basics * update test * cleanup * fix leading space * fix accidental comment * save changes * update secret in gsm, fix bug * fix linter issue * cleanup * use defaultClient	2023-08-30 18:18:17 -04:00
Cody Rose	a2c0abbfd6	Unify S3 client creation logic (#1657 ) This PR unifies some code paths within the S3 source. This is being done to better support a future implementation of S3 source validation; less code that runs means less code to validate. The logical change is to move the handling of "role-less" operation down the call tree, which allows for a single code path for more of the S3 code. This PR also fixes a bug that would occur in the (rare) case that the source couldn't create a regional S3 client. Before, an error would be logged, but it would be followed by a panic. Now the bucket in question is skipped.	2023-08-30 17:49:37 -04:00
Miccah	522b2fab29	Add a cancel cause to job cancellation (#1728 )	2023-08-30 12:00:44 -07:00
Miccah	c77c1172c8	Add the 'Cause' family of functions to the context wrapper library (#1725 ) Go 1.20 introduced `WithCancelCause`, `WithTimeoutCause`, and `WithDeadlineCause` to allow adding a reason to context cancellations. Adding it to our wrapper will allow us to use these features.	2023-08-30 07:57:45 -07:00
ahrav	d6afca682b	remove fmt.Print (#1727 )	2023-08-30 07:17:38 -07:00
ahrav	4dc5eb7912	Optimize read to max (#1714 ) * optimize the ReadToMax. * add comment. * remove dumb comment. * update comment. * fix test. * lint. * address comments. * use limit reader. * update equality check. * update test.' * use custom limit reader. * address comments. * revert fun.	2023-08-29 17:31:40 -07:00
Miccah	7ba880f47a	Add AvailableCapacity method to SourceManager (#1665 )	2023-08-29 12:36:44 -07:00
ahrav	2b1b1b5ad0	Add jobID to chunk. (#1721 )	2023-08-29 12:02:30 -07:00
ahrav	c51e8f8af5	buffer channel. (#1718 )	2023-08-28 18:08:31 -07:00
Zubair Khan	66271aed7b	add detectors that were missed (#1716 )	2023-08-28 18:02:11 -07:00
Zachary Rice	d1b1ac408e	Expired invite link fix (#1713 )	2023-08-28 14:21:13 -05:00
ahrav	0932ea224b	[chore] - Prevent nil deref panic (#1709 )	2023-08-26 20:39:50 -07:00