Compare commits

..

1 Commits

Author SHA1 Message Date
Ahrav Dutta e608afea5f PR 7 of 40 2023-08-25 21:40:09 -07:00
74 changed files with 636 additions and 2794 deletions
+1 -1
View File
@@ -32,7 +32,7 @@ Have questions? Feedback? Jump in slack or discord and hang out with us
Join our [Slack Community](https://join.slack.com/t/trufflehog-community/shared_invite/zt-pw2qbi43-Aa86hkiimstfdKH9UCpPzQ)
Join the [Secret Scanning Discord](https://discord.gg/8Hzbrnkr7E)
Join the [Secret Scanning Discord](https://discord.gg/sydS6AHTUP)
# :tv: Demo
+3 -3
View File
@@ -1,6 +1,6 @@
module github.com/trufflesecurity/trufflehog/v3
go 1.21
go 1.20
replace github.com/jpillora/overseer => github.com/trufflesecurity/overseer v1.1.7-custom5
@@ -50,7 +50,6 @@ require (
github.com/joho/godotenv v1.5.1
github.com/jpillora/overseer v1.1.6
github.com/kylelemons/godebug v1.1.0
github.com/launchdarkly/go-server-sdk/v6 v6.1.0
github.com/lib/pq v1.10.9
github.com/lrstanley/bubblezone v0.0.0-20221222153816-e95291e2243e
github.com/mattn/go-isatty v0.0.18
@@ -171,8 +170,8 @@ require (
github.com/google/uuid v1.3.0 // indirect
github.com/googleapis/enterprise-certificate-proxy v0.2.5 // indirect
github.com/gorilla/css v1.0.0 // indirect
github.com/gregjones/httpcache v0.0.0-20171119193500-2bcd89a1743f // indirect
github.com/gsterjov/go-libsecret v0.0.0-20161001094733-a6f4afe4910c // indirect
github.com/gregjones/httpcache v0.0.0-20171119193500-2bcd89a1743f // indirect
github.com/h2non/parth v0.0.0-20190131123155-b4df798d6542 // indirect
github.com/hashicorp/errwrap v1.0.0 // indirect
github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
@@ -193,6 +192,7 @@ require (
github.com/launchdarkly/go-sdk-events/v2 v2.0.1 // indirect
github.com/launchdarkly/go-semver v1.0.2 // indirect
github.com/launchdarkly/go-server-sdk-evaluation/v2 v2.0.2 // indirect
github.com/launchdarkly/go-server-sdk/v6 v6.1.0 // indirect
github.com/lucasb-eyer/go-colorful v1.2.0 // indirect
github.com/mailru/easyjson v0.7.7 // indirect
github.com/mattn/go-colorable v0.1.13 // indirect
-18
View File
@@ -39,7 +39,6 @@ github.com/Azure/azure-sdk-for-go/sdk/azcore v1.4.0 h1:rTnT/Jrcm+figWlYz4Ixzt0SJ
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.4.0/go.mod h1:ON4tFdPTwRcgWEaVDrN3584Ef+b7GgSJaXxe5fW9t4M=
github.com/Azure/azure-sdk-for-go/sdk/azidentity v0.11.0/go.mod h1:HcM1YX14R7CJcghJGOYCgdezslRSVzqwLf/q+4Y2r/0=
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.1.0 h1:QkAcEIAKbNL4KoFr4SathZPhDhF4mVwpBMFlYjyAqy8=
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.1.0/go.mod h1:bhXu1AjYL+wutSL/kpSq6s7733q2Rb0yuot9Zgfqa/0=
github.com/Azure/azure-sdk-for-go/sdk/internal v0.7.0/go.mod h1:yqy467j36fJxcRV2TzfVZ1pCb5vxm4BtZPUdYWe/Xo8=
github.com/Azure/azure-sdk-for-go/sdk/internal v1.1.2 h1:+5VZ72z0Qan5Bog5C+ZkgSqUbeVUd9wgtHOrIKuc5b8=
github.com/Azure/azure-sdk-for-go/sdk/internal v1.1.2/go.mod h1:eWRD7oawr1Mu1sLCawqVc0CUiF43ia3qQMxLscsKQ9w=
@@ -66,7 +65,6 @@ github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBp
github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 h1:mFRzDkZVAjdal+s7s0MwaRv9igoPqLRdzOLzw/8Xvq8=
github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358/go.mod h1:chxPXzSsl7ZWRAuOIE23GDNzjWuZquvFlgA8xmpunjU=
github.com/AzureAD/microsoft-authentication-library-for-go v0.5.1 h1:BWe8a+f/t+7KY7zH2mqygeUD0t8hNFXe08p1Pb3/jKE=
github.com/AzureAD/microsoft-authentication-library-for-go v0.5.1/go.mod h1:Vt9sXTKwMyGcOxSmLDMnGPgqsUg7m8pe215qMLrDXw4=
github.com/BobuSumisu/aho-corasick v1.0.3 h1:uuf+JHwU9CHP2Vx+wAy6jcksJThhJS9ehR8a+4nPE9g=
github.com/BobuSumisu/aho-corasick v1.0.3/go.mod h1:hm4jLcvZKI2vRF2WDU1N4p/jpWtpOzp3nLmi9AzX/XE=
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
@@ -95,14 +93,12 @@ github.com/alexbrainman/sspi v0.0.0-20210105120005-909beea2cc74/go.mod h1:cEWa1L
github.com/andybalholm/brotli v1.0.5 h1:8uQZIdzKmjc/iuPu7O2ioW48L81FgatrcpfFmiq/cCs=
github.com/andybalholm/brotli v1.0.5/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig=
github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be h1:9AeTilPcZAjCFIImctFaOjnTIavg87rW78vTPkQqLI8=
github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be/go.mod h1:ySMOLuWl6zY27l47sB3qLNK6tF2fkHG55UZxx8oIVo4=
github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY=
github.com/apache/arrow/go/v12 v12.0.1 h1:JsR2+hzYYjgSUkBSaahpqCetqZMr76djX80fF/DiJbg=
github.com/apache/arrow/go/v12 v12.0.1/go.mod h1:weuTY7JvTG/HDPtMQxEUp7pU73vkLWMLpY67QwZ/WWw=
github.com/apache/thrift v0.16.0 h1:qEy6UW60iVOlUy+b9ZR0d5WzUWYGOo4HfopoyBaNmoY=
github.com/apache/thrift v0.16.0/go.mod h1:PHK3hniurgQaNMZYaCLEqXKsYK8upmhPbmdP2FXSqgU=
github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio=
github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs=
github.com/atotto/clipboard v0.1.4 h1:EH0zSVneZPSuFR11BlR9YppQTVDbh5+16AmcJi4g1z4=
github.com/atotto/clipboard v0.1.4/go.mod h1:ZY9tmq7sm5xIbd9bOK4onWV4S6X0u6GY7Vn0Yu86PYI=
github.com/aws/aws-sdk-go v1.44.83 h1:7+Rtc2Eio6EKUNoZeMV/IVxzVrY5oBQcNPtCcgIHYJA=
@@ -241,7 +237,6 @@ github.com/dsnet/golib v0.0.0-20171103203638-1ea166775780/go.mod h1:Lj+Z9rebOhdf
github.com/dvsekhvalnov/jose2go v1.5.0 h1:3j8ya4Z4kMCwT5nXIKFSV84YS+HdqSSO0VsTQxaLAeM=
github.com/dvsekhvalnov/jose2go v1.5.0/go.mod h1:QsHjhyTlD/lAVqn/NSbVZmSCGeDehTB/mPZadG+mhXU=
github.com/elazarl/goproxy v0.0.0-20221015165544-a0805db90819 h1:RIB4cRk+lBqKK3Oy0r2gRX4ui7tuhiZq2SuTtTCi0/0=
github.com/elazarl/goproxy v0.0.0-20221015165544-a0805db90819/go.mod h1:Ro8st/ElPeALwNFlcTpWmkr6IoMFfkjXAvTHpevnDsM=
github.com/emirpasic/gods v1.18.1 h1:FXtiHYKDGKCW2KzwZKx0iC0PQmdlorYgdFG9jPXJ1Bc=
github.com/emirpasic/gods v1.18.1/go.mod h1:8tpGGwCnJ5H4r6BWwaV6OrWmMoPhUl5jm/FMNAnJvWQ=
github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
@@ -267,7 +262,6 @@ github.com/getsentry/sentry-go v0.22.0 h1:XNX9zKbv7baSEI65l+H1GEJgSeIC1c7EN5kluW
github.com/getsentry/sentry-go v0.22.0/go.mod h1:lc76E2QywIyW8WuBnwl8Lc4bkmQH4+w1gwTf25trprY=
github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
github.com/gliderlabs/ssh v0.3.5 h1:OcaySEmAQJgyYcArR+gGGTHCyE7nvhEMTlYY+Dp8CpY=
github.com/gliderlabs/ssh v0.3.5/go.mod h1:8XB4KraRrX39qHhT6yxPsHedjA08I/uBVwj4xC+/+z4=
github.com/go-asn1-ber/asn1-ber v1.5.4 h1:vXT6d/FNDiELJnLb6hGNa309LMsrCoYFvpwHDF0+Y1A=
github.com/go-asn1-ber/asn1-ber v1.5.4/go.mod h1:hEBeB/ic+5LoWskz+yKT7vGhhPYkProFKoKdwZRWMe0=
github.com/go-errors/errors v1.4.2 h1:J6MZopCL4uSllY1OfXM374weqZFFItUbrImctkmUxIA=
@@ -277,7 +271,6 @@ github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376/go.mod h1:an3vInlBmS
github.com/go-git/go-billy/v5 v5.4.1 h1:Uwp5tDRkPr+l/TnbHOQzp+tmJfLceOlbVucgpTz8ix4=
github.com/go-git/go-billy/v5 v5.4.1/go.mod h1:vjbugF6Fz7JIflbVpl1hJsGjSHNltrSw45YK/ukIvQg=
github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20230305113008-0c11038e723f h1:Pz0DHeFij3XFhoBRGUDPzSJ+w2UcK5/0JvF8DRI58r8=
github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20230305113008-0c11038e723f/go.mod h1:8LHG1a3SRW71ettAD/jW13h8c6AqjVSeL11RAdgaqpo=
github.com/go-git/go-git/v5 v5.8.1 h1:Zo79E4p7TRk0xoRgMq0RShiTHGKcKI4+DI6BfJc/Q+A=
github.com/go-git/go-git/v5 v5.8.1/go.mod h1:FHFuoD6yGz5OSKEBK+aWN9Oah0q54Jxl0abmj6GnqAo=
github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
@@ -373,7 +366,6 @@ github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17
github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPgecwXBIDzw5no=
github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
github.com/google/martian/v3 v3.3.2 h1:IqNFLAmvJOgVlpdEBiQbDc2EwKW77amAycfTuWKdfvw=
github.com/google/martian/v3 v3.3.2/go.mod h1:oBOf6HBosgwRXnUGWUB05QECsc6uvmMiJ3+6W4l/CUk=
github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
@@ -435,7 +427,6 @@ github.com/jpillora/s3 v1.1.4 h1:YCCKDWzb/Ye9EBNd83ATRF/8wPEy0xd43Rezb6u6fzc=
github.com/jpillora/s3 v1.1.4/go.mod h1:yedE603V+crlFi1Kl/5vZJaBu9pUzE9wvKegU/lF2zs=
github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk=
github.com/karlseguin/expect v1.0.2-0.20190806010014-778a5f0c6003 h1:vJ0Snvo+SLMY72r5J4sEfkuE7AFbixEP2qRbEcum/wA=
github.com/karlseguin/expect v1.0.2-0.20190806010014-778a5f0c6003/go.mod h1:zNBxMY8P21owkeogJELCLeHIt+voOSduHYTFUbwRAV8=
github.com/kevinburke/ssh_config v1.2.0 h1:x584FjTGwHzMwvHx18PXxbBVzfnxogHaAReU4gf13a4=
github.com/kevinburke/ssh_config v1.2.0/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM=
@@ -454,7 +445,6 @@ github.com/klauspost/pgzip v1.2.5/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQ
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
@@ -477,10 +467,7 @@ github.com/launchdarkly/go-server-sdk-evaluation/v2 v2.0.2 h1:PAM0GvE0nIUBeOkjdi
github.com/launchdarkly/go-server-sdk-evaluation/v2 v2.0.2/go.mod h1:Mztipcz+7ZMatXVun3k/IfPa8IOgUnAqiZawtFh2MRg=
github.com/launchdarkly/go-server-sdk/v6 v6.1.0 h1:pp/VBIon5JNbtSw7ih7I5MXN9mXHfo6T5xikKVy52dI=
github.com/launchdarkly/go-server-sdk/v6 v6.1.0/go.mod h1:3TfS6vNsNksPT4LGeGuGKtT3zTjTbbOsCUK3nrj1neA=
github.com/launchdarkly/go-test-helpers/v2 v2.2.0 h1:L3kGILP/6ewikhzhdNkHy1b5y4zs50LueWenVF0sBbs=
github.com/launchdarkly/go-test-helpers/v2 v2.2.0/go.mod h1:L7+th5govYp5oKU9iN7To5PgznBuIjBPn+ejqKR0avw=
github.com/launchdarkly/go-test-helpers/v3 v3.0.2 h1:rh0085g1rVJM5qIukdaQ8z1XTWZztbJ49vRZuveqiuU=
github.com/launchdarkly/go-test-helpers/v3 v3.0.2/go.mod h1:u2ZvJlc/DDJTFrshWW50tWMZHLVYXofuSHUfTU/eIwM=
github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw=
github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
github.com/lrstanley/bubblezone v0.0.0-20221222153816-e95291e2243e h1:XoxHx8K6ZKoMtjzWOMDuM69LCdjDDsTOtTfWGrT/fns=
@@ -561,7 +548,6 @@ github.com/paulbellamy/ratecounter v0.2.0/go.mod h1:Hfx1hDpSGoqxkVVpBi/IlYD7kChl
github.com/pierrec/lz4/v4 v4.1.17 h1:kV4Ip+/hUBC+8T6+2EgburRtkE9ef4nbY3f4dFhGjMc=
github.com/pierrec/lz4/v4 v4.1.17/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
github.com/pingcap/errors v0.11.4 h1:lFuQV/oaUMGcD2tqt+01ROSmJs75VG1ToEOkZIZ4nE4=
github.com/pingcap/errors v0.11.4/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8=
github.com/pjbgf/sha1cd v0.3.0 h1:4D5XXmUUBUl/xQ6IjCkEAbqXskkq/4O7LmGn0AqMDs4=
github.com/pjbgf/sha1cd v0.3.0/go.mod h1:nZ1rrWOcGJ5uZgEEVL1VUM9iRQiZvWdbZjkKyFzPPsI=
github.com/pkg/browser v0.0.0-20180916011732-0a3d74bf9ce4/go.mod h1:4OwLy04Bl9Ef3GJJCoec+30X3LQs/0/m4HFRt/2LUSA=
@@ -640,7 +626,6 @@ github.com/ulikunitz/xz v0.5.10/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0o
github.com/urfave/cli v1.22.12/go.mod h1:sSBEIC79qR6OvcmsD4U3KABeOTxDqQtdDnaFuUN30b8=
github.com/vbatts/tar-split v0.11.3 h1:hLFqsOLQ1SsppQNTMpkpPXClLDfC2A3Zgy9OUU+RVck=
github.com/vbatts/tar-split v0.11.3/go.mod h1:9QlHN18E+fEH7RdG+QAJJcuya3rqT7eXSTY7wGrAokY=
github.com/wsxiaoys/terminal v0.0.0-20160513160801-0940f3fc43a0 h1:3UeQBvD0TFrlVjOeLOBz+CPAI8dnbqNSVwUwRrkp7vQ=
github.com/wsxiaoys/terminal v0.0.0-20160513160801-0940f3fc43a0/go.mod h1:IXCdmsXIht47RaVFLEdVnh1t+pgYtTAhQGj73kz+2DM=
github.com/xanzy/go-gitlab v0.88.0 h1:9GHBrxyCUNZZNuAsbJ1NbEH6XAYsKyTn6NfE0wYO5SY=
github.com/xanzy/go-gitlab v0.88.0/go.mod h1:5ryv+MnpZStBH8I/77HuQBsMbBGANtVpLWC15qOjWAw=
@@ -664,7 +649,6 @@ github.com/yuin/goldmark-emoji v1.0.1/go.mod h1:2w1E6FEWLcDQkoTE+7HU6QF1F6SLlNGj
github.com/yusufpapurcu/wmi v1.2.2 h1:KBNDSne4vP5mbSWnJbO+51IMOXJB67QiYCSBrubbPRg=
github.com/yusufpapurcu/wmi v1.2.2/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0=
github.com/zeebo/assert v1.3.0 h1:g7C04CbJuIDKNPFHmsk4hwZDO5O+kntRxzaUoNXj+IQ=
github.com/zeebo/assert v1.3.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0=
github.com/zeebo/xxh3 v1.0.2 h1:xZmwmqxHZA8AI603jOQ0tMqmBr9lPeFwGg6d+xy9DC0=
github.com/zeebo/xxh3 v1.0.2/go.mod h1:5NWz9Sef7zIDm2JHfFlcQvNekmcEl9ekUZQQKCYaDcA=
go.mongodb.org/mongo-driver v1.12.0 h1:aPx33jmn/rQuJXPQLZQ8NtfPQG8CaqgLThFtqRb0PiE=
@@ -901,7 +885,6 @@ golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8T
golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 h1:H2TDz8ibqkAF6YGhCdN3jS9O0/s90v0rJh3X/OLHEUk=
golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8=
gonum.org/v1/gonum v0.11.0 h1:f1IJhK4Km5tBJmaiJXtk/PkL4cdVX6J+tGiM187uT5E=
gonum.org/v1/gonum v0.11.0/go.mod h1:fSG4YDCxxUZQJ7rKsQrj0gMOg00Il0Z96/qMA4bVQhA=
google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE=
google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M=
google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg=
@@ -997,7 +980,6 @@ gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gotest.tools/v3 v3.0.3 h1:4AuOwCGf4lLR9u3YOe2awrHygurzhO/HeQ6laiA6Sx0=
gotest.tools/v3 v3.0.3/go.mod h1:Z7Lb0S5l+klDB31fvDQX8ss/FlKDxtlFlw3Oa8Ymbl8=
honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+2 -15
View File
@@ -44,18 +44,6 @@ If you think that something should be included outside of these guidelines, plea
- Go 1.17+
- Make
### Adding New Token Formats to an Existing Scanner
In some instances, services will update their token format, requiring a new regex to properly detect secrets in addition to supporting the previous token format. Accomodating this can be done without adding a net-new detector. [We provide a `Versioner` interface](https://github.com/trufflesecurity/trufflehog/blob/e18cfd5e0af1469a9f05b8d5732bcc94c39da49c/pkg/detectors/detectors.go#L30) that can be implemented.
1. Create a copy of the package and append `_v2` to the package and file names. Ex: `<packagename>/` -> `<packagename>_v2`, `<packagename>.go` -> `<packagename>_v2.go`
Note: Be sure to update the tests to reference the new secret values in GSM, or the tests will fail.
2. Implement the `Versioner` interface. [GitHub example implementation.](/pkg/detectors/github_old/github_old.go#L22)
3. Proceed from step 3 of [Creating a new Secret Scanner](#creating-a-new-secret-scanner)
### Creating a new Secret Scanner
1. Identify the Secret Detector name from the [/proto/detectors.proto](/proto/detectors.proto) `DetectorType` enum.
@@ -108,7 +96,7 @@ Do not embed test credentials in the test code. Instead, use GCP Secrets Manager
Note: `/tmp/s` is a valid path on Linux. You will need to change that for Windows or OSX, otherwise you will see an error. On Windows you will also need to install [WSL](https://docs.microsoft.com/en-us/windows/wsl/install).
```bash
gcloud secrets versions access --project trufflehog-testing --secret detectors5 latest > /tmp/s
gcloud secrets versions access --project trufflehog-testing --secret detectors3 latest > /tmp/s
```
2. Add the secret that you need for testing.
@@ -125,9 +113,8 @@ Do not embed test credentials in the test code. Instead, use GCP Secrets Manager
3. Update the secret version with your modification.
```bash
gcloud secrets versions add --project trufflehog-testing detectors5 --data-file /tmp/s
gcloud secrets versions add --project trufflehog-testing detectors3 --data-file /tmp/s
```
Note: We increment the detectors file name `detectors(n+1)` once the previous one exceeds the max size allowed by GSM (65kb).
4. Access the secret value as shown in the [example code](pkg/detectors/heroku/heroku_test.go).
-12
View File
@@ -40,18 +40,6 @@ If you think that something should be included outside of these guidelines, plea
- Go 1.17+
- Make
### Adding New Token Formats to an Existing Scanner
In some instances, services will update their token format, requiring a new regex to properly detect secrets in addition to supporting the previous token format. Accomodating this can be done without adding a net-new detector. [We provide a `Versioner` interface](https://github.com/trufflesecurity/trufflehog/blob/e18cfd5e0af1469a9f05b8d5732bcc94c39da49c/pkg/detectors/detectors.go#L30) that can be implemented.
1. Create a copy of the package and append `_v2` to the package and file names. Ex: `<packagename>/` -> `<packagename>_v2`, `<packagename>.go` -> `<packagename>_v2.go`
Note: Be sure to update the tests to reference the new secret values in GSM, or the tests will fail.
2. Implement the `Versioner` interface. [GitHub example implementation.](/pkg/detectors/github_old/github_old.go#L22)
3. Proceed from step 3 of [Creating a new Secret Scanner](#creating-a-new-secret-scanner)
### Creating a new Secret Scanner
1. Identify the Secret Detector name from the [/proto/detectors.proto](/proto/detectors.proto) `DetectorType` enum. If necessary, run `make protos` when adding new ones.
-10
View File
@@ -136,12 +136,6 @@ var (
circleCiScan = cli.Command("circleci", "Scan CircleCI")
circleCiScanToken = circleCiScan.Flag("token", "CircleCI token. Can also be provided with environment variable").Envar("CIRCLECI_TOKEN").Required().String()
trelloScan = cli.Command("trello", "Scan Trello")
trelloApiKey = trelloScan.Flag("apiKey", "Trello API Key. Can also be provided with environment variable").Envar("TRELLO_API_KEY").Required().String()
trelloToken = trelloScan.Flag("token", "Trello token. Can also be provided with environment variable").Envar("TRELLO_TOKEN").Required().String()
trelloBoardIDs = trelloScan.Flag("boardIDs", "Trello board IDs to scan. Can also be provided with environment variable").Envar("TRELLO_BOARD_IDS").Required().Strings()
dockerScan = cli.Command("docker", "Scan Docker Image")
dockerScanImages = dockerScan.Flag("image", "Docker image to scan. Use the file:// prefix to point to a local tarball, otherwise a image registry is assumed.").Required().Strings()
)
@@ -498,10 +492,6 @@ func run(state overseer.State) {
if err := e.ScanCircleCI(ctx, *circleCiScanToken); err != nil {
logFatal(err, "Failed to scan CircleCI.")
}
case trelloScan.FullCommand():
if err := e.ScanTrello(ctx, *trelloBoardIDs, *trelloApiKey, *trelloToken); err != nil {
logFatal(err, "Failed to scan Trello.")
}
case gcsScan.FullCommand():
cfg := sources.GCSConfig{
ProjectID: *gcsProjectID,
+2 -40
View File
@@ -26,10 +26,9 @@ type Context interface {
Logger() logr.Logger
}
// CancelFunc and CancelCauseFunc are type aliases to allow use as if they are
// the same types as the standard library variants.
// CancelFunc is a type alias to context.CancelFunc to allow use as if they are
// the same types.
type CancelFunc = context.CancelFunc
type CancelCauseFunc = context.CancelCauseFunc
// logCtx implements Context.
type logCtx struct {
@@ -69,16 +68,6 @@ func WithCancel(parent Context) (Context, context.CancelFunc) {
return lCtx, cancel
}
// WithCancelCause returns context.WithCancelCause with the log object propagated.
func WithCancelCause(parent Context) (Context, context.CancelCauseFunc) {
ctx, cancel := context.WithCancelCause(parent)
lCtx := logCtx{
log: parent.Logger(),
Context: ctx,
}
return lCtx, cancel
}
// WithDeadline returns context.WithDeadline with the log object propagated and
// the deadline added to the structured log values.
func WithDeadline(parent Context, d time.Time) (Context, context.CancelFunc) {
@@ -90,17 +79,6 @@ func WithDeadline(parent Context, d time.Time) (Context, context.CancelFunc) {
return lCtx, cancel
}
// WithDeadlineCause returns context.WithDeadlineCause with the log object
// propagated and the deadline added to the structured log values.
func WithDeadlineCause(parent Context, d time.Time, cause error) (Context, context.CancelFunc) {
ctx, cancel := context.WithDeadlineCause(parent, d, cause)
lCtx := logCtx{
log: parent.Logger().WithValues("deadline", d),
Context: ctx,
}
return lCtx, cancel
}
// WithTimeout returns context.WithTimeout with the log object propagated and
// the timeout added to the structured log values.
func WithTimeout(parent Context, timeout time.Duration) (Context, context.CancelFunc) {
@@ -112,22 +90,6 @@ func WithTimeout(parent Context, timeout time.Duration) (Context, context.Cancel
return lCtx, cancel
}
// WithTimeoutCause returns context.WithTimeoutCause with the log object
// propagated and the timeout added to the structured log values.
func WithTimeoutCause(parent Context, timeout time.Duration, cause error) (Context, context.CancelFunc) {
ctx, cancel := context.WithTimeoutCause(parent, timeout, cause)
lCtx := logCtx{
log: parent.Logger().WithValues("timeout", timeout),
Context: ctx,
}
return lCtx, cancel
}
// Cause returns the context.Cause of the context.
func Cause(ctx context.Context) error {
return context.Cause(ctx)
}
// WithValue returns context.WithValue with the log object propagated and
// the value added to the structured log values (if the key is a string).
func WithValue(parent Context, key, val any) Context {
-8
View File
@@ -3,7 +3,6 @@ package context
import (
"bytes"
"context"
"fmt"
"strings"
"testing"
"time"
@@ -178,10 +177,3 @@ func TestRace(t *testing.T) {
cancel()
_ = ctx.Err()
}
func TestCause(t *testing.T) {
ctx, cancel := WithCancelCause(Background())
err := fmt.Errorf("oh no")
cancel(err)
assert.Equal(t, err, Cause(ctx))
}
+7 -16
View File
@@ -1,11 +1,11 @@
package cloverly
import (
"bytes"
"context"
"fmt"
"net/http"
"regexp"
"strings"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
@@ -14,37 +14,29 @@ import (
type Scanner struct{}
// Ensure the Scanner satisfies the interface at compile time.
var _ detectors.Detector = (*Scanner)(nil)
var (
client = common.SaneHttpClient()
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"cloverly"}) + `\b([a-z0-9:_]{28})\b`)
)
// Keywords are used for efficiently pre-filtering chunks.
// Use identifiers in the secret preferably, or the provider name.
func (s Scanner) Keywords() []string {
return []string{"cloverly"}
func (s Scanner) Keywords() [][]byte {
return [][]byte{[]byte("cloverly")}
}
// FromData will find and optionally verify Cloverly secrets in a given set of bytes.
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
dataStr := string(data)
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
matches := keyPat.FindAllSubmatch(data, -1)
for _, match := range matches {
if len(match) != 2 {
continue
}
resMatch := strings.TrimSpace(match[1])
resMatch := bytes.TrimSpace(match[1])
s1 := detectors.Result{
DetectorType: detectorspb.DetectorType_Cloverly,
Raw: []byte(resMatch),
Raw: resMatch,
}
if verify {
@@ -52,14 +44,13 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
if err != nil {
continue
}
req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", resMatch))
req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", string(resMatch)))
res, err := client.Do(req)
if err == nil {
defer res.Body.Close()
if res.StatusCode >= 200 && res.StatusCode < 300 {
s1.Verified = true
} else {
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
continue
}
+10 -20
View File
@@ -1,11 +1,11 @@
package cloze
import (
"bytes"
"context"
"net/http"
"net/url"
"regexp"
"strings"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
@@ -14,52 +14,44 @@ import (
type Scanner struct{}
// Ensure the Scanner satisfies the interface at compile time.
var _ detectors.Detector = (*Scanner)(nil)
var (
client = common.SaneHttpClient()
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"cloze"}) + `\b([0-9a-f]{32})\b`)
emailPat = regexp.MustCompile(detectors.PrefixRegex([]string{"cloze"}) + `\b([\w\.-]+@[\w-]+\.[\w\.-]{2,5})\b`)
)
// Keywords are used for efficiently pre-filtering chunks.
// Use identifiers in the secret preferably, or the provider name.
func (s Scanner) Keywords() []string {
return []string{"cloze"}
func (s Scanner) Keywords() [][]byte {
return [][]byte{[]byte("cloze")}
}
// FromData will find and optionally verify Cloze secrets in a given set of bytes.
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
dataStr := string(data)
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
emailMatches := emailPat.FindAllStringSubmatch(dataStr, -1)
matches := keyPat.FindAllSubmatch(data, -1)
emailMatches := emailPat.FindAllSubmatch(data, -1)
for _, emailMatch := range emailMatches {
if len(emailMatch) != 2 {
continue
}
resEmailMatch := strings.TrimSpace(emailMatch[1])
resEmailMatch := bytes.TrimSpace(emailMatch[1])
for _, match := range matches {
if len(match) != 2 {
continue
}
resMatch := strings.TrimSpace(match[1])
resMatch := bytes.TrimSpace(match[1])
s1 := detectors.Result{
DetectorType: detectorspb.DetectorType_Cloze,
Raw: []byte(resMatch),
Raw: resMatch,
}
if verify {
payload := url.Values{}
payload.Add("user", resEmailMatch)
payload.Add("api_key", resMatch)
payload.Add("user", string(resEmailMatch))
payload.Add("api_key", string(resMatch))
req, err := http.NewRequest("GET", "https://api.cloze.com/v1/profile?"+payload.Encode(), nil)
if err != nil {
@@ -71,11 +63,9 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
if res.StatusCode >= 200 && res.StatusCode < 300 {
s1.Verified = true
} else {
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
continue
}
}
}
}
+7 -16
View File
@@ -1,11 +1,11 @@
package clustdoc
import (
"bytes"
"context"
"fmt"
"net/http"
"regexp"
"strings"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
@@ -14,37 +14,29 @@ import (
type Scanner struct{}
// Ensure the Scanner satisfies the interface at compile time.
var _ detectors.Detector = (*Scanner)(nil)
var (
client = common.SaneHttpClient()
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"clustdoc"}) + `\b([0-9a-zA-Z]{60})\b`)
)
// Keywords are used for efficiently pre-filtering chunks.
// Use identifiers in the secret preferably, or the provider name.
func (s Scanner) Keywords() []string {
return []string{"clustdoc"}
func (s Scanner) Keywords() [][]byte {
return [][]byte{[]byte("clustdoc")}
}
// FromData will find and optionally verify ClustDoc secrets in a given set of bytes.
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
dataStr := string(data)
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
matches := keyPat.FindAllSubmatch(data, -1)
for _, match := range matches {
if len(match) != 2 {
continue
}
resMatch := strings.TrimSpace(match[1])
resMatch := bytes.TrimSpace(match[1])
s1 := detectors.Result{
DetectorType: detectorspb.DetectorType_ClustDoc,
Raw: []byte(resMatch),
Raw: resMatch,
}
if verify {
@@ -53,14 +45,13 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
continue
}
req.Header.Add("Accept", "application/json")
req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", resMatch))
req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", string(resMatch)))
res, err := client.Do(req)
if err == nil {
defer res.Body.Close()
if res.StatusCode >= 200 && res.StatusCode < 300 {
s1.Verified = true
} else {
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
continue
}
+7 -16
View File
@@ -1,10 +1,10 @@
package codacy
import (
"bytes"
"context"
"net/http"
"regexp"
"strings"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
@@ -13,37 +13,29 @@ import (
type Scanner struct{}
// Ensure the Scanner satisfies the interface at compile time.
var _ detectors.Detector = (*Scanner)(nil)
var (
client = common.SaneHttpClient()
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"codacy"}) + `\b([0-9A-Za-z]{20})\b`)
)
// Keywords are used for efficiently pre-filtering chunks.
// Use identifiers in the secret preferably, or the provider name.
func (s Scanner) Keywords() []string {
return []string{"codacy"}
func (s Scanner) Keywords() [][]byte {
return [][]byte{[]byte("codacy")}
}
// FromData will find and optionally verify Codacy secrets in a given set of bytes.
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
dataStr := string(data)
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
matches := keyPat.FindAllSubmatch(data, -1)
for _, match := range matches {
if len(match) != 2 {
continue
}
resMatch := strings.TrimSpace(match[1])
resMatch := bytes.TrimSpace(match[1])
s1 := detectors.Result{
DetectorType: detectorspb.DetectorType_Codacy,
Raw: []byte(resMatch),
Raw: resMatch,
}
if verify {
@@ -52,14 +44,13 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
continue
}
req.Header.Add("Accept", "application/json")
req.Header.Add("api-token", resMatch)
req.Header.Add("api-token", string(resMatch))
res, err := client.Do(req)
if err == nil {
defer res.Body.Close()
if res.StatusCode >= 200 && res.StatusCode < 300 {
s1.Verified = true
} else {
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
continue
}
+6 -15
View File
@@ -1,11 +1,11 @@
package codeclimate
import (
"bytes"
"context"
"fmt"
"net/http"
"regexp"
"strings"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
@@ -14,37 +14,29 @@ import (
type Scanner struct{}
// Ensure the Scanner satisfies the interface at compile time
var _ detectors.Detector = (*Scanner)(nil)
var (
client = common.SaneHttpClient()
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"codeclimate"}) + `\b([a-f0-9]{40})\b`)
)
// Keywords are used for efficiently pre-filtering chunks.
// Use identifiers in the secret preferably, or the provider name.
func (s Scanner) Keywords() []string {
return []string{"codeclimate"}
func (s Scanner) Keywords() [][]byte {
return [][]byte{[]byte("codeclimate")}
}
// FromData will find and optionally verify Codeclimate secrets in a given set of bytes.
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
dataStr := string(data)
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
matches := keyPat.FindAllSubmatch(data, -1)
for _, match := range matches {
if len(match) != 2 {
continue
}
resMatch := strings.TrimSpace(match[1])
resMatch := bytes.TrimSpace(match[1])
s1 := detectors.Result{
DetectorType: detectorspb.DetectorType_Codeclimate,
Raw: []byte(resMatch),
Raw: resMatch,
}
if verify {
@@ -60,7 +52,6 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
if res.StatusCode >= 200 && res.StatusCode < 300 {
s1.Verified = true
} else {
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
continue
}
+7 -16
View File
@@ -1,10 +1,10 @@
package codemagic
import (
"bytes"
"context"
"net/http"
"regexp"
"strings"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
@@ -13,37 +13,29 @@ import (
type Scanner struct{}
// Ensure the Scanner satisfies the interface at compile time
var _ detectors.Detector = (*Scanner)(nil)
var (
client = common.SaneHttpClient()
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"codemagic"}) + common.BuildRegex(common.AlphaNumPattern, "_", 43))
)
// Keywords are used for efficiently pre-filtering chunks.
// Use identifiers in the secret preferably, or the provider name.
func (s Scanner) Keywords() []string {
return []string{"codemagic"}
func (s Scanner) Keywords() [][]byte {
return [][]byte{[]byte("codemagic")}
}
// FromData will find and optionally verify Codemagic secrets in a given set of bytes.
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
dataStr := string(data)
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
matches := keyPat.FindAllSubmatch(data, -1)
for _, match := range matches {
if len(match) != 2 {
continue
}
resMatch := strings.TrimSpace(match[1])
resMatch := bytes.TrimSpace(match[1])
s1 := detectors.Result{
DetectorType: detectorspb.DetectorType_Codemagic,
Raw: []byte(resMatch),
Raw: resMatch,
}
if verify {
@@ -52,14 +44,13 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
continue
}
req.Header.Add("Content-Type", "application/json")
req.Header.Add("x-auth-token", resMatch)
req.Header.Add("x-auth-token", string(resMatch))
res, err := client.Do(req)
if err == nil {
defer res.Body.Close()
if res.StatusCode >= 200 && res.StatusCode < 300 {
s1.Verified = true
} else {
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
continue
}
+9 -16
View File
@@ -1,10 +1,10 @@
package codequiry
import (
"bytes"
"context"
"net/http"
"regexp"
"strings"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
@@ -13,53 +13,46 @@ import (
type Scanner struct{}
// Ensure the Scanner satisfies the interface at compile time.
var _ detectors.Detector = (*Scanner)(nil)
var (
client = common.SaneHttpClient()
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"codequiry"}) + `\b([a-zA-Z-0-9]{64})\b`)
)
// Keywords are used for efficiently pre-filtering chunks.
// Use identifiers in the secret preferably, or the provider name.
func (s Scanner) Keywords() []string {
return []string{"codequiry"}
func (s Scanner) Keywords() [][]byte {
return [][]byte{[]byte("codequiry")}
}
// FromData will find and optionally verify Codequiry secrets in a given set of bytes.
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
dataStr := string(data)
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
matches := keyPat.FindAllSubmatch(data, -1)
for _, match := range matches {
if len(match) != 2 {
continue
}
resMatch := strings.TrimSpace(match[1])
resMatch := bytes.TrimSpace(match[1])
s1 := detectors.Result{
DetectorType: detectorspb.DetectorType_Codequiry,
Raw: []byte(resMatch),
Raw: resMatch,
}
if verify {
req, err := http.NewRequestWithContext(ctx, "GET", "https://codequiry.com/api/v1/checks", nil)
if err != nil {
continue
}
req.Header.Add("apikey", resMatch)
req.Header.Add("apikey", string(resMatch))
res, err := client.Do(req)
if err == nil {
defer res.Body.Close()
if res.StatusCode >= 200 && res.StatusCode < 300 {
s1.Verified = true
} else {
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
continue
}
+7 -17
View File
@@ -1,10 +1,10 @@
package coinapi
import (
"bytes"
"context"
"net/http"
"regexp"
"strings"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
@@ -13,37 +13,30 @@ import (
type Scanner struct{}
// Ensure the Scanner satisfies the interface at compile time.
var _ detectors.Detector = (*Scanner)(nil)
var (
client = common.SaneHttpClient()
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"coinapi"}) + `\b([A-Z0-9-]{36})\b`)
)
// Keywords are used for efficiently pre-filtering chunks.
// Use identifiers in the secret preferably, or the provider name.
func (s Scanner) Keywords() []string {
return []string{"coinapi"}
func (s Scanner) Keywords() [][]byte {
return [][]byte{[]byte("coinapi")}
}
// FromData will find and optionally verify CoinApi secrets in a given set of bytes.
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
dataStr := string(data)
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
matches := keyPat.FindAllSubmatch(data, -1)
for _, match := range matches {
if len(match) != 2 {
continue
}
resMatch := strings.TrimSpace(match[1])
resMatch := bytes.TrimSpace(match[1])
s1 := detectors.Result{
DetectorType: detectorspb.DetectorType_CoinApi,
Raw: []byte(resMatch),
Raw: resMatch,
}
if verify {
@@ -51,24 +44,21 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
if err != nil {
continue
}
req.Header.Add("X-CoinAPI-Key", resMatch)
req.Header.Add("X-CoinAPI-Key", string(resMatch))
res, err := client.Do(req)
if err == nil {
defer res.Body.Close()
if res.StatusCode >= 200 && res.StatusCode < 300 {
s1.Verified = true
} else {
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
continue
}
}
}
}
results = append(results, s1)
}
return results, nil
}
+8 -11
View File
@@ -1,11 +1,11 @@
package coinbase
import (
"bytes"
"context"
"fmt"
"net/http"
"regexp"
"strings"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
@@ -21,30 +21,28 @@ var (
client = common.SaneHttpClient()
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"coinbase"}) + `\b([a-zA-Z-0-9]{64})\b`)
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"coinbase"}) + `\b([a-zA-Z0-9]{64})\b`)
)
// Keywords are used for efficiently pre-filtering chunks.
// Use identifiers in the secret preferably, or the provider name.
func (s Scanner) Keywords() []string {
return []string{"coinbase"}
func (s Scanner) Keywords() [][]byte {
return [][]byte{[]byte("coinbase")}
}
// FromData will find and optionally verify Coinbase secrets in a given set of bytes.
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
dataStr := string(data)
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
matches := keyPat.FindAllSubmatch(data, -1)
for _, match := range matches {
if len(match) != 2 {
continue
}
resMatch := strings.TrimSpace(match[1])
resMatch := bytes.TrimSpace(match[1])
s1 := detectors.Result{
DetectorType: detectorspb.DetectorType_Coinbase,
Raw: []byte(resMatch),
Raw: resMatch,
}
if verify {
@@ -52,7 +50,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
if err != nil {
continue
}
req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", resMatch))
req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", string(resMatch)))
res, err := client.Do(req)
if err == nil {
defer res.Body.Close()
@@ -66,7 +64,6 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
}
}
}
results = append(results, s1)
}
+30 -38
View File
@@ -1,12 +1,12 @@
package coinlayer
import (
"bytes"
"context"
"fmt"
"io"
"net/http"
"regexp"
"strings"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
@@ -15,70 +15,62 @@ import (
type Scanner struct{}
// Ensure the Scanner satisfies the interface at compile time.
var _ detectors.Detector = (*Scanner)(nil)
var (
client = common.SaneHttpClient()
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"coinlayer"}) + `\b([a-z0-9]{32})\b`)
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"coinlayer"}) + `\b([0-9a-f]{32})\b`)
)
// Keywords are used for efficiently pre-filtering chunks.
// Use identifiers in the secret preferably, or the provider name.
func (s Scanner) Keywords() []string {
return []string{"coinlayer"}
func (s Scanner) Keywords() [][]byte {
return [][]byte{[]byte("coinlayer")}
}
// FromData will find and optionally verify Coinlayer secrets in a given set of bytes.
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
dataStr := string(data)
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) ([]detectors.Result, error) {
matches := keyPat.FindAllSubmatch(data, -1)
var results []detectors.Result
for _, match := range matches {
if len(match) != 2 {
continue
}
resMatch := strings.TrimSpace(match[1])
resMatch := bytes.TrimSpace(match[1])
s1 := detectors.Result{
DetectorType: detectorspb.DetectorType_Coinlayer,
Raw: []byte(resMatch),
Raw: resMatch,
}
if verify {
req, err := http.NewRequestWithContext(ctx, "GET", fmt.Sprintf("https://api.coinlayer.com/api/livelive?access_key=%s", resMatch), nil)
url := fmt.Sprintf("https://api.coinlayer.com/api/live?access_key=%s", string(resMatch))
req, err := http.NewRequestWithContext(ctx, "GET", url, nil)
if err != nil {
continue
}
res, err := client.Do(req)
if err == nil {
bodyBytes, err := io.ReadAll(res.Body)
if err == nil {
bodyString := string(bodyBytes)
validResponse := strings.Contains(bodyString, `"success": true`) || strings.Contains(bodyString, `"info":"Access Restricted - Your current Subscription Plan does not support HTTPS Encryption."`)
defer res.Body.Close()
if res.StatusCode >= 200 && res.StatusCode < 300 {
if validResponse {
s1.Verified = true
} else {
s1.Verified = false
}
} else {
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
continue
}
}
resp, err := client.Do(req)
if err != nil {
continue
}
defer resp.Body.Close()
if resp.StatusCode >= 200 && resp.StatusCode < 300 {
v1 := []byte(`"success": true`)
v2 := []byte(`"info":"Access Restricted - Your current Subscription Plan does not support HTTPS Encryption."`)
bodyBytes, err := io.ReadAll(resp.Body)
if err != nil || (bytes.Contains(bodyBytes, v1) || bytes.Contains(bodyBytes, v2)) {
s1.Verified = true
}
}
}
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
continue
}
}
results = append(results, s1)
}
return results, nil
}
+7 -10
View File
@@ -1,11 +1,11 @@
package coinlib
import (
"bytes"
"context"
"fmt"
"net/http"
"regexp"
"strings"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
@@ -20,31 +20,29 @@ var _ detectors.Detector = (*Scanner)(nil)
var (
client = common.SaneHttpClient()
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
// Keeping the regexp patterns unchanged
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"coinlib"}) + `\b([a-z0-9]{16})\b`)
)
// Keywords are used for efficiently pre-filtering chunks.
// Use identifiers in the secret preferably, or the provider name.
func (s Scanner) Keywords() []string {
return []string{"coinlib"}
func (s Scanner) Keywords() [][]byte {
return [][]byte{[]byte("coinlib")}
}
// FromData will find and optionally verify Coinlib secrets in a given set of bytes.
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
dataStr := string(data)
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
matches := keyPat.FindAllSubmatch(data, -1)
for _, match := range matches {
if len(match) != 2 {
continue
}
resMatch := strings.TrimSpace(match[1])
resMatch := bytes.TrimSpace(match[1])
s1 := detectors.Result{
DetectorType: detectorspb.DetectorType_Coinlib,
Raw: []byte(resMatch),
Raw: resMatch,
}
if verify {
@@ -58,7 +56,6 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
if res.StatusCode >= 200 && res.StatusCode < 300 {
s1.Verified = true
} else {
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
continue
}
+7 -15
View File
@@ -1,10 +1,10 @@
package coinmarketcap
import (
"bytes"
"context"
"net/http"
"regexp"
"strings"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
@@ -13,37 +13,30 @@ import (
type Scanner struct{}
// Ensure the Scanner satisfies the interface at compile time.
var _ detectors.Detector = (*Scanner)(nil)
var (
client = common.SaneHttpClient()
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"coinmarketcap", "CMC"}) + `\b([0-9Aa-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})\b`)
)
// Keywords are used for efficiently pre-filtering chunks.
// Use identifiers in the secret preferably, or the provider name.
func (s Scanner) Keywords() []string {
return []string{"coinmarketcap", "CMC"}
func (s Scanner) Keywords() [][]byte {
return [][]byte{[]byte("coinmarketcap"), []byte("CMC")}
}
// FromData will find and optionally verify Coinmarketcap secrets in a given set of bytes.
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
dataStr := string(data)
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
matches := keyPat.FindAllSubmatch(data, -1)
for _, match := range matches {
if len(match) != 2 {
continue
}
resMatch := strings.TrimSpace(match[1])
resMatch := bytes.TrimSpace(match[1])
s1 := detectors.Result{
DetectorType: detectorspb.DetectorType_CoinMarketCap,
Raw: []byte(resMatch),
Raw: resMatch,
}
if verify {
@@ -52,14 +45,13 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
continue
}
req.Header.Add("Accept", "application/json")
req.Header.Add("X-CMC_PRO_API_KEY", resMatch)
req.Header.Add("X-CMC_PRO_API_KEY", string(resMatch))
res, err := client.Do(req)
if err == nil {
defer res.Body.Close()
if res.StatusCode >= 200 && res.StatusCode < 300 {
s1.Verified = true
} else {
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
continue
}
+7 -15
View File
@@ -1,11 +1,11 @@
package collect2
import (
"bytes"
"context"
"fmt"
"net/http"
"regexp"
"strings"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
@@ -14,41 +14,34 @@ import (
type Scanner struct{}
// Ensure the Scanner satisfies the interface at compile time
var _ detectors.Detector = (*Scanner)(nil)
var (
client = common.SaneHttpClient()
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"collect2"}) + `\b([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})\b`)
)
// Keywords are used for efficiently pre-filtering chunks.
// Use identifiers in the secret preferably, or the provider name.
func (s Scanner) Keywords() []string {
return []string{"collect2"}
func (s Scanner) Keywords() [][]byte {
return [][]byte{[]byte("collect2")}
}
// FromData will find and optionally verify Collect2 secrets in a given set of bytes.
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
dataStr := string(data)
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
matches := keyPat.FindAllSubmatch(data, -1)
for _, match := range matches {
if len(match) != 2 {
continue
}
resMatch := strings.TrimSpace(match[1])
resMatch := bytes.TrimSpace(match[1])
s1 := detectors.Result{
DetectorType: detectorspb.DetectorType_Collect2,
Raw: []byte(resMatch),
Raw: resMatch,
}
if verify {
req, err := http.NewRequestWithContext(ctx, "GET", fmt.Sprintf("https://collect2.com/api/%s/datarecord/", resMatch), nil)
req, err := http.NewRequestWithContext(ctx, "GET", fmt.Sprintf("https://collect2.com/api/%s/datarecord/", string(resMatch)), nil)
if err != nil {
continue
}
@@ -59,7 +52,6 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
if res.StatusCode >= 200 && res.StatusCode < 300 {
s1.Verified = true
} else {
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
continue
}
+10 -24
View File
@@ -1,13 +1,11 @@
package column
import (
"bytes"
"context"
"encoding/base64"
"fmt"
"log"
"net/http"
"regexp"
"strings"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
@@ -16,38 +14,29 @@ import (
type Scanner struct{}
// Ensure the Scanner satisfies the interface at compile time.
var _ detectors.Detector = (*Scanner)(nil)
var (
client = common.SaneHttpClient()
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"column"}) + `\b((?:test|live)_[a-zA-Z0-9]{27})\b`)
)
// Keywords are used for efficiently pre-filtering chunks.
// Use identifiers in the secret preferably, or the provider name.
func (s Scanner) Keywords() []string {
return []string{"column"}
func (s Scanner) Keywords() [][]byte {
return [][]byte{[]byte("column")}
}
// FromData will find and optionally verify Column secrets in a given set of bytes.
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
dataStr := string(data)
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
matches := keyPat.FindAllSubmatch(data, -1)
for _, match := range matches {
if len(match) != 2 {
continue
}
resMatch := strings.TrimSpace(match[1])
log.Println("resmatch: " + resMatch)
resMatch := bytes.TrimSpace(match[1])
s1 := detectors.Result{
DetectorType: detectorspb.DetectorType_Column,
Raw: []byte(resMatch),
Raw: resMatch,
}
if verify {
@@ -55,19 +44,16 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
if err != nil {
continue
}
// req.SetBasicAuth(resMatch, "")
req.Header.Add("Authorization", fmt.Sprintf("Basic %s", base64.StdEncoding.EncodeToString([]byte(resMatch))))
req.Header.Add("Authorization", "Basic "+base64.StdEncoding.EncodeToString(resMatch))
res, err := client.Do(req)
if err == nil {
defer res.Body.Close()
if res.StatusCode >= 200 && res.StatusCode < 300 {
s1.Verified = true
} else {
s1.Verified = false
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
// if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
// continue
// }
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
continue
}
}
}
}
+7 -14
View File
@@ -1,10 +1,10 @@
package commercejs
import (
"bytes"
"context"
"net/http"
"regexp"
"strings"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
@@ -19,31 +19,25 @@ var _ detectors.Detector = (*Scanner)(nil)
var (
client = common.SaneHttpClient()
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"commercejs"}) + `\b([a-z0-9_]{48})\b`)
)
// Keywords are used for efficiently pre-filtering chunks.
// Use identifiers in the secret preferably, or the provider name.
func (s Scanner) Keywords() []string {
return []string{"commercejs"}
func (s Scanner) Keywords() [][]byte {
return [][]byte{[]byte("commercejs")}
}
// FromData will find and optionally verify CommerceJS secrets in a given set of bytes.
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
dataStr := string(data)
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
matches := keyPat.FindAllSubmatch(data, -1)
for _, match := range matches {
if len(match) != 2 {
continue
}
resMatch := strings.TrimSpace(match[1])
resMatch := bytes.TrimSpace(match[1])
s1 := detectors.Result{
DetectorType: detectorspb.DetectorType_CommerceJS,
Raw: []byte(resMatch),
Raw: resMatch,
}
if verify {
@@ -51,14 +45,13 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
if err != nil {
continue
}
req.Header.Add("X-Authorization", resMatch)
req.Header.Add("X-Authorization", string(resMatch))
res, err := client.Do(req)
if err == nil {
defer res.Body.Close()
if res.StatusCode >= 200 && res.StatusCode < 300 {
s1.Verified = true
} else {
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
continue
}
+8 -17
View File
@@ -1,11 +1,11 @@
package commodities
import (
"bytes"
"context"
"io"
"net/http"
"regexp"
"strings"
"time"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
@@ -15,42 +15,35 @@ import (
type Scanner struct{}
// Ensure the Scanner satisfies the interface at compile time.
var _ detectors.Detector = (*Scanner)(nil)
var (
client = common.SaneHttpClient()
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"commodities"}) + `\b([a-zA-Z0-9]{60})\b`)
)
// Keywords are used for efficiently pre-filtering chunks.
// Use identifiers in the secret preferably, or the provider name.
func (s Scanner) Keywords() []string {
return []string{"commodities"}
func (s Scanner) Keywords() [][]byte {
return [][]byte{[]byte("commodities")}
}
// FromData will find and optionally verify Commodities secrets in a given set of bytes.
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
dataStr := string(data)
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
matches := keyPat.FindAllSubmatch(data, -1)
for _, match := range matches {
if len(match) != 2 {
continue
}
resMatch := strings.TrimSpace(match[1])
resMatch := bytes.TrimSpace(match[1])
s1 := detectors.Result{
DetectorType: detectorspb.DetectorType_Commodities,
Raw: []byte(resMatch),
Raw: resMatch,
}
if verify {
client.Timeout = 5 * time.Second
req, err := http.NewRequestWithContext(ctx, "GET", "https://commodities-api.com/api/latest?access_key="+resMatch, nil)
req, err := http.NewRequestWithContext(ctx, "GET", string(bytes.Split(resMatch, []byte("="))[0]), nil)
if err != nil {
continue
}
@@ -60,8 +53,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
if err != nil {
continue
}
bodyString := string(bodyBytes)
validResponse := strings.Contains(bodyString, `"success":true`)
validResponse := bytes.Contains(bodyBytes, []byte(`"success":true`))
defer res.Body.Close()
if res.StatusCode >= 200 && res.StatusCode < 300 {
if validResponse {
@@ -70,7 +62,6 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
s1.Verified = false
}
} else {
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
continue
}
+18 -26
View File
@@ -1,11 +1,10 @@
package companyhub
import (
"bytes"
"context"
"fmt"
"net/http"
"regexp"
"strings"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
@@ -14,46 +13,40 @@ import (
type Scanner struct{}
// Ensure the Scanner satisfies the interface at compile time.
var _ detectors.Detector = (*Scanner)(nil)
var (
client = common.SaneHttpClient()
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"companyhub"}) + `\b([0-9a-zA-Z]{20})\b`)
idPat = regexp.MustCompile(detectors.PrefixRegex([]string{"companyhub"}) + `\b([a-zA-Z0-9$%^=-]{4,32})\b`)
)
// Keywords are used for efficiently pre-filtering chunks.
// Use identifiers in the secret preferably, or the provider name.
func (s Scanner) Keywords() []string {
return []string{"companyhub"}
func (s Scanner) Keywords() [][]byte {
return [][]byte{[]byte("companyhub")}
}
// FromData will find and optionally verify CompanyHub secrets in a given set of bytes.
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
dataStr := string(data)
keyMatches := keyPat.FindAllSubmatch(data, -1)
idMatches := idPat.FindAllSubmatch(data, -1)
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
idmatches := idPat.FindAllStringSubmatch(dataStr, -1)
for _, match := range matches {
if len(match) != 2 {
for _, keyMatch := range keyMatches {
if len(keyMatch) != 2 {
continue
}
resMatch := strings.TrimSpace(match[1])
for _, idmatch := range idmatches {
if len(idmatch) != 2 {
resKeyMatch := bytes.TrimSpace(keyMatch[1])
for _, idMatch := range idMatches {
if len(idMatch) != 2 {
continue
}
resIdMatch := strings.TrimSpace(idmatch[1])
resIdMatch := bytes.TrimSpace(idMatch[1])
s1 := detectors.Result{
DetectorType: detectorspb.DetectorType_CompanyHub,
Raw: []byte(resMatch),
RawV2: []byte(resMatch + resIdMatch),
Raw: resKeyMatch,
RawV2: append(resKeyMatch, resIdMatch...),
}
if verify {
@@ -61,16 +54,16 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
if err != nil {
continue
}
req.Header.Add("Authorization", fmt.Sprintf("%s %s", resIdMatch, resMatch))
req.Header.Add("Authorization", string(bytes.Join([][]byte{resIdMatch, resKeyMatch}, []byte(" "))))
req.Header.Add("Content-Type", "application/json")
res, err := client.Do(req)
if err == nil {
defer res.Body.Close()
if res.StatusCode >= 200 && res.StatusCode < 300 {
s1.Verified = true
} else {
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
if detectors.IsKnownFalsePositive(resKeyMatch, detectors.DefaultFalsePositives, true) {
continue
}
}
@@ -79,7 +72,6 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
results = append(results, s1)
}
}
return results, nil
+15 -12
View File
@@ -1,12 +1,12 @@
package confluent
import (
"bytes"
"context"
b64 "encoding/base64"
"fmt"
"net/http"
"regexp"
"strings"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
@@ -28,44 +28,47 @@ var (
// Keywords are used for efficiently pre-filtering chunks.
// Use identifiers in the secret preferably, or the provider name.
func (s Scanner) Keywords() []string {
return []string{"confluent"}
func (s Scanner) Keywords() [][]byte {
return [][]byte{[]byte("confluent")}
}
// FromData will find and optionally verify Confluent secrets in a given set of bytes.
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
dataStr := string(data)
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
secretMatches := secretPat.FindAllStringSubmatch(dataStr, -1)
matches := keyPat.FindAllSubmatch(data, -1)
secretMatches := secretPat.FindAllSubmatch(data, -1)
for _, match := range matches {
if len(match) != 2 {
continue
}
resMatch := strings.TrimSpace(match[1])
resMatch := bytes.TrimSpace(match[1])
for _, secret := range secretMatches {
if len(secret) != 2 {
continue
}
resSecret := strings.TrimSpace(secret[1])
resSecret := bytes.TrimSpace(secret[1])
s1 := detectors.Result{
DetectorType: detectorspb.DetectorType_Confluent,
Raw: []byte(resMatch),
RawV2: []byte(resMatch + resSecret),
Raw: resMatch,
RawV2: bytes.Join([][]byte{resMatch, resSecret}, []byte{}),
}
if verify {
data := fmt.Sprintf("%s:%s", resMatch, resSecret)
data := fmt.Sprintf("%s:%s", string(resMatch), string(resSecret))
sEnc := b64.StdEncoding.EncodeToString([]byte(data))
req, err := http.NewRequestWithContext(ctx, "GET", "https://api.confluent.cloud/iam/v2/api-keys/"+resMatch, nil)
req, err := http.NewRequestWithContext(ctx, "GET", "https://api.confluent.cloud/iam/v2/api-keys/"+string(resMatch), nil)
if err != nil {
continue
}
req.Header.Add("Authorization", fmt.Sprintf("Basic %s", sEnc))
res, err := client.Do(req)
if err == nil {
defer res.Body.Close()
if res.StatusCode >= 200 && res.StatusCode < 300 {
@@ -1,11 +1,11 @@
package contentfulpersonalaccesstoken
import (
"bytes"
"context"
"fmt"
"net/http"
"regexp"
"strings"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
@@ -22,27 +22,22 @@ var (
keyPat = regexp.MustCompile(`\b(CFPAT-[a-zA-Z0-9_\-]{43})\b`)
)
// Keywords are used for efficiently pre-filtering chunks.
// Use identifiers in the secret preferably, or the provider name.
func (s Scanner) Keywords() []string {
return []string{"CFPAT-"}
func (s Scanner) Keywords() [][]byte {
return [][]byte{[]byte("CFPAT-")}
}
// FromData will find and optionally verify ContentfulDelivery secrets in a given set of bytes.
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
dataStr := string(data)
keyMatches := keyPat.FindAllStringSubmatch(dataStr, -1)
keyMatches := keyPat.FindAllSubmatch(data, -1)
for _, match := range keyMatches {
if len(match) != 2 {
continue
}
keyRes := strings.TrimSpace(match[1])
keyRes := bytes.TrimSpace(match[1])
s1 := detectors.Result{
DetectorType: detectorspb.DetectorType_ContentfulPersonalAccessToken,
Raw: []byte(keyRes),
Raw: keyRes,
}
if verify {
@@ -50,7 +45,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
if err != nil {
continue
}
req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", keyRes))
req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", string(keyRes)))
res, err := client.Do(req)
if err == nil {
defer res.Body.Close()
@@ -65,7 +60,6 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
}
results = append(results, s1)
}
return results, nil
@@ -1,11 +1,11 @@
package conversiontools
import (
"bytes"
"context"
"fmt"
"net/http"
"regexp"
"strings"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
@@ -14,41 +14,33 @@ import (
type Scanner struct{}
// Ensure the Scanner satisfies the interface at compile time.
var _ detectors.Detector = (*Scanner)(nil)
var (
client = common.SaneHttpClient()
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"conversiontools"}) + `\b(ey[a-zA-Z0-9_.]{157,165})\b`)
)
// Keywords are used for efficiently pre-filtering chunks.
// Use identifiers in the secret preferably, or the provider name.
func (s Scanner) Keywords() []string {
return []string{"conversiontools"}
func (s Scanner) Keywords() [][]byte {
return [][]byte{[]byte("conversiontools")}
}
// FromData will find and optionally verify ConversionTools secrets in a given set of bytes.
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
dataStr := string(data)
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
matches := keyPat.FindAllSubmatch(data, -1)
for _, match := range matches {
if len(match) != 2 {
continue
}
resMatch := strings.TrimSpace(match[1])
resMatch := bytes.TrimSpace(match[1])
s1 := detectors.Result{
DetectorType: detectorspb.DetectorType_ConversionTools,
Raw: []byte(resMatch),
Raw: resMatch,
}
if verify {
payload := strings.NewReader(`{ "type": "convert.website_to_jpg", "options": { "url": "http://google.com", "images": "yes" }}`)
payload := bytes.NewBuffer([]byte(`{ "type": "convert.website_to_jpg", "options": { "url": "http://google.com", "images": "yes" }}`))
req, err := http.NewRequestWithContext(ctx, "POST", "https://api.conversiontools.io/v1/tasks", payload)
if err != nil {
continue
@@ -61,7 +53,6 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
if res.StatusCode >= 200 && res.StatusCode < 300 {
s1.Verified = true
} else {
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
continue
}
+4 -20
View File
@@ -13,18 +13,13 @@ import (
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
)
type Scanner struct {
client *http.Client
}
type Scanner struct{}
// Ensure the Scanner satisfies the interface at compile time.
var _ detectors.Detector = (*Scanner)(nil)
var _ detectors.Versioner = (*Scanner)(nil)
func (Scanner) Version() int { return 1 }
var (
defaultClient = common.SaneHttpClient()
client = common.SaneHttpClient()
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
tokenPat = regexp.MustCompile(detectors.PrefixRegex([]string{"jira"}) + `\b([a-zA-Z-0-9]{24})\b`)
@@ -75,11 +70,6 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
}
if verify {
client := s.client
if client == nil {
client = defaultClient
}
data := fmt.Sprintf("%s:%s", resEmail, resToken)
sEnc := b64.StdEncoding.EncodeToString([]byte(data))
req, err := http.NewRequestWithContext(ctx, "GET", "https://"+resDomain+"/rest/api/3/dashboard", nil)
@@ -95,15 +85,9 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
// If the request is successful and the login reason is not failed authentication, then the token is valid.
// This is because Jira returns a 200 status code even if the token is invalid.
// Jira returns a default dashboard page.
if res.StatusCode >= 200 && res.StatusCode < 300 {
if res.Header.Get(loginReasonHeaderKey) != failedAuth {
s1.Verified = true
}
} else {
s1.VerificationError = fmt.Errorf("unexpected HTTP response status %d", res.StatusCode)
if (res.StatusCode >= 200 && res.StatusCode < 300) && res.Header.Get(loginReasonHeaderKey) != failedAuth {
s1.Verified = true
}
} else {
s1.VerificationError = err
}
}
+20 -60
View File
@@ -9,8 +9,7 @@ import (
"testing"
"time"
"github.com/google/go-cmp/cmp"
"github.com/google/go-cmp/cmp/cmpopts"
"github.com/kylelemons/godebug/pretty"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
@@ -20,12 +19,12 @@ import (
func TestJiraToken_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")
testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
secret := testSecrets.MustGetField("JIRA_TOKEN")
inactiveSecret := testSecrets.MustGetField("JIRA_INACTIVE")
token := testSecrets.MustGetField("JIRA_TOKEN")
inactiveToken := testSecrets.MustGetField("JIRA_INACTIVE")
email := testSecrets.MustGetField("JIRA_EMAIL")
domain := testSecrets.MustGetField("JIRA_DOMAIN")
@@ -35,19 +34,18 @@ func TestJiraToken_FromChunk(t *testing.T) {
verify bool
}
tests := []struct {
name string
s Scanner
args args
want []detectors.Result
wantErr bool
wantVerificationErr bool
name string
s Scanner
args args
want []detectors.Result
wantErr bool
}{
{
name: "found, verified",
s: Scanner{},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a jira secret %s within jira %s with jira %s", secret, email, domain)),
data: []byte(fmt.Sprintf("You can find a jira secret %s within jira %s with jira %s", token, email, domain)),
verify: true,
},
want: []detectors.Result{
@@ -56,15 +54,14 @@ func TestJiraToken_FromChunk(t *testing.T) {
Verified: true,
},
},
wantErr: false,
wantVerificationErr: false,
wantErr: false,
},
{
name: "found, unverified",
s: Scanner{},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a jira secret %s within jira %s but not jira %s valid", inactiveSecret, email, domain)), // the secret would satisfy the regex but not pass validation
data: []byte(fmt.Sprintf("You can find a jira secret %s within jira %s but not jira %s valid", inactiveToken, email, domain)), // the secret would satisfy the regex but not pass validation
verify: true,
},
want: []detectors.Result{
@@ -73,8 +70,7 @@ func TestJiraToken_FromChunk(t *testing.T) {
Verified: false,
},
},
wantErr: false,
wantVerificationErr: false,
wantErr: false,
},
{
name: "not found",
@@ -84,48 +80,14 @@ func TestJiraToken_FromChunk(t *testing.T) {
data: []byte("You cannot find the secret within"),
verify: true,
},
want: nil,
wantErr: false,
wantVerificationErr: false,
},
{
name: "found, would be verified if not for timeout",
s: Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a jira secret %s within jira %s with jira %s", secret, email, domain)),
verify: true,
},
want: []detectors.Result{
{
DetectorType: detectorspb.DetectorType_JiraToken,
Verified: false,
},
},
wantErr: false,
wantVerificationErr: true,
},
{
name: "found, verified but unexpected api surface",
s: Scanner{client: common.ConstantResponseHttpClient(404, "")},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a jira secret %s within jira %s with jira %s", secret, email, domain)),
verify: true,
},
want: []detectors.Result{
{
DetectorType: detectorspb.DetectorType_JiraToken,
Verified: false,
},
},
wantErr: false,
wantVerificationErr: true,
want: nil,
wantErr: false,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
s := Scanner{}
got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
if (err != nil) != tt.wantErr {
t.Errorf("JiraToken.FromData() error = %v, wantErr %v", err, tt.wantErr)
return
@@ -134,12 +96,10 @@ func TestJiraToken_FromChunk(t *testing.T) {
if len(got[i].Raw) == 0 {
t.Fatalf("no raw secret present: \n %+v", got[i])
}
if (got[i].VerificationError != nil) != tt.wantVerificationErr {
t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError)
}
got[i].Raw = nil
got[i].RawV2 = nil
}
ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "RawV2", "VerificationError")
if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {
if diff := pretty.Compare(got, tt.want); diff != "" {
t.Errorf("JiraToken.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
}
})
-129
View File
@@ -1,129 +0,0 @@
package jiratoken_v2
import (
"context"
b64 "encoding/base64"
"fmt"
"net/http"
"regexp"
"strings"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
)
type Scanner struct {
client *http.Client
}
// Ensure the Scanner satisfies the interface at compile time.
var _ detectors.Detector = (*Scanner)(nil)
var _ detectors.Versioner = (*Scanner)(nil)
func (Scanner) Version() int { return 2 }
var (
defaultClient = common.SaneHttpClient()
// https://support.atlassian.com/atlassian-account/docs/manage-api-tokens-for-your-atlassian-account/
// Tokens created after Jan 18 2023 use a variable length
tokenPat = regexp.MustCompile(detectors.PrefixRegex([]string{"jira"}) + `\b([A-Za-z0-9+/=_-]+=[A-Za-z0-9]{8})\b`)
domainPat = regexp.MustCompile(detectors.PrefixRegex([]string{"jira"}) + `\b([a-zA-Z-0-9]{5,24}\.[a-zA-Z-0-9]{3,16}\.[a-zA-Z-0-9]{3,16})\b`)
emailPat = regexp.MustCompile(detectors.PrefixRegex([]string{"jira"}) + `\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Z|a-z]{2,}\b`)
)
const (
failedAuth = "AUTHENTICATED_FAILED"
loginReasonHeaderKey = "X-Seraph-LoginReason"
)
// Keywords are used for efficiently pre-filtering chunks.
// Use identifiers in the secret preferably, or the provider name.
func (s Scanner) Keywords() []string {
return []string{"jira"}
}
// FromData will find and optionally verify JiraToken secrets in a given set of bytes.
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
dataStr := string(data)
tokens := tokenPat.FindAllStringSubmatch(dataStr, -1)
domains := domainPat.FindAllStringSubmatch(dataStr, -1)
emails := emailPat.FindAllStringSubmatch(dataStr, -1)
for _, email := range emails {
email = strings.Split(email[0], " ")
if len(email) != 2 {
continue
}
resEmail := strings.TrimSpace(email[1])
for _, token := range tokens {
if len(token) != 2 {
continue
}
resToken := strings.TrimSpace(token[1])
for _, domain := range domains {
if len(domain) != 2 {
continue
}
resDomain := strings.TrimSpace(domain[1])
s1 := detectors.Result{
DetectorType: detectorspb.DetectorType_JiraToken,
Raw: []byte(resToken),
RawV2: []byte(fmt.Sprintf("%s:%s:%s", resEmail, resToken, resDomain)),
}
if verify {
client := s.client
if client == nil {
client = defaultClient
}
data := fmt.Sprintf("%s:%s", resEmail, resToken)
sEnc := b64.StdEncoding.EncodeToString([]byte(data))
req, err := http.NewRequestWithContext(ctx, "GET", "https://"+resDomain+"/rest/api/3/dashboard", nil)
if err != nil {
continue
}
req.Header.Add("Accept", "application/json")
req.Header.Add("Authorization", fmt.Sprintf("Basic %s", sEnc))
res, err := client.Do(req)
if err == nil {
defer res.Body.Close()
// If the request is successful and the login reason is not failed authentication, then the token is valid.
// This is because Jira returns a 200 status code even if the token is invalid.
// Jira returns a default dashboard page.
if res.StatusCode >= 200 && res.StatusCode < 300 {
if res.Header.Get(loginReasonHeaderKey) != failedAuth {
s1.Verified = true
}
} else {
s1.VerificationError = fmt.Errorf("unexpected HTTP response status %d", res.StatusCode)
}
} else {
s1.VerificationError = err
}
}
if !s1.Verified {
if detectors.IsKnownFalsePositive(string(s1.Raw), detectors.DefaultFalsePositives, true) {
continue
}
}
results = append(results, s1)
}
}
}
return results, nil
}
func (s Scanner) Type() detectorspb.DetectorType {
return detectorspb.DetectorType_JiraToken
}
@@ -1,163 +0,0 @@
//go:build detectors
// +build detectors
package jiratoken_v2
import (
"context"
"fmt"
"testing"
"time"
"github.com/google/go-cmp/cmp"
"github.com/google/go-cmp/cmp/cmpopts"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
)
func TestJiraToken_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
secret := testSecrets.MustGetField("JIRA_V2_TOKEN")
inactiveSecret := testSecrets.MustGetField("JIRA_V2_INACTIVE")
email := testSecrets.MustGetField("JIRA_V2_EMAIL")
domain := testSecrets.MustGetField("JIRA_V2_DOMAIN")
type args struct {
ctx context.Context
data []byte
verify bool
}
tests := []struct {
name string
s Scanner
args args
want []detectors.Result
wantErr bool
wantVerificationErr bool
}{
{
name: "found, verified",
s: Scanner{},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a jira secret %s within jira %s with jira %s", secret, email, domain)),
verify: true,
},
want: []detectors.Result{
{
DetectorType: detectorspb.DetectorType_JiraToken,
Verified: true,
},
},
wantErr: false,
wantVerificationErr: false,
},
{
name: "found, unverified",
s: Scanner{},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a jira secret %s within jira %s but not jira %s valid", inactiveSecret, email, domain)), // the secret would satisfy the regex but not pass validation
verify: true,
},
want: []detectors.Result{
{
DetectorType: detectorspb.DetectorType_JiraToken,
Verified: false,
},
},
wantErr: false,
wantVerificationErr: false,
},
{
name: "not found",
s: Scanner{},
args: args{
ctx: context.Background(),
data: []byte("You cannot find the secret within"),
verify: true,
},
want: nil,
wantErr: false,
wantVerificationErr: false,
},
{
name: "found, would be verified if not for timeout",
s: Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a jira secret %s within jira %s with jira %s", secret, email, domain)),
verify: true,
},
want: []detectors.Result{
{
DetectorType: detectorspb.DetectorType_JiraToken,
Verified: false,
},
},
wantErr: false,
wantVerificationErr: true,
},
{
name: "found, verified but unexpected api surface",
s: Scanner{client: common.ConstantResponseHttpClient(404, "")},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a jira secret %s within jira %s with jira %s", secret, email, domain)),
verify: true,
},
want: []detectors.Result{
{
DetectorType: detectorspb.DetectorType_JiraToken,
Verified: false,
},
},
wantErr: false,
wantVerificationErr: true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
if (err != nil) != tt.wantErr {
t.Errorf("JiraToken.FromData() error = %v, wantErr %v", err, tt.wantErr)
return
}
for i := range got {
if len(got[i].Raw) == 0 {
t.Fatalf("no raw secret present: \n %+v", got[i])
}
if (got[i].VerificationError != nil) != tt.wantVerificationErr {
t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError)
}
}
ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "RawV2", "VerificationError")
if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {
t.Errorf("JiraToken.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
}
})
}
}
func BenchmarkFromData(benchmark *testing.B) {
ctx := context.Background()
s := Scanner{}
for name, data := range detectors.MustGetBenchmarkData() {
benchmark.Run(name, func(b *testing.B) {
b.ResetTimer()
for n := 0; n < b.N; n++ {
_, err := s.FromData(ctx, false, data)
if err != nil {
b.Fatal(err)
}
}
})
}
}
@@ -21,13 +21,13 @@ var (
client = common.SaneHttpClient()
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"pagerduty", "pd"}) + `\b([a-zA-Z0-9_+-]{20})\b`)
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"pagerduty"}) + `\b([a-z]{1}\+[a-zA-Z]{9}\-[a-z]{2}\-[a-z0-9]{5})\b`)
)
// Keywords are used for efficiently pre-filtering chunks.
// Use identifiers in the secret preferably, or the provider name.
func (s Scanner) Keywords() []string {
return []string{"pagerduty", "pd"}
return []string{"pagerduty"}
}
// FromData will find and optionally verify PagerDutyApiKey secrets in a given set of bytes.
+2
View File
@@ -43,6 +43,8 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
instanceMatches := instancePat.FindAllStringSubmatch(dataStr, -1)
tokenMatches := accessTokenPat.FindAllStringSubmatch(dataStr, -1)
fmt.Printf("instanceMatches: %v\n", instanceMatches)
for _, instance := range instanceMatches {
if len(instance) != 1 {
continue
+6 -33
View File
@@ -2,7 +2,6 @@ package sendbird
import (
"context"
"encoding/json"
"fmt"
"net/http"
"regexp"
@@ -13,25 +12,19 @@ import (
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
)
type Scanner struct {
client *http.Client
}
type Scanner struct{}
// Ensure the Scanner satisfies the interface at compile time.
var _ detectors.Detector = (*Scanner)(nil)
var (
defaultClient = common.SaneHttpClient()
client = common.SaneHttpClient()
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"sendbird"}) + `\b([0-9a-f]{40})\b`)
appIdPat = regexp.MustCompile(detectors.PrefixRegex([]string{"sendbird"}) + `\b([0-9A-F]{8}-[0-9A-F]{4}-[0-9A-F]{4}-[0-9A-F]{4}-[0-9A-F]{12})\b`)
)
type userResp struct {
Code int `json:"code"`
}
// Keywords are used for efficiently pre-filtering chunks.
// Use identifiers in the secret preferably, or the provider name.
func (s Scanner) Keywords() []string {
@@ -68,40 +61,20 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
continue
}
req.Header.Add("Api-Token", resMatch)
client := s.client
if client == nil {
client = defaultClient
}
res, err := client.Do(req)
if err == nil {
defer res.Body.Close()
if res.StatusCode >= 200 && res.StatusCode < 300 {
s1.Verified = true
} else if res.StatusCode == 400 { // Sendbird returns 400 for all errors
var userResp userResp
err := json.NewDecoder(res.Body).Decode(&userResp)
if err != nil {
s1.VerificationError = fmt.Errorf("error decoding json response body: %w", err)
} else if userResp.Code != 400401 {
// https://sendbird.com/docs/chat/platform-api/v3/error-codes
// Sendbird always includes its own error codes with 400 responses
// 400401 (InvalidApiToken) is the only one that indicates a bad token
s1.VerificationError = fmt.Errorf("unexpected response code: %d", userResp.Code)
}
} else {
s1.VerificationError = fmt.Errorf("unexpected HTTP response status %d", res.StatusCode)
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
continue
}
}
} else {
s1.VerificationError = err
}
}
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
if !s1.Verified && detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
continue
}
results = append(results, s1)
}
}
+16 -90
View File
@@ -9,8 +9,8 @@ import (
"testing"
"time"
"github.com/google/go-cmp/cmp"
"github.com/google/go-cmp/cmp/cmpopts"
"github.com/kylelemons/godebug/pretty"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
@@ -19,7 +19,7 @@ import (
func TestSendbird_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")
testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
@@ -33,12 +33,11 @@ func TestSendbird_FromChunk(t *testing.T) {
verify bool
}
tests := []struct {
name string
s Scanner
args args
want []detectors.Result
wantErr bool
wantVerificationErr bool
name string
s Scanner
args args
want []detectors.Result
wantErr bool
}{
{
name: "found, verified",
@@ -54,8 +53,7 @@ func TestSendbird_FromChunk(t *testing.T) {
Verified: true,
},
},
wantErr: false,
wantVerificationErr: false,
wantErr: false,
},
{
name: "found, unverified",
@@ -71,8 +69,7 @@ func TestSendbird_FromChunk(t *testing.T) {
Verified: false,
},
},
wantErr: false,
wantVerificationErr: false,
wantErr: false,
},
{
name: "not found",
@@ -82,82 +79,14 @@ func TestSendbird_FromChunk(t *testing.T) {
data: []byte("You cannot find the secret within"),
verify: true,
},
want: nil,
wantErr: false,
wantVerificationErr: false,
},
{
name: "found, would be verified if not for timeout",
s: Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a sendbird application ID %s with sendbird secret %s within", appId, secret)),
verify: true,
},
want: []detectors.Result{
{
DetectorType: detectorspb.DetectorType_Sendbird,
Verified: false,
},
},
wantErr: false,
wantVerificationErr: true,
},
{
name: "found, verified but unexpected api surface",
s: Scanner{client: common.ConstantResponseHttpClient(404, "")},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a sendbird application ID %s with sendbird secret %s within", appId, secret)),
verify: true,
},
want: []detectors.Result{
{
DetectorType: detectorspb.DetectorType_Sendbird,
Verified: false,
},
},
wantErr: false,
wantVerificationErr: true,
},
{
name: "found, verified but unexpected error code",
s: Scanner{client: common.ConstantResponseHttpClient(400, `{"code":400402}`)},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a sendbird application ID %s with sendbird secret %s within", appId, secret)),
verify: true,
},
want: []detectors.Result{
{
DetectorType: detectorspb.DetectorType_Sendbird,
Verified: false,
},
},
wantErr: false,
wantVerificationErr: true,
},
{
name: "found, verified but failed to decode json",
s: Scanner{client: common.ConstantResponseHttpClient(400, `{`)},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a sendbird application ID %s with sendbird secret %s within", appId, secret)),
verify: true,
},
want: []detectors.Result{
{
DetectorType: detectorspb.DetectorType_Sendbird,
Verified: false,
},
},
wantErr: false,
wantVerificationErr: true,
want: nil,
wantErr: false,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
s := Scanner{}
got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
if (err != nil) != tt.wantErr {
t.Errorf("Sendbird.FromData() error = %v, wantErr %v", err, tt.wantErr)
return
@@ -166,12 +95,9 @@ func TestSendbird_FromChunk(t *testing.T) {
if len(got[i].Raw) == 0 {
t.Fatalf("no raw secret present: \n %+v", got[i])
}
if (got[i].VerificationError != nil) != tt.wantVerificationErr {
t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError)
}
got[i].Raw = nil
}
ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "VerificationError")
if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {
if diff := pretty.Compare(got, tt.want); diff != "" {
t.Errorf("Sendbird.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
}
})
+18 -71
View File
@@ -2,10 +2,7 @@ package sentrytoken
import (
"context"
"encoding/json"
"errors"
"fmt"
"io"
"net/http"
"regexp"
"strings"
@@ -15,20 +12,16 @@ import (
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
)
type Scanner struct {
client *http.Client
}
type Scanner struct{}
// Ensure the Scanner satisfies the interface at compile time.
var _ detectors.Detector = (*Scanner)(nil)
var (
defaultClient = common.SaneHttpClient()
client = common.SaneHttpClient()
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"sentry"}) + `\b([a-f0-9]{64})\b`)
errUnauthorized = fmt.Errorf("token unauthorized")
)
// Keywords are used for efficiently pre-filtering chunks.
@@ -54,78 +47,32 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
}
if verify {
client := s.client
if client == nil {
client = defaultClient
req, err := http.NewRequestWithContext(ctx, "GET", "https://sentry.io/api/0/projects/", nil)
if err != nil {
continue
}
isVerified, verificationErr := verifyToken(ctx, client, resMatch)
switch {
case errors.Is(verificationErr, errUnauthorized):
s1.Verified = false
case isVerified:
s1.Verified = true
default:
s1.VerificationError = verificationErr
req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", resMatch))
// req.Header.Add("Cookie", "sc=g2ceDPcsBwPLGXugHUv0coGBi09TQYKqmtp0u4JYdIvOTFd28yVvn1yiuXgmJJa8; session=.eJwNyE0OwiAQQGGsPwtP4kqoGQa4gUdgZ2CGiaZJCUqjGxOPbnfve7_h29QpHpVSvbw61To9StvEwzre9TkVbkPcr7jN5dPbNl7X1jUt_a4vLpEHgUycQSgFZ0YBRkGxnthxwIyBAgdngcF4YxBssiMjekGv2245_wHyySMG:1m0J5S:FjpRlBVuFx577UQp6GcSGWTq0uk")
res, err := client.Do(req)
if err == nil {
defer res.Body.Close()
if res.StatusCode >= 200 && res.StatusCode < 300 {
s1.Verified = true
} else {
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
continue
}
}
}
}
if !s1.Verified && detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
continue
}
results = append(results, s1)
}
return results, nil
}
type Response []Project
type Project struct {
Organization Organization `json:"organization"`
}
type Organization struct {
ID string `json:"id"`
Name string `json:"name"`
}
func verifyToken(ctx context.Context, client *http.Client, token string) (bool, error) {
req, err := http.NewRequestWithContext(ctx, http.MethodGet, "https://sentry.io/api/0/projects/", nil)
if err != nil {
return false, err
}
req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", token))
res, err := client.Do(req)
if err != nil {
return false, err
}
defer res.Body.Close()
var isVerified bool
switch res.StatusCode {
case http.StatusOK, http.StatusForbidden:
isVerified = true
case http.StatusUnauthorized:
return false, errUnauthorized
default:
return false, fmt.Errorf("unexpected HTTP response status %d", res.StatusCode)
}
bytes, readErr := io.ReadAll(res.Body)
if readErr != nil {
return false, readErr
}
var resp Response
if err = json.Unmarshal(bytes, &resp); err != nil {
return false, err
}
return isVerified, err
}
func (s Scanner) Type() detectorspb.DetectorType {
return detectorspb.DetectorType_SentryToken
}
+13 -85
View File
@@ -9,9 +9,7 @@ import (
"testing"
"time"
"github.com/google/go-cmp/cmp"
"github.com/google/go-cmp/cmp/cmpopts"
"github.com/kylelemons/godebug/pretty"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
@@ -34,12 +32,11 @@ func TestSentryToken_FromChunk(t *testing.T) {
verify bool
}
tests := []struct {
name string
s Scanner
args args
want []detectors.Result
wantErr bool
wantVerificationErr bool
name string
s Scanner
args args
want []detectors.Result
wantErr bool
}{
{
name: "found, verified",
@@ -73,56 +70,6 @@ func TestSentryToken_FromChunk(t *testing.T) {
},
wantErr: false,
},
{
name: "found, would be verified but for timeout",
s: Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a sentry super secret %s within", secret)),
verify: true,
},
want: []detectors.Result{
{
DetectorType: detectorspb.DetectorType_SentryToken,
Verified: false,
},
},
wantErr: false,
wantVerificationErr: true,
},
{
name: "found and valid but unexpected api response",
s: Scanner{client: common.ConstantResponseHttpClient(500, "")},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a sentry super secret %s within", secret)),
verify: true,
},
want: []detectors.Result{
{
DetectorType: detectorspb.DetectorType_SentryToken,
Verified: false,
},
},
wantErr: false,
wantVerificationErr: true,
},
{
name: "found, good key but wrong scope",
s: Scanner{client: common.ConstantResponseHttpClient(403, responseBody403)},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a sentry super secret %s within", secret)),
verify: true,
},
want: []detectors.Result{
{
DetectorType: detectorspb.DetectorType_SentryToken,
Verified: true,
},
},
wantErr: false,
},
{
name: "not found",
s: Scanner{},
@@ -135,46 +82,27 @@ func TestSentryToken_FromChunk(t *testing.T) {
wantErr: false,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
s := Scanner{}
got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
if (err != nil) != tt.wantErr {
t.Errorf("Gitlab.FromData() error = %v, wantErr %v", err, tt.wantErr)
t.Errorf("SentryToken.FromData() error = %v, wantErr %v", err, tt.wantErr)
return
}
for i := range got {
if len(got[i].Raw) == 0 {
t.Fatal("no raw secret present")
}
if (got[i].VerificationError != nil) != tt.wantVerificationErr {
t.Fatalf("wantVerificationError = %v, verification error = %v,", tt.wantVerificationErr, got[i].VerificationError)
t.Fatalf("no raw secret present: \n %+v", got[i])
}
got[i].Raw = nil
}
opts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "VerificationError")
if diff := cmp.Diff(got, tt.want, opts); diff != "" {
t.Errorf("Gitlab.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
if diff := pretty.Compare(got, tt.want); diff != "" {
t.Errorf("SentryToken.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
}
})
}
}
const responseBody403 = `
[
{
"organization": {
"id": "911964",
"slug": "wigslap",
"status": {
"id": "active",
"name": "active"
},
"name": "wigslap"
}
}
]
`
func BenchmarkFromData(benchmark *testing.B) {
ctx := context.Background()
s := Scanner{}
+4 -21
View File
@@ -2,7 +2,6 @@ package slackwebhook
import (
"context"
"fmt"
"io"
"net/http"
"regexp"
@@ -13,15 +12,14 @@ import (
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
)
type Scanner struct {
client *http.Client
}
type Scanner struct{}
// Ensure the Scanner satisfies the interface at compile time.
var _ detectors.Detector = (*Scanner)(nil)
var (
defaultClient = common.SaneHttpClient()
client = common.SaneHttpClient()
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
keyPat = regexp.MustCompile(`(https://hooks\.slack\.com/services/T[A-Z0-9]+/B[A-Z0-9]+/[A-Za-z0-9]{23,25})`)
)
@@ -50,12 +48,6 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
}
if verify {
client := s.client
if client == nil {
client = defaultClient
}
payload := strings.NewReader(`{"text": ""}`)
req, err := http.NewRequestWithContext(ctx, "POST", resMatch, payload)
if err != nil {
@@ -70,18 +62,9 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
continue
}
body := string(bodyBytes)
defer res.Body.Close()
if res.StatusCode >= 200 && res.StatusCode < 300 || (res.StatusCode == 400 && (strings.Contains(body, "no_text") || strings.Contains(body, "missing_text"))) {
if (res.StatusCode >= 200 && res.StatusCode < 300) || (res.StatusCode == 400 && (strings.Contains(body, "no_text") || strings.Contains(body, "missing_text"))) {
s1.Verified = true
} else if res.StatusCode == 401 || res.StatusCode == 403 {
// The secret is determinately not verified (nothing to do)
} else {
s1.VerificationError = fmt.Errorf("unexpected HTTP response status %d", res.StatusCode)
}
} else {
s1.VerificationError = err
}
}
+14 -55
View File
@@ -6,11 +6,10 @@ package slackwebhook
import (
"context"
"fmt"
"github.com/google/go-cmp/cmp"
"github.com/google/go-cmp/cmp/cmpopts"
"testing"
"time"
"github.com/kylelemons/godebug/pretty"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
@@ -33,12 +32,11 @@ func TestSlackWebhook_FromChunk(t *testing.T) {
verify bool
}
tests := []struct {
name string
s Scanner
args args
want []detectors.Result
wantErr bool
wantVerificationErr bool
name string
s Scanner
args args
want []detectors.Result
wantErr bool
}{
{
name: "found, verified",
@@ -54,8 +52,7 @@ func TestSlackWebhook_FromChunk(t *testing.T) {
Verified: true,
},
},
wantErr: false,
wantVerificationErr: false,
wantErr: false,
},
{
name: "found, unverified",
@@ -71,8 +68,7 @@ func TestSlackWebhook_FromChunk(t *testing.T) {
Verified: false,
},
},
wantErr: false,
wantVerificationErr: false,
wantErr: false,
},
{
name: "not found",
@@ -82,48 +78,14 @@ func TestSlackWebhook_FromChunk(t *testing.T) {
data: []byte("You cannot find the secret within"),
verify: true,
},
want: nil,
wantErr: false,
wantVerificationErr: false,
},
{
name: "found, would be verified if not for timeout",
s: Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a slackwebhook secret %s within", secret)),
verify: true,
},
want: []detectors.Result{
{
DetectorType: detectorspb.DetectorType_SlackWebhook,
Verified: false,
},
},
wantErr: false,
wantVerificationErr: true,
},
{
name: "found, verified but unexpected api surface",
s: Scanner{client: common.ConstantResponseHttpClient(404, "")},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a slackwebhook secret %s within", secret)),
verify: true,
},
want: []detectors.Result{
{
DetectorType: detectorspb.DetectorType_SlackWebhook,
Verified: false,
},
},
wantErr: false,
wantVerificationErr: true,
want: nil,
wantErr: false,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
s := Scanner{}
got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
if (err != nil) != tt.wantErr {
t.Errorf("SlackWebhook.FromData() error = %v, wantErr %v", err, tt.wantErr)
return
@@ -132,12 +94,9 @@ func TestSlackWebhook_FromChunk(t *testing.T) {
if len(got[i].Raw) == 0 {
t.Fatalf("no raw secret present: \n %+v", got[i])
}
if (got[i].VerificationError != nil) != tt.wantVerificationErr {
t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError)
}
got[i].Raw = nil
}
ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "VerificationError")
if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {
if diff := pretty.Compare(got, tt.want); diff != "" {
t.Errorf("SlackWebhook.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
}
})
-94
View File
@@ -1,94 +0,0 @@
package sourcegraph
import (
"context"
"fmt"
"net/http"
"regexp"
"strings"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
)
type Scanner struct {
client *http.Client
}
// Ensure the Scanner satisfies the interface at compile time.
var _ detectors.Detector = (*Scanner)(nil)
var (
defaultClient = common.SaneHttpClient()
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
keyPat = regexp.MustCompile(`\b(sgp_[a-f0-9]{40})\b`)
)
// Keywords are used for efficiently pre-filtering chunks.
// Use identifiers in the secret preferably, or the provider name.
func (s Scanner) Keywords() []string {
return []string{"sgp_"}
}
// FromData will find and optionally verify Sourcegraph secrets in a given set of bytes.
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
dataStr := string(data)
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
for _, match := range matches {
if len(match) != 2 {
continue
}
resMatch := strings.TrimSpace(match[1])
s1 := detectors.Result{
DetectorType: detectorspb.DetectorType_Sourcegraph,
Raw: []byte(resMatch),
}
if verify {
client := s.client
if client == nil {
client = defaultClient
}
payload := strings.NewReader("{\"query\": \"query { currentUser { username } }\"}")
req, err := http.NewRequestWithContext(ctx, "POST", "https://sourcegraph.com/.api/graphql", payload)
req.Header.Add("Authorization", "token "+resMatch)
req.Header.Add("Content-Type", "application/json")
if err != nil {
continue
}
res, err := client.Do(req)
if err == nil {
defer res.Body.Close()
if res.StatusCode >= 200 && res.StatusCode < 300 {
s1.Verified = true
} else if res.StatusCode == 401 {
// The secret is determinately not verified (nothing to do)
} else {
s1.VerificationError = fmt.Errorf("unexpected HTTP response status %d", res.StatusCode)
}
} else {
s1.VerificationError = err
}
}
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
if !s1.Verified && detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
continue
}
results = append(results, s1)
}
return results, nil
}
func (s Scanner) Type() detectorspb.DetectorType {
return detectorspb.DetectorType_Sourcegraph
}
@@ -1,161 +0,0 @@
//go:build detectors
// +build detectors
package sourcegraph
import (
"context"
"fmt"
"github.com/google/go-cmp/cmp"
"github.com/google/go-cmp/cmp/cmpopts"
"testing"
"time"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
)
func TestSourcegraph_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors4")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
secret := testSecrets.MustGetField("SOURCEGRAPH")
inactiveSecret := testSecrets.MustGetField("SOURCEGRAPH_INACTIVE")
type args struct {
ctx context.Context
data []byte
verify bool
}
tests := []struct {
name string
s Scanner
args args
want []detectors.Result
wantErr bool
wantVerificationErr bool
}{
{
name: "found, verified",
s: Scanner{},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a sourcegraph secret %s within", secret)),
verify: true,
},
want: []detectors.Result{
{
DetectorType: detectorspb.DetectorType_Sourcegraph,
Verified: true,
},
},
wantErr: false,
wantVerificationErr: false,
},
{
name: "found, unverified",
s: Scanner{},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a sourcegraph secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
verify: true,
},
want: []detectors.Result{
{
DetectorType: detectorspb.DetectorType_Sourcegraph,
Verified: false,
},
},
wantErr: false,
wantVerificationErr: false,
},
{
name: "not found",
s: Scanner{},
args: args{
ctx: context.Background(),
data: []byte("You cannot find the secret within"),
verify: true,
},
want: nil,
wantErr: false,
wantVerificationErr: false,
},
{
name: "found, would be verified if not for timeout",
s: Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a sourcegraph secret %s within", secret)),
verify: true,
},
want: []detectors.Result{
{
DetectorType: detectorspb.DetectorType_Sourcegraph,
Verified: false,
},
},
wantErr: false,
wantVerificationErr: true,
},
{
name: "found, verified but unexpected api surface",
s: Scanner{client: common.ConstantResponseHttpClient(404, "")},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a sourcegraph secret %s within", secret)),
verify: true,
},
want: []detectors.Result{
{
DetectorType: detectorspb.DetectorType_Sourcegraph,
Verified: false,
},
},
wantErr: false,
wantVerificationErr: true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
if (err != nil) != tt.wantErr {
t.Errorf("Sourcegraph.FromData() error = %v, wantErr %v", err, tt.wantErr)
return
}
for i := range got {
if len(got[i].Raw) == 0 {
t.Fatalf("no raw secret present: \n %+v", got[i])
}
if (got[i].VerificationError != nil) != tt.wantVerificationErr {
t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError)
}
}
ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "VerificationError")
if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {
t.Errorf("Sourcegraph.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
}
})
}
}
func BenchmarkFromData(benchmark *testing.B) {
ctx := context.Background()
s := Scanner{}
for name, data := range detectors.MustGetBenchmarkData() {
benchmark.Run(name, func(b *testing.B) {
b.ResetTimer()
for n := 0; n < b.N; n++ {
_, err := s.FromData(ctx, false, data)
if err != nil {
b.Fatal(err)
}
}
})
}
}
+1 -16
View File
@@ -2,7 +2,6 @@ package twilio
import (
"context"
"fmt"
"net/http"
"regexp"
@@ -11,15 +10,12 @@ import (
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
)
type Scanner struct {
client *http.Client
}
type Scanner struct{}
// Ensure the Scanner satisfies the interface at compile time.
var _ detectors.Detector = (*Scanner)(nil)
var (
defaultClient = common.SaneHttpClient()
identifierPat = regexp.MustCompile(`(?i)sid.{0,20}AC[0-9a-f]{32}`) // Should we have this? Seems restrictive.
sidPat = regexp.MustCompile(`\bAC[0-9a-f]{32}\b`)
keyPat = regexp.MustCompile(`\b[0-9a-f]{32}\b`)
@@ -55,11 +51,6 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
}
if verify {
client = s.client
if client == nil {
client = defaultClient
}
req, err := http.NewRequestWithContext(
ctx, "GET", "https://verify.twilio.com/v2/Services", nil)
if err != nil {
@@ -74,13 +65,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
if res.StatusCode >= 200 && res.StatusCode < 300 {
s1.Verified = true
} else if res.StatusCode == 401 || res.StatusCode == 403 {
// The secret is determinately not verified (nothing to do)
} else {
s1.VerificationError = fmt.Errorf("unexpected HTTP response status %d", res.StatusCode)
}
} else {
s1.VerificationError = err
}
}
+13 -55
View File
@@ -6,11 +6,11 @@ package twilio
import (
"context"
"fmt"
"github.com/google/go-cmp/cmp"
"github.com/google/go-cmp/cmp/cmpopts"
"testing"
"time"
"github.com/kylelemons/godebug/pretty"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
@@ -33,12 +33,11 @@ func TestTwilio_FromChunk(t *testing.T) {
verify bool
}
tests := []struct {
name string
s Scanner
args args
want []detectors.Result
wantErr bool
wantVerificationErr bool
name string
s Scanner
args args
want []detectors.Result
wantErr bool
}{
{
name: "found, verified",
@@ -53,7 +52,6 @@ func TestTwilio_FromChunk(t *testing.T) {
DetectorType: detectorspb.DetectorType_Twilio,
Verified: true,
Redacted: id,
RawV2: []byte(id + secret),
},
},
wantErr: false,
@@ -71,7 +69,6 @@ func TestTwilio_FromChunk(t *testing.T) {
DetectorType: detectorspb.DetectorType_Twilio,
Verified: false,
Redacted: id,
RawV2: []byte(id + secretInactive),
},
},
wantErr: false,
@@ -87,62 +84,23 @@ func TestTwilio_FromChunk(t *testing.T) {
want: nil,
wantErr: false,
},
{
name: "found, would be verified if not for timeout",
s: Scanner{client: common.SaneHttpClientTimeOut(100 * time.Microsecond)},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a twillio secret %s within awsId %s", secret, id)),
verify: true,
},
want: []detectors.Result{
{
DetectorType: detectorspb.DetectorType_Twilio,
Verified: false,
Redacted: id,
RawV2: []byte(id + secret),
},
},
wantErr: false,
wantVerificationErr: true,
},
{
name: "found, verified but unexpected api surface",
s: Scanner{client: common.ConstantResponseHttpClient(404, "")},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a twillio secret %s within awsId %s", secret, id)),
verify: true,
},
want: []detectors.Result{
{
DetectorType: detectorspb.DetectorType_Twilio,
Verified: false,
Redacted: id,
RawV2: []byte(id + secret),
},
},
wantErr: false,
wantVerificationErr: true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
s := Scanner{}
got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
if (err != nil) != tt.wantErr {
t.Errorf("Twilio.FromData() error = %v, wantErr %v", err, tt.wantErr)
return
}
for i := range got {
if len(got[i].Raw) == 0 {
t.Fatalf("no raw secret present: \n %+v", got[i])
}
if (got[i].VerificationError != nil) != tt.wantVerificationErr {
t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError)
t.Fatal("no raw secret present")
}
got[i].Raw = nil
got[i].RawV2 = nil
}
ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "VerificationError")
if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {
if diff := pretty.Compare(got, tt.want); diff != "" {
t.Errorf("Twilio.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
}
})
+1 -11
View File
@@ -2,11 +2,6 @@ package engine
import (
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/envoyapikey"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/huggingface"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/salesforce"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/snowflake"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/sourcegraph"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/trufflehogenterprise"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/abbysale"
@@ -292,7 +287,6 @@ import (
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/getresponse"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/getsandbox"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/github"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/github_oauth2"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/github_old"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/githubapp"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/gitlab"
@@ -562,6 +556,7 @@ import (
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/satismeterwritekey"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/saucelabs"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/scalewaykey"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/github_oauth2"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/scalr"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/scrapeowl"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/scraperapi"
@@ -1537,11 +1532,6 @@ func DefaultDetectors() []detectors.Detector {
couchbase.Scanner{},
envoyapikey.Scanner{},
github_oauth2.Scanner{},
snowflake.Scanner{},
huggingface.Scanner{},
trufflehogenterprise.Scanner{},
salesforce.Scanner{},
sourcegraph.Scanner{},
}
}
-50
View File
@@ -1,50 +0,0 @@
package engine
import (
"runtime"
"google.golang.org/protobuf/proto"
"google.golang.org/protobuf/types/known/anypb"
"github.com/trufflesecurity/trufflehog/v3/pkg/context"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
"github.com/trufflesecurity/trufflehog/v3/pkg/sources/trello"
)
// ScanTrello scans Trello boards for secrets.
func (e *Engine) ScanTrello(ctx context.Context, apiKey string) error {
trelloSource := trello.Source{}
boardIDs, err := trelloSource.GetBoardIDs(ctx, apiKey)
if err != nil {
ctx.Logger().Error(err, "failed to get Trello board IDs")
return err
}
connection := &sourcespb.Trello{
Auth: &sourcespb.Trello_TrelloAuth{
ApiKey: apiKey,
},
Boards: boardIDs,
}
var conn anypb.Any
err = anypb.MarshalFrom(&conn, connection, proto.MarshalOptions{})
if err != nil {
ctx.Logger().Error(err, "failed to marshal Trello connection")
return err
}
handle, err := e.sourceManager.Enroll(ctx, "trufflehog - Trello", new(trello.Source).Type(),
func(ctx context.Context, jobID, sourceID int64) (sources.Source, error) {
if err := trelloSource.Init(ctx, "trufflehog - Trello", jobID, sourceID, true, &conn, runtime.NumCPU()); err != nil {
return nil, err
}
return &trelloSource, nil
})
if err != nil {
return err
}
_, err = e.sourceManager.ScheduleRun(ctx, handle)
return err
}
+2 -2
View File
@@ -132,9 +132,9 @@ func GenerateLink(repo, commit, file string, line int64) string {
return baseLink
case providerAzure:
baseLink := repo + "/commit/" + commit + "/" + file
baseLink := repo + "?path=" + file + "&version=GB" + commit
if line > 0 {
baseLink += "?line=" + strconv.FormatInt(line, 10)
baseLink += "&line=" + strconv.FormatInt(line, 10)
}
return baseLink
+2 -2
View File
@@ -148,7 +148,7 @@ func TestGenerateLink(t *testing.T) {
commit: "abcdef",
file: "main.go",
},
want: "https://dev.azure.com/org/project/_git/repo/commit/abcdef/main.go",
want: "https://dev.azure.com/org/project/_git/repo?path=main.go&version=GBabcdef",
},
{
name: "Azure link gen with line",
@@ -158,7 +158,7 @@ func TestGenerateLink(t *testing.T) {
file: "main.go",
line: int64(20),
},
want: "https://dev.azure.com/org/project/_git/repo/commit/abcdef/main.go?line=20",
want: "https://dev.azure.com/org/project/_git/repo?path=main.go&version=GBabcdef&line=20",
},
}
for _, tt := range tests {
+58 -78
View File
@@ -17,23 +17,18 @@ import (
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
logContext "github.com/trufflesecurity/trufflehog/v3/pkg/context"
"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
)
type ctxKey int
const (
depthKey ctxKey = iota
errMaxArchiveDepthReached = "max archive depth reached"
)
var (
maxDepth = 5
maxSize = 250 * 1024 * 1024 // 20MB
maxTimeout = time.Duration(30) * time.Second
defaultBufferSize = 512
)
// Ensure the Archive satisfies the interfaces at compile time.
@@ -67,7 +62,7 @@ func SetArchiveMaxTimeout(timeout time.Duration) {
// FromFile extracts the files from an archive.
func (a *Archive) FromFile(originalCtx context.Context, data io.Reader) chan []byte {
archiveChan := make(chan []byte, defaultBufferSize)
archiveChan := make(chan []byte, 512)
go func() {
ctx, cancel := context.WithTimeout(originalCtx, maxTimeout)
logger := logContext.AddLogger(ctx).Logger()
@@ -84,66 +79,51 @@ func (a *Archive) FromFile(originalCtx context.Context, data io.Reader) chan []b
return archiveChan
}
type decompressorInfo struct {
depth int
reader io.Reader
archiveChan chan []byte
archiver archiver.Decompressor
}
// openArchive takes a reader and extracts the contents up to the maximum depth.
func (a *Archive) openArchive(ctx context.Context, depth int, reader io.Reader, archiveChan chan []byte) error {
if depth >= maxDepth {
return fmt.Errorf(errMaxArchiveDepthReached)
return fmt.Errorf("max archive depth reached")
}
format, arReader, err := archiver.Identify("", reader)
if errors.Is(err, archiver.ErrNoMatch) && depth > 0 {
return a.handleNonArchiveContent(ctx, arReader, archiveChan)
format, reader, err := archiver.Identify("", reader)
if err != nil {
if errors.Is(err, archiver.ErrNoMatch) && depth > 0 {
chunkSize := 10 * 1024
for {
chunk := make([]byte, chunkSize)
n, _ := reader.Read(chunk)
archiveChan <- chunk
if n < chunkSize {
break
}
}
return nil
}
return err
}
switch archive := format.(type) {
case archiver.Decompressor:
info := decompressorInfo{depth: depth, reader: arReader, archiveChan: archiveChan, archiver: archive}
return a.handleDecompressor(ctx, info)
case archiver.Extractor:
return archive.Extract(context.WithValue(ctx, depthKey, depth+1), reader, nil, a.extractorHandler(archiveChan))
default:
return fmt.Errorf("unknown archive type: %s", format.Name())
}
}
func (a *Archive) handleNonArchiveContent(ctx context.Context, reader io.Reader, archiveChan chan []byte) error {
aCtx := logContext.AddLogger(ctx)
chunkReader := sources.NewChunkReader()
chunkResChan := chunkReader(aCtx, reader)
for data := range chunkResChan {
if err := data.Error(); err != nil {
aCtx.Logger().Error(err, "error reading chunk")
continue
}
if err := common.CancellableWrite(ctx, archiveChan, data.Bytes()); err != nil {
compReader, err := archive.OpenReader(reader)
if err != nil {
return err
}
fileBytes, err := a.ReadToMax(ctx, compReader)
if err != nil {
return err
}
newReader := bytes.NewReader(fileBytes)
return a.openArchive(ctx, depth+1, newReader, archiveChan)
case archiver.Extractor:
err := archive.Extract(context.WithValue(ctx, depthKey, depth+1), reader, nil, a.extractorHandler(archiveChan))
if err != nil {
return err
}
return nil
}
return nil
}
func (a *Archive) handleDecompressor(ctx context.Context, info decompressorInfo) error {
compReader, err := info.archiver.OpenReader(info.reader)
if err != nil {
return err
}
fileBytes, err := a.ReadToMax(ctx, compReader)
if err != nil {
return err
}
newReader := bytes.NewReader(fileBytes)
return a.openArchive(ctx, info.depth+1, newReader, info.archiveChan)
return fmt.Errorf("Unknown archive type: %s", format.Name())
}
// IsFiletype returns true if the provided reader is an archive.
func (a *Archive) IsFiletype(_ context.Context, reader io.Reader) (io.Reader, bool) {
func (a *Archive) IsFiletype(ctx context.Context, reader io.Reader) (io.Reader, bool) {
format, readerB, err := archiver.Identify("", reader)
if err != nil {
return readerB, false
@@ -153,9 +133,8 @@ func (a *Archive) IsFiletype(_ context.Context, reader io.Reader) (io.Reader, bo
return readerB, true
case archiver.Decompressor:
return readerB, true
default:
return readerB, false
}
return readerB, false
}
// extractorHandler is applied to each file in an archiver.Extractor file.
@@ -199,33 +178,34 @@ func (a *Archive) ReadToMax(ctx context.Context, reader io.Reader) (data []byte,
if e, ok := r.(error); ok {
err = e
} else {
err = fmt.Errorf("panic occurred: %v", r)
err = fmt.Errorf("Panic occurred: %v", r)
}
logger.Error(err, "Panic occurred when reading archive")
}
}()
if common.IsDone(ctx) {
return nil, ctx.Err()
fileContent := bytes.Buffer{}
logger.V(5).Info("Remaining buffer capacity", "bytes", maxSize-a.size)
for i := 0; i <= maxSize/512; i++ {
if common.IsDone(ctx) {
return nil, ctx.Err()
}
fileChunk := make([]byte, 512)
bRead, err := reader.Read(fileChunk)
if err != nil && !errors.Is(err, io.EOF) {
return []byte{}, err
}
a.size += bRead
if len(fileChunk) > 0 {
fileContent.Write(fileChunk[0:bRead])
}
if bRead < 512 {
return fileContent.Bytes(), nil
}
if a.size >= maxSize && bRead == 512 {
logger.V(2).Info("Max archive size reached.")
return fileContent.Bytes(), nil
}
}
var fileContent bytes.Buffer
// Create a limited reader to ensure we don't read more than the max size.
lr := io.LimitReader(reader, int64(maxSize))
// Using io.CopyBuffer for performance advantages. Though buf is mandatory
// for the method, due to the internal implementation of io.CopyBuffer, when
// *bytes.Buffer implements io.WriterTo or io.ReaderFrom, the provided buf
// is simply ignored. Thus, we can pass nil for the buf parameter.
_, err = io.CopyBuffer(&fileContent, lr, nil)
if err != nil && !errors.Is(err, io.EOF) {
return nil, err
}
if fileContent.Len() == maxSize {
logger.V(2).Info("Max archive size reached.")
}
return fileContent.Bytes(), nil
}
@@ -302,7 +282,7 @@ func (a *Archive) HandleSpecialized(ctx logContext.Context, reader io.Reader) (i
// The caller is responsible for closing the returned reader.
func (a *Archive) extractDebContent(ctx logContext.Context, file io.Reader) (io.ReadCloser, error) {
if a.currentDepth >= maxDepth {
return nil, fmt.Errorf(errMaxArchiveDepthReached)
return nil, fmt.Errorf("max archive depth reached")
}
tmpEnv, err := a.createTempEnv(ctx, file)
-53
View File
@@ -1,7 +1,6 @@
package handlers
import (
"bytes"
"context"
"io"
"net/http"
@@ -127,58 +126,6 @@ func TestHandleFile(t *testing.T) {
assert.Equal(t, 1, len(ch))
}
func TestReadToMax(t *testing.T) {
tests := []struct {
name string
input []byte
expected []byte
}{
{
name: "read full content within maxSize",
input: []byte("abcdefg"),
expected: []byte("abcdefg"),
},
{
name: "read content larger than maxSize",
input: make([]byte, maxSize+10), // this creates a byte slice 10 bytes larger than maxSize
expected: make([]byte, maxSize),
},
{
name: "empty input",
input: []byte(""),
expected: []byte(""),
},
}
a := &Archive{}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
reader := bytes.NewReader(tt.input)
output, err := a.ReadToMax(context.Background(), reader)
assert.Nil(t, err)
assert.Equal(t, tt.expected, output)
})
}
}
func BenchmarkReadToMax(b *testing.B) {
data := bytes.Repeat([]byte("a"), 1024*1000) // 1MB of data.
reader := bytes.NewReader(data)
a := &Archive{}
b.ResetTimer()
for i := 0; i < b.N; i++ {
b.StartTimer()
_, _ = a.ReadToMax(context.Background(), reader)
b.StopTimer()
_, _ = reader.Seek(0, 0) // Reset the reader position.
a.size = 0 // Reset archive size.
}
}
func TestExtractDebContent(t *testing.T) {
// Open the sample .deb file from the testdata folder.
file, err := os.Open("testdata/test.deb")
+1 -1
View File
@@ -133,7 +133,7 @@ func BranchHeads(repo *gogit.Repository) (map[string]*object.Commit, error) {
}
headCommit, err := repo.CommitObject(*headHash)
if err != nil {
logger.Error(err, "unable to get commit", "head_hash", headHash.String())
logger.Error(err, "unable to get commit", "commit", headCommit.String())
return nil
}
branches[branchName] = headCommit
+6 -11
View File
@@ -999,7 +999,6 @@ const (
DetectorType_Salesforce DetectorType = 925
DetectorType_HuggingFace DetectorType = 926
DetectorType_Snowflake DetectorType = 927
DetectorType_Sourcegraph DetectorType = 928
)
// Enum value maps for DetectorType.
@@ -1929,7 +1928,6 @@ var (
925: "Salesforce",
926: "HuggingFace",
927: "Snowflake",
928: "Sourcegraph",
}
DetectorType_value = map[string]int32{
"Alibaba": 0,
@@ -2856,7 +2854,6 @@ var (
"Salesforce": 925,
"HuggingFace": 926,
"Snowflake": 927,
"Sourcegraph": 928,
}
)
@@ -3235,7 +3232,7 @@ var file_detectors_proto_rawDesc = []byte{
0x44, 0x65, 0x63, 0x6f, 0x64, 0x65, 0x72, 0x54, 0x79, 0x70, 0x65, 0x12, 0x0b, 0x0a, 0x07, 0x55,
0x4e, 0x4b, 0x4e, 0x4f, 0x57, 0x4e, 0x10, 0x00, 0x12, 0x09, 0x0a, 0x05, 0x50, 0x4c, 0x41, 0x49,
0x4e, 0x10, 0x01, 0x12, 0x0a, 0x0a, 0x06, 0x42, 0x41, 0x53, 0x45, 0x36, 0x34, 0x10, 0x02, 0x12,
0x09, 0x0a, 0x05, 0x55, 0x54, 0x46, 0x31, 0x36, 0x10, 0x03, 0x2a, 0x9e, 0x74, 0x0a, 0x0c, 0x44,
0x09, 0x0a, 0x05, 0x55, 0x54, 0x46, 0x31, 0x36, 0x10, 0x03, 0x2a, 0x8c, 0x74, 0x0a, 0x0c, 0x44,
0x65, 0x74, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x54, 0x79, 0x70, 0x65, 0x12, 0x0b, 0x0a, 0x07, 0x41,
0x6c, 0x69, 0x62, 0x61, 0x62, 0x61, 0x10, 0x00, 0x12, 0x08, 0x0a, 0x04, 0x41, 0x4d, 0x51, 0x50,
0x10, 0x01, 0x12, 0x07, 0x0a, 0x03, 0x41, 0x57, 0x53, 0x10, 0x02, 0x12, 0x09, 0x0a, 0x05, 0x41,
@@ -4164,13 +4161,11 @@ var file_detectors_proto_rawDesc = []byte{
0x74, 0x68, 0x32, 0x10, 0x9c, 0x07, 0x12, 0x0f, 0x0a, 0x0a, 0x53, 0x61, 0x6c, 0x65, 0x73, 0x66,
0x6f, 0x72, 0x63, 0x65, 0x10, 0x9d, 0x07, 0x12, 0x10, 0x0a, 0x0b, 0x48, 0x75, 0x67, 0x67, 0x69,
0x6e, 0x67, 0x46, 0x61, 0x63, 0x65, 0x10, 0x9e, 0x07, 0x12, 0x0e, 0x0a, 0x09, 0x53, 0x6e, 0x6f,
0x77, 0x66, 0x6c, 0x61, 0x6b, 0x65, 0x10, 0x9f, 0x07, 0x12, 0x10, 0x0a, 0x0b, 0x53, 0x6f, 0x75,
0x72, 0x63, 0x65, 0x67, 0x72, 0x61, 0x70, 0x68, 0x10, 0xa0, 0x07, 0x42, 0x3d, 0x5a, 0x3b, 0x67,
0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x74, 0x72, 0x75, 0x66, 0x66, 0x6c,
0x65, 0x73, 0x65, 0x63, 0x75, 0x72, 0x69, 0x74, 0x79, 0x2f, 0x74, 0x72, 0x75, 0x66, 0x66, 0x6c,
0x65, 0x68, 0x6f, 0x67, 0x2f, 0x76, 0x33, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x70, 0x62, 0x2f, 0x64,
0x65, 0x74, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x73, 0x70, 0x62, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74,
0x6f, 0x33,
0x77, 0x66, 0x6c, 0x61, 0x6b, 0x65, 0x10, 0x9f, 0x07, 0x42, 0x3d, 0x5a, 0x3b, 0x67, 0x69, 0x74,
0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x74, 0x72, 0x75, 0x66, 0x66, 0x6c, 0x65, 0x73,
0x65, 0x63, 0x75, 0x72, 0x69, 0x74, 0x79, 0x2f, 0x74, 0x72, 0x75, 0x66, 0x66, 0x6c, 0x65, 0x68,
0x6f, 0x67, 0x2f, 0x76, 0x33, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x70, 0x62, 0x2f, 0x64, 0x65, 0x74,
0x65, 0x63, 0x74, 0x6f, 0x72, 0x73, 0x70, 0x62, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
}
var (
+1 -82
View File
@@ -60,7 +60,6 @@ const (
SourceType_SOURCE_TYPE_SHAREPOINT SourceType = 29
SourceType_SOURCE_TYPE_GCS_UNAUTHED SourceType = 30
SourceType_SOURCE_TYPE_AZURE_REPOS SourceType = 31
SourceType_SOURCE_TYPE_TRELLO SourceType = 32
)
// Enum value maps for SourceType.
@@ -98,7 +97,6 @@ var (
29: "SOURCE_TYPE_SHAREPOINT",
30: "SOURCE_TYPE_GCS_UNAUTHED",
31: "SOURCE_TYPE_AZURE_REPOS",
32: "SOURCE_TYPE_AZURE_TRELLO",
}
SourceType_value = map[string]int32{
"SOURCE_TYPE_AZURE_STORAGE": 0,
@@ -133,7 +131,6 @@ var (
"SOURCE_TYPE_SHAREPOINT": 29,
"SOURCE_TYPE_GCS_UNAUTHED": 30,
"SOURCE_TYPE_AZURE_REPOS": 31,
"SOURCE_TYPE_AZURE_TRELLO": 32,
}
)
@@ -526,80 +523,6 @@ func (*Bitbucket_Oauth) isBitbucket_Credential() {}
func (*Bitbucket_BasicAuth) isBitbucket_Credential() {}
type Trello struct {
state protoimpl.MessageState
sizeCache protoimpl.SizeCache
unknownFields protoimpl.UnknownFields
Auth *Trello_TrelloAuth `protobuf:"bytes,1,opt,name=auth,proto3,oneof"`
Boards []string `protobuf:"bytes,2,rep,name=boards,proto3" json:"boards,omitempty"`
}
type Trello_TrelloAuth struct {
ApiKey string `protobuf:"bytes,1,opt,name=api_key,json=apiKey,proto3,oneof"`
Token string `protobuf:"bytes,2,opt,name=token,proto3,oneof"`
}
func (x *Trello) Reset() {
*x = Trello{}
if protoimpl.UnsafeEnabled {
mi := &file_sources_proto_msgTypes[4]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
}
func (x *Trello) String() string {
return protoimpl.X.MessageStringOf(x)
}
func (*Trello) ProtoMessage() {}
func (x *Trello) ProtoReflect() protoreflect.Message {
mi := &file_sources_proto_msgTypes[4]
if protoimpl.UnsafeEnabled && x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
ms.StoreMessageInfo(mi)
}
return ms
}
return mi.MessageOf(x)
}
// Deprecated: Use Trello.ProtoReflect.Descriptor instead.
func (*Trello) Descriptor() ([]byte, []int) {
return file_sources_proto_rawDescGZIP(), []int{4} // Assuming Trello is the 5th message in source.proto file
}
func (x *Trello) GetAuth() *Trello_TrelloAuth {
if x != nil {
return x.Auth
}
return nil
}
func (t *Trello_TrelloAuth) GetApiKey() string {
if t != nil && t.ApiKey != nil {
return t.ApiKey
}
return ""
}
func (t *Trello_TrelloAuth) GetToken() string {
if t != nil && t.Token != nil {
return t.Token
}
return ""
}
func (x *Trello) GetBoards() []string {
if x != nil {
return x.Boards
}
return nil
}
type CircleCI struct {
state protoimpl.MessageState
sizeCache protoimpl.SizeCache
@@ -3770,7 +3693,6 @@ var file_sources_proto_goTypes = []interface{}{
(*SlackRealtime)(nil), // 28: sources.SlackRealtime
(*Sharepoint)(nil), // 29: sources.Sharepoint
(*AzureRepos)(nil), // 30: sources.AzureRepos
(*Trello)(nil), // 45: sources.Trello
(*durationpb.Duration)(nil), // 31: google.protobuf.Duration
(*anypb.Any)(nil), // 32: google.protobuf.Any
(*credentialspb.BasicAuth)(nil), // 33: credentials.BasicAuth
@@ -4304,16 +4226,13 @@ func file_sources_proto_init() {
(*AzureRepos_Token)(nil),
(*AzureRepos_Oauth)(nil),
}
file_sources_proto_msgTypes[29].OneofWrappers = []interface{}{
(*Trello_TrelloAuth)(nil),
}
type x struct{}
out := protoimpl.TypeBuilder{
File: protoimpl.DescBuilder{
GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
RawDescriptor: file_sources_proto_rawDesc,
NumEnums: 2,
NumMessages: 30,
NumMessages: 29,
NumExtensions: 0,
NumServices: 0,
},
+1 -1
View File
@@ -20,7 +20,7 @@ const (
// Chunker takes a chunk and splits it into chunks of ChunkSize.
func Chunker(originalChunk *Chunk) chan *Chunk {
chunkChan := make(chan *Chunk, 1)
chunkChan := make(chan *Chunk)
go func() {
defer close(chunkChan)
if len(originalChunk.Data) <= TotalChunkSize {
-1
View File
@@ -238,7 +238,6 @@ func (s *Source) chunkAction(ctx context.Context, proj project, bld build, act a
SourceType: s.Type(),
SourceName: s.name,
SourceID: s.SourceID(),
JobID: s.JobID(),
Data: removeCircleSha1Line(data.Bytes()),
SourceMetadata: &source_metadatapb.MetaData{
Data: &source_metadatapb.MetaData_Circleci{
+138 -220
View File
@@ -2,6 +2,7 @@ package docker
import (
"archive/tar"
"bytes"
"compress/gzip"
"errors"
"fmt"
@@ -13,11 +14,9 @@ import (
v1 "github.com/google/go-containerregistry/pkg/v1"
"github.com/google/go-containerregistry/pkg/v1/remote"
"github.com/google/go-containerregistry/pkg/v1/tarball"
"golang.org/x/sync/errgroup"
"google.golang.org/protobuf/proto"
"google.golang.org/protobuf/types/known/anypb"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/context"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/source_metadatapb"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
@@ -25,16 +24,17 @@ import (
)
type Source struct {
name string
sourceId int64
jobId int64
verify bool
concurrency int
conn sourcespb.Docker
name string
sourceId int64
jobId int64
verify bool
conn sourcespb.Docker
sources.Progress
sources.CommonSourceUnitUnmarshaller
}
var FilesizeLimitBytes int64 = 50 * 1024 * 1024 // 50MB
// Ensure the Source satisfies the interfaces at compile time.
var _ sources.Source = (*Source)(nil)
var _ sources.SourceUnitUnmarshaller = (*Source)(nil)
@@ -59,7 +59,6 @@ func (s *Source) Init(_ context.Context, name string, jobId, sourceId int64, ver
s.sourceId = sourceId
s.jobId = jobId
s.verify = verify
s.concurrency = concurrency
// Reset metrics for this source at initialization time.
dockerImagesScanned.WithLabelValues(s.name).Set(0)
@@ -72,209 +71,165 @@ func (s *Source) Init(_ context.Context, name string, jobId, sourceId int64, ver
return nil
}
type imageInfo struct {
image v1.Image
base string
tag string
}
type layerInfo struct {
digest v1.Hash
base string
tag string
}
// Chunks emits data over a channel that is decoded and scanned for secrets.
func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) error {
ctx = context.WithValues(ctx, "source_type", s.Type(), "source_name", s.name)
workers := new(errgroup.Group)
workers.SetLimit(s.concurrency)
scanErrs := sources.NewScanErrors()
for _, image := range s.conn.GetImages() {
image := image
workers.Go(func() error {
if common.IsDone(ctx) {
return nil
}
imgInfo, err := s.processImage(ctx, image)
if err != nil {
scanErrs.Add(err)
return nil
}
ctx = context.WithValues(ctx, "image", imgInfo.base, "tag", imgInfo.tag)
ctx.Logger().V(2).Info("scanning image")
layers, err := imgInfo.image.Layers()
if err != nil {
scanErrs.Add(err)
return nil
}
for _, layer := range layers {
if err := s.processLayer(ctx, layer, imgInfo, chunksChan); err != nil {
scanErrs.Add(err)
return nil
}
dockerLayersScanned.WithLabelValues(s.name).Inc()
}
dockerImagesScanned.WithLabelValues(s.name).Inc()
return nil
})
}
_ = workers.Wait()
if scanErrs.Count() > 0 {
ctx.Logger().V(2).Info("scan errors", "errors", scanErrs.String())
}
return nil
}
// processImage processes an individual image and prepares it for further processing.
func (s *Source) processImage(ctx context.Context, image string) (imageInfo, error) {
var (
imgInfo imageInfo
hasDigest bool
imageName name.Reference
)
remoteOpts, err := s.remoteOpts()
if err != nil {
return imgInfo, err
return err
}
const filePrefix = "file://"
if strings.HasPrefix(image, filePrefix) {
image = strings.TrimPrefix(image, filePrefix)
imgInfo.base = image
imgInfo.image, err = tarball.ImageFromPath(image, nil)
if err != nil {
return imgInfo, err
}
} else {
imgInfo.base, imgInfo.tag, hasDigest = baseAndTagFromImage(image)
logger := ctx.Logger().WithValues("source_type", s.Type(), "source_name", s.name)
if hasDigest {
imageName, err = name.NewDigest(image)
for _, image := range s.conn.GetImages() {
var img v1.Image
var err error
var base, tag string
var hasDigest bool
var imageName name.Reference
if strings.HasPrefix(image, "file://") {
image = strings.TrimPrefix(image, "file://")
base = image
img, err = tarball.ImageFromPath(image, nil)
if err != nil {
return err
}
} else {
imageName, err = name.NewTag(image)
}
if err != nil {
return imgInfo, err
base, tag, hasDigest = baseAndTagFromImage(image)
if hasDigest {
imageName, err = name.NewDigest(image)
if err != nil {
return err
}
} else {
imageName, err = name.NewTag(image)
if err != nil {
return err
}
}
img, err = remote.Image(imageName, remoteOpts...)
if err != nil {
return err
}
}
imgInfo.image, err = remote.Image(imageName, remoteOpts...)
if err != nil {
return imgInfo, err
}
}
logger = logger.WithValues("image", base, "tag", tag)
logger.V(2).Info("scanning image")
ctx.Logger().WithValues("image", imgInfo.base, "tag", imgInfo.tag).V(2).Info("scanning image")
return imgInfo, nil
}
// processLayer processes an individual layer of an image.
func (s *Source) processLayer(ctx context.Context, layer v1.Layer, imgInfo imageInfo, chunksChan chan *sources.Chunk) error {
layerInfo := layerInfo{
base: imgInfo.base,
tag: imgInfo.tag,
}
var err error
layerInfo.digest, err = layer.Digest()
if err != nil {
return err
}
ctx.Logger().WithValues("layer", layerInfo.digest.String()).V(2).Info("scanning layer")
rc, err := layer.Compressed()
if err != nil {
return err
}
defer rc.Close()
gzipReader, err := gzip.NewReader(rc)
if err != nil {
return err
}
defer gzipReader.Close()
tarReader := tar.NewReader(gzipReader)
for {
header, err := tarReader.Next()
if errors.Is(err, io.EOF) {
break
}
layers, err := img.Layers()
if err != nil {
return err
}
info := chunkProcessingInfo{size: header.Size, name: header.Name, reader: tarReader, layer: layerInfo}
if err := s.processChunk(ctx, info, chunksChan); err != nil {
return err
}
}
for _, layer := range layers {
return nil
}
digest, err := layer.Digest()
if err != nil {
return err
}
type chunkProcessingInfo struct {
size int64
name string
reader io.Reader
layer layerInfo
}
logger = logger.WithValues("layer", digest.String())
logger.V(2).Info("scanning layer")
// processChunk processes an individual chunk of a layer.
func (s *Source) processChunk(ctx context.Context, info chunkProcessingInfo, chunksChan chan *sources.Chunk) error {
const filesizeLimitBytes int64 = 50 * 1024 * 1024 // 50MB
if info.size > filesizeLimitBytes {
ctx.Logger().V(4).Info("skipping large file", "file", info.name, "size", info.size)
return nil
}
rc, err := layer.Compressed()
if err != nil {
return err
}
chunkReader := sources.NewChunkReader()
chunkResChan := chunkReader(ctx, info.reader)
defer rc.Close()
for data := range chunkResChan {
if err := data.Error(); err != nil {
ctx.Logger().Error(err, "error reading chunk.")
continue
}
gzipReader, err := gzip.NewReader(rc)
if err != nil {
return err
}
chunk := &sources.Chunk{
SourceType: s.Type(),
SourceName: s.name,
SourceID: s.SourceID(),
SourceMetadata: &source_metadatapb.MetaData{
Data: &source_metadatapb.MetaData_Docker{
Docker: &source_metadatapb.Docker{
File: "/" + info.name,
Image: info.layer.base,
Tag: info.layer.tag,
Layer: info.layer.digest.String(),
defer gzipReader.Close()
tarReader := tar.NewReader(gzipReader)
for {
header, err := tarReader.Next()
if errors.Is(err, io.EOF) {
break // End of archive
}
if err != nil {
return err
}
// Skip files larger than FilesizeLimitBytes
if header.Size > FilesizeLimitBytes {
continue
}
file := bytes.NewBuffer(make([]byte, 0, header.Size))
_, err = io.Copy(file, tarReader)
if err != nil {
return err
}
chunk := &sources.Chunk{
SourceType: s.Type(),
SourceName: s.name,
SourceID: s.SourceID(),
Data: file.Bytes(),
SourceMetadata: &source_metadatapb.MetaData{
Data: &source_metadatapb.MetaData_Docker{
Docker: &source_metadatapb.Docker{
File: "/" + header.Name, // Prepend /, because the file is relative to the root of the image
Image: base,
Tag: tag,
Layer: digest.String(),
},
},
},
},
},
Verify: s.verify,
}
chunk.Data = data.Bytes()
Verify: s.verify,
}
if err := common.CancellableWrite(ctx, chunksChan, chunk); err != nil {
return err
select {
case chunksChan <- chunk:
case <-ctx.Done():
return ctx.Err()
}
}
dockerLayersScanned.WithLabelValues(s.name).Inc()
}
dockerImagesScanned.WithLabelValues(s.name).Inc()
}
return nil
}
func baseAndTagFromImage(image string) (base, tag string, hasDigest bool) {
regRepoDelimiter := "/"
tagDelim := ":"
digestDelim := "@"
// Split on digest first, if present.
parts := strings.SplitN(image, digestDelim, 2)
if len(parts) > 1 {
base = parts[0]
tag = parts[1]
hasDigest = true
return
}
parts = strings.Split(image, tagDelim)
// Verify that we aren't confusing a tag for a hostname w/ port for the purposes of weak validation.
if len(parts) > 1 && !strings.Contains(parts[len(parts)-1], regRepoDelimiter) {
base = strings.Join(parts[:len(parts)-1], tagDelim)
tag = parts[len(parts)-1]
} else {
base = image
tag = "latest"
}
return
}
func (s *Source) remoteOpts() ([]remote.Option, error) {
switch s.conn.GetCredential().(type) {
case *sourcespb.Docker_Unauthenticated:
@@ -300,40 +255,3 @@ func (s *Source) remoteOpts() ([]remote.Option, error) {
return nil, fmt.Errorf("unknown credential type: %T", s.conn.Credential)
}
}
func baseAndTagFromImage(image string) (base, tag string, hasDigest bool) {
if base, tag, hasDigest = extractDigest(image); hasDigest {
return base, tag, true
}
base, tag = extractTagOrUseDefault(image)
return base, tag, false
}
// extractDigest tries to split the image string on the digest delimiter.
// If successful, it means the image has a digest.
func extractDigest(image string) (base, tag string, hasDigest bool) {
const digestDelim = "@"
if parts := strings.SplitN(image, digestDelim, 2); len(parts) > 1 {
return parts[0], parts[1], true
}
return "", "", false
}
// extractTagOrUseDefault extracts the tag from the image string.
// If no tag is found, it defaults to "latest".
func extractTagOrUseDefault(image string) (base, tag string) {
const (
tagDelim = ":"
regRepoDelim = "/"
)
parts := strings.Split(image, tagDelim)
// Check if the last part is not a hostname with a port (for weak validation)
if len(parts) > 1 && !strings.Contains(parts[len(parts)-1], regRepoDelim) {
return strings.Join(parts[:len(parts)-1], tagDelim), parts[len(parts)-1]
}
return image, "latest"
}
-2
View File
@@ -161,7 +161,6 @@ func (s *Source) scanFile(ctx context.Context, path string, chunksChan chan *sou
SourceType: s.Type(),
SourceName: s.name,
SourceID: s.SourceID(),
JobID: s.JobID(),
SourceMetadata: &source_metadatapb.MetaData{
Data: &source_metadatapb.MetaData_Filesystem{
Filesystem: &source_metadatapb.Filesystem{
@@ -192,7 +191,6 @@ func (s *Source) scanFile(ctx context.Context, path string, chunksChan chan *sou
SourceType: s.Type(),
SourceName: s.name,
SourceID: s.SourceID(),
JobID: s.JobID(),
Data: data.Bytes(),
SourceMetadata: &source_metadatapb.MetaData{
Data: &source_metadatapb.MetaData_Filesystem{
-1
View File
@@ -324,7 +324,6 @@ func (s *Source) processObject(ctx context.Context, o object) error {
chunkSkel := &sources.Chunk{
SourceName: s.name,
SourceType: s.Type(),
JobID: s.JobID(),
SourceID: s.sourceId,
Verify: s.verify,
SourceMetadata: &source_metadatapb.MetaData{
-7
View File
@@ -446,7 +446,6 @@ func (s *Git) ScanCommits(ctx context.Context, repo *git.Repository, path string
chunkSkel := &sources.Chunk{
SourceName: s.sourceName,
SourceID: s.sourceID,
JobID: s.jobID,
SourceType: s.sourceType,
SourceMetadata: metadata,
Verify: s.verify,
@@ -465,7 +464,6 @@ func (s *Git) ScanCommits(ctx context.Context, repo *git.Repository, path string
chunksChan <- &sources.Chunk{
SourceName: s.sourceName,
SourceID: s.sourceID,
JobID: s.jobID,
SourceType: s.sourceType,
SourceMetadata: metadata,
Data: diff.Content.Bytes(),
@@ -492,7 +490,6 @@ func (s *Git) gitChunk(ctx context.Context, diff gitparse.Diff, fileName, email,
chunksChan <- &sources.Chunk{
SourceName: s.sourceName,
SourceID: s.sourceID,
JobID: s.jobID,
SourceType: s.sourceType,
SourceMetadata: metadata,
Data: append([]byte{}, newChunkBuffer.Bytes()...),
@@ -507,7 +504,6 @@ func (s *Git) gitChunk(ctx context.Context, diff gitparse.Diff, fileName, email,
chunksChan <- &sources.Chunk{
SourceName: s.sourceName,
SourceID: s.sourceID,
JobID: s.jobID,
SourceType: s.sourceType,
SourceMetadata: metadata,
Data: line,
@@ -527,7 +523,6 @@ func (s *Git) gitChunk(ctx context.Context, diff gitparse.Diff, fileName, email,
chunksChan <- &sources.Chunk{
SourceName: s.sourceName,
SourceID: s.sourceID,
JobID: s.jobID,
SourceType: s.sourceType,
SourceMetadata: metadata,
Data: append([]byte{}, newChunkBuffer.Bytes()...),
@@ -593,7 +588,6 @@ func (s *Git) ScanStaged(ctx context.Context, repo *git.Repository, path string,
chunkSkel := &sources.Chunk{
SourceName: s.sourceName,
SourceID: s.sourceID,
JobID: s.jobID,
SourceType: s.sourceType,
SourceMetadata: metadata,
Verify: s.verify,
@@ -608,7 +602,6 @@ func (s *Git) ScanStaged(ctx context.Context, repo *git.Repository, path string,
chunksChan <- &sources.Chunk{
SourceName: s.sourceName,
SourceID: s.sourceID,
JobID: s.jobID,
SourceType: s.sourceType,
SourceMetadata: metadata,
Data: diff.Content.Bytes(),
+2 -7
View File
@@ -149,16 +149,14 @@ func (s *Source) newFilteredRepoCache(c cache.Cache, include, exclude []string)
for _, ig := range include {
g, err := glob.Compile(ig)
if err != nil {
s.log.V(1).Info("invalid include glob", "include_value", ig, "err", err)
continue
s.log.V(1).Info("invalid include glob", "glob", g, "err", err)
}
includeGlobs = append(includeGlobs, g)
}
for _, eg := range exclude {
g, err := glob.Compile(eg)
if err != nil {
s.log.V(1).Info("invalid exclude glob", "exclude_value", eg, "err", err)
continue
s.log.V(1).Info("invalid exclude glob", "glob", g, "err", err)
}
excludeGlobs = append(excludeGlobs, g)
}
@@ -1225,7 +1223,6 @@ func (s *Source) chunkIssueComments(ctx context.Context, repo, repoPath string,
chunk := &sources.Chunk{
SourceName: s.name,
SourceID: s.SourceID(),
JobID: s.JobID(),
SourceType: s.Type(),
SourceMetadata: &source_metadatapb.MetaData{
Data: &source_metadatapb.MetaData_Github{
@@ -1259,7 +1256,6 @@ func (s *Source) chunkPullRequestComments(ctx context.Context, repo string, comm
SourceName: s.name,
SourceID: s.SourceID(),
SourceType: s.Type(),
JobID: s.JobID(),
SourceMetadata: &source_metadatapb.MetaData{
Data: &source_metadatapb.MetaData_Github{
Github: &source_metadatapb.Github{
@@ -1291,7 +1287,6 @@ func (s *Source) chunkGistComments(ctx context.Context, gistUrl string, comments
SourceName: s.name,
SourceID: s.SourceID(),
SourceType: s.Type(),
JobID: s.JobID(),
SourceMetadata: &source_metadatapb.MetaData{
Data: &source_metadatapb.MetaData_Github{
Github: &source_metadatapb.Github{
+4 -4
View File
@@ -71,11 +71,11 @@ func (r *JobProgressRef) Done() <-chan struct{} {
// CancelRun requests that the job this is referencing is cancelled and stops
// running. This method will have no effect if the job does not allow
// cancellation.
func (r *JobProgressRef) CancelRun(cause error) {
func (r *JobProgressRef) CancelRun() {
if r.jobProgress == nil || r.jobProgress.jobCancel == nil {
return
}
r.jobProgress.jobCancel(cause)
r.jobProgress.jobCancel()
}
// Fatal is a wrapper around error to differentiate non-fatal errors from fatal
@@ -108,7 +108,7 @@ type JobProgress struct {
ctx context.Context
cancel context.CancelFunc
// Requests to cancel the job.
jobCancel context.CancelCauseFunc
jobCancel context.CancelFunc
// Metrics.
metrics JobProgressMetrics
metricsLock sync.Mutex
@@ -150,7 +150,7 @@ func WithHooks(hooks ...JobProgressHook) func(*JobProgress) {
}
// WithCancel allows cancelling the job by the JobProgressRef.
func WithCancel(cancel context.CancelCauseFunc) func(*JobProgress) {
func WithCancel(cancel context.CancelFunc) func(*JobProgress) {
return func(jp *JobProgress) { jp.jobCancel = cancel }
}
+103 -148
View File
@@ -31,7 +31,6 @@ import (
)
const (
defaultAWSRegion = "us-east-1"
defaultMaxObjectSize = 250 * 1024 * 1024 // 250 MiB
maxObjectSizeLimit = 250 * 1024 * 1024 // 250 MiB
)
@@ -54,7 +53,6 @@ type Source struct {
// Ensure the Source satisfies the interfaces at compile time
var _ sources.Source = (*Source)(nil)
var _ sources.SourceUnitUnmarshaller = (*Source)(nil)
var _ sources.Validator = (*Source)(nil)
// Type returns the type of source
func (s *Source) Type() sourcespb.SourceType {
@@ -95,23 +93,6 @@ func (s *Source) Init(aCtx context.Context, name string, jobId, sourceId int64,
return nil
}
func (s *Source) Validate(ctx context.Context) []error {
var errors []error
visitor := func(c context.Context, defaultRegionClient *s3.S3, roleArn string, buckets []string) {
roleErrs := s.validateBucketAccess(c, defaultRegionClient, roleArn, buckets)
if len(roleErrs) > 0 {
errors = append(errors, roleErrs...)
}
}
err := s.visitRoles(ctx, visitor)
if err != nil {
errors = append(errors, err)
}
return errors
}
// setMaxObjectSize sets the maximum size of objects that will be scanned. If
// not set, set to a negative number, or set larger than the
// maxObjectSizeLimit, the defaultMaxObjectSizeLimit will be used.
@@ -123,34 +104,22 @@ func (s *Source) setMaxObjectSize(maxObjectSize int64) {
}
}
func (s *Source) newClient(region, roleArn string) (*s3.S3, error) {
func (s *Source) newClient(region string) (*s3.S3, error) {
cfg := aws.NewConfig()
cfg.CredentialsChainVerboseErrors = aws.Bool(true)
cfg.Region = aws.String(region)
if roleArn == "" {
switch cred := s.conn.GetCredential().(type) {
case *sourcespb.S3_SessionToken:
cfg.Credentials = credentials.NewStaticCredentials(cred.SessionToken.Key, cred.SessionToken.Secret, cred.SessionToken.SessionToken)
case *sourcespb.S3_AccessKey:
cfg.Credentials = credentials.NewStaticCredentials(cred.AccessKey.Key, cred.AccessKey.Secret, "")
case *sourcespb.S3_Unauthenticated:
cfg.Credentials = credentials.AnonymousCredentials
case *sourcespb.S3_CloudEnvironment:
// Nothing needs to be done!
default:
return nil, errors.Errorf("invalid configuration given for %s source", s.name)
}
} else {
sess, err := session.NewSession(cfg)
if err != nil {
return nil, err
}
stsClient := sts.New(sess)
cfg.Credentials = stscreds.NewCredentialsWithClient(stsClient, roleArn, func(p *stscreds.AssumeRoleProvider) {
p.RoleSessionName = "trufflehog"
})
switch cred := s.conn.GetCredential().(type) {
case *sourcespb.S3_SessionToken:
cfg.Credentials = credentials.NewStaticCredentials(cred.SessionToken.Key, cred.SessionToken.Secret, cred.SessionToken.SessionToken)
case *sourcespb.S3_AccessKey:
cfg.Credentials = credentials.NewStaticCredentials(cred.AccessKey.Key, cred.AccessKey.Secret, "")
case *sourcespb.S3_Unauthenticated:
cfg.Credentials = credentials.AnonymousCredentials
case *sourcespb.S3_CloudEnvironment:
// Nothing needs to be done!
default:
return nil, errors.Errorf("invalid configuration given for %s source", s.name)
}
sess, err := session.NewSessionWithOptions(session.Options{
@@ -164,6 +133,40 @@ func (s *Source) newClient(region, roleArn string) (*s3.S3, error) {
return s3.New(sess), nil
}
// Separate role assumption functionality into a different newClient function
func (s *Source) newRoleClient(region string, roleArn string) (*s3.S3, error) {
cfg := aws.NewConfig()
cfg.CredentialsChainVerboseErrors = aws.Bool(true)
cfg.Region = aws.String(region)
var baseCredentials *credentials.Credentials
sess, err := session.NewSession(cfg)
if err != nil {
return nil, err
}
stsClient := sts.New(sess)
baseCredentials = stscreds.NewCredentialsWithClient(stsClient, roleArn, func(p *stscreds.AssumeRoleProvider) {
p.RoleSessionName = "trufflehog"
})
cfg.Credentials = baseCredentials
sess, err = session.NewSessionWithOptions(session.Options{
SharedConfigState: session.SharedConfigEnable,
Config: *cfg,
})
if err != nil {
return nil, err
}
// Create and return the S3 client with the final configuration
return s3.New(sess), nil
}
// IAM identity needs s3:ListBuckets permission
func (s *Source) getBucketsToScan(client *s3.S3) ([]string, error) {
if len(s.conn.Buckets) > 0 {
@@ -172,7 +175,7 @@ func (s *Source) getBucketsToScan(client *s3.S3) ([]string, error) {
res, err := client.ListBuckets(&s3.ListBucketsInput{})
if err != nil {
return nil, err
return nil, fmt.Errorf("could not list s3 buckets: %w", err)
}
var bucketsToScan []string
@@ -182,30 +185,37 @@ func (s *Source) getBucketsToScan(client *s3.S3) ([]string, error) {
return bucketsToScan, nil
}
func (s *Source) scanBuckets(ctx context.Context, client *s3.S3, role string, bucketsToScan []string, chunksChan chan *sources.Chunk) {
func (s *Source) scanBuckets(ctx context.Context, client *s3.S3, role string, bucketsToScan []string, chunksChan chan *sources.Chunk) error {
const defaultAWSRegion = "us-east-1"
objectCount := uint64(0)
logger := s.log
if role != "" {
logger = logger.WithValues("roleArn", role)
}
for i, bucket := range bucketsToScan {
logger := logger.WithValues("bucket", bucket)
if common.IsDone(ctx) {
return
return nil
}
s.SetProgressComplete(i, len(bucketsToScan), fmt.Sprintf("Bucket: %s", bucket), "")
logger.Info("Scanning bucket")
regionalClient, err := s.getRegionalClientForBucket(ctx, client, role, bucket)
s.log.Info("Scanning bucket", "bucket", bucket)
region, err := s3manager.GetBucketRegionWithClient(ctx, client, bucket)
if err != nil {
logger.Error(err, "could not get regional client for bucket")
s.log.Error(err, "could not get s3 region for bucket", "bucket: ", bucket)
continue
}
var regionalClient *s3.S3
if region != defaultAWSRegion {
if role != "" {
regionalClient, err = s.newRoleClient(region, role)
} else {
regionalClient, err = s.newClient(region)
}
if err != nil {
s.log.Error(err, "could not make regional s3 client")
}
} else {
regionalClient = client
}
errorCount := sync.Map{}
err = regionalClient.ListObjectsV2PagesWithContext(
@@ -216,46 +226,51 @@ func (s *Source) scanBuckets(ctx context.Context, client *s3.S3, role string, bu
})
if err != nil {
if role == "" {
logger.Error(err, "could not list objects in bucket")
} else {
// Our documentation blesses specifying a role to assume without specifying buckets to scan, which will
// often cause this to happen a lot (because in that case the scanner tries to scan every bucket in the
// account, but the role probably doesn't have access to all of them). This makes it expected behavior
// and therefore not an error.
logger.V(3).Info("could not list objects in bucket",
"err", err)
}
s.log.Error(err, "could not list objects in s3 bucket", "bucket: ", bucket)
continue
}
}
s.SetProgressComplete(len(bucketsToScan), len(bucketsToScan), fmt.Sprintf("Completed scanning source %s. %d objects scanned.", s.name, objectCount), "")
return nil
}
// Chunks emits chunks of bytes over a channel.
func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) error {
visitor := func(c context.Context, defaultRegionClient *s3.S3, roleArn string, buckets []string) {
s.scanBuckets(c, defaultRegionClient, roleArn, buckets, chunksChan)
const defaultAWSRegion = "us-east-1"
if len(s.conn.Roles) > 0 {
for _, role := range s.conn.Roles {
client, err := s.newRoleClient(defaultAWSRegion, role)
if err != nil {
return errors.WrapPrefix(err, "could not create s3 client", 0)
}
bucketsToScan, err := s.getBucketsToScan(client)
if err != nil {
return err
}
if err := s.scanBuckets(ctx, client, role, bucketsToScan, chunksChan); err != nil {
return err
}
}
} else {
client, err := s.newClient(defaultAWSRegion)
if err != nil {
return errors.WrapPrefix(err, "could not create s3 client", 0)
}
bucketsToScan, err := s.getBucketsToScan(client)
if err != nil {
return err
}
if err := s.scanBuckets(ctx, client, "", bucketsToScan, chunksChan); err != nil {
return err
}
}
return s.visitRoles(ctx, visitor)
}
func (s *Source) getRegionalClientForBucket(ctx context.Context, defaultRegionClient *s3.S3, role, bucket string) (*s3.S3, error) {
region, err := s3manager.GetBucketRegionWithClient(ctx, defaultRegionClient, bucket)
if err != nil {
return nil, errors.WrapPrefix(err, "could not get s3 region for bucket", 0)
}
if region == defaultAWSRegion {
return defaultRegionClient, nil
}
regionalClient, err := s.newClient(region, role)
if err != nil {
return nil, errors.WrapPrefix(err, "could not create regional s3 client", 0)
}
return regionalClient, nil
return nil
}
// pageChunker emits chunks onto the given channel from a page
@@ -361,7 +376,6 @@ func (s *Source) pageChunker(ctx context.Context, client *s3.S3, chunksChan chan
SourceType: s.Type(),
SourceName: s.name,
SourceID: s.SourceID(),
JobID: s.JobID(),
SourceMetadata: &source_metadatapb.MetaData{
Data: &source_metadatapb.MetaData_S3{
S3: &source_metadatapb.S3{
@@ -417,65 +431,6 @@ func (s *Source) pageChunker(ctx context.Context, client *s3.S3, chunksChan chan
_ = s.jobPool.Wait()
}
func (s *Source) validateBucketAccess(ctx context.Context, client *s3.S3, roleArn string, buckets []string) []error {
shouldHaveAccessToAllBuckets := roleArn == ""
wasAbleToListAnyBucket := false
var errors []error
for _, bucket := range buckets {
if common.IsDone(ctx) {
return append(errors, ctx.Err())
}
regionalClient, err := s.getRegionalClientForBucket(ctx, client, roleArn, bucket)
if err != nil {
errors = append(errors, fmt.Errorf("could not get regional client for bucket %q: %w", bucket, err))
continue
}
_, err = regionalClient.ListObjectsV2(&s3.ListObjectsV2Input{Bucket: &bucket})
if err == nil {
wasAbleToListAnyBucket = true
} else if shouldHaveAccessToAllBuckets {
errors = append(errors, fmt.Errorf("could not list objects in bucket %q: %w", bucket, err))
}
}
if !wasAbleToListAnyBucket {
if roleArn == "" {
errors = append(errors, fmt.Errorf("could not list objects in any bucket"))
} else {
errors = append(errors, fmt.Errorf("role %q could not list objects in any bucket", roleArn))
}
}
return errors
}
func (s *Source) visitRoles(ctx context.Context, f func(c context.Context, defaultRegionClient *s3.S3, roleArn string, buckets []string)) error {
roles := s.conn.Roles
if len(roles) == 0 {
roles = []string{""}
}
for _, role := range roles {
client, err := s.newClient(defaultAWSRegion, role)
if err != nil {
return errors.WrapPrefix(err, "could not create s3 client", 0)
}
bucketsToScan, err := s.getBucketsToScan(client)
if err != nil {
return fmt.Errorf("role %q could not list any s3 buckets for scanning: %w", role, err)
}
f(ctx, client, role, bucketsToScan)
}
return nil
}
// S3 links currently have the general format of:
// https://[bucket].s3[.region unless us-east-1].amazonaws.com/[key]
func makeS3Link(bucket, region, key string) string {
-128
View File
@@ -4,14 +4,10 @@
package s3
import (
"fmt"
"sync"
"testing"
"time"
"github.com/stretchr/testify/assert"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/credentialspb"
"google.golang.org/protobuf/types/known/anypb"
"github.com/trufflesecurity/trufflehog/v3/pkg/context"
@@ -49,127 +45,3 @@ func TestSource_ChunksCount(t *testing.T) {
}
assert.Greater(t, got, wantChunkCount)
}
func TestSource_Validate(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*15)
defer cancel()
secret, err := common.GetTestSecret(ctx)
if err != nil {
t.Fatal(fmt.Errorf("failed to access secret: %v", err))
}
s3key := secret.MustGetField("AWS_S3_KEY")
s3secret := secret.MustGetField("AWS_S3_SECRET")
tests := []struct {
name string
roles []string
buckets []string
wantErrCount int
}{
{
name: "buckets without roles, can access all buckets",
buckets: []string{
"truffletestbucket-s3-tests",
},
wantErrCount: 0,
},
{
name: "buckets without roles, one error per inaccessible bucket",
buckets: []string{
"truffletestbucket-s3-tests",
"truffletestbucket-s3-role-assumption",
"truffletestbucket-no-access",
},
wantErrCount: 2,
},
{
name: "roles without buckets, all can access at least one account bucket",
roles: []string{
"arn:aws:iam::619888638459:role/s3-test-assume-role",
},
wantErrCount: 0,
},
{
name: "roles without buckets, one error per role that cannot access any account buckets",
roles: []string{
"arn:aws:iam::619888638459:role/s3-test-assume-role",
"arn:aws:iam::619888638459:role/test-no-access",
},
wantErrCount: 1,
},
{
name: "role and buckets, can access at least one bucket",
roles: []string{
"arn:aws:iam::619888638459:role/s3-test-assume-role",
},
buckets: []string{
"truffletestbucket-s3-role-assumption",
"truffletestbucket-no-access",
},
wantErrCount: 0,
},
{
name: "roles and buckets, one error per role that cannot access at least one bucket",
roles: []string{
"arn:aws:iam::619888638459:role/s3-test-assume-role",
"arn:aws:iam::619888638459:role/test-no-access",
},
buckets: []string{
"truffletestbucket-s3-role-assumption",
"truffletestbucket-no-access",
},
wantErrCount: 1,
},
{
name: "role and buckets, a bucket doesn't even exist",
roles: []string{
"arn:aws:iam::619888638459:role/s3-test-assume-role",
},
buckets: []string{
"truffletestbucket-s3-role-assumption",
"not-a-real-bucket-asljdhmglasjgvklhsdaljfh", // need a bucket name that nobody is likely to ever create
},
wantErrCount: 1,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*15)
var cancelOnce sync.Once
defer cancelOnce.Do(cancel)
// These are used by the tests that assume roles
t.Setenv("AWS_ACCESS_KEY_ID", s3key)
t.Setenv("AWS_SECRET_ACCESS_KEY", s3secret)
s := &Source{}
conn, err := anypb.New(&sourcespb.S3{
// These are used by the tests that don't assume roles
Credential: &sourcespb.S3_AccessKey{
AccessKey: &credentialspb.KeySecret{
Key: s3key,
Secret: s3secret,
},
},
Buckets: tt.buckets,
Roles: tt.roles,
})
if err != nil {
t.Fatal(err)
}
err = s.Init(ctx, tt.name, 0, 0, false, conn, 0)
if err != nil {
t.Fatal(err)
}
errs := s.Validate(ctx)
assert.Equal(t, tt.wantErrCount, len(errs))
})
}
}
+4 -28
View File
@@ -36,10 +36,7 @@ type SourceManager struct {
handles map[handle]sourceInfo
handlesLock sync.Mutex
// Pool limiting the amount of concurrent sources running.
pool errgroup.Group
poolLimit int
currentRunningCount int32
// Max number of units to scan concurrently per source.
pool errgroup.Group
concurrentUnits int
// Run the sources using source unit enumeration / chunking if available.
useSourceUnits bool
@@ -71,10 +68,7 @@ func WithReportHook(hook JobProgressHook) func(*SourceManager) {
// WithConcurrentSources limits the concurrent number of sources a manager can run.
func WithConcurrentSources(concurrency int) func(*SourceManager) {
return func(mgr *SourceManager) {
mgr.pool.SetLimit(concurrency)
mgr.poolLimit = concurrency
}
return func(mgr *SourceManager) { mgr.pool.SetLimit(concurrency) }
}
// WithBufferedOutput sets the size of the buffer used for the Chunks() channel.
@@ -170,17 +164,15 @@ func (s *SourceManager) asyncRun(ctx context.Context, handle handle) (JobProgres
return JobProgressRef{SourceID: int64(handle), SourceName: sourceName}, err
}
// Create a JobProgress object for tracking progress.
ctx, cancel := context.WithCancelCause(ctx)
ctx, cancel := context.WithCancel(ctx)
progress := NewJobProgress(jobID, int64(handle), sourceName, WithHooks(s.hooks...), WithCancel(cancel))
s.pool.Go(func() error {
atomic.AddInt32(&s.currentRunningCount, 1)
defer atomic.AddInt32(&s.currentRunningCount, -1)
ctx := context.WithValues(ctx,
"job_id", jobID,
"source_manager_worker_id", common.RandomID(5),
)
defer common.Recover(ctx)
defer cancel(nil)
defer cancel()
return s.run(ctx, handle, jobID, progress)
})
return progress.Ref(), nil
@@ -215,16 +207,6 @@ func (s *SourceManager) ScanChunk(chunk *Chunk) {
s.outputChunks <- chunk
}
// AvailableCapacity returns the number of concurrent jobs the manager can
// accommodate at this time. If there is no limit, -1 is returned.
func (s *SourceManager) AvailableCapacity() int {
if s.poolLimit == 0 {
return -1
}
runCount := atomic.LoadInt32(&s.currentRunningCount)
return s.poolLimit - int(runCount)
}
// preflightChecks is a helper method to check the Manager or the context isn't
// done and that the handle is valid.
func (s *SourceManager) preflightChecks(ctx context.Context, handle handle) error {
@@ -247,12 +229,6 @@ func (s *SourceManager) run(ctx context.Context, handle handle, jobID int64, rep
report.Start(time.Now())
defer func() { report.End(time.Now()) }()
defer func() {
if err := context.Cause(ctx); err != nil {
report.ReportError(Fatal{err})
}
}()
// Initialize the source.
sourceInfo, ok := s.getSourceInfo(handle)
if !ok {
+1 -24
View File
@@ -312,31 +312,8 @@ func TestSourceManagerCancelRun(t *testing.T) {
ref, err := mgr.ScheduleRun(context.Background(), handle)
assert.NoError(t, err)
cancelErr := fmt.Errorf("abort! abort!")
ref.CancelRun(cancelErr)
ref.CancelRun()
<-ref.Done()
assert.Error(t, ref.Snapshot().FatalError())
assert.True(t, errors.Is(ref.Snapshot().FatalError(), returnedErr))
assert.True(t, errors.Is(ref.Snapshot().FatalErrors(), cancelErr))
}
func TestSourceManagerAvailableCapacity(t *testing.T) {
mgr := NewManager(WithConcurrentSources(1337))
start, end := make(chan struct{}), make(chan struct{})
handle, err := enrollDummy(mgr, callbackChunker{func(context.Context, chan *Chunk) error {
start <- struct{}{} // Send start signal.
<-end // Wait for end signal.
return nil
}})
assert.NoError(t, err)
assert.Equal(t, 1337, mgr.AvailableCapacity())
ref, err := mgr.ScheduleRun(context.Background(), handle)
assert.NoError(t, err)
<-start // Wait for start signal.
assert.Equal(t, 1336, mgr.AvailableCapacity())
end <- struct{}{} // Send end signal.
<-ref.Done() // Wait for the job to finish.
assert.Equal(t, 1337, mgr.AvailableCapacity())
}
-2
View File
@@ -272,7 +272,6 @@ func (s *Source) monitorConnection(ctx context.Context, conn net.Conn, chunksCha
SourceName: s.syslog.sourceName,
SourceID: s.syslog.sourceID,
SourceType: s.syslog.sourceType,
JobID: s.JobID(),
SourceMetadata: metadata,
Data: input,
Verify: s.verify,
@@ -314,7 +313,6 @@ func (s *Source) acceptUDPConnections(ctx context.Context, netListener net.Packe
chunksChan <- &sources.Chunk{
SourceName: s.syslog.sourceName,
SourceID: s.syslog.sourceID,
JobID: s.JobID(),
SourceType: s.syslog.sourceType,
SourceMetadata: metadata,
Data: input,
-239
View File
@@ -1,239 +0,0 @@
package trello
import (
"encoding/json"
"fmt"
"net/http"
"sync/atomic"
"github.com/go-errors/errors"
"google.golang.org/protobuf/proto"
"google.golang.org/protobuf/types/known/anypb"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/context"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/source_metadatapb"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
)
const baseURL = "https://api.trello.com/1/"
type Source struct {
name string
apiKey string
token string
boardIDs []string
sourceId int64
jobId int64
verify bool
jobPool *errgroup.Group
sources.Progress
client *http.Client
sources.CommonSourceUnitUnmarshaller
}
// Type returns the type of the source.
func (s *Source) Type() sourcespb.SourceType {
return sourcespb.SourceType_SOURCE_TYPE_TRELLO
}
// Init returns an initialized Source.
func (s *Source) Init(_ context.Context, name string, jobId, sourceId int64, verify bool, connection *anypb.Any, concurrency int) error {
s.name = name
s.sourceId = sourceId
s.jobId = jobId
s.verify = verify
s.jobPool = &errgroup.Group{}
s.jobPool.SetLimit(concurrency)
s.client = common.RetryableHttpClientTimeout(3)
var conn sourcespb.Trello
if err := anypb.UnmarshalTo(connection, &conn, proto.UnmarshalOptions{}); err != nil {
return errors.WrapPrefix(err, "error unmarshalling connection", 0)
}
s.apiKey = conn.GetAuth().GetApiKey()
s.token = conn.GetAuth().GetToken()
s.boardIDs = conn.GetBoards()
return nil
}
// Chunks emits chunks of bytes over a channel.
func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) error {
scanErrs := sources.NewScanErrors()
for _, boardID := range s.boardIDs {
board, err := s.getBoard(ctx, boardID)
if err != nil {
scanErrs.Add(err)
return nil
}
if err = s.chunkBoard(ctx, board, chunksChan); err != nil {
scanErrs.Add(err)
return nil
}
}
_ = s.jobPool.Wait()
if scanErrs.Count() > 0 {
ctx.Logger().V(2).Info("encountered errors while scanning", "count", scanErrs.Count(), "errors", scanErrs)
}
return nil
}
type board struct {
Name string `json:"name"`
Desc string `json:"desc"`
ID string `json:"id"`
URL string `json:"url"`
}
type card struct {
Name string `json:"name"`
Desc string `json:"desc"`
ID string `json:"id"`
URL string `json:"url"`
}
type action struct {
Type string `json:"type"`
Data struct {
Text string `json:"text"`
} `json:"data"`
}
// Add these two methods to Source
func (s *Source) getCards(_ context.Context, boardID string) ([]card, error) {
reqURL := fmt.Sprintf("%sboards/%s/cards?key=%s&token=%s", baseURL, boardID, s.apiKey, s.token)
req, err := http.NewRequest("GET", reqURL, nil)
if err != nil {
return nil, err
}
req.Header.Set("Accept", "application/json")
res, err := s.client.Do(req)
if err != nil {
return nil, err
}
defer res.Body.Close()
var cards []card
if err := json.NewDecoder(res.Body).Decode(&cards); err != nil {
return nil, err
}
return cards, nil
}
func (s *Source) getComments(_ context.Context, cardID string) ([]action, error) {
reqURL := fmt.Sprintf("%scards/%s/actions?key=%s&token=%s", baseURL, cardID, s.apiKey, s.token)
req, err := http.NewRequest("GET", reqURL, nil)
if err != nil {
return nil, err
}
req.Header.Set("Accept", "application/json")
res, err := s.client.Do(req)
if err != nil {
return nil, err
}
defer res.Body.Close()
var actions []action
if err := json.NewDecoder(res.Body).Decode(&actions); err != nil {
return nil, err
}
return actions, nil
}
func (s *Source) getBoard(_ context.Context, boardID string) (*board, error) {
reqURL := fmt.Sprintf("%sboards/%s?key=%s&token=%s", baseURL, boardID, s.apiKey, s.token)
req, err := http.NewRequest("GET", reqURL, nil)
if err != nil {
return nil, err
}
req.Header.Set("Accept", "application/json")
res, err := s.client.Do(req)
if err != nil {
return nil, err
}
defer res.Body.Close()
var b board
if err := json.NewDecoder(res.Body).Decode(&b); err != nil {
return nil, err
}
return &b, nil
}
func (s *Source) chunkBoard(ctx context.Context, board *board, chunksChan chan *sources.Chunk) error {
err := s.chunkItem(ctx, chunksChan, board.ID, board.Desc, board.URL, source_metadatapb.ItemType_ITEM_TYPE_BOARD, board.Name)
if err != nil {
return err
}
cards, err := s.getCards(ctx, board.ID)
if err != nil {
return err
}
for _, card := range cards {
err := s.chunkItem(ctx, chunksChan, card.ID, card.Desc, card.URL, source_metadatapb.ItemType_ITEM_TYPE_CARD, card.Name)
if err != nil {
return err
}
comments, err := s.getComments(ctx, card.ID)
if err != nil {
return err
}
for _, comment := range comments {
if comment.Type != "commentCard" {
continue
}
err := s.chunkItem(ctx, chunksChan, card.ID, comment.Data.Text, card.URL, source_metadatapb.ItemType_ITEM_TYPE_COMMENT, "")
if err != nil {
return err
}
}
}
return nil
}
func (s *Source) chunkItem(ctx context.Context, chunksChan chan *sources.Chunk, id, text, url string, itemType source_metadatapb.ItemType, name string) error {
data := []byte(text)
chunk := &sources.Chunk{
SourceType: s.Type(),
SourceName: s.name,
SourceID: s.sourceId,
JobID: s.jobId,
Data: data,
SourceMetadata: &source_metadatapb.MetaData{
Data: &source_metadatapb.MetaData_Trello{
Trello: &source_metadatapb.Trello{
Id: id,
Url: url,
Type: itemType,
Name: name
},
},
},
Verify: s.verify,
}
atomic.AddInt64(&s.scannedBytes, int64(len(data)))
if err := common.CancellableWrite(ctx, chunksChan, chunk); err != nil {
return err
}
return nil
}
-94
View File
@@ -1,94 +0,0 @@
package trello
import (
"context"
"fmt"
"testing"
"time"
"github.com/kylelemons/godebug/pretty"
"google.golang.org/protobuf/types/known/anypb"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/context"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/source_metadatapb"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
)
func TestSource_Scan(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*3)
defer cancel()
secret, err := common.GetTestSecret(ctx)
if err != nil {
t.Fatal(fmt.Errorf("failed to access secret: %v", err))
}
apiKey := secret.MustGetField("TRELLO_API_KEY")
token := secret.MustGetField("TRELLO_TOKEN")
type init struct {
name string
verify bool
connection *sourcespb.Trello
}
tests := []struct {
name string
init init
wantSourceMetadata *source_metadatapb.MetaData
wantErr bool
}{
{
name: "get a chunk",
init: init{
name: "example board",
connection: &sourcespb.Trello{
Auth: &sourcespb.Trello_TrelloAuth{
ApiKey: apiKey,
Token: token,
},
Boards: []string{"boardID1"},
},
verify: true,
},
wantSourceMetadata: &source_metadatapb.MetaData{
Data: &source_metadatapb.MetaData_Trello{
Trello: &source_metadatapb.Trello{
BoardId: "boardID1",
BoardName: "Example Board",
CardId: "cardID1",
},
},
},
wantErr: false,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
s := Source{}
conn, err := anypb.New(tt.init.connection)
if err != nil {
t.Fatal(err)
}
if err := s.Init(ctx, tt.init.name, 0, 0, tt.init.verify, conn, 5); (err != nil) != tt.wantErr {
t.Errorf("Source.Init() error = %v, wantErr %v", err, tt.wantErr)
return
}
chunksCh := make(chan *sources.Chunk, 1)
go func() {
if err := s.Chunks(ctx, chunksCh); (err != nil) != tt.wantErr {
t.Errorf("Source.Chunks() error = %v, wantErr %v", err, tt.wantErr)
return
}
}()
gotChunk := <-chunksCh
if diff := pretty.Compare(gotChunk.SourceMetadata, tt.wantSourceMetadata); diff != "" {
t.Errorf("Source.Chunks() %s diff: (-got +want)\n%s", tt.name, diff)
}
})
}
}
@@ -68,7 +68,6 @@ func New(c common.Common) *SourceSelect {
OssItem("Filesystem", "Scan your filesystem by selecting what directories to scan."),
OssItem("AWS S3", "Scan Amazon S3 buckets."),
OssItem("CircleCI", "Scan CircleCI, a CI/CD platform."),
OssItem("Trello", "Scan Trello, a ticket system."),
OssItem("Syslog", "Scan syslog, event data logs."),
OssItem("Docker", "Scan a Docker instance, a containerized application."),
OssItem("GCS (Google Cloud Storage)", "Scan a Google Cloud Storage instance."),
-3
View File
@@ -5,7 +5,6 @@ import (
tea "github.com/charmbracelet/bubbletea"
"github.com/trufflesecurity/trufflehog/v3/pkg/tui/sources/circleci"
"github.com/trufflesecurity/trufflehog/v3/pkg/tui/sources/trello"
"github.com/trufflesecurity/trufflehog/v3/pkg/tui/sources/docker"
"github.com/trufflesecurity/trufflehog/v3/pkg/tui/sources/filesystem"
"github.com/trufflesecurity/trufflehog/v3/pkg/tui/sources/gcs"
@@ -53,8 +52,6 @@ func GetSourceFields(sourceName string) CmdModel {
return syslog.GetFields()
case "circleci":
return circleci.GetFields()
case "trello":
return trello.GetFields()
case "docker":
return docker.GetFields()
}
-64
View File
@@ -1,64 +0,0 @@
package trello
import (
"strings"
"github.com/trufflesecurity/trufflehog/v3/pkg/tui/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/tui/components/textinputs"
)
type trelloCmdModel struct {
textinputs.Model
}
func GetFields() trelloCmdModel {
apiKey := textinputs.InputConfig{
Label: "API Key",
Key: "api_key",
Required: true,
Placeholder: "Your Trello API key",
}
token := textinputs.InputConfig{
Label: "Token",
Key: "token",
Required: true,
Placeholder: "Your Trello token",
}
boardIds := textinputs.InputConfig{
Label: "Board IDs",
Key: "board_ids",
Required: true,
Placeholder: "Comma-separated list of Trello board ID(s)",
}
return trelloCmdModel{textinputs.New([]textinputs.InputConfig{apiKey, token, boardIds})}
}
func (m trelloCmdModel) Cmd() string {
var command []string
command = append(command, "trufflehog", "trello")
inputs := m.GetInputs()
if inputs["api_key"] != "" {
command = append(command, "--api-key="+inputs["api_key"])
}
if inputs["token"] != "" {
command = append(command, "--token="+inputs["token"])
}
if inputs["board_ids"] != "" {
command = append(command, "--board-ids="+inputs["board_ids"])
}
return strings.Join(command, " ")
}
func (m trelloCmdModel) Summary() string {
inputs := m.GetInputs()
labels := m.GetLabels()
keys := []string{"api_key", "token", "board_ids"}
return common.SummarizeSource(keys, inputs, labels)
}
-1
View File
@@ -936,7 +936,6 @@ enum DetectorType {
Salesforce = 925;
HuggingFace = 926;
Snowflake = 927;
Sourcegraph = 928;
}
message Result {
-12
View File
@@ -186,17 +186,6 @@ message Test {
string file = 1;
}
message Trello {
string board_id = 1;
string card_id = 2;
string list_id = 3;
string action_id = 4;
string user_id = 5;
string username = 6;
string timestamp = 7;
string link = 8;
}
message Jenkins {
string project_name = 1;
int64 build_number = 2;
@@ -305,6 +294,5 @@ message MetaData {
SharePoint sharepoint = 25;
GoogleDrive googleDrive = 26;
AzureRepos azureRepos = 27;
Trello trello = 28;
}
}
-12
View File
@@ -44,7 +44,6 @@ enum SourceType {
SOURCE_TYPE_SHAREPOINT = 29;
SOURCE_TYPE_GCS_UNAUTHED = 30;
SOURCE_TYPE_AZURE_REPOS = 31;
SOURCE_TYPE_TRELLO = 32;
}
message LocalSource {
@@ -266,17 +265,6 @@ message Jenkins {
}
}
message Trello {
message TrelloAuth {
string api_key = 1;
string token = 2;
}
TrelloAuth auth = 1;
repeated string boards = 2;
}
message Teams {
string endpoint = 1 [(validate.rules).string.uri_ref = true];
oneof credential {