Compare commits
16 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 Ahrav Dutta
|
514c9f8e40 | ||
|
Ahrav Dutta
|
9894b14b02 | ||
|
Zachary Rice
|
4a77688097 | ||
|
trufflesteeeve
|
11bff81def | ||
|
dependabot[bot]
|
06f2d3a162 | ||
|
Zubair Khan
|
f52946b996 | ||
|
Miccah
|
945c27cb82 | ||
|
Brendan Shaklovitz
|
da5301ea1e | ||
|
dependabot[bot]
|
7cefea6562 | ||
|
Chris Atkin
|
6e6895b48e | ||
|
dependabot[bot]
|
146ddb351b | ||
|
dependabot[bot]
|
ab84f1fbc6 | ||
|
dependabot[bot]
|
3ea766e8c2 | ||
|
Zubair Khan
|
cd67f6bf16 | ||
|
Dustin Decker
|
eeefde1ec9 | ||
|
Miccah
|
f3152b6885 |
@@ -6,7 +6,7 @@ replace github.com/jpillora/overseer => github.com/trufflesecurity/overseer v1.1
|
||||
|
||||
require (
|
||||
cloud.google.com/go/secretmanager v1.11.0
|
||||
cloud.google.com/go/storage v1.29.0
|
||||
cloud.google.com/go/storage v1.30.1
|
||||
github.com/Azure/go-autorest/autorest/azure/auth v0.5.11
|
||||
github.com/TheZeroSlave/zapsentry v1.15.0
|
||||
github.com/aws/aws-sdk-go v1.44.83
|
||||
@@ -14,6 +14,7 @@ require (
|
||||
github.com/bill-rich/go-syslog v0.0.0-20220413021637-49edb52a574c
|
||||
github.com/bitfinexcom/bitfinex-api-go v0.0.0-20210608095005-9e0b26f200fb
|
||||
github.com/bradleyfalzon/ghinstallation/v2 v2.4.0
|
||||
github.com/couchbase/gocb/v2 v2.6.3
|
||||
github.com/crewjam/rfc5424 v0.1.0
|
||||
github.com/denisenkom/go-mssqldb v0.12.3
|
||||
github.com/envoyproxy/protoc-gen-validate v1.0.1
|
||||
@@ -22,7 +23,7 @@ require (
|
||||
github.com/getsentry/sentry-go v0.22.0
|
||||
github.com/go-errors/errors v1.4.2
|
||||
github.com/go-git/go-git/v5 v5.7.0
|
||||
github.com/go-ldap/ldap/v3 v3.4.4
|
||||
github.com/go-ldap/ldap/v3 v3.4.5
|
||||
github.com/go-logr/logr v1.2.4
|
||||
github.com/go-logr/zapr v1.2.4
|
||||
github.com/go-redis/redis v6.15.9+incompatible
|
||||
@@ -50,13 +51,13 @@ require (
|
||||
github.com/sergi/go-diff v1.3.1
|
||||
github.com/stretchr/testify v1.8.4
|
||||
github.com/tailscale/depaware v0.0.0-20210622194025-720c4b409502
|
||||
github.com/xanzy/go-gitlab v0.85.0
|
||||
go.mongodb.org/mongo-driver v1.11.6
|
||||
github.com/xanzy/go-gitlab v0.86.0
|
||||
go.mongodb.org/mongo-driver v1.12.0
|
||||
go.uber.org/zap v1.24.0
|
||||
golang.org/x/crypto v0.10.0
|
||||
golang.org/x/exp v0.0.0-20221018205818-5c77f4b2bbd7
|
||||
golang.org/x/oauth2 v0.9.0
|
||||
golang.org/x/sync v0.2.0
|
||||
golang.org/x/sync v0.3.0
|
||||
golang.org/x/text v0.10.0
|
||||
google.golang.org/api v0.128.0
|
||||
google.golang.org/protobuf v1.30.0
|
||||
@@ -77,7 +78,7 @@ require (
|
||||
github.com/Azure/go-autorest/autorest/date v0.3.0 // indirect
|
||||
github.com/Azure/go-autorest/logger v0.2.1 // indirect
|
||||
github.com/Azure/go-autorest/tracing v0.6.0 // indirect
|
||||
github.com/Azure/go-ntlmssp v0.0.0-20220621081337-cb9428e4ac1e // indirect
|
||||
github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 // indirect
|
||||
github.com/Microsoft/go-winio v0.6.1 // indirect
|
||||
github.com/ProtonMail/go-crypto v0.0.0-20230518184743-7afd39499903 // indirect
|
||||
github.com/acomagu/bufpipe v1.0.4 // indirect
|
||||
@@ -93,6 +94,7 @@ require (
|
||||
github.com/cloudflare/circl v1.3.3 // indirect
|
||||
github.com/connesc/cipherio v0.2.1 // indirect
|
||||
github.com/containerd/stargz-snapshotter/estargz v0.14.3 // indirect
|
||||
github.com/couchbase/gocbcore/v10 v10.2.3 // indirect
|
||||
github.com/davecgh/go-spew v1.1.1 // indirect
|
||||
github.com/dimchansky/utfbom v1.1.1 // indirect
|
||||
github.com/docker/cli v23.0.5+incompatible // indirect
|
||||
@@ -153,8 +155,8 @@ require (
|
||||
github.com/vbatts/tar-split v0.11.3 // indirect
|
||||
github.com/xanzy/ssh-agent v0.3.3 // indirect
|
||||
github.com/xdg-go/pbkdf2 v1.0.0 // indirect
|
||||
github.com/xdg-go/scram v1.1.1 // indirect
|
||||
github.com/xdg-go/stringprep v1.0.3 // indirect
|
||||
github.com/xdg-go/scram v1.1.2 // indirect
|
||||
github.com/xdg-go/stringprep v1.0.4 // indirect
|
||||
github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d // indirect
|
||||
github.com/yusufpapurcu/wmi v1.2.2 // indirect
|
||||
go.opencensus.io v0.24.0 // indirect
|
||||
|
||||
@@ -25,8 +25,8 @@ cloud.google.com/go/secretmanager v1.11.0 h1:tNzxvs3fP57iTrhaS1XdEPwg4LSPKiDNgp+
|
||||
cloud.google.com/go/secretmanager v1.11.0/go.mod h1:qeQq0/jyJqrGeULu0GkRsVSPKTvf98AEqJnuEIQiJwA=
|
||||
cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw=
|
||||
cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos=
|
||||
cloud.google.com/go/storage v1.29.0 h1:6weCgzRvMg7lzuUurI4697AqIRPU1SvzHhynwpW31jI=
|
||||
cloud.google.com/go/storage v1.29.0/go.mod h1:4puEjyTKnku6gfKoTfNOU/W+a9JyuVNxjpS5GBrB8h4=
|
||||
cloud.google.com/go/storage v1.30.1 h1:uOdMxAs8HExqBlnLtnQyP0YkvbiDpdGShGKtx6U/oNM=
|
||||
cloud.google.com/go/storage v1.30.1/go.mod h1:NfxhC0UJE1aXSx7CIIbCf7y9HKT7BiccwkR7+P7gN8E=
|
||||
dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/azcore v0.19.0/go.mod h1:h6H6c8enJmmocHUbLiiGY6sx7f9i+X3m1CHdd5c6Rdw=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/azidentity v0.11.0/go.mod h1:HcM1YX14R7CJcghJGOYCgdezslRSVzqwLf/q+4Y2r/0=
|
||||
@@ -49,8 +49,8 @@ github.com/Azure/go-autorest/logger v0.2.1 h1:IG7i4p/mDa2Ce4TRyAO8IHnVhAVF3RFU+Z
|
||||
github.com/Azure/go-autorest/logger v0.2.1/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZmbF5NWuPV8+WeEW8=
|
||||
github.com/Azure/go-autorest/tracing v0.6.0 h1:TYi4+3m5t6K48TGI9AUdb+IzbnSxvnvUMfuitfgcfuo=
|
||||
github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU=
|
||||
github.com/Azure/go-ntlmssp v0.0.0-20220621081337-cb9428e4ac1e h1:NeAW1fUYUEWhft7pkxDf6WoUvEZJ/uOKsvtpjLnn8MU=
|
||||
github.com/Azure/go-ntlmssp v0.0.0-20220621081337-cb9428e4ac1e/go.mod h1:chxPXzSsl7ZWRAuOIE23GDNzjWuZquvFlgA8xmpunjU=
|
||||
github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 h1:mFRzDkZVAjdal+s7s0MwaRv9igoPqLRdzOLzw/8Xvq8=
|
||||
github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358/go.mod h1:chxPXzSsl7ZWRAuOIE23GDNzjWuZquvFlgA8xmpunjU=
|
||||
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
|
||||
github.com/BurntSushi/toml v1.2.1/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ=
|
||||
github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
|
||||
@@ -68,6 +68,8 @@ github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 h1:JYp7IbQjafo
|
||||
github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
|
||||
github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 h1:s6gZFSlWYmbqAuRjVTiNNhvNRfY2Wxp9nhfyel4rklc=
|
||||
github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137/go.mod h1:OMCwj8VM1Kc9e19TLln2VL61YJF0x1XFtfdL4JdbSyE=
|
||||
github.com/alexbrainman/sspi v0.0.0-20210105120005-909beea2cc74 h1:Kk6a4nehpJ3UuJRqlA3JxYxBZEqCeOmATOvrbT4p9RA=
|
||||
github.com/alexbrainman/sspi v0.0.0-20210105120005-909beea2cc74/go.mod h1:cEWa1LVoE5KvSD9ONXsZrj0z6KqySlCCNKHlLzbqAt4=
|
||||
github.com/andybalholm/brotli v1.0.5 h1:8uQZIdzKmjc/iuPu7O2ioW48L81FgatrcpfFmiq/cCs=
|
||||
github.com/andybalholm/brotli v1.0.5/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig=
|
||||
github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be h1:9AeTilPcZAjCFIImctFaOjnTIavg87rW78vTPkQqLI8=
|
||||
@@ -113,6 +115,13 @@ github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWH
|
||||
github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
|
||||
github.com/connesc/cipherio v0.2.1 h1:FGtpTPMbKNNWByNrr9aEBtaJtXjqOzkIXNYJp6OEycw=
|
||||
github.com/connesc/cipherio v0.2.1/go.mod h1:ukY0MWJDFnJEbXMQtOcn2VmTpRfzcTz4OoVrWGGJZcA=
|
||||
github.com/couchbase/gocb/v2 v2.6.3 h1:5RsMo+RRfK0mVxHLAfpBz3/tHlgXZb1WBNItLk9Ab+c=
|
||||
github.com/couchbase/gocb/v2 v2.6.3/go.mod h1:yF5F6BHTZ/ZowhEuZbySbXrlI4rHd1TIhm5azOaMbJU=
|
||||
github.com/couchbase/gocbcore/v10 v10.2.3 h1:PEkRSNSkKjUBXx82Ucr094+anoiCG5GleOOQZOHo6D4=
|
||||
github.com/couchbase/gocbcore/v10 v10.2.3/go.mod h1:lYQIIk+tzoMcwtwU5GzPbDdqEkwkH3isI2rkSpfL0oM=
|
||||
github.com/couchbaselabs/gocaves/client v0.0.0-20230307083111-cc3960c624b1/go.mod h1:AVekAZwIY2stsJOMWLAS/0uA/+qdp7pjO8EHnl61QkY=
|
||||
github.com/couchbaselabs/gocaves/client v0.0.0-20230404095311-05e3ba4f0259 h1:2TXy68EGEzIMHOx9UvczR5ApVecwCfQZ0LjkmwMI6g4=
|
||||
github.com/couchbaselabs/gocaves/client v0.0.0-20230404095311-05e3ba4f0259/go.mod h1:AVekAZwIY2stsJOMWLAS/0uA/+qdp7pjO8EHnl61QkY=
|
||||
github.com/containerd/stargz-snapshotter/estargz v0.14.3 h1:OqlDCK3ZVUO6C3B/5FSkDwbkEETK84kQgEeFwDC+62k=
|
||||
github.com/containerd/stargz-snapshotter/estargz v0.14.3/go.mod h1:KY//uOCIkSuNAHhJogcZtrNHdKrA99/FCCRjE3HD36o=
|
||||
github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
|
||||
@@ -173,8 +182,8 @@ github.com/go-git/go-git/v5 v5.7.0 h1:t9AudWVLmqzlo+4bqdf7GY+46SUuRsx59SboFxkq2a
|
||||
github.com/go-git/go-git/v5 v5.7.0/go.mod h1:coJHKEOk5kUClpsNlXrUvPrDxY3w3gjHvhcZd8Fodw8=
|
||||
github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
|
||||
github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
|
||||
github.com/go-ldap/ldap/v3 v3.4.4 h1:qPjipEpt+qDa6SI/h1fzuGWoRUY+qqQ9sOZq67/PYUs=
|
||||
github.com/go-ldap/ldap/v3 v3.4.4/go.mod h1:fe1MsuN5eJJ1FeLT/LEBVdWfNWKh459R7aXgXtJC+aI=
|
||||
github.com/go-ldap/ldap/v3 v3.4.5 h1:ekEKmaDrpvR2yf5Nc/DClsGG9lAmdDixe44mLzlW5r8=
|
||||
github.com/go-ldap/ldap/v3 v3.4.5/go.mod h1:bMGIq3AGbytbaMwf8wdv5Phdxz0FWHTIYMSzyrYgnQs=
|
||||
github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ=
|
||||
github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
|
||||
github.com/go-logr/zapr v1.2.4 h1:QHVo+6stLbfJmYGkQ7uGHUCu5hnAFAj6mDe6Ea0SeOo=
|
||||
@@ -419,6 +428,7 @@ github.com/smartystreets/gunit v1.1.3 h1:32x+htJCu3aMswhPw3teoJ+PnWPONqdNgaGs6Qt
|
||||
github.com/smartystreets/gunit v1.1.3/go.mod h1:EH5qMBab2UclzXUcpR8b93eHsIlp9u+pDQIRp5DZNzQ=
|
||||
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||
github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
|
||||
github.com/stretchr/objx v0.5.0 h1:1zr/of2m5FGMsad5YfcqgdqdWrIhu+EBEJRhR1U7z/c=
|
||||
github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
|
||||
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
|
||||
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
|
||||
@@ -427,17 +437,15 @@ github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5
|
||||
github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
||||
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
||||
github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
||||
github.com/stretchr/testify v1.7.2/go.mod h1:R6va5+xMeoiuVRoj+gSkQ7d3FALtqAAGI1FQKckRals=
|
||||
github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
|
||||
github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
|
||||
github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
|
||||
github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
|
||||
github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
|
||||
github.com/tailscale/depaware v0.0.0-20210622194025-720c4b409502 h1:34icjjmqJ2HPjrSuJYEkdZ+0ItmGQAQ75cRHIiftIyE=
|
||||
github.com/tailscale/depaware v0.0.0-20210622194025-720c4b409502/go.mod h1:p9lPsd+cx33L3H9nNoecRRxPssFKUwwI50I3pZ0yT+8=
|
||||
github.com/therootcompany/xz v1.0.1 h1:CmOtsn1CbtmyYiusbfmhmkpAAETj0wBIH6kCYaX+xzw=
|
||||
github.com/therootcompany/xz v1.0.1/go.mod h1:3K3UH1yCKgBneZYhuQUvJ9HPD19UEXEI0BWbMn8qNMY=
|
||||
github.com/tidwall/pretty v1.0.0 h1:HsD+QiTn7sK6flMKIvNmpqz1qrpP3Ps6jOKIKMooyg4=
|
||||
github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk=
|
||||
github.com/trufflesecurity/overseer v1.1.7-custom5 h1:xu+Fg6fkSRifUPzUCl7N8HmobJ6WGOkIApGnM7mJS6w=
|
||||
github.com/trufflesecurity/overseer v1.1.7-custom5/go.mod h1:nT9w37AiO1Nop2VhVhNfzAFaPjthvxgpDV3XKsxYkcI=
|
||||
github.com/ulikunitz/xz v0.5.6/go.mod h1:2bypXElzHzzJZwzH67Y6wb67pO62Rzfn7BSiF4ABRW8=
|
||||
@@ -446,16 +454,16 @@ github.com/ulikunitz/xz v0.5.10/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0o
|
||||
github.com/urfave/cli v1.22.12/go.mod h1:sSBEIC79qR6OvcmsD4U3KABeOTxDqQtdDnaFuUN30b8=
|
||||
github.com/vbatts/tar-split v0.11.3 h1:hLFqsOLQ1SsppQNTMpkpPXClLDfC2A3Zgy9OUU+RVck=
|
||||
github.com/vbatts/tar-split v0.11.3/go.mod h1:9QlHN18E+fEH7RdG+QAJJcuya3rqT7eXSTY7wGrAokY=
|
||||
github.com/xanzy/go-gitlab v0.85.0 h1:E/wjnsd/mM5kV6O9y5+i6zxjx+wfAwa97sgcT1ETNwk=
|
||||
github.com/xanzy/go-gitlab v0.85.0/go.mod h1:5ryv+MnpZStBH8I/77HuQBsMbBGANtVpLWC15qOjWAw=
|
||||
github.com/xanzy/go-gitlab v0.86.0 h1:jR8V9cK9jXRQDb46KOB20NCF3ksY09luaG0IfXE6p7w=
|
||||
github.com/xanzy/go-gitlab v0.86.0/go.mod h1:5ryv+MnpZStBH8I/77HuQBsMbBGANtVpLWC15qOjWAw=
|
||||
github.com/xanzy/ssh-agent v0.3.3 h1:+/15pJfg/RsTxqYcX6fHqOXZwwMP+2VyYWJeWM2qQFM=
|
||||
github.com/xanzy/ssh-agent v0.3.3/go.mod h1:6dzNDKs0J9rVPHPhaGCukekBHKqfl+L3KghI1Bc68Uw=
|
||||
github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c=
|
||||
github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI=
|
||||
github.com/xdg-go/scram v1.1.1 h1:VOMT+81stJgXW3CpHyqHN3AXDYIMsx56mEFrB37Mb/E=
|
||||
github.com/xdg-go/scram v1.1.1/go.mod h1:RaEWvsqvNKKvBPvcKeFjrG2cJqOkHTiyTpzz23ni57g=
|
||||
github.com/xdg-go/stringprep v1.0.3 h1:kdwGpVNwPFtjs98xCGkHjQtGKh86rDcRZN17QEMCOIs=
|
||||
github.com/xdg-go/stringprep v1.0.3/go.mod h1:W3f5j4i+9rC0kuIEJL0ky1VpHXQU3ocBgklLGvcBnW8=
|
||||
github.com/xdg-go/scram v1.1.2 h1:FHX5I5B4i4hKRVRBCFRxq1iQRej7WO3hhBuJf+UUySY=
|
||||
github.com/xdg-go/scram v1.1.2/go.mod h1:RT/sEzTbU5y00aCK8UOx6R7YryM0iF1N2MOmC3kKLN4=
|
||||
github.com/xdg-go/stringprep v1.0.4 h1:XLI/Ng3O1Atzq0oBs3TWm+5ZVgkq2aqdlvP9JtoZ6c8=
|
||||
github.com/xdg-go/stringprep v1.0.4/go.mod h1:mPGuuIYwz7CmR2bT9j4GbQqutWS1zV24gijq1dTyGkM=
|
||||
github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d h1:splanxYIlg+5LfHAM6xpdFEAYOk8iySO56hMFq6uLyA=
|
||||
github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA=
|
||||
github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
|
||||
@@ -463,8 +471,8 @@ github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1
|
||||
github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
|
||||
github.com/yusufpapurcu/wmi v1.2.2 h1:KBNDSne4vP5mbSWnJbO+51IMOXJB67QiYCSBrubbPRg=
|
||||
github.com/yusufpapurcu/wmi v1.2.2/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0=
|
||||
go.mongodb.org/mongo-driver v1.11.6 h1:XM7G6PjiGAO5betLF13BIa5TlLUUE3uJ/2Ox3Lz1K+o=
|
||||
go.mongodb.org/mongo-driver v1.11.6/go.mod h1:G9TgswdsWjX4tmDA5zfs2+6AEPpYJwqblyjsfuh8oXY=
|
||||
go.mongodb.org/mongo-driver v1.12.0 h1:aPx33jmn/rQuJXPQLZQ8NtfPQG8CaqgLThFtqRb0PiE=
|
||||
go.mongodb.org/mongo-driver v1.12.0/go.mod h1:AZkxhPnFJUoH7kZlFkVKucV20K387miPfm7oimrSmK0=
|
||||
go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
|
||||
go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
|
||||
go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
|
||||
@@ -578,8 +586,8 @@ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJ
|
||||
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.2.0 h1:PUR+T4wwASmuSTYdKjYHI5TD22Wy5ogLU5qZCOLxBrI=
|
||||
golang.org/x/sync v0.2.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.3.0 h1:ftCYgMx6zT/asHUrPw8BLLscYtGznsLAnjq5RH9P66E=
|
||||
golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y=
|
||||
golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
|
||||
@@ -476,7 +476,7 @@ func run(state overseer.State) {
|
||||
}
|
||||
}
|
||||
// asynchronously wait for scanning to finish and cleanup
|
||||
go e.Finish(ctx)
|
||||
go e.Finish(ctx, logFatal)
|
||||
|
||||
if !*jsonLegacy && !*jsonOut {
|
||||
fmt.Fprintf(os.Stderr, "🐷🔑🐷 TruffleHog. Unearth your secrets. 🐷🔑🐷\n\n")
|
||||
|
||||
+33
-1
@@ -2,6 +2,7 @@ package common
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
)
|
||||
@@ -15,7 +16,11 @@ const RegexPattern = "0-9a-z"
|
||||
const AlphaNumPattern = "0-9a-zA-Z"
|
||||
const HexPattern = "0-9a-f"
|
||||
|
||||
//Custom Regex functions
|
||||
type RegexState struct {
|
||||
compiledRegex *regexp.Regexp
|
||||
}
|
||||
|
||||
// Custom Regex functions
|
||||
func BuildRegex(pattern string, specialChar string, length int) string {
|
||||
return fmt.Sprintf(`\b([%s%s]{%s})\b`, pattern, specialChar, strconv.Itoa(length))
|
||||
}
|
||||
@@ -37,3 +42,30 @@ func RangeValidation(rangeInput string) bool {
|
||||
func ToUpperCase(input string) string {
|
||||
return strings.ToUpper(input)
|
||||
}
|
||||
|
||||
func (r RegexState) Matches(data []byte) []string {
|
||||
matches := r.compiledRegex.FindAllStringSubmatch(string(data), -1)
|
||||
|
||||
res := make([]string, 0, len(matches))
|
||||
|
||||
// trim off spaces and different quote types ('").
|
||||
for i := range matches {
|
||||
res = append(res, strings.Trim(matches[i][1], `"' )`))
|
||||
}
|
||||
|
||||
return res
|
||||
}
|
||||
|
||||
// UsernameRegexCheck constructs an username usernameRegex pattern from a given pattern of excluded characters.
|
||||
func UsernameRegexCheck(pattern string) RegexState {
|
||||
raw := fmt.Sprintf(`(?im)(?:user|usr)\S{0,40}?[:=\s]{1,3}[ '"=]{0,1}([^:%+v]{4,40})\b`, pattern)
|
||||
|
||||
return RegexState{regexp.MustCompile(raw)}
|
||||
}
|
||||
|
||||
// PasswordRegexCheck constructs an username usernameRegex pattern from a given pattern of excluded characters.
|
||||
func PasswordRegexCheck(pattern string) RegexState {
|
||||
raw := fmt.Sprintf(`(?im)(?:pass|password)\S{0,40}?[:=\s]{1,3}[ '"=]{0,1}([^:%+v]{4,40})`, pattern)
|
||||
|
||||
return RegexState{regexp.MustCompile(raw)}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,59 @@
|
||||
package common
|
||||
|
||||
import (
|
||||
"github.com/stretchr/testify/assert"
|
||||
"regexp"
|
||||
"testing"
|
||||
)
|
||||
|
||||
const (
|
||||
usernamePattern = `?()/\+=\s\n`
|
||||
passwordPattern = `^<>;.*&|£\n\s`
|
||||
usernameRegex = `(?im)(?:user|usr)\S{0,40}?[:=\s]{1,3}[ '"=]{0,1}([^:?()/\+=\s\n]{4,40})\b`
|
||||
passwordRegex = `(?im)(?:pass|password)\S{0,40}?[:=\s]{1,3}[ '"=]{0,1}([^:^<>;.*&|£\n\s]{4,40})`
|
||||
)
|
||||
|
||||
func TestUsernameRegexCheck(t *testing.T) {
|
||||
usernameRegexPat := UsernameRegexCheck(usernamePattern)
|
||||
|
||||
expectedRegexPattern := regexp.MustCompile(usernameRegex)
|
||||
|
||||
if usernameRegexPat.compiledRegex.String() != expectedRegexPattern.String() {
|
||||
t.Errorf("\n got %v \n want %v", usernameRegexPat.compiledRegex, expectedRegexPattern)
|
||||
}
|
||||
|
||||
testString := `username = "johnsmith123"
|
||||
username='johnsmith123'
|
||||
username:="johnsmith123"
|
||||
username = johnsmith123
|
||||
username=johnsmith123`
|
||||
|
||||
expectedStr := []string{"johnsmith123", "johnsmith123", "johnsmith123", "johnsmith123", "johnsmith123"}
|
||||
|
||||
usernameRegexMatches := usernameRegexPat.Matches([]byte(testString))
|
||||
|
||||
assert.Exactly(t, usernameRegexMatches, expectedStr)
|
||||
|
||||
}
|
||||
|
||||
func TestPasswordRegexCheck(t *testing.T) {
|
||||
passwordRegexPat := PasswordRegexCheck(passwordPattern)
|
||||
|
||||
expectedRegexPattern := regexp.MustCompile(passwordRegex)
|
||||
assert.Equal(t, passwordRegexPat.compiledRegex, expectedRegexPattern)
|
||||
|
||||
testString := `password = "johnsmith123$!"
|
||||
password='johnsmith123$!'
|
||||
password:="johnsmith123$!"
|
||||
password = johnsmith123$!
|
||||
password=johnsmith123$!
|
||||
PasswordAuthenticator(username, "johnsmith123$!")`
|
||||
|
||||
expectedStr := []string{"johnsmith123$!", "johnsmith123$!", "johnsmith123$!", "johnsmith123$!", "johnsmith123$!",
|
||||
"johnsmith123$!"}
|
||||
|
||||
passwordRegexMatches := passwordRegexPat.Matches([]byte(testString))
|
||||
|
||||
assert.Exactly(t, passwordRegexMatches, expectedStr)
|
||||
|
||||
}
|
||||
@@ -0,0 +1,152 @@
|
||||
package couchbase
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"log"
|
||||
"regexp"
|
||||
"strings"
|
||||
"time"
|
||||
"unicode"
|
||||
|
||||
"github.com/couchbase/gocb/v2"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
|
||||
)
|
||||
|
||||
type Scanner struct{}
|
||||
|
||||
// Ensure the Scanner satisfies the interface at compile time.
|
||||
var _ detectors.Detector = (*Scanner)(nil)
|
||||
|
||||
var (
|
||||
|
||||
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
|
||||
connectionStringPat = regexp.MustCompile(`\bcb\.[a-z0-9]+\.cloud\.couchbase\.com\b`)
|
||||
usernamePat = `?()/\+=\s\n`
|
||||
passwordPat = `^<>;.*&|£\n\s`
|
||||
//passwordPat = regexp.MustCompile(`(?i)(?:pass|pwd)(?:.|[\n\r]){0,15}(\b[^<>;.*&|£\n\s]{8,100}$)`)
|
||||
// passwordPat = regexp.MustCompile(`(?im)(?:pass|pwd)\S{0,40}?[:=\s]{1,3}[ '"=]{0,1}([^:?()/\+=\s\n]{4,40})\b`)
|
||||
)
|
||||
|
||||
func meetsCouchbasePasswordRequirements(password string) (string, bool) {
|
||||
var hasLower, hasUpper, hasNumber, hasSpecialChar bool
|
||||
for _, char := range password {
|
||||
switch {
|
||||
case unicode.IsLower(char):
|
||||
hasLower = true
|
||||
case unicode.IsUpper(char):
|
||||
hasUpper = true
|
||||
case unicode.IsNumber(char):
|
||||
hasNumber = true
|
||||
case unicode.IsPunct(char) || unicode.IsSymbol(char):
|
||||
hasSpecialChar = true
|
||||
}
|
||||
|
||||
if hasLower && hasUpper && hasNumber && hasSpecialChar {
|
||||
return password, true
|
||||
}
|
||||
}
|
||||
|
||||
return "", false
|
||||
}
|
||||
|
||||
// Keywords are used for efficiently pre-filtering chunks.
|
||||
// Use identifiers in the secret preferably, or the provider name.
|
||||
func (s Scanner) Keywords() []string {
|
||||
return []string{"couchbase://", "couchbases://"}
|
||||
}
|
||||
|
||||
// FromData will find and optionally verify Couchbase secrets in a given set of bytes.
|
||||
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
|
||||
dataStr := string(data)
|
||||
|
||||
connectionStringMatches := connectionStringPat.FindAllStringSubmatch(dataStr, -1)
|
||||
|
||||
// prepend 'couchbases://' to the connection string as the connection
|
||||
// string format is couchbases://cb.stuff.cloud.couchbase.com but the
|
||||
// cb.stuff.cloud.couchbase.com may be separated from the couchbases:// in codebases.
|
||||
for i, connectionStringMatch := range connectionStringMatches {
|
||||
connectionStringMatches[i][0] = "couchbases://" + connectionStringMatch[0]
|
||||
}
|
||||
|
||||
usernameRegexState := common.UsernameRegexCheck(usernamePat)
|
||||
usernameMatches := usernameRegexState.Matches(data)
|
||||
|
||||
passwordRegexState := common.PasswordRegexCheck(passwordPat)
|
||||
passwordMatches := passwordRegexState.Matches(data)
|
||||
|
||||
for _, connectionStringMatch := range connectionStringMatches {
|
||||
resConnectionStringMatch := strings.TrimSpace(connectionStringMatch[0])
|
||||
|
||||
for _, resUsernameMatch := range usernameMatches {
|
||||
|
||||
for _, resPasswordMatch := range passwordMatches {
|
||||
_, metPasswordRequirements := meetsCouchbasePasswordRequirements(resPasswordMatch)
|
||||
|
||||
if !metPasswordRequirements {
|
||||
continue
|
||||
}
|
||||
|
||||
s1 := detectors.Result{
|
||||
DetectorType: detectorspb.DetectorType_Couchbase,
|
||||
Raw: []byte(fmt.Sprintf("%s:%s@%s", resUsernameMatch, resPasswordMatch, resConnectionStringMatch)),
|
||||
}
|
||||
|
||||
if verify {
|
||||
|
||||
options := gocb.ClusterOptions{
|
||||
Authenticator: gocb.PasswordAuthenticator{
|
||||
Username: resUsernameMatch,
|
||||
Password: resPasswordMatch,
|
||||
},
|
||||
}
|
||||
|
||||
// Sets a pre-configured profile called "wan-development" to help avoid latency issues
|
||||
// when accessing Capella from a different Wide Area Network
|
||||
// or Availability Zone (e.g. your laptop).
|
||||
if err := options.ApplyProfile(gocb.ClusterConfigProfileWanDevelopment); err != nil {
|
||||
log.Fatal("apply profile err", err)
|
||||
}
|
||||
|
||||
// Initialize the Connection
|
||||
cluster, err := gocb.Connect(resConnectionStringMatch, options)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
|
||||
// We'll ping the KV nodes in our cluster.
|
||||
pings, err := cluster.Ping(&gocb.PingOptions{
|
||||
Timeout: time.Second * 5,
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
|
||||
for _, ping := range pings.Services {
|
||||
for _, pingEndpoint := range ping {
|
||||
if pingEndpoint.State == gocb.PingStateOk {
|
||||
s1.Verified = true
|
||||
break
|
||||
} else {
|
||||
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
|
||||
if detectors.IsKnownFalsePositive(resPasswordMatch, detectors.DefaultFalsePositives, true) {
|
||||
continue
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
results = append(results, s1)
|
||||
}
|
||||
}
|
||||
}
|
||||
return results, nil
|
||||
}
|
||||
|
||||
func (s Scanner) Type() detectorspb.DetectorType {
|
||||
return detectorspb.DetectorType_Couchbase
|
||||
}
|
||||
@@ -0,0 +1,122 @@
|
||||
//go:build detectors
|
||||
// +build detectors
|
||||
|
||||
package couchbase
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/kylelemons/godebug/pretty"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
|
||||
)
|
||||
|
||||
func TestCouchbase_FromChunk(t *testing.T) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
|
||||
defer cancel()
|
||||
testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")
|
||||
if err != nil {
|
||||
t.Fatalf("could not get test secrets from GCP: %s", err)
|
||||
}
|
||||
endpoint := testSecrets.MustGetField("COUCHBASE_ENDPOINT")
|
||||
username := testSecrets.MustGetField("COUCHBASE_USERNAME")
|
||||
password := testSecrets.MustGetField("COUCHBASE_PASSWORD")
|
||||
inactiveSecret := testSecrets.MustGetField("COUCHBASE_INACTIVE_PASSWORD")
|
||||
|
||||
type args struct {
|
||||
ctx context.Context
|
||||
data []byte
|
||||
verify bool
|
||||
}
|
||||
tests := []struct {
|
||||
name string
|
||||
s Scanner
|
||||
args args
|
||||
want []detectors.Result
|
||||
wantErr bool
|
||||
}{
|
||||
{
|
||||
name: "found, verified",
|
||||
s: Scanner{},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("db uri: %s \n username = %s \n password = %s", endpoint, username, password)),
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_Couchbase,
|
||||
Verified: true,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "found, unverified",
|
||||
s: Scanner{},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("db uri: %s \n username = %s \n password = %s", endpoint, username, inactiveSecret)), // the secret would satisfy the regex but not pass validation
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_Couchbase,
|
||||
Verified: false,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "not found",
|
||||
s: Scanner{},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte("You cannot find the secret within"),
|
||||
verify: true,
|
||||
},
|
||||
want: nil,
|
||||
wantErr: false,
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
s := Scanner{}
|
||||
|
||||
got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
|
||||
if (err != nil) != tt.wantErr {
|
||||
t.Errorf("Couchbase.FromData() error = %v, wantErr %v", err, tt.wantErr)
|
||||
return
|
||||
}
|
||||
for i := range got {
|
||||
if len(got[i].Raw) == 0 {
|
||||
t.Fatalf("no raw secret present: \n %+v", got[i])
|
||||
}
|
||||
got[i].Raw = nil
|
||||
}
|
||||
if diff := pretty.Compare(got, tt.want); diff != "" {
|
||||
t.Errorf("Couchbase.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkFromData(benchmark *testing.B) {
|
||||
ctx := context.Background()
|
||||
s := Scanner{}
|
||||
for name, data := range detectors.MustGetBenchmarkData() {
|
||||
benchmark.Run(name, func(b *testing.B) {
|
||||
for n := 0; n < b.N; n++ {
|
||||
_, err := s.FromData(ctx, false, data)
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -39,6 +39,8 @@ type EndpointCustomizer interface {
|
||||
DefaultEndpoint() string
|
||||
}
|
||||
|
||||
// Result is a secret that has been found by a Detector and optionally verified.
|
||||
// It contains the raw secret identification data, redacted data, and extra data.
|
||||
type Result struct {
|
||||
// DetectorType is the type of Detector.
|
||||
DetectorType detectorspb.DetectorType
|
||||
@@ -46,7 +48,15 @@ type Result struct {
|
||||
DetectorName string
|
||||
// DecoderType is the type of Decoder.
|
||||
DecoderType detectorspb.DecoderType
|
||||
Verified bool
|
||||
// Verified is true if the secret has been confirmed to be a valid secret.
|
||||
Verified bool
|
||||
// HasVerificationIndeterminate is true if the secret could NOT be verified.
|
||||
// This is used to differentiate between secrets that are NOT verified and secrets that COULD NOT be verified.
|
||||
// Ex: A 429 response from a service, or a timeout would set this to true.
|
||||
HasVerificationIndeterminate bool
|
||||
// VerificationIndeterminateError holds the error if the secret could NOT be verified.
|
||||
// This is used specifically for cases that resulted in indeterminate resolution, such as a 429 or timeout.
|
||||
VerificationIndeterminateError error
|
||||
// Raw contains the raw secret identifier data. Prefer IDs over secrets since it is used for deduping after hashing.
|
||||
Raw []byte
|
||||
// RawV2 contains the raw secret identifier that is a combination of both the ID and the secret.
|
||||
@@ -59,6 +69,17 @@ type Result struct {
|
||||
StructuredData *detectorspb.StructuredData
|
||||
}
|
||||
|
||||
// IsVerified checks if a result is guaranteed to be verified.
|
||||
func (r Result) IsVerified() bool {
|
||||
return r.Verified && !r.HasVerificationIndeterminate
|
||||
}
|
||||
|
||||
// IsUnverified checks if a result is guaranteed to be unverified.
|
||||
// This is used to differentiate between secrets that are NOT verified and secrets that COULD NOT be verified.
|
||||
func (r Result) IsUnverified() bool {
|
||||
return !r.Verified && !r.HasVerificationIndeterminate
|
||||
}
|
||||
|
||||
type ResultWithMetadata struct {
|
||||
// SourceMetadata contains source-specific contextual information.
|
||||
SourceMetadata *source_metadatapb.MetaData
|
||||
|
||||
@@ -3,7 +3,9 @@
|
||||
|
||||
package detectors
|
||||
|
||||
import "testing"
|
||||
import (
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestPrefixRegex(t *testing.T) {
|
||||
tests := []struct {
|
||||
@@ -37,3 +39,101 @@ func BenchmarkPrefixRegex(b *testing.B) {
|
||||
PrefixRegex(kws)
|
||||
}
|
||||
}
|
||||
|
||||
func TestIsVerified(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
result Result
|
||||
want bool
|
||||
}{
|
||||
{
|
||||
name: "Verified true, HasVerificationIndeterminate false",
|
||||
result: Result{
|
||||
Verified: true,
|
||||
HasVerificationIndeterminate: false,
|
||||
},
|
||||
want: true,
|
||||
},
|
||||
{
|
||||
name: "Verified false, HasVerificationIndeterminate false",
|
||||
result: Result{
|
||||
Verified: false,
|
||||
HasVerificationIndeterminate: false,
|
||||
},
|
||||
want: false,
|
||||
},
|
||||
{
|
||||
name: "Verified true, HasVerificationIndeterminate true",
|
||||
result: Result{
|
||||
Verified: true,
|
||||
HasVerificationIndeterminate: true,
|
||||
},
|
||||
want: false,
|
||||
},
|
||||
{
|
||||
name: "Verified false, HasVerificationIndeterminate true",
|
||||
result: Result{
|
||||
Verified: false,
|
||||
HasVerificationIndeterminate: true,
|
||||
},
|
||||
want: false,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
if got := tt.result.IsVerified(); got != tt.want {
|
||||
t.Errorf("IsVerified() = %v, want %v", got, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestIsUnverified(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
result Result
|
||||
want bool
|
||||
}{
|
||||
{
|
||||
name: "Verified true, HasVerificationIndeterminate false",
|
||||
result: Result{
|
||||
Verified: true,
|
||||
HasVerificationIndeterminate: false,
|
||||
},
|
||||
want: false,
|
||||
},
|
||||
{
|
||||
name: "Verified false, HasVerificationIndeterminate false",
|
||||
result: Result{
|
||||
Verified: false,
|
||||
HasVerificationIndeterminate: false,
|
||||
},
|
||||
want: true,
|
||||
},
|
||||
{
|
||||
name: "Verified true, HasVerificationIndeterminate true",
|
||||
result: Result{
|
||||
Verified: true,
|
||||
HasVerificationIndeterminate: true,
|
||||
},
|
||||
want: false,
|
||||
},
|
||||
{
|
||||
name: "Verified false, HasVerificationIndeterminate true",
|
||||
result: Result{
|
||||
Verified: false,
|
||||
HasVerificationIndeterminate: true,
|
||||
},
|
||||
want: false,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
if got := tt.result.IsUnverified(); got != tt.want {
|
||||
t.Errorf("IsUnverified() = %v, want %v", got, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -6,7 +6,7 @@ import (
|
||||
"unicode"
|
||||
)
|
||||
|
||||
var DefaultFalsePositives = []FalsePositive{"example", "xxxxxx", "aaaaaa", "abcde", "00000", "sample"}
|
||||
var DefaultFalsePositives = []FalsePositive{"example", "xxxxxx", "aaaaaa", "abcde", "00000", "sample", "www"}
|
||||
|
||||
type FalsePositive string
|
||||
|
||||
@@ -31,9 +31,9 @@ var FalsePositiveWordlists = Wordlists{
|
||||
programmingBookWords: bytesToCleanWordList(programmingBookWords),
|
||||
}
|
||||
|
||||
//IsKnownFalsePositives will not return a valid secret finding if any of the disqualifying conditions are met
|
||||
//Currently that includes: No number, english word in key, or matches common example pattens.
|
||||
//Only the secret key material should be passed into this function
|
||||
// IsKnownFalsePositives will not return a valid secret finding if any of the disqualifying conditions are met
|
||||
// Currently that includes: No number, english word in key, or matches common example pattens.
|
||||
// Only the secret key material should be passed into this function
|
||||
func IsKnownFalsePositive(match string, falsePositives []FalsePositive, wordCheck bool) bool {
|
||||
|
||||
for _, fp := range falsePositives {
|
||||
|
||||
@@ -38,19 +38,18 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
keyMatches := keyPat.FindAllString(dataStr, -1)
|
||||
|
||||
for _, key := range keyMatches {
|
||||
secMatches := secretPat.FindAllString(dataStr, -1)
|
||||
|
||||
if verify {
|
||||
secMatches := secretPat.FindAllString(dataStr, -1)
|
||||
for _, secret := range secMatches {
|
||||
|
||||
for _, secret := range secMatches {
|
||||
|
||||
s1 := detectors.Result{
|
||||
DetectorType: detectorspb.DetectorType_RazorPay,
|
||||
Raw: []byte(key),
|
||||
RawV2: []byte(key + secret),
|
||||
Redacted: key,
|
||||
}
|
||||
s1 := detectors.Result{
|
||||
DetectorType: detectorspb.DetectorType_RazorPay,
|
||||
Raw: []byte(key),
|
||||
RawV2: []byte(key + secret),
|
||||
Redacted: key,
|
||||
}
|
||||
|
||||
if verify {
|
||||
req, err := http.NewRequest("GET", "https://api.razorpay.com/v1/items?count=1", nil)
|
||||
if err != nil {
|
||||
continue
|
||||
@@ -66,25 +65,21 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
if res.StatusCode >= 200 && res.StatusCode < 300 {
|
||||
if json.Valid(bodyBytes) {
|
||||
s1.Verified = true
|
||||
} else {
|
||||
s1.Verified = false
|
||||
}
|
||||
} else {
|
||||
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
|
||||
if detectors.IsKnownFalsePositive(key, detectors.DefaultFalsePositives, true) {
|
||||
continue
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
results = append(results, s1)
|
||||
}
|
||||
|
||||
if !s1.Verified && detectors.IsKnownFalsePositive(key, detectors.DefaultFalsePositives, true) {
|
||||
continue
|
||||
}
|
||||
|
||||
results = append(results, s1)
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
results = detectors.CleanResults(results)
|
||||
return
|
||||
return detectors.CleanResults(results), nil
|
||||
}
|
||||
|
||||
func (s Scanner) Type() detectorspb.DetectorType {
|
||||
|
||||
@@ -62,7 +62,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
continue
|
||||
}
|
||||
body := string(bodyBytes)
|
||||
if (res.StatusCode >= 200 && res.StatusCode < 300) || (res.StatusCode == 400 && strings.Contains(body, "no_text")) {
|
||||
if (res.StatusCode >= 200 && res.StatusCode < 300) || (res.StatusCode == 400 && (strings.Contains(body, "no_text") || strings.Contains(body, "missing_text"))) {
|
||||
s1.Verified = true
|
||||
}
|
||||
}
|
||||
|
||||
@@ -21,6 +21,7 @@ var (
|
||||
identifierPat = regexp.MustCompile(`(?i)sid.{0,20}AC[0-9a-f]{32}`) // Should we have this? Seems restrictive.
|
||||
sidPat = regexp.MustCompile(`\bAC[0-9a-f]{32}\b`)
|
||||
keyPat = regexp.MustCompile(`\b[0-9a-f]{32}\b`)
|
||||
client = common.SaneHttpClient()
|
||||
)
|
||||
|
||||
// Keywords are used for efficiently pre-filtering chunks.
|
||||
@@ -43,15 +44,15 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
sidMatches := sidPat.FindAllString(dataStr, -1)
|
||||
|
||||
for _, sid := range sidMatches {
|
||||
s := detectors.Result{
|
||||
DetectorType: detectorspb.DetectorType_Twilio,
|
||||
Raw: []byte(sid),
|
||||
Redacted: sid,
|
||||
}
|
||||
for _, key := range keyMatches {
|
||||
s1 := detectors.Result{
|
||||
DetectorType: detectorspb.DetectorType_Twilio,
|
||||
Raw: []byte(sid),
|
||||
RawV2: []byte(sid + key),
|
||||
Redacted: sid,
|
||||
}
|
||||
|
||||
if verify {
|
||||
client := common.SaneHttpClient()
|
||||
for _, key := range keyMatches {
|
||||
if verify {
|
||||
|
||||
form := url.Values{}
|
||||
form.Add("FriendlyName", "MyServiceName")
|
||||
@@ -71,22 +72,22 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
res.Body.Close() // The request body is unused.
|
||||
|
||||
if res.StatusCode >= 200 && res.StatusCode < 300 {
|
||||
s.Verified = true
|
||||
s1.Verified = true
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if !s.Verified && detectors.IsKnownFalsePositive(string(s.Raw), detectors.DefaultFalsePositives, true) {
|
||||
continue
|
||||
}
|
||||
if !s1.Verified && detectors.IsKnownFalsePositive(string(s1.Raw), detectors.DefaultFalsePositives, true) {
|
||||
continue
|
||||
}
|
||||
|
||||
if len(keyMatches) > 0 {
|
||||
results = append(results, s)
|
||||
if len(keyMatches) > 0 {
|
||||
results = append(results, s1)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return
|
||||
return detectors.CleanResults(results), nil
|
||||
}
|
||||
|
||||
func (s Scanner) Type() detectorspb.DetectorType {
|
||||
|
||||
@@ -98,6 +98,7 @@ func TestTwilio_FromChunk(t *testing.T) {
|
||||
t.Fatal("no raw secret present")
|
||||
}
|
||||
got[i].Raw = nil
|
||||
got[i].RawV2 = nil
|
||||
}
|
||||
if diff := pretty.Compare(got, tt.want); diff != "" {
|
||||
t.Errorf("Twilio.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
package engine
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"runtime"
|
||||
|
||||
"github.com/go-errors/errors"
|
||||
@@ -38,14 +39,13 @@ func (e *Engine) ScanCircleCI(ctx context.Context, token string) error {
|
||||
return errors.WrapPrefix(err, "failed to init Circle CI source", 0)
|
||||
}
|
||||
|
||||
e.sourcesWg.Add(1)
|
||||
go func() {
|
||||
e.sourcesWg.Go(func() error {
|
||||
defer common.RecoverWithExit(ctx)
|
||||
defer e.sourcesWg.Done()
|
||||
err := circleSource.Chunks(ctx, e.ChunksChan())
|
||||
if err != nil {
|
||||
ctx.Logger().Error(err, "error scanning Circle CI")
|
||||
return fmt.Errorf("error scanning CircleCI: %w", err)
|
||||
}
|
||||
}()
|
||||
return nil
|
||||
})
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
package engine
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"runtime"
|
||||
|
||||
"github.com/go-errors/errors"
|
||||
@@ -25,14 +26,13 @@ func (e *Engine) ScanDocker(ctx context.Context, conn *anypb.Any) error {
|
||||
return errors.WrapPrefix(err, "could not init docker source", 0)
|
||||
}
|
||||
|
||||
e.sourcesWg.Add(1)
|
||||
go func() {
|
||||
e.sourcesWg.Go(func() error {
|
||||
defer common.RecoverWithExit(ctx)
|
||||
defer e.sourcesWg.Done()
|
||||
err := dockerSource.Chunks(ctx, e.ChunksChan())
|
||||
if err != nil {
|
||||
ctx.Logger().Error(err, "error scanning docker image")
|
||||
return fmt.Errorf("error scanning docker image: %w", err)
|
||||
}
|
||||
}()
|
||||
return nil
|
||||
})
|
||||
return nil
|
||||
}
|
||||
|
||||
+13
-3
@@ -10,6 +10,7 @@ import (
|
||||
"time"
|
||||
|
||||
ahocorasick "github.com/petar-dambovaliev/aho-corasick"
|
||||
"golang.org/x/sync/errgroup"
|
||||
"google.golang.org/protobuf/proto"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
@@ -32,7 +33,7 @@ type Engine struct {
|
||||
chunksScanned uint64
|
||||
bytesScanned uint64
|
||||
detectorAvgTime sync.Map
|
||||
sourcesWg sync.WaitGroup
|
||||
sourcesWg *errgroup.Group
|
||||
workersWg sync.WaitGroup
|
||||
// filterUnverified is used to reduce the number of unverified results.
|
||||
// If there are multiple unverified results for the same chunk for the same detector,
|
||||
@@ -123,6 +124,11 @@ func Start(ctx context.Context, options ...EngineOption) *Engine {
|
||||
}
|
||||
ctx.Logger().V(2).Info("engine started", "workers", e.concurrency)
|
||||
|
||||
sourcesWg, egCtx := errgroup.WithContext(ctx)
|
||||
sourcesWg.SetLimit(e.concurrency)
|
||||
e.sourcesWg = sourcesWg
|
||||
ctx.SetParent(egCtx)
|
||||
|
||||
if len(e.decoders) == 0 {
|
||||
e.decoders = decoders.DefaultDecoders()
|
||||
}
|
||||
@@ -188,10 +194,14 @@ func Start(ctx context.Context, options ...EngineOption) *Engine {
|
||||
// Finish waits for running sources to complete and workers to finish scanning
|
||||
// chunks before closing their respective channels. Once Finish is called, no
|
||||
// more sources may be scanned by the engine.
|
||||
func (e *Engine) Finish(ctx context.Context) {
|
||||
func (e *Engine) Finish(ctx context.Context, logFunc func(error, string, ...any)) {
|
||||
defer common.RecoverWithExit(ctx)
|
||||
// wait for the sources to finish putting chunks onto the chunks channel
|
||||
e.sourcesWg.Wait()
|
||||
sourceErr := e.sourcesWg.Wait()
|
||||
if sourceErr != nil {
|
||||
logFunc(sourceErr, "error occurred while collecting chunks")
|
||||
}
|
||||
|
||||
close(e.chunks)
|
||||
// wait for the workers to finish processing all of the chunks and putting
|
||||
// results onto the results channel
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
package engine
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"runtime"
|
||||
|
||||
"github.com/go-errors/errors"
|
||||
@@ -37,14 +38,13 @@ func (e *Engine) ScanFileSystem(ctx context.Context, c sources.FilesystemConfig)
|
||||
return errors.WrapPrefix(err, "could not init filesystem source", 0)
|
||||
}
|
||||
fileSystemSource.WithFilter(c.Filter)
|
||||
e.sourcesWg.Add(1)
|
||||
go func() {
|
||||
e.sourcesWg.Go(func() error {
|
||||
defer common.RecoverWithExit(ctx)
|
||||
defer e.sourcesWg.Done()
|
||||
err := fileSystemSource.Chunks(ctx, e.ChunksChan())
|
||||
if err != nil {
|
||||
ctx.Logger().Error(err, "error scanning filesystem")
|
||||
return fmt.Errorf("error scanning filesystem: %w", err)
|
||||
}
|
||||
}()
|
||||
return nil
|
||||
})
|
||||
return nil
|
||||
}
|
||||
|
||||
+4
-5
@@ -54,14 +54,13 @@ func (e *Engine) ScanGCS(ctx context.Context, c sources.GCSConfig) error {
|
||||
return fmt.Errorf("failed to initialize GCS source: %w", err)
|
||||
}
|
||||
|
||||
e.sourcesWg.Add(1)
|
||||
go func() {
|
||||
e.sourcesWg.Go(func() error {
|
||||
defer common.RecoverWithExit(ctx)
|
||||
defer e.sourcesWg.Done()
|
||||
if err := source.Chunks(ctx, e.ChunksChan()); err != nil {
|
||||
ctx.Logger().Error(err, "could not scan GCS")
|
||||
return fmt.Errorf("could not scan GCS: %w", err)
|
||||
}
|
||||
}()
|
||||
return nil
|
||||
})
|
||||
return nil
|
||||
}
|
||||
|
||||
|
||||
+21
-2
@@ -5,6 +5,7 @@ import (
|
||||
"testing"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/context"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/decoders"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
|
||||
)
|
||||
|
||||
@@ -55,12 +56,30 @@ func TestScanGCS(t *testing.T) {
|
||||
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
e := &Engine{}
|
||||
err := e.ScanGCS(context.Background(), test.gcsConfig)
|
||||
ctx, cancel := context.WithCancel(context.TODO())
|
||||
defer cancel()
|
||||
|
||||
e := Start(ctx,
|
||||
WithConcurrency(1),
|
||||
WithDecoders(decoders.DefaultDecoders()...),
|
||||
WithDetectors(false, DefaultDetectors()...),
|
||||
)
|
||||
go func() {
|
||||
resultCount := 0
|
||||
for range e.ResultsChan() {
|
||||
resultCount++
|
||||
}
|
||||
}()
|
||||
|
||||
err := e.ScanGCS(ctx, test.gcsConfig)
|
||||
if err != nil && !test.wantErr && !strings.Contains(err.Error(), "googleapi: Error 400: Bad Request") {
|
||||
t.Errorf("ScanGCS() got: %v, want: %v", err, nil)
|
||||
return
|
||||
}
|
||||
logFatalFunc := func(_ error, _ string, _ ...any) {
|
||||
t.Fatalf("error logging function should not have been called")
|
||||
}
|
||||
e.Finish(ctx, logFatalFunc)
|
||||
|
||||
if err == nil && test.wantErr {
|
||||
t.Errorf("ScanGCS() got: %v, want: %v", err, "error")
|
||||
|
||||
+4
-5
@@ -66,14 +66,13 @@ func (e *Engine) ScanGit(ctx context.Context, c sources.GitConfig) error {
|
||||
"source_type", sourcespb.SourceType_SOURCE_TYPE_GIT.String(),
|
||||
"source_name", "git",
|
||||
)
|
||||
e.sourcesWg.Add(1)
|
||||
go func() {
|
||||
e.sourcesWg.Go(func() error {
|
||||
defer common.RecoverWithExit(ctx)
|
||||
defer e.sourcesWg.Done()
|
||||
err := gitSource.ScanRepo(ctx, repo, c.RepoPath, scanOptions, e.ChunksChan())
|
||||
if err != nil {
|
||||
ctx.Logger().Error(err, "could not scan repo")
|
||||
return fmt.Errorf("could not scan repo: %w", err)
|
||||
}
|
||||
}()
|
||||
return nil
|
||||
})
|
||||
return nil
|
||||
}
|
||||
|
||||
+41
-32
@@ -53,39 +53,45 @@ func TestGitEngine(t *testing.T) {
|
||||
base: "2f251b8c1e72135a375b659951097ec7749d4af9",
|
||||
},
|
||||
} {
|
||||
e := Start(ctx,
|
||||
WithConcurrency(1),
|
||||
WithDecoders(decoders.DefaultDecoders()...),
|
||||
WithDetectors(false, DefaultDetectors()...),
|
||||
)
|
||||
cfg := sources.GitConfig{
|
||||
RepoPath: path,
|
||||
HeadRef: tTest.branch,
|
||||
BaseRef: tTest.base,
|
||||
MaxDepth: tTest.maxDepth,
|
||||
Filter: tTest.filter,
|
||||
}
|
||||
if err := e.ScanGit(ctx, cfg); err != nil {
|
||||
return
|
||||
}
|
||||
go e.Finish(ctx)
|
||||
resultCount := 0
|
||||
for result := range e.ResultsChan() {
|
||||
switch meta := result.SourceMetadata.GetData().(type) {
|
||||
case *source_metadatapb.MetaData_Git:
|
||||
if tTest.expected[meta.Git.Commit].B != string(result.Raw) {
|
||||
t.Errorf("%s: unexpected result. Got: %s, Expected: %s", tName, string(result.Raw), tTest.expected[meta.Git.Commit].B)
|
||||
}
|
||||
if tTest.expected[meta.Git.Commit].LineNumber != result.SourceMetadata.GetGit().Line {
|
||||
t.Errorf("%s: unexpected line number. Got: %d, Expected: %d", tName, result.SourceMetadata.GetGit().Line, tTest.expected[meta.Git.Commit].LineNumber)
|
||||
}
|
||||
t.Run(tName, func(t *testing.T) {
|
||||
e := Start(ctx,
|
||||
WithConcurrency(1),
|
||||
WithDecoders(decoders.DefaultDecoders()...),
|
||||
WithDetectors(false, DefaultDetectors()...),
|
||||
)
|
||||
cfg := sources.GitConfig{
|
||||
RepoPath: path,
|
||||
HeadRef: tTest.branch,
|
||||
BaseRef: tTest.base,
|
||||
MaxDepth: tTest.maxDepth,
|
||||
Filter: tTest.filter,
|
||||
}
|
||||
if err := e.ScanGit(ctx, cfg); err != nil {
|
||||
return
|
||||
}
|
||||
resultCount++
|
||||
|
||||
}
|
||||
if resultCount != len(tTest.expected) {
|
||||
t.Errorf("%s: unexpected number of results. Got: %d, Expected: %d", tName, resultCount, len(tTest.expected))
|
||||
}
|
||||
logFatalFunc := func(_ error, _ string, _ ...any) {
|
||||
t.Fatalf("error logging function should not have been called")
|
||||
}
|
||||
go e.Finish(ctx, logFatalFunc)
|
||||
resultCount := 0
|
||||
for result := range e.ResultsChan() {
|
||||
switch meta := result.SourceMetadata.GetData().(type) {
|
||||
case *source_metadatapb.MetaData_Git:
|
||||
if tTest.expected[meta.Git.Commit].B != string(result.Raw) {
|
||||
t.Errorf("%s: unexpected result. Got: %s, Expected: %s", tName, string(result.Raw), tTest.expected[meta.Git.Commit].B)
|
||||
}
|
||||
if tTest.expected[meta.Git.Commit].LineNumber != result.SourceMetadata.GetGit().Line {
|
||||
t.Errorf("%s: unexpected line number. Got: %d, Expected: %d", tName, result.SourceMetadata.GetGit().Line, tTest.expected[meta.Git.Commit].LineNumber)
|
||||
}
|
||||
}
|
||||
resultCount++
|
||||
|
||||
}
|
||||
if resultCount != len(tTest.expected) {
|
||||
t.Errorf("%s: unexpected number of results. Got: %d, Expected: %d", tName, resultCount, len(tTest.expected))
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -124,5 +130,8 @@ func BenchmarkGitEngine(b *testing.B) {
|
||||
return
|
||||
}
|
||||
}
|
||||
e.Finish(ctx)
|
||||
logFatalFunc := func(_ error, _ string, _ ...any) {
|
||||
b.Fatalf("error logging function should not have been called")
|
||||
}
|
||||
e.Finish(ctx, logFatalFunc)
|
||||
}
|
||||
|
||||
@@ -1,6 +1,8 @@
|
||||
package engine
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
gogit "github.com/go-git/go-git/v5"
|
||||
"google.golang.org/protobuf/proto"
|
||||
"google.golang.org/protobuf/types/known/anypb"
|
||||
@@ -58,14 +60,13 @@ func (e *Engine) ScanGitHub(ctx context.Context, c sources.GithubConfig) error {
|
||||
scanOptions := git.NewScanOptions(opts...)
|
||||
source.WithScanOptions(scanOptions)
|
||||
|
||||
e.sourcesWg.Add(1)
|
||||
go func() {
|
||||
e.sourcesWg.Go(func() error {
|
||||
defer common.RecoverWithExit(ctx)
|
||||
defer e.sourcesWg.Done()
|
||||
err := source.Chunks(ctx, e.ChunksChan())
|
||||
if err != nil {
|
||||
ctx.Logger().Error(err, "could not scan github")
|
||||
return fmt.Errorf("could not scan github: %w", err)
|
||||
}
|
||||
}()
|
||||
return nil
|
||||
})
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -63,14 +63,13 @@ func (e *Engine) ScanGitLab(ctx context.Context, c sources.GitlabConfig) error {
|
||||
}
|
||||
gitlabSource.WithScanOptions(scanOptions)
|
||||
|
||||
e.sourcesWg.Add(1)
|
||||
go func() {
|
||||
e.sourcesWg.Go(func() error {
|
||||
defer common.RecoverWithExit(ctx)
|
||||
defer e.sourcesWg.Done()
|
||||
err := gitlabSource.Chunks(ctx, e.ChunksChan())
|
||||
if err != nil {
|
||||
ctx.Logger().Error(err, "error scanning GitLab")
|
||||
return fmt.Errorf("error scanning GitLab: %w", err)
|
||||
}
|
||||
}()
|
||||
return nil
|
||||
})
|
||||
return nil
|
||||
}
|
||||
|
||||
+4
-5
@@ -65,14 +65,13 @@ func (e *Engine) ScanS3(ctx context.Context, c sources.S3Config) error {
|
||||
return errors.WrapPrefix(err, "failed to init S3 source", 0)
|
||||
}
|
||||
|
||||
e.sourcesWg.Add(1)
|
||||
go func() {
|
||||
e.sourcesWg.Go(func() error {
|
||||
defer common.RecoverWithExit(ctx)
|
||||
defer e.sourcesWg.Done()
|
||||
err := s3Source.Chunks(ctx, e.ChunksChan())
|
||||
if err != nil {
|
||||
ctx.Logger().Error(err, "error scanning S3")
|
||||
return fmt.Errorf("error scanning S3: %w", err)
|
||||
}
|
||||
}()
|
||||
return nil
|
||||
})
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
package engine
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
|
||||
"github.com/go-errors/errors"
|
||||
@@ -53,14 +54,13 @@ func (e *Engine) ScanSyslog(ctx context.Context, c sources.SyslogConfig) error {
|
||||
return err
|
||||
}
|
||||
|
||||
e.sourcesWg.Add(1)
|
||||
go func() {
|
||||
e.sourcesWg.Go(func() error {
|
||||
defer common.RecoverWithExit(ctx)
|
||||
defer e.sourcesWg.Done()
|
||||
err := source.Chunks(ctx, e.ChunksChan())
|
||||
if err != nil {
|
||||
ctx.Logger().Error(err, "could not scan syslog")
|
||||
return fmt.Errorf("could not scan syslog: %w", err)
|
||||
}
|
||||
}()
|
||||
return nil
|
||||
})
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -991,6 +991,7 @@ const (
|
||||
DetectorType_Aiven DetectorType = 917
|
||||
DetectorType_Prefect DetectorType = 918
|
||||
DetectorType_Docusign DetectorType = 919
|
||||
DetectorType_Couchbase DetectorType = 920
|
||||
)
|
||||
|
||||
// Enum value maps for DetectorType.
|
||||
@@ -1912,6 +1913,7 @@ var (
|
||||
917: "Aiven",
|
||||
918: "Prefect",
|
||||
919: "Docusign",
|
||||
920: "Couchbase",
|
||||
}
|
||||
DetectorType_value = map[string]int32{
|
||||
"Alibaba": 0,
|
||||
@@ -2830,6 +2832,7 @@ var (
|
||||
"Aiven": 917,
|
||||
"Prefect": 918,
|
||||
"Docusign": 919,
|
||||
"Couchbase": 920,
|
||||
}
|
||||
)
|
||||
|
||||
@@ -3194,7 +3197,7 @@ var file_detectors_proto_rawDesc = []byte{
|
||||
0x0a, 0x0b, 0x44, 0x65, 0x63, 0x6f, 0x64, 0x65, 0x72, 0x54, 0x79, 0x70, 0x65, 0x12, 0x0b, 0x0a,
|
||||
0x07, 0x55, 0x4e, 0x4b, 0x4e, 0x4f, 0x57, 0x4e, 0x10, 0x00, 0x12, 0x09, 0x0a, 0x05, 0x50, 0x4c,
|
||||
0x41, 0x49, 0x4e, 0x10, 0x01, 0x12, 0x0a, 0x0a, 0x06, 0x42, 0x41, 0x53, 0x45, 0x36, 0x34, 0x10,
|
||||
0x02, 0x12, 0x09, 0x0a, 0x05, 0x55, 0x54, 0x46, 0x31, 0x36, 0x10, 0x03, 0x2a, 0xf9, 0x72, 0x0a,
|
||||
0x02, 0x12, 0x09, 0x0a, 0x05, 0x55, 0x54, 0x46, 0x31, 0x36, 0x10, 0x03, 0x2a, 0x89, 0x73, 0x0a,
|
||||
0x0c, 0x44, 0x65, 0x74, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x54, 0x79, 0x70, 0x65, 0x12, 0x0b, 0x0a,
|
||||
0x07, 0x41, 0x6c, 0x69, 0x62, 0x61, 0x62, 0x61, 0x10, 0x00, 0x12, 0x08, 0x0a, 0x04, 0x41, 0x4d,
|
||||
0x51, 0x50, 0x10, 0x01, 0x12, 0x07, 0x0a, 0x03, 0x41, 0x57, 0x53, 0x10, 0x02, 0x12, 0x09, 0x0a,
|
||||
@@ -4114,7 +4117,8 @@ var file_detectors_proto_rawDesc = []byte{
|
||||
0x0b, 0x4e, 0x75, 0x47, 0x65, 0x74, 0x41, 0x70, 0x69, 0x4b, 0x65, 0x79, 0x10, 0x94, 0x07, 0x12,
|
||||
0x0a, 0x0a, 0x05, 0x41, 0x69, 0x76, 0x65, 0x6e, 0x10, 0x95, 0x07, 0x12, 0x0c, 0x0a, 0x07, 0x50,
|
||||
0x72, 0x65, 0x66, 0x65, 0x63, 0x74, 0x10, 0x96, 0x07, 0x12, 0x0d, 0x0a, 0x08, 0x44, 0x6f, 0x63,
|
||||
0x75, 0x73, 0x69, 0x67, 0x6e, 0x10, 0x97, 0x07, 0x42, 0x3d, 0x5a, 0x3b, 0x67, 0x69, 0x74, 0x68,
|
||||
0x75, 0x73, 0x69, 0x67, 0x6e, 0x10, 0x97, 0x07, 0x12, 0x0e, 0x0a, 0x09, 0x43, 0x6f, 0x75, 0x63,
|
||||
0x68, 0x62, 0x61, 0x73, 0x65, 0x10, 0x98, 0x07, 0x42, 0x3d, 0x5a, 0x3b, 0x67, 0x69, 0x74, 0x68,
|
||||
0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x74, 0x72, 0x75, 0x66, 0x66, 0x6c, 0x65, 0x73, 0x65,
|
||||
0x63, 0x75, 0x72, 0x69, 0x74, 0x79, 0x2f, 0x74, 0x72, 0x75, 0x66, 0x66, 0x6c, 0x65, 0x68, 0x6f,
|
||||
0x67, 0x2f, 0x76, 0x33, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x70, 0x62, 0x2f, 0x64, 0x65, 0x74, 0x65,
|
||||
|
||||
@@ -31,10 +31,12 @@ type Source struct {
|
||||
jobPool *errgroup.Group
|
||||
sources.Progress
|
||||
client *http.Client
|
||||
sources.CommonSourceUnitUnmarshaller
|
||||
}
|
||||
|
||||
// Ensure the Source satisfies the interface at compile time.
|
||||
// Ensure the Source satisfies the interfaces at compile time.
|
||||
var _ sources.Source = (*Source)(nil)
|
||||
var _ sources.SourceUnitUnmarshaller = (*Source)(nil)
|
||||
|
||||
// Type returns the type of source.
|
||||
// It is used for matching source types in configuration and job input.
|
||||
|
||||
@@ -30,12 +30,14 @@ type Source struct {
|
||||
verify bool
|
||||
conn sourcespb.Docker
|
||||
sources.Progress
|
||||
sources.CommonSourceUnitUnmarshaller
|
||||
}
|
||||
|
||||
var FilesizeLimitBytes int64 = 50 * 1024 * 1024 // 50MB
|
||||
|
||||
// Ensure the Source satisfies the interface at compile time.
|
||||
// Ensure the Source satisfies the interfaces at compile time.
|
||||
var _ sources.Source = (*Source)(nil)
|
||||
var _ sources.SourceUnitUnmarshaller = (*Source)(nil)
|
||||
|
||||
// Type returns the type of source.
|
||||
// It is used for matching source types in configuration and job input.
|
||||
|
||||
@@ -39,10 +39,12 @@ type Source struct {
|
||||
log logr.Logger
|
||||
filter *common.Filter
|
||||
sources.Progress
|
||||
sources.CommonSourceUnitUnmarshaller
|
||||
}
|
||||
|
||||
// Ensure the Source satisfies the interface at compile time
|
||||
// Ensure the Source satisfies the interfaces at compile time
|
||||
var _ sources.Source = (*Source)(nil)
|
||||
var _ sources.SourceUnitUnmarshaller = (*Source)(nil)
|
||||
|
||||
// Type returns the type of source.
|
||||
// It is used for matching source types in configuration and job input.
|
||||
|
||||
@@ -30,8 +30,9 @@ import (
|
||||
|
||||
const defaultCachePersistIncrement = 2500
|
||||
|
||||
// Ensure the Source satisfies the interface at compile time.
|
||||
// Ensure the Source satisfies the interfaces at compile time.
|
||||
var _ sources.Source = (*Source)(nil)
|
||||
var _ sources.SourceUnitUnmarshaller = (*Source)(nil)
|
||||
|
||||
// Type returns the type of source.
|
||||
// It is used for matching source types in configuration and job input.
|
||||
@@ -69,6 +70,7 @@ type Source struct {
|
||||
|
||||
mu sync.Mutex
|
||||
sources.Progress // progress is not thread safe
|
||||
sources.CommonSourceUnitUnmarshaller
|
||||
}
|
||||
|
||||
// persistableCache is a wrapper around cache.Cache that allows
|
||||
|
||||
@@ -67,8 +67,9 @@ func NewGit(sourceType sourcespb.SourceType, jobID, sourceID int64, sourceName s
|
||||
}
|
||||
}
|
||||
|
||||
// Ensure the Source satisfies the interface at compile time.
|
||||
// Ensure the Source satisfies the interfaces at compile time.
|
||||
var _ sources.Source = (*Source)(nil)
|
||||
var _ sources.SourceUnitUnmarshaller = (*Source)(nil)
|
||||
|
||||
// Type returns the type of source.
|
||||
// It is used for matching source types in configuration and job input.
|
||||
@@ -738,7 +739,7 @@ func PrepareRepoSinceCommit(ctx context.Context, uriString, commitHash string) (
|
||||
var path string
|
||||
switch {
|
||||
case uri.User != nil:
|
||||
ctx.Logger().V(1).Info("cloning repo with authentication", "uri", uri)
|
||||
ctx.Logger().V(1).Info("cloning repo with authentication", "uri", uri.Redacted())
|
||||
password, ok := uri.User.Password()
|
||||
if !ok {
|
||||
return "", true, fmt.Errorf("password must be included in Git repo URL when username is provided")
|
||||
@@ -776,7 +777,7 @@ func PrepareRepo(ctx context.Context, uriString string) (string, bool, error) {
|
||||
remote = true
|
||||
switch {
|
||||
case uri.User != nil:
|
||||
ctx.Logger().V(1).Info("cloning repo with authentication", "uri", uri)
|
||||
ctx.Logger().V(1).Info("cloning repo with authentication", "uri", uri.Redacted())
|
||||
password, ok := uri.User.Password()
|
||||
if !ok {
|
||||
return "", remote, fmt.Errorf("password must be included in Git repo URL when username is provided")
|
||||
@@ -875,3 +876,7 @@ func handleBinary(ctx context.Context, repo *git.Repository, chunksChan chan *so
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Source) UnmarshalSourceUnit(data []byte) (sources.SourceUnit, error) {
|
||||
return UnmarshalUnit(data)
|
||||
}
|
||||
|
||||
@@ -0,0 +1,40 @@
|
||||
package git
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
|
||||
)
|
||||
|
||||
const (
|
||||
UnitRepo string = "repo"
|
||||
UnitDir string = "dir"
|
||||
)
|
||||
|
||||
// Ensure SourceUnit implements the interface at compile time.
|
||||
var _ sources.SourceUnit = SourceUnit{}
|
||||
|
||||
// A git source unit can be two kinds of units: either a local directory path
|
||||
// or a remote repository.
|
||||
type SourceUnit struct {
|
||||
Kind string `json:"kind"`
|
||||
ID string `json:"id"`
|
||||
}
|
||||
|
||||
// Implement sources.SourceUnit interface.
|
||||
func (u SourceUnit) SourceUnitID() string {
|
||||
return u.ID
|
||||
}
|
||||
|
||||
// Helper function to unmarshal raw bytes into our SourceUnit struct.
|
||||
func UnmarshalUnit(data []byte) (sources.SourceUnit, error) {
|
||||
var unit SourceUnit
|
||||
if err := json.Unmarshal(data, &unit); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if unit.ID == "" || (unit.Kind != UnitRepo && unit.Kind != UnitDir) {
|
||||
return nil, fmt.Errorf("not a git.SourceUnit")
|
||||
}
|
||||
return unit, nil
|
||||
}
|
||||
@@ -69,14 +69,17 @@ type Source struct {
|
||||
mu sync.Mutex
|
||||
publicMap map[string]source_metadatapb.Visibility
|
||||
sources.Progress
|
||||
sources.CommonSourceUnitUnmarshaller
|
||||
}
|
||||
|
||||
func (s *Source) WithScanOptions(scanOptions *git.ScanOptions) {
|
||||
s.scanOptions = scanOptions
|
||||
}
|
||||
|
||||
// Ensure the Source satisfies the interface at compile time
|
||||
// Ensure the Source satisfies the interfaces at compile time
|
||||
var _ sources.Source = (*Source)(nil)
|
||||
var _ sources.SourceUnitUnmarshaller = (*Source)(nil)
|
||||
|
||||
var endsWithGithub = regexp.MustCompile(`github\.com/?$`)
|
||||
|
||||
// Type returns the type of source.
|
||||
|
||||
@@ -47,10 +47,12 @@ type Source struct {
|
||||
resumeInfoMutex sync.Mutex
|
||||
sources.Progress
|
||||
jobPool *errgroup.Group
|
||||
sources.CommonSourceUnitUnmarshaller
|
||||
}
|
||||
|
||||
// Ensure the Source satisfies the interface at compile time.
|
||||
// Ensure the Source satisfies the interfaces at compile time.
|
||||
var _ sources.Source = (*Source)(nil)
|
||||
var _ sources.SourceUnitUnmarshaller = (*Source)(nil)
|
||||
|
||||
// Type returns the type of source.
|
||||
// It is used for matching source types in configuration and job input.
|
||||
|
||||
@@ -46,10 +46,12 @@ type Source struct {
|
||||
conn *sourcespb.S3
|
||||
jobPool *errgroup.Group
|
||||
maxObjectSize int64
|
||||
sources.CommonSourceUnitUnmarshaller
|
||||
}
|
||||
|
||||
// Ensure the Source satisfies the interface at compile time
|
||||
// Ensure the Source satisfies the interfaces at compile time
|
||||
var _ sources.Source = (*Source)(nil)
|
||||
var _ sources.SourceUnitUnmarshaller = (*Source)(nil)
|
||||
|
||||
// Type returns the type of source
|
||||
func (s *Source) Type() sourcespb.SourceType {
|
||||
@@ -195,7 +197,10 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) err
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
s.log.Error(err, "could not list objects in s3 bucket", "bucket", bucket)
|
||||
return fmt.Errorf(
|
||||
"could not list objects in s3 bucket: bucket %s: %w",
|
||||
bucket,
|
||||
err)
|
||||
}
|
||||
}
|
||||
s.SetProgressComplete(len(bucketsToScan), len(bucketsToScan), fmt.Sprintf("Completed scanning source %s. %d objects scanned.", s.name, objectCount), "")
|
||||
|
||||
@@ -1,12 +1,37 @@
|
||||
package sources
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
)
|
||||
|
||||
// Ensure CommonSourceUnit implements SourceUnit at compile time.
|
||||
var _ SourceUnit = CommonSourceUnit{}
|
||||
|
||||
// CommonSourceUnit is a common implementation of SourceUnit that Sources can
|
||||
// use instead of implementing their own types.
|
||||
type CommonSourceUnit struct {
|
||||
ID string `json:"source_unit_id"`
|
||||
}
|
||||
|
||||
// Implement the SourceUnit interface.
|
||||
// SourceUnitID implements the SourceUnit interface.
|
||||
func (c CommonSourceUnit) SourceUnitID() string {
|
||||
return c.ID
|
||||
}
|
||||
|
||||
// CommonSourceUnitUnmarshaller is an implementation of SourceUnitUnmarshaller
|
||||
// for the CommonSourceUnit. A source can embed this struct to gain the
|
||||
// functionality of converting []byte to a CommonSourceUnit.
|
||||
type CommonSourceUnitUnmarshaller struct{}
|
||||
|
||||
// UnmarshalSourceUnit implements the SourceUnitUnmarshaller interface.
|
||||
func (c CommonSourceUnitUnmarshaller) UnmarshalSourceUnit(data []byte) (SourceUnit, error) {
|
||||
var unit CommonSourceUnit
|
||||
if err := json.Unmarshal(data, &unit); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if unit.ID == "" {
|
||||
return nil, fmt.Errorf("not a CommonSourceUnit")
|
||||
}
|
||||
return unit, nil
|
||||
}
|
||||
|
||||
@@ -928,6 +928,7 @@ enum DetectorType {
|
||||
Aiven = 917;
|
||||
Prefect = 918;
|
||||
Docusign = 919;
|
||||
Couchbase = 920;
|
||||
}
|
||||
|
||||
message Result {
|
||||
|
||||
Reference in New Issue
Block a user