Compare commits
5 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 Bill Rich
|
0a3679671a | ||
|
Bill Rich
|
3b49183caf | ||
|
Bill Rich
|
809b35a843 | ||
|
Bill Rich
|
6dd13a0b7c | ||
|
Bill Rich
|
bd1539d22a |
@@ -135,7 +135,7 @@ Flags:
|
||||
--regex No-op flag for backwards compat.
|
||||
|
||||
Args:
|
||||
<uri> Git repository URL. https://, file://, or ssh:// schema expected.
|
||||
<uri> Git repository URL. https:// or file:// schema expected.
|
||||
```
|
||||
|
||||
For example, to scan a `git` repository, start with
|
||||
|
||||
@@ -8,12 +8,14 @@ replace github.com/zricethezav/gitleaks/v8 => github.com/trufflesecurity/gitleak
|
||||
|
||||
replace github.com/gitleaks/go-gitdiff => github.com/trufflesecurity/go-gitdiff v0.7.6-zombies2
|
||||
|
||||
replace github.com/bill-rich/gitparser => /home/hrich/go/src/github.com/bill-rich/gitparse
|
||||
|
||||
require (
|
||||
cloud.google.com/go/secretmanager v1.5.0
|
||||
github.com/Azure/go-autorest/autorest/azure/auth v0.5.11
|
||||
github.com/TheZeroSlave/zapsentry v1.11.0
|
||||
github.com/aws/aws-sdk-go v1.44.83
|
||||
github.com/bill-rich/disk-buffer-reader v0.1.3
|
||||
github.com/aws/aws-sdk-go v1.44.61
|
||||
github.com/bill-rich/disk-buffer-reader v0.1.2
|
||||
github.com/bill-rich/gitparser v0.0.0-00010101000000-000000000000
|
||||
github.com/bill-rich/go-syslog v0.0.0-20220413021637-49edb52a574c
|
||||
github.com/bitfinexcom/bitfinex-api-go v0.0.0-20210608095005-9e0b26f200fb
|
||||
github.com/bradleyfalzon/ghinstallation/v2 v2.1.0
|
||||
@@ -21,11 +23,8 @@ require (
|
||||
github.com/envoyproxy/protoc-gen-validate v0.6.7
|
||||
github.com/fatih/color v1.13.0
|
||||
github.com/felixge/fgprof v0.9.2
|
||||
github.com/getsentry/sentry-go v0.13.0
|
||||
github.com/go-errors/errors v1.4.2
|
||||
github.com/go-git/go-git/v5 v5.4.2
|
||||
github.com/go-logr/logr v1.2.3
|
||||
github.com/go-logr/zapr v1.2.3
|
||||
github.com/golang-jwt/jwt v3.2.2+incompatible
|
||||
github.com/google/go-github/v42 v42.0.0
|
||||
github.com/gorilla/mux v1.8.0
|
||||
@@ -34,7 +33,7 @@ require (
|
||||
github.com/joho/godotenv v1.4.0
|
||||
github.com/jpillora/overseer v1.1.6
|
||||
github.com/kylelemons/godebug v1.1.0
|
||||
github.com/mattn/go-colorable v0.1.13
|
||||
github.com/mattn/go-colorable v0.1.12
|
||||
github.com/mholt/archiver/v4 v4.0.0-alpha.7
|
||||
github.com/paulbellamy/ratecounter v0.2.0
|
||||
github.com/pkg/errors v0.9.1
|
||||
@@ -44,11 +43,11 @@ require (
|
||||
github.com/sirupsen/logrus v1.9.0
|
||||
github.com/stretchr/testify v1.8.0
|
||||
github.com/tailscale/depaware v0.0.0-20210622194025-720c4b409502
|
||||
github.com/xanzy/go-gitlab v0.73.1
|
||||
go.uber.org/zap v1.22.0
|
||||
github.com/xanzy/go-gitlab v0.69.0
|
||||
github.com/zricethezav/gitleaks/v8 v8.10.3
|
||||
golang.org/x/crypto v0.0.0-20211215165025-cf75a172585e
|
||||
golang.org/x/net v0.0.0-20220805013720-a33c5aa5df48
|
||||
golang.org/x/oauth2 v0.0.0-20220722155238-128564f6959c
|
||||
golang.org/x/net v0.0.0-20220617184016-355a448f1bc9
|
||||
golang.org/x/oauth2 v0.0.0-20220608161450-d0670ef3b1eb
|
||||
golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f
|
||||
google.golang.org/genproto v0.0.0-20220617124728-180714bec0ad
|
||||
google.golang.org/protobuf v1.28.1
|
||||
@@ -73,11 +72,11 @@ require (
|
||||
github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 // indirect
|
||||
github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 // indirect
|
||||
github.com/andybalholm/brotli v1.0.4 // indirect
|
||||
github.com/benbjohnson/clock v1.1.0 // indirect
|
||||
github.com/davecgh/go-spew v1.1.1 // indirect
|
||||
github.com/dimchansky/utfbom v1.1.1 // indirect
|
||||
github.com/dsnet/compress v0.0.1 // indirect
|
||||
github.com/emirpasic/gods v1.12.0 // indirect
|
||||
github.com/gitleaks/go-gitdiff v0.7.6 // indirect
|
||||
github.com/go-git/gcfg v1.5.0 // indirect
|
||||
github.com/go-git/go-billy/v5 v5.3.1 // indirect
|
||||
github.com/go-ole/go-ole v1.2.6 // indirect
|
||||
@@ -88,7 +87,7 @@ require (
|
||||
github.com/google/go-cmp v0.5.8 // indirect
|
||||
github.com/google/go-github/v45 v45.2.0 // indirect
|
||||
github.com/google/go-querystring v1.1.0 // indirect
|
||||
github.com/google/pprof v0.0.0-20211214055906-6f57359322fd // indirect
|
||||
github.com/google/pprof v0.0.0-20220729232143-a41b82acbcb1 // indirect
|
||||
github.com/googleapis/enterprise-certificate-proxy v0.1.0 // indirect
|
||||
github.com/googleapis/gax-go/v2 v2.4.0 // indirect
|
||||
github.com/h2non/parth v0.0.0-20190131123155-b4df798d6542 // indirect
|
||||
@@ -100,7 +99,7 @@ require (
|
||||
github.com/kevinburke/ssh_config v0.0.0-20201106050909-4977a11b4351 // indirect
|
||||
github.com/klauspost/compress v1.15.5 // indirect
|
||||
github.com/klauspost/pgzip v1.2.5 // indirect
|
||||
github.com/mattn/go-isatty v0.0.16 // indirect
|
||||
github.com/mattn/go-isatty v0.0.14 // indirect
|
||||
github.com/mitchellh/go-homedir v1.1.0 // indirect
|
||||
github.com/nwaples/rardecode/v2 v2.0.0-beta.2 // indirect
|
||||
github.com/pierrec/lz4/v4 v4.1.14 // indirect
|
||||
@@ -111,18 +110,15 @@ require (
|
||||
github.com/xanzy/ssh-agent v0.3.0 // indirect
|
||||
github.com/yusufpapurcu/wmi v1.2.2 // indirect
|
||||
go.opencensus.io v0.23.0 // indirect
|
||||
go.uber.org/atomic v1.7.0 // indirect
|
||||
go.uber.org/multierr v1.6.0 // indirect
|
||||
golang.org/x/mod v0.5.0 // indirect
|
||||
golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab // indirect
|
||||
golang.org/x/text v0.3.7 // indirect
|
||||
golang.org/x/time v0.0.0-20220722155302-e5dcc9cfc0b9 // indirect
|
||||
golang.org/x/time v0.0.0-20220411224347-583f2d630306 // indirect
|
||||
golang.org/x/tools v0.1.7 // indirect
|
||||
golang.org/x/xerrors v0.0.0-20220609144429-65e65417b02f // indirect
|
||||
google.golang.org/api v0.85.0 // indirect
|
||||
google.golang.org/appengine v1.6.7 // indirect
|
||||
google.golang.org/grpc v1.47.0 // indirect
|
||||
gopkg.in/warnings.v0 v0.1.2 // indirect
|
||||
gopkg.in/yaml.v2 v2.4.0 // indirect
|
||||
gopkg.in/yaml.v3 v3.0.1 // indirect
|
||||
)
|
||||
|
||||
@@ -85,8 +85,6 @@ github.com/Microsoft/go-winio v0.4.16/go.mod h1:XB6nPKklQyQ7GC9LdcBEcBl8PF76WugX
|
||||
github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
|
||||
github.com/ProtonMail/go-crypto v0.0.0-20210428141323-04723f9f07d7 h1:YoJbenK9C67SkzkDfmQuVln04ygHj3vjZfd9FL+GmQQ=
|
||||
github.com/ProtonMail/go-crypto v0.0.0-20210428141323-04723f9f07d7/go.mod h1:z4/9nQmJSSwwds7ejkxaJwO37dru3geImFUdJlaLzQo=
|
||||
github.com/TheZeroSlave/zapsentry v1.11.0 h1:QtmkinFhsfKehvxE/tm/VkrBe/K8iQZXjp+ieOevsQI=
|
||||
github.com/TheZeroSlave/zapsentry v1.11.0/go.mod h1:00uO/VpPrSJG/XigAfTi0F4WMFIw2DmP/IDVUhPBvNw=
|
||||
github.com/acomagu/bufpipe v1.0.3 h1:fxAGrHZTgQ9w5QqVItgzwj235/uYZYgbXitB+dLupOk=
|
||||
github.com/acomagu/bufpipe v1.0.3/go.mod h1:mxdxdup/WdsKVreO5GpW4+M/1CE2sMG4jeGJ2sYmHc4=
|
||||
github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 h1:JYp7IbQjafoB+tBA3gMyHYHrpOtNuDiK/uB5uXxq5wM=
|
||||
@@ -100,12 +98,10 @@ github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYU
|
||||
github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY=
|
||||
github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio=
|
||||
github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs=
|
||||
github.com/aws/aws-sdk-go v1.44.83 h1:7+Rtc2Eio6EKUNoZeMV/IVxzVrY5oBQcNPtCcgIHYJA=
|
||||
github.com/aws/aws-sdk-go v1.44.83/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo=
|
||||
github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8=
|
||||
github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA=
|
||||
github.com/bill-rich/disk-buffer-reader v0.1.3 h1:8RUWjBrLB52kwJUjVDdrU0S/7z07DrMshgudfiy6QFs=
|
||||
github.com/bill-rich/disk-buffer-reader v0.1.3/go.mod h1:VVzzsK1Ac2AnpOfp/5r9JlIFaFkZ9uSf7zisZayCt0Y=
|
||||
github.com/aws/aws-sdk-go v1.44.61 h1:NcpLSS3Z0MiVQIYugx4I40vSIEEAXT0baO684ExNRco=
|
||||
github.com/aws/aws-sdk-go v1.44.61/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo=
|
||||
github.com/bill-rich/disk-buffer-reader v0.1.2 h1:pN9K5JoacTcNWp2SCd3n7mPouSwMP9ouTS66Qa+5IPY=
|
||||
github.com/bill-rich/disk-buffer-reader v0.1.2/go.mod h1:VVzzsK1Ac2AnpOfp/5r9JlIFaFkZ9uSf7zisZayCt0Y=
|
||||
github.com/bill-rich/go-syslog v0.0.0-20220413021637-49edb52a574c h1:tSME5FDS02qQll3JYodI6RZR/g4EKOHApGv1wMZT+Z0=
|
||||
github.com/bill-rich/go-syslog v0.0.0-20220413021637-49edb52a574c/go.mod h1:+sCc6hztur+oZCLOsNk6wCCy+GLrnSNHSRmTnnL+8iQ=
|
||||
github.com/bitfinexcom/bitfinex-api-go v0.0.0-20210608095005-9e0b26f200fb h1:9v7Bzlg+1EBYi2IYcUmOwHReBEfqBbYIj3ZCi9cIe1Q=
|
||||
@@ -159,8 +155,6 @@ github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYF
|
||||
github.com/felixge/fgprof v0.9.2 h1:tAMHtWMyl6E0BimjVbFt7fieU6FpjttsZN7j0wT5blc=
|
||||
github.com/felixge/fgprof v0.9.2/go.mod h1:+VNi+ZXtHIQ6wIw6bUT8nXQRefQflWECoFyRealT5sg=
|
||||
github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc=
|
||||
github.com/getsentry/sentry-go v0.13.0 h1:20dgTiUSfxRB/EhMPtxcL9ZEbM1ZdR+W/7f7NWD+xWo=
|
||||
github.com/getsentry/sentry-go v0.13.0/go.mod h1:EOsfu5ZdvKPfeHYV6pTVQnsjfp30+XA7//UooKNumH0=
|
||||
github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
|
||||
github.com/gliderlabs/ssh v0.2.2 h1:6zsha5zo/TWhRhwqCD3+EarCAgZ2yN28ipRnGPnwkI0=
|
||||
github.com/gliderlabs/ssh v0.2.2/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aevW3Awn0=
|
||||
@@ -178,11 +172,6 @@ github.com/go-git/go-git/v5 v5.4.2/go.mod h1:gQ1kArt6d+n+BGd+/B/I74HwRTLhth2+zti
|
||||
github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
|
||||
github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
|
||||
github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
|
||||
github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
|
||||
github.com/go-logr/logr v1.2.3 h1:2DntVwHkVopvECVRSlL5PSo9eG+cAkDCuckLubN+rq0=
|
||||
github.com/go-logr/logr v1.2.3/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
|
||||
github.com/go-logr/zapr v1.2.3 h1:a9vnzlIBPQBBkeaR9IuMUfmVOrQlkoC4YfPoFkX3T7A=
|
||||
github.com/go-logr/zapr v1.2.3/go.mod h1:eIauM6P8qSvTw5o2ez6UEAfGjQKrxQTl5EoK+Qa2oG4=
|
||||
github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY=
|
||||
github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0=
|
||||
github.com/gobwas/httphead v0.0.0-20200921212729-da3d93bc3c58/go.mod h1:L0fX3K22YWvt/FAX9NnzrNzcI4wNYi9Yku4O0LKYflo=
|
||||
@@ -271,8 +260,9 @@ github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLe
|
||||
github.com/google/pprof v0.0.0-20210601050228-01bbb1931b22/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
|
||||
github.com/google/pprof v0.0.0-20210609004039-a478d1d731e9/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
|
||||
github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
|
||||
github.com/google/pprof v0.0.0-20211214055906-6f57359322fd h1:1FjCyPC+syAzJ5/2S8fqdZK1R22vvA0J7JZKcuOIQ7Y=
|
||||
github.com/google/pprof v0.0.0-20211214055906-6f57359322fd/go.mod h1:KgnwoLYCZ8IQu3XUZ8Nc/bM9CCZFOyjUNOSygVozoDg=
|
||||
github.com/google/pprof v0.0.0-20220729232143-a41b82acbcb1 h1:8pyqKJvrJqUYaKS851Ule26pwWvey6IDMiczaBLDKLQ=
|
||||
github.com/google/pprof v0.0.0-20220729232143-a41b82acbcb1/go.mod h1:gSuNB+gJaOiQKLEZ+q+PK9Mq3SOzhRcw2GsGS/FhYDk=
|
||||
github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
|
||||
github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
||||
github.com/googleapis/enterprise-certificate-proxy v0.0.0-20220520183353-fd19c99a87aa/go.mod h1:17drOmN3MwGY7t0e+Ei9b45FFGA3fBs3x36SsCg1hq8=
|
||||
@@ -347,13 +337,11 @@ github.com/lyft/protoc-gen-star v0.6.0/go.mod h1:TGAoBVkt8w7MPG72TrKIu85MIdXwDuz
|
||||
github.com/matryer/is v1.2.0 h1:92UTHpy8CDwaJ08GqLDzhhuixiBUUD1p3AU6PHddz4A=
|
||||
github.com/matryer/is v1.2.0/go.mod h1:2fLPjFQM9rhQ15aVEtbuwhJinnOqrmgXPNdZsdwlWXA=
|
||||
github.com/mattn/go-colorable v0.1.9/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc=
|
||||
github.com/mattn/go-colorable v0.1.12 h1:jF+Du6AlPIjs2BiUiQlKOX0rt3SujHxPnksPKZbaA40=
|
||||
github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4=
|
||||
github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=
|
||||
github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg=
|
||||
github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
|
||||
github.com/mattn/go-isatty v0.0.14 h1:yVuAays6BHfxijgZPzw+3Zlu5yQgKGP2/hcQbHb7S9Y=
|
||||
github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94=
|
||||
github.com/mattn/go-isatty v0.0.16 h1:bq3VjFmv/sOjHtdEhmkEV4x1AJtvUvOJ2PFAZ5+peKQ=
|
||||
github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
|
||||
github.com/mholt/archiver/v4 v4.0.0-alpha.7 h1:xzByj8G8tj0Oq7ZYYU4+ixL/CVb5ruWCm0EZQ1PjOkE=
|
||||
github.com/mholt/archiver/v4 v4.0.0-alpha.7/go.mod h1:Fs8qUkO74HHaidabihzYephJH8qmGD/nCP6tE5xC9BM=
|
||||
github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y=
|
||||
@@ -368,7 +356,6 @@ github.com/paulbellamy/ratecounter v0.2.0 h1:2L/RhJq+HA8gBQImDXtLPrDXK5qAj6ozWVK
|
||||
github.com/paulbellamy/ratecounter v0.2.0/go.mod h1:Hfx1hDpSGoqxkVVpBi/IlYD7kChlfo5C6hzIHwPqfFE=
|
||||
github.com/pierrec/lz4/v4 v4.1.14 h1:+fL8AQEZtz/ijeNnpduH0bROTu0O3NZAlPjQxGn8LwE=
|
||||
github.com/pierrec/lz4/v4 v4.1.14/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
|
||||
github.com/pingcap/errors v0.11.4 h1:lFuQV/oaUMGcD2tqt+01ROSmJs75VG1ToEOkZIZ4nE4=
|
||||
github.com/pkg/diff v0.0.0-20200914180035-5b29258ca4f7 h1:+/+DxvQaYifJ+grD4klzrS5y+KJXldn/2YTl5JG+vZ8=
|
||||
github.com/pkg/diff v0.0.0-20200914180035-5b29258ca4f7/go.mod h1:zO8QMzTeZd5cpnIkz/Gn6iK0jDfGicM1nynOkkPIl28=
|
||||
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
|
||||
@@ -403,7 +390,6 @@ github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+
|
||||
github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||
github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
|
||||
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
|
||||
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
|
||||
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
|
||||
github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
|
||||
github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
||||
@@ -415,13 +401,17 @@ github.com/tailscale/depaware v0.0.0-20210622194025-720c4b409502 h1:34icjjmqJ2HP
|
||||
github.com/tailscale/depaware v0.0.0-20210622194025-720c4b409502/go.mod h1:p9lPsd+cx33L3H9nNoecRRxPssFKUwwI50I3pZ0yT+8=
|
||||
github.com/therootcompany/xz v1.0.1 h1:CmOtsn1CbtmyYiusbfmhmkpAAETj0wBIH6kCYaX+xzw=
|
||||
github.com/therootcompany/xz v1.0.1/go.mod h1:3K3UH1yCKgBneZYhuQUvJ9HPD19UEXEI0BWbMn8qNMY=
|
||||
github.com/trufflesecurity/gitleaks/v8 v8.6.1-custom10 h1:QuGZ5bJcQpVz+3sfvKKPDkQwdUueiBg0V+2eMHzkryo=
|
||||
github.com/trufflesecurity/gitleaks/v8 v8.6.1-custom10/go.mod h1:2iZpX4Epnmx7VK2atbIMEjHW9rivie5RRe0ZhPWUFvM=
|
||||
github.com/trufflesecurity/go-gitdiff v0.7.6-zombies2 h1:srCJzbE3b44+ZIPcgJSfvinHCOQlkMwVghtKf23un6o=
|
||||
github.com/trufflesecurity/go-gitdiff v0.7.6-zombies2/go.mod h1:pKz0X4YzCKZs30BL+weqBIG7mx0jl4tF1uXV9ZyNvrA=
|
||||
github.com/trufflesecurity/overseer v1.1.7-custom5 h1:xu+Fg6fkSRifUPzUCl7N8HmobJ6WGOkIApGnM7mJS6w=
|
||||
github.com/trufflesecurity/overseer v1.1.7-custom5/go.mod h1:nT9w37AiO1Nop2VhVhNfzAFaPjthvxgpDV3XKsxYkcI=
|
||||
github.com/ulikunitz/xz v0.5.6/go.mod h1:2bypXElzHzzJZwzH67Y6wb67pO62Rzfn7BSiF4ABRW8=
|
||||
github.com/ulikunitz/xz v0.5.10 h1:t92gobL9l3HE202wg3rlk19F6X+JOxl9BBrCCMYEYd8=
|
||||
github.com/ulikunitz/xz v0.5.10/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14=
|
||||
github.com/xanzy/go-gitlab v0.73.1 h1:UMagqUZLJdjss1SovIC+kJCH4k2AZWXl58gJd38Y/hI=
|
||||
github.com/xanzy/go-gitlab v0.73.1/go.mod h1:d/a0vswScO7Agg1CZNz15Ic6SSvBG9vfw8egL99t4kA=
|
||||
github.com/xanzy/go-gitlab v0.69.0 h1:sPci9xHzlX+lcJvPqNu3y3BQpePuR2R694Bal4AeyB8=
|
||||
github.com/xanzy/go-gitlab v0.69.0/go.mod h1:o4yExCtdaqlM8YGdDJWuZoBmfxBsmA9TPEjs9mx1UO4=
|
||||
github.com/xanzy/ssh-agent v0.3.0 h1:wUMzuKtKilRgBAD1sUb8gOwwRr2FGoBVumcjoOACClI=
|
||||
github.com/xanzy/ssh-agent v0.3.0/go.mod h1:3s9xbODqPuuhK9JV1R321M/FlMZSBvE5aY6eAcqrDh0=
|
||||
github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
|
||||
@@ -440,15 +430,6 @@ go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk=
|
||||
go.opencensus.io v0.23.0 h1:gqCw0LfLxScz8irSi8exQc7fyQ0fKQU/qnC/X8+V/1M=
|
||||
go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E=
|
||||
go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI=
|
||||
go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw=
|
||||
go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
|
||||
go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A=
|
||||
go.uber.org/goleak v1.1.11 h1:wy28qYRKZgnJTxGxvye5/wgWr1EKjmUDGYox5mGlRlI=
|
||||
go.uber.org/multierr v1.6.0 h1:y6IPFStTAIT5Ytl7/XYmHvzXQ7S3g/IeZW9hyZ5thw4=
|
||||
go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU=
|
||||
go.uber.org/zap v1.19.0/go.mod h1:xg/QME4nWcxGxrpdeYfq7UvYrLh66cuVKdrbD1XF/NI=
|
||||
go.uber.org/zap v1.22.0 h1:Zcye5DUgBloQ9BaT4qc9BnjOFog5TvBSAGkJ3Nf70c0=
|
||||
go.uber.org/zap v1.22.0/go.mod h1:H4siCOZOrAolnUPJEkfaSjDqyP+BDS0DdDWzwcgt3+U=
|
||||
golang.org/x/crypto v0.0.0-20190219172222-a4c6cb3142f2/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
|
||||
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
|
||||
golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
|
||||
@@ -544,9 +525,8 @@ golang.org/x/net v0.0.0-20220325170049-de3da57026de/go.mod h1:CfG3xpIq0wQ8r1q4Su
|
||||
golang.org/x/net v0.0.0-20220412020605-290c469a71a5/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
|
||||
golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
|
||||
golang.org/x/net v0.0.0-20220607020251-c690dde0001d/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
|
||||
golang.org/x/net v0.0.0-20220617184016-355a448f1bc9 h1:Yqz/iviulwKwAREEeUd3nbBFn0XuyJqkoft2IlrvOhc=
|
||||
golang.org/x/net v0.0.0-20220617184016-355a448f1bc9/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
|
||||
golang.org/x/net v0.0.0-20220805013720-a33c5aa5df48 h1:N9Vc/rorQUDes6B9CNdIxAn5jODGj2wzfrei2x4wNj4=
|
||||
golang.org/x/net v0.0.0-20220805013720-a33c5aa5df48/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk=
|
||||
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
|
||||
golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
|
||||
golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
|
||||
@@ -566,9 +546,8 @@ golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8/go.mod h1:KelEdhl1UZF7XfJ
|
||||
golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc=
|
||||
golang.org/x/oauth2 v0.0.0-20220309155454-6242fa91716a/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc=
|
||||
golang.org/x/oauth2 v0.0.0-20220411215720-9780585627b5/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc=
|
||||
golang.org/x/oauth2 v0.0.0-20220608161450-d0670ef3b1eb h1:8tDJ3aechhddbdPAxpycgXHJRMLpk/Ab+aa4OgdN5/g=
|
||||
golang.org/x/oauth2 v0.0.0-20220608161450-d0670ef3b1eb/go.mod h1:jaDAt6Dkxork7LmZnYtzbRWj0W47D86a3TGe0YHBvmE=
|
||||
golang.org/x/oauth2 v0.0.0-20220722155238-128564f6959c h1:q3gFqPqH7NVofKo3c3yETAP//pPI+G5mvB7qqj1Y5kY=
|
||||
golang.org/x/oauth2 v0.0.0-20220722155238-128564f6959c/go.mod h1:h4gKUeWbJ4rQPri7E0u6Gs4e9Ri2zaLxzw5DI5XGrYg=
|
||||
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
@@ -668,8 +647,8 @@ golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
|
||||
golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
|
||||
golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
|
||||
golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
|
||||
golang.org/x/time v0.0.0-20220722155302-e5dcc9cfc0b9 h1:ftMN5LMiBFjbzleLqtoBZk7KdJwhuybIU+FckUHgoyQ=
|
||||
golang.org/x/time v0.0.0-20220722155302-e5dcc9cfc0b9/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
|
||||
golang.org/x/time v0.0.0-20220411224347-583f2d630306 h1:+gHMid33q6pen7kv9xvT+JRinntgeXO2AeZVd0AWD3w=
|
||||
golang.org/x/time v0.0.0-20220411224347-583f2d630306/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
|
||||
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
|
||||
@@ -685,7 +664,6 @@ golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgw
|
||||
golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
|
||||
golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
|
||||
golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
|
||||
golang.org/x/tools v0.0.0-20191108193012-7d206e10da11/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
|
||||
golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
|
||||
golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
|
||||
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
|
||||
@@ -930,9 +908,7 @@ gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||
gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||
gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||
gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
|
||||
gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
|
||||
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
|
||||
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
|
||||
|
||||
@@ -46,7 +46,7 @@ var (
|
||||
fail = cli.Flag("fail", "Exit with code 183 if results are found.").Bool()
|
||||
|
||||
gitScan = cli.Command("git", "Find credentials in git repositories.")
|
||||
gitScanURI = gitScan.Arg("uri", "Git repository URL. https://, file://, or ssh:// schema expected.").Required().String()
|
||||
gitScanURI = gitScan.Arg("uri", "Git repository URL. https:// or file:// schema expected.").Required().String()
|
||||
gitScanIncludePaths = gitScan.Flag("include-paths", "Path to file with newline separated regexes for files to include in scan.").Short('i').String()
|
||||
gitScanExcludePaths = gitScan.Flag("exclude-paths", "Path to file with newline separated regexes for files to exclude in scan.").Short('x').String()
|
||||
gitScanSinceCommit = gitScan.Flag("since-commit", "Commit to start scan from.").String()
|
||||
@@ -192,15 +192,15 @@ func run(state overseer.State) {
|
||||
defer os.RemoveAll(repoPath)
|
||||
}
|
||||
|
||||
g := func(c *sources.Config) {
|
||||
c.RepoPath = repoPath
|
||||
c.HeadRef = *gitScanBranch
|
||||
c.BaseRef = *gitScanSinceCommit
|
||||
c.MaxDepth = *gitScanMaxDepth
|
||||
c.Filter = filter
|
||||
config := sources.Config{
|
||||
RepoPath: repoPath,
|
||||
HeadRef: *gitScanBranch,
|
||||
BaseRef: *gitScanSinceCommit,
|
||||
MaxDepth: *gitScanMaxDepth,
|
||||
Filter: filter,
|
||||
}
|
||||
|
||||
if err = e.ScanGit(ctx, sources.NewConfig(g)); err != nil {
|
||||
if err = e.ScanGit(ctx, config); err != nil {
|
||||
logrus.WithError(err).Fatal("Failed to scan Git.")
|
||||
}
|
||||
case githubScan.FullCommand():
|
||||
@@ -208,7 +208,7 @@ func run(state overseer.State) {
|
||||
log.Fatal("You must specify at least one organization or repository.")
|
||||
}
|
||||
|
||||
github := func(c *sources.Config) {
|
||||
github := func(c sources.Config) {
|
||||
c.Endpoint = *githubScanEndpoint
|
||||
c.Repos = *githubScanRepos
|
||||
c.Orgs = *githubScanOrgs
|
||||
@@ -222,7 +222,7 @@ func run(state overseer.State) {
|
||||
logrus.WithError(err).Fatal("Failed to scan Github.")
|
||||
}
|
||||
case gitlabScan.FullCommand():
|
||||
gitlab := func(c *sources.Config) {
|
||||
gitlab := func(c sources.Config) {
|
||||
c.Endpoint = *gitlabScanEndpoint
|
||||
c.Token = *gitlabScanToken
|
||||
c.Repos = *gitlabScanRepos
|
||||
@@ -232,7 +232,7 @@ func run(state overseer.State) {
|
||||
logrus.WithError(err).Fatal("Failed to scan GitLab.")
|
||||
}
|
||||
case filesystemScan.FullCommand():
|
||||
fs := func(c *sources.Config) {
|
||||
fs := func(c sources.Config) {
|
||||
c.Directories = *filesystemDirectories
|
||||
}
|
||||
|
||||
@@ -240,7 +240,7 @@ func run(state overseer.State) {
|
||||
logrus.WithError(err).Fatal("Failed to scan filesystem")
|
||||
}
|
||||
case s3Scan.FullCommand():
|
||||
s3 := func(c *sources.Config) {
|
||||
s3 := func(c sources.Config) {
|
||||
c.Key = *s3ScanKey
|
||||
c.Secret = *s3ScanSecret
|
||||
c.Buckets = *s3ScanBuckets
|
||||
@@ -250,7 +250,7 @@ func run(state overseer.State) {
|
||||
logrus.WithError(err).Fatal("Failed to scan S3.")
|
||||
}
|
||||
case syslogScan.FullCommand():
|
||||
syslog := func(c *sources.Config) {
|
||||
syslog := func(c sources.Config) {
|
||||
c.Address = *syslogAddress
|
||||
c.Protocol = *syslogProtocol
|
||||
c.CertPath = *syslogTLSCert
|
||||
|
||||
@@ -86,9 +86,6 @@ func FilterRulesFromFile(source string) (*FilterRuleSet, error) {
|
||||
|
||||
// Pass returns true if the include FilterRuleSet matches the pattern and the exclude FilterRuleSet does not match.
|
||||
func (filter *Filter) Pass(object string) bool {
|
||||
if filter == nil {
|
||||
return true
|
||||
}
|
||||
excluded := filter.exclude.Matches(object)
|
||||
included := filter.include.Matches(object)
|
||||
return !excluded && included
|
||||
|
||||
@@ -1,125 +0,0 @@
|
||||
package context
|
||||
|
||||
import (
|
||||
"context"
|
||||
"time"
|
||||
|
||||
"github.com/go-logr/logr"
|
||||
)
|
||||
|
||||
var (
|
||||
// defaultLogger can be set via SetDefaultLogger.
|
||||
defaultLogger logr.Logger = logr.Discard()
|
||||
)
|
||||
|
||||
// Context wraps context.Context and includes an additional Logger() method.
|
||||
type Context interface {
|
||||
context.Context
|
||||
Logger() logr.Logger
|
||||
}
|
||||
|
||||
type CancelFunc context.CancelFunc
|
||||
|
||||
// logCtx implements Context.
|
||||
type logCtx struct {
|
||||
// Embed context.Context to get all methods for free.
|
||||
context.Context
|
||||
log logr.Logger
|
||||
}
|
||||
|
||||
// Logger returns a structured logger.
|
||||
func (l logCtx) Logger() logr.Logger {
|
||||
return l.log
|
||||
}
|
||||
|
||||
// Background returns context.Background with a default logger.
|
||||
func Background() Context {
|
||||
return logCtx{
|
||||
log: defaultLogger,
|
||||
Context: context.Background(),
|
||||
}
|
||||
}
|
||||
|
||||
// TODO returns context.TODO with a default logger.
|
||||
func TODO() Context {
|
||||
return logCtx{
|
||||
log: defaultLogger,
|
||||
Context: context.TODO(),
|
||||
}
|
||||
}
|
||||
|
||||
// WithCancel returns context.WithCancel with the log object propagated.
|
||||
func WithCancel(parent Context) (Context, context.CancelFunc) {
|
||||
ctx, cancel := context.WithCancel(parent)
|
||||
return logCtx{
|
||||
log: parent.Logger(),
|
||||
Context: ctx,
|
||||
}, cancel
|
||||
}
|
||||
|
||||
// WithDeadline returns context.WithDeadline with the log object propagated and
|
||||
// the deadline added to the structured log values.
|
||||
func WithDeadline(parent Context, d time.Time) (Context, context.CancelFunc) {
|
||||
ctx, cancel := context.WithDeadline(parent, d)
|
||||
return logCtx{
|
||||
log: parent.Logger().WithValues("deadline", d),
|
||||
Context: ctx,
|
||||
}, cancel
|
||||
}
|
||||
|
||||
// WithTimeout returns context.WithTimeout with the log object propagated and
|
||||
// the timeout added to the structured log values.
|
||||
func WithTimeout(parent Context, timeout time.Duration) (Context, context.CancelFunc) {
|
||||
ctx, cancel := context.WithTimeout(parent, timeout)
|
||||
return logCtx{
|
||||
log: parent.Logger().WithValues("timeout", timeout),
|
||||
Context: ctx,
|
||||
}, cancel
|
||||
}
|
||||
|
||||
// WithValue returns context.WithValue with the log object propagated and
|
||||
// the value added to the structured log values (if the key is a string).
|
||||
func WithValue(parent Context, key, val any) Context {
|
||||
logger := parent.Logger()
|
||||
if k, ok := key.(string); ok {
|
||||
logger = logger.WithValues(k, val)
|
||||
}
|
||||
return logCtx{
|
||||
log: logger,
|
||||
Context: context.WithValue(parent, key, val),
|
||||
}
|
||||
}
|
||||
|
||||
// WithValues returns context.WithValue with the log object propagated and
|
||||
// the values added to the structured log values (if the key is a string).
|
||||
func WithValues(parent Context, keyAndVals ...any) Context {
|
||||
ctx := parent
|
||||
for i := 0; i < len(keyAndVals)-1; i += 2 {
|
||||
ctx = WithValue(ctx, keyAndVals[i], keyAndVals[i+1])
|
||||
}
|
||||
return ctx
|
||||
}
|
||||
|
||||
// WithLogger converts a context.Context into a Context by adding a logger.
|
||||
func WithLogger(parent context.Context, logger logr.Logger) Context {
|
||||
return logCtx{
|
||||
log: logger,
|
||||
Context: parent,
|
||||
}
|
||||
}
|
||||
|
||||
// AddLogger converts a context.Context into a Context. If the underlying type
|
||||
// is already a Context, that will be returned, otherwise a default logger will
|
||||
// be added.
|
||||
func AddLogger(parent context.Context) Context {
|
||||
if loggerCtx, ok := parent.(Context); ok {
|
||||
return loggerCtx
|
||||
}
|
||||
return WithLogger(parent, defaultLogger)
|
||||
}
|
||||
|
||||
// SetupDefaultLogger sets the package-level global default logger that will be
|
||||
// used for Background and TODO contexts.
|
||||
func SetDefaultLogger(l logr.Logger) {
|
||||
defaultLogger = l
|
||||
}
|
||||
@@ -1,171 +0,0 @@
|
||||
package context
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/go-logr/logr"
|
||||
"github.com/go-logr/zapr"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/log"
|
||||
"go.uber.org/zap"
|
||||
"go.uber.org/zap/zapcore"
|
||||
"go.uber.org/zap/zaptest"
|
||||
)
|
||||
|
||||
// testLogger is a helper function to create a logger with a closure callback.
|
||||
func testLogger(t *testing.T, f func(zapcore.Entry)) logr.Logger {
|
||||
return zapr.NewLogger(zaptest.NewLogger(t,
|
||||
zaptest.WrapOptions(zap.Hooks(func(e zapcore.Entry) error {
|
||||
f(e)
|
||||
return nil
|
||||
}))))
|
||||
}
|
||||
|
||||
// infoCounterContext is a helper function to create a Context that will count
|
||||
// the number of Info messages logged.
|
||||
func infoCounterContext(t *testing.T) (Context, *int) {
|
||||
var infoCount int
|
||||
logger := testLogger(t, func(e zapcore.Entry) {
|
||||
if e.Level == zap.InfoLevel {
|
||||
infoCount++
|
||||
}
|
||||
})
|
||||
return WithLogger(context.Background(), logger), &infoCount
|
||||
}
|
||||
|
||||
func TestWithCancel(t *testing.T) {
|
||||
parentCtx, infoCount := infoCounterContext(t)
|
||||
ctx, cancel := WithCancel(parentCtx)
|
||||
cancel()
|
||||
assert.Equal(t, 0, *infoCount)
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
ctx.Logger().Info("yay")
|
||||
case <-time.After(1 * time.Second):
|
||||
assert.Fail(t, "context should be done")
|
||||
}
|
||||
assert.Equal(t, 1, *infoCount)
|
||||
}
|
||||
|
||||
func TestWithTimeout(t *testing.T) {
|
||||
parentCtx, infoCount := infoCounterContext(t)
|
||||
ctx, cancel := WithTimeout(parentCtx, 10*time.Millisecond)
|
||||
defer cancel()
|
||||
|
||||
assert.Equal(t, 0, *infoCount)
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
ctx.Logger().Info("yay")
|
||||
case <-time.After(1 * time.Second):
|
||||
assert.Fail(t, "context should be done")
|
||||
}
|
||||
assert.Equal(t, 1, *infoCount)
|
||||
|
||||
ctx, cancel = WithTimeout(parentCtx, 1*time.Second)
|
||||
defer cancel()
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
assert.Fail(t, "context should not be done")
|
||||
case <-time.After(10 * time.Millisecond):
|
||||
ctx.Logger().Info("yay")
|
||||
}
|
||||
assert.Equal(t, 2, *infoCount)
|
||||
}
|
||||
|
||||
func TestWithLogger(t *testing.T) {
|
||||
var infoCount int
|
||||
logger := testLogger(t, func(e zapcore.Entry) {
|
||||
if e.Level == zap.InfoLevel {
|
||||
infoCount++
|
||||
}
|
||||
})
|
||||
|
||||
ctx := WithLogger(context.Background(), logger)
|
||||
assert.Equal(t, logger, ctx.Logger())
|
||||
|
||||
assert.Equal(t, 0, infoCount)
|
||||
ctx.Logger().Info("yay")
|
||||
assert.Equal(t, 1, infoCount)
|
||||
}
|
||||
|
||||
func TestAsContext(t *testing.T) {
|
||||
var gotValue any
|
||||
normalFuncThatTakesContext := func(ctx context.Context) {
|
||||
if logCtx, ok := ctx.(Context); ok {
|
||||
logCtx.Logger().Info("yay")
|
||||
}
|
||||
gotValue = ctx.Value("key")
|
||||
}
|
||||
parentCtx, infoCount := infoCounterContext(t)
|
||||
ctx := WithValue(parentCtx, "key", "value")
|
||||
|
||||
assert.Equal(t, 0, *infoCount)
|
||||
normalFuncThatTakesContext(ctx)
|
||||
assert.Equal(t, 1, *infoCount)
|
||||
assert.Equal(t, "value", gotValue)
|
||||
}
|
||||
|
||||
func TestWithValues(t *testing.T) {
|
||||
var buffer bytes.Buffer
|
||||
logger, sync := log.New("test",
|
||||
log.WithConsoleSink(&buffer),
|
||||
)
|
||||
defer func(prevLogger logr.Logger) {
|
||||
defaultLogger = prevLogger
|
||||
}(defaultLogger)
|
||||
SetDefaultLogger(logger)
|
||||
|
||||
{
|
||||
ctx1 := Background()
|
||||
ctx1.Logger().Info("only a", "a", 0)
|
||||
|
||||
ctx2 := WithValue(ctx1, "b", 1)
|
||||
ctx2.Logger().Info("only b")
|
||||
assert.Equal(t, 1, ctx2.Value("b"))
|
||||
|
||||
ctx3 := WithLogger(ctx2, ctx2.Logger().WithValues("c", 2, "d", 3))
|
||||
ctx3.Logger().Info("bcd")
|
||||
|
||||
ctx2.Logger().Info("only b again")
|
||||
|
||||
type customKey string
|
||||
ctx4 := WithValue(Background(), customKey("foo"), "bar")
|
||||
// foo:bar shouldn't be added to the logger because the key isn't a string
|
||||
ctx4.Logger().Info("foo")
|
||||
|
||||
ctx5 := WithValues(ctx2, "e", 4, "f", 5, 6, "six")
|
||||
ctx5.Logger().Info("bef")
|
||||
assert.Equal(t, "six", ctx5.Value(6))
|
||||
|
||||
ctx6 := WithValues(ctx2, "what does this do?")
|
||||
ctx6.Logger().Info("silently fail I suppose")
|
||||
}
|
||||
assert.Nil(t, sync())
|
||||
logs := strings.Split(strings.TrimSpace(buffer.String()), "\n")
|
||||
|
||||
assert.Equal(t, 7, len(logs))
|
||||
assert.Contains(t, logs[0], `{"a": 0}`)
|
||||
assert.Contains(t, logs[1], `{"b": 1}`)
|
||||
assert.Contains(t, logs[2], `{"b": 1, "c": 2, "d": 3}`)
|
||||
assert.Contains(t, logs[3], `{"b": 1}`)
|
||||
assert.NotContains(t, logs[4], `{"foo": "bar"}`)
|
||||
assert.Contains(t, logs[5], `{"b": 1, "e": 4, "f": 5}`)
|
||||
assert.Contains(t, logs[6], `silently fail`)
|
||||
assert.NotContains(t, logs[6], `what does this do?`)
|
||||
}
|
||||
|
||||
func TestDiscardLogger(t *testing.T) {
|
||||
var panicked bool
|
||||
defer func() {
|
||||
if r := recover(); r != nil {
|
||||
panicked = true
|
||||
}
|
||||
assert.False(t, panicked)
|
||||
}()
|
||||
ctx := Background()
|
||||
ctx.Logger().Info("this shouldn't panic")
|
||||
}
|
||||
@@ -1,72 +0,0 @@
|
||||
package apilayer
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"strings"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
|
||||
)
|
||||
|
||||
type Scanner struct{}
|
||||
|
||||
// Ensure the Scanner satisfies the interface at compile time
|
||||
var _ detectors.Detector = (*Scanner)(nil)
|
||||
|
||||
var (
|
||||
client = common.SaneHttpClient()
|
||||
|
||||
//Make sure that your group is surrounded in boundry characters such as below to reduce false positives
|
||||
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"apilayer"}) + `\b([a-zA-Z0-9]{32})\b`)
|
||||
)
|
||||
|
||||
// Keywords are used for efficiently pre-filtering chunks.
|
||||
// Use identifiers in the secret preferably, or the provider name.
|
||||
func (s Scanner) Keywords() []string {
|
||||
return []string{"apilayer"}
|
||||
}
|
||||
|
||||
// FromData will find and optionally verify Apilayer secrets in a given set of bytes.
|
||||
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
|
||||
dataStr := string(data)
|
||||
|
||||
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
|
||||
|
||||
for _, match := range matches {
|
||||
if len(match) != 2 {
|
||||
continue
|
||||
}
|
||||
resMatch := strings.TrimSpace(match[1])
|
||||
|
||||
s1 := detectors.Result{
|
||||
DetectorType: detectorspb.DetectorType_Apilayer,
|
||||
Raw: []byte(resMatch),
|
||||
}
|
||||
|
||||
if verify {
|
||||
req, err := http.NewRequestWithContext(ctx, "GET", "https://api.apilayer.com/number_verification/countries", nil)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
req.Header.Add("apikey", resMatch)
|
||||
res, err := client.Do(req)
|
||||
if err == nil {
|
||||
defer res.Body.Close()
|
||||
if res.StatusCode >= 200 && res.StatusCode < 300 {
|
||||
s1.Verified = true
|
||||
} else {
|
||||
//This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key
|
||||
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
|
||||
continue
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
results = append(results, s1)
|
||||
}
|
||||
|
||||
return detectors.CleanResults(results), nil
|
||||
}
|
||||
@@ -1,116 +0,0 @@
|
||||
package apilayer
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/kylelemons/godebug/pretty"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
|
||||
)
|
||||
|
||||
func TestApilayer_FromChunk(t *testing.T) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
|
||||
defer cancel()
|
||||
testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")
|
||||
if err != nil {
|
||||
t.Fatalf("could not get test secrets from GCP: %s", err)
|
||||
}
|
||||
secret := testSecrets.MustGetField("APILAYER")
|
||||
inactiveSecret := testSecrets.MustGetField("APILAYER_INACTIVE")
|
||||
|
||||
type args struct {
|
||||
ctx context.Context
|
||||
data []byte
|
||||
verify bool
|
||||
}
|
||||
tests := []struct {
|
||||
name string
|
||||
s Scanner
|
||||
args args
|
||||
want []detectors.Result
|
||||
wantErr bool
|
||||
}{
|
||||
{
|
||||
name: "found, verified",
|
||||
s: Scanner{},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a apilayer secret %s within", secret)),
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_Apilayer,
|
||||
Verified: true,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "found, unverified",
|
||||
s: Scanner{},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a apilayer secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_Apilayer,
|
||||
Verified: false,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "not found",
|
||||
s: Scanner{},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte("You cannot find the secret within"),
|
||||
verify: true,
|
||||
},
|
||||
want: nil,
|
||||
wantErr: false,
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
s := Scanner{}
|
||||
got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
|
||||
if (err != nil) != tt.wantErr {
|
||||
t.Errorf("Apilayer.FromData() error = %v, wantErr %v", err, tt.wantErr)
|
||||
return
|
||||
}
|
||||
for i := range got {
|
||||
if len(got[i].Raw) == 0 {
|
||||
t.Fatalf("no raw secret present: \n %+v", got[i])
|
||||
}
|
||||
got[i].Raw = nil
|
||||
}
|
||||
if diff := pretty.Compare(got, tt.want); diff != "" {
|
||||
t.Errorf("Apilayer.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkFromData(benchmark *testing.B) {
|
||||
ctx := context.Background()
|
||||
s := Scanner{}
|
||||
for name, data := range detectors.MustGetBenchmarkData() {
|
||||
benchmark.Run(name, func(b *testing.B) {
|
||||
for n := 0; n < b.N; n++ {
|
||||
_, err := s.FromData(ctx, false, data)
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -1,78 +0,0 @@
|
||||
package appointedd
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"strings"
|
||||
"io/ioutil"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
|
||||
)
|
||||
|
||||
type Scanner struct{}
|
||||
|
||||
// Ensure the Scanner satisfies the interface at compile time
|
||||
var _ detectors.Detector = (*Scanner)(nil)
|
||||
|
||||
var (
|
||||
client = common.SaneHttpClient()
|
||||
|
||||
//Make sure that your group is surrounded in boundry characters such as below to reduce false positives
|
||||
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"appointedd"}) + `\b([a-zA-Z0-9=+]{88})`)
|
||||
)
|
||||
|
||||
// Keywords are used for efficiently pre-filtering chunks.
|
||||
// Use identifiers in the secret preferably, or the provider name.
|
||||
func (s Scanner) Keywords() []string {
|
||||
return []string{"appointedd"}
|
||||
}
|
||||
|
||||
// FromData will find and optionally verify appointedd secrets in a given set of bytes.
|
||||
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
|
||||
dataStr := string(data)
|
||||
|
||||
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
|
||||
|
||||
for _, match := range matches {
|
||||
if len(match) != 2 {
|
||||
continue
|
||||
}
|
||||
resMatch := strings.TrimSpace(match[1])
|
||||
|
||||
s1 := detectors.Result{
|
||||
DetectorType: detectorspb.DetectorType_Appointedd,
|
||||
Raw: []byte(resMatch),
|
||||
}
|
||||
if verify {
|
||||
req, err := http.NewRequestWithContext(ctx, "GET", "https://api.appointedd.com/v1/availability/slots", nil)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
req.Header.Add("X-API-KEY", resMatch)
|
||||
res, err := client.Do(req)
|
||||
if err == nil {
|
||||
defer res.Body.Close()
|
||||
bodyBytes, err := ioutil.ReadAll(res.Body)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
body := string(bodyBytes)
|
||||
|
||||
if strings.Contains(body, "total") {
|
||||
s1.Verified = true
|
||||
} else {
|
||||
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
|
||||
continue
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
results = append(results, s1)
|
||||
}
|
||||
|
||||
return detectors.CleanResults(results), nil
|
||||
}
|
||||
@@ -1,116 +0,0 @@
|
||||
package appointedd
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/kylelemons/godebug/pretty"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
|
||||
)
|
||||
|
||||
func TestAppointedd_FromChunk(t *testing.T) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
|
||||
defer cancel()
|
||||
testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
|
||||
if err != nil {
|
||||
t.Fatalf("could not get test secrets from GCP: %s", err)
|
||||
}
|
||||
secret := testSecrets.MustGetField("APPOINTEDD")
|
||||
inactiveSecret := testSecrets.MustGetField("APPOINTEDD_INACTIVE")
|
||||
|
||||
type args struct {
|
||||
ctx context.Context
|
||||
data []byte
|
||||
verify bool
|
||||
}
|
||||
tests := []struct {
|
||||
name string
|
||||
s Scanner
|
||||
args args
|
||||
want []detectors.Result
|
||||
wantErr bool
|
||||
}{
|
||||
{
|
||||
name: "found, verified",
|
||||
s: Scanner{},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a appointedd secret %s within", secret)),
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_Appointedd,
|
||||
Verified: true,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "found, unverified",
|
||||
s: Scanner{},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a appointedd secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_Appointedd,
|
||||
Verified: false,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "not found",
|
||||
s: Scanner{},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte("You cannot find the secret within"),
|
||||
verify: true,
|
||||
},
|
||||
want: nil,
|
||||
wantErr: false,
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
s := Scanner{}
|
||||
got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
|
||||
if (err != nil) != tt.wantErr {
|
||||
t.Errorf("Appointedd.FromData() error = %v, wantErr %v", err, tt.wantErr)
|
||||
return
|
||||
}
|
||||
for i := range got {
|
||||
if len(got[i].Raw) == 0 {
|
||||
t.Fatalf("no raw secret present: \n %+v", got[i])
|
||||
}
|
||||
got[i].Raw = nil
|
||||
}
|
||||
if diff := pretty.Compare(got, tt.want); diff != "" {
|
||||
t.Errorf("Appointedd.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkFromData(benchmark *testing.B) {
|
||||
ctx := context.Background()
|
||||
s := Scanner{}
|
||||
for name, data := range detectors.MustGetBenchmarkData() {
|
||||
benchmark.Run(name, func(b *testing.B) {
|
||||
for n := 0; n < b.N; n++ {
|
||||
_, err := s.FromData(ctx, false, data)
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -1,95 +0,0 @@
|
||||
package braintreepayments
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io/ioutil"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
|
||||
)
|
||||
|
||||
type Scanner struct{}
|
||||
|
||||
// Ensure the Scanner satisfies the interface at compile time
|
||||
var _ detectors.Detector = (*Scanner)(nil)
|
||||
|
||||
var (
|
||||
client = common.SaneHttpClient()
|
||||
|
||||
//Make sure that your group is surrounded in boundry characters such as below to reduce false positives
|
||||
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"braintree"}) + `\b([0-9a-f]{32})\b`)
|
||||
idPat = regexp.MustCompile(detectors.PrefixRegex([]string{"braintree"}) + `\b([0-9a-z]{16})\b`)
|
||||
)
|
||||
|
||||
// Keywords are used for efficiently pre-filtering chunks.
|
||||
// Use identifiers in the secret preferably, or the provider name.
|
||||
func (s Scanner) Keywords() []string {
|
||||
return []string{"braintree"}
|
||||
}
|
||||
|
||||
// FromData will find and optionally verify BraintreePayments secrets in a given set of bytes.
|
||||
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
|
||||
dataStr := string(data)
|
||||
|
||||
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
|
||||
idMatches := idPat.FindAllStringSubmatch(dataStr, -1)
|
||||
|
||||
for _, match := range matches {
|
||||
if len(match) != 2 {
|
||||
continue
|
||||
}
|
||||
resMatch := strings.TrimSpace(match[1])
|
||||
|
||||
for _, idMatch := range idMatches {
|
||||
if len(idMatch) != 2 {
|
||||
continue
|
||||
}
|
||||
|
||||
resIdMatch := strings.TrimSpace(idMatch[1])
|
||||
|
||||
s1 := detectors.Result{
|
||||
DetectorType: detectorspb.DetectorType_BraintreePayments,
|
||||
Raw: []byte(resMatch),
|
||||
}
|
||||
|
||||
if verify {
|
||||
payload := strings.NewReader(`{"query": "query { ping }"}`)
|
||||
req, err := http.NewRequestWithContext(ctx, "POST", "https://payments.sandbox.braintree-api.com/graphql", payload)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
req.Header.Add("Content-Type", "application/json")
|
||||
req.Header.Add("Braintree-Version", "2019-01-01")
|
||||
req.SetBasicAuth(resIdMatch, resMatch)
|
||||
res, err := client.Do(req)
|
||||
if err == nil {
|
||||
bodyBytes, err := ioutil.ReadAll(res.Body)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
|
||||
bodyString := string(bodyBytes)
|
||||
validResponse := strings.Contains(bodyString, `"data":{`)
|
||||
|
||||
defer res.Body.Close()
|
||||
if res.StatusCode >= 200 && res.StatusCode < 300 && validResponse {
|
||||
s1.Verified = true
|
||||
} else {
|
||||
//This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key
|
||||
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
|
||||
continue
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
results = append(results, s1)
|
||||
}
|
||||
}
|
||||
|
||||
return detectors.CleanResults(results), nil
|
||||
}
|
||||
@@ -1,117 +0,0 @@
|
||||
package braintreepayments
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/kylelemons/godebug/pretty"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
|
||||
)
|
||||
|
||||
func TestBraintreePayments_FromChunk(t *testing.T) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
|
||||
defer cancel()
|
||||
testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
|
||||
if err != nil {
|
||||
t.Fatalf("could not get test secrets from GCP: %s", err)
|
||||
}
|
||||
secret := testSecrets.MustGetField("BRAINTREEPAYMENTS")
|
||||
id := testSecrets.MustGetField("BRAINTREEPAYMENTS_USER")
|
||||
inactiveSecret := testSecrets.MustGetField("BRAINTREEPAYMENTS_INACTIVE")
|
||||
|
||||
type args struct {
|
||||
ctx context.Context
|
||||
data []byte
|
||||
verify bool
|
||||
}
|
||||
tests := []struct {
|
||||
name string
|
||||
s Scanner
|
||||
args args
|
||||
want []detectors.Result
|
||||
wantErr bool
|
||||
}{
|
||||
{
|
||||
name: "found, verified",
|
||||
s: Scanner{},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a braintree secret %s within braintree %s", secret, id)),
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_BraintreePayments,
|
||||
Verified: true,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "found, unverified",
|
||||
s: Scanner{},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a braintree secret %s within braintree %s but not valid", inactiveSecret, id)), // the secret would satisfy the regex but not pass validation
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_BraintreePayments,
|
||||
Verified: false,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "not found",
|
||||
s: Scanner{},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte("You cannot find the secret within"),
|
||||
verify: true,
|
||||
},
|
||||
want: nil,
|
||||
wantErr: false,
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
s := Scanner{}
|
||||
got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
|
||||
if (err != nil) != tt.wantErr {
|
||||
t.Errorf("BraintreePayments.FromData() error = %v, wantErr %v", err, tt.wantErr)
|
||||
return
|
||||
}
|
||||
for i := range got {
|
||||
if len(got[i].Raw) == 0 {
|
||||
t.Fatalf("no raw secret present: \n %+v", got[i])
|
||||
}
|
||||
got[i].Raw = nil
|
||||
}
|
||||
if diff := pretty.Compare(got, tt.want); diff != "" {
|
||||
t.Errorf("BraintreePayments.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkFromData(benchmark *testing.B) {
|
||||
ctx := context.Background()
|
||||
s := Scanner{}
|
||||
for name, data := range detectors.MustGetBenchmarkData() {
|
||||
benchmark.Run(name, func(b *testing.B) {
|
||||
for n := 0; n < b.N; n++ {
|
||||
_, err := s.FromData(ctx, false, data)
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -1,74 +0,0 @@
|
||||
package codeclimate
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"strings"
|
||||
"fmt"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
|
||||
)
|
||||
|
||||
type Scanner struct{}
|
||||
|
||||
// Ensure the Scanner satisfies the interface at compile time
|
||||
var _ detectors.Detector = (*Scanner)(nil)
|
||||
|
||||
var (
|
||||
client = common.SaneHttpClient()
|
||||
|
||||
//Make sure that your group is surrounded in boundry characters such as below to reduce false positives
|
||||
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"codeclimate"}) + `\b([a-f0-9]{40})\b`)
|
||||
)
|
||||
|
||||
// Keywords are used for efficiently pre-filtering chunks.
|
||||
// Use identifiers in the secret preferably, or the provider name.
|
||||
func (s Scanner) Keywords() []string {
|
||||
return []string{"codeclimate"}
|
||||
}
|
||||
|
||||
// FromData will find and optionally verify Codeclimate secrets in a given set of bytes.
|
||||
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
|
||||
dataStr := string(data)
|
||||
|
||||
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
|
||||
|
||||
for _, match := range matches {
|
||||
if len(match) != 2 {
|
||||
continue
|
||||
}
|
||||
resMatch := strings.TrimSpace(match[1])
|
||||
|
||||
s1 := detectors.Result{
|
||||
DetectorType: detectorspb.DetectorType_Codeclimate,
|
||||
Raw: []byte(resMatch),
|
||||
}
|
||||
|
||||
if verify {
|
||||
req, err := http.NewRequestWithContext(ctx, "GET", "https://api.codeclimate.com/v1/user", nil)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
req.Header.Add("Accept", "application/vnd.api+json")
|
||||
req.Header.Add("Authorization", fmt.Sprintf("Token token=%s", resMatch))
|
||||
res, err := client.Do(req)
|
||||
if err == nil {
|
||||
defer res.Body.Close()
|
||||
if res.StatusCode >= 200 && res.StatusCode < 300 {
|
||||
s1.Verified = true
|
||||
} else {
|
||||
//This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key
|
||||
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
|
||||
continue
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
results = append(results, s1)
|
||||
}
|
||||
|
||||
return detectors.CleanResults(results), nil
|
||||
}
|
||||
@@ -1,116 +0,0 @@
|
||||
package codeclimate
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/kylelemons/godebug/pretty"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
|
||||
)
|
||||
|
||||
func TestCodeclimate_FromChunk(t *testing.T) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
|
||||
defer cancel()
|
||||
testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors4")
|
||||
if err != nil {
|
||||
t.Fatalf("could not get test secrets from GCP: %s", err)
|
||||
}
|
||||
secret := testSecrets.MustGetField("CODECLIMATE")
|
||||
inactiveSecret := testSecrets.MustGetField("CODECLIMATE_INACTIVE")
|
||||
|
||||
type args struct {
|
||||
ctx context.Context
|
||||
data []byte
|
||||
verify bool
|
||||
}
|
||||
tests := []struct {
|
||||
name string
|
||||
s Scanner
|
||||
args args
|
||||
want []detectors.Result
|
||||
wantErr bool
|
||||
}{
|
||||
{
|
||||
name: "found, verified",
|
||||
s: Scanner{},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a codeclimate secret %s within", secret)),
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_Codeclimate,
|
||||
Verified: true,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "found, unverified",
|
||||
s: Scanner{},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a codeclimate secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_Codeclimate,
|
||||
Verified: false,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "not found",
|
||||
s: Scanner{},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte("You cannot find the secret within"),
|
||||
verify: true,
|
||||
},
|
||||
want: nil,
|
||||
wantErr: false,
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
s := Scanner{}
|
||||
got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
|
||||
if (err != nil) != tt.wantErr {
|
||||
t.Errorf("Codeclimate.FromData() error = %v, wantErr %v", err, tt.wantErr)
|
||||
return
|
||||
}
|
||||
for i := range got {
|
||||
if len(got[i].Raw) == 0 {
|
||||
t.Fatalf("no raw secret present: \n %+v", got[i])
|
||||
}
|
||||
got[i].Raw = nil
|
||||
}
|
||||
if diff := pretty.Compare(got, tt.want); diff != "" {
|
||||
t.Errorf("Codeclimate.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkFromData(benchmark *testing.B) {
|
||||
ctx := context.Background()
|
||||
s := Scanner{}
|
||||
for name, data := range detectors.MustGetBenchmarkData() {
|
||||
benchmark.Run(name, func(b *testing.B) {
|
||||
for n := 0; n < b.N; n++ {
|
||||
_, err := s.FromData(ctx, false, data)
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -1,82 +0,0 @@
|
||||
package demio
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"strings"
|
||||
"fmt"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
|
||||
)
|
||||
|
||||
type Scanner struct{}
|
||||
|
||||
// Ensure the Scanner satisfies the interface at compile time
|
||||
var _ detectors.Detector = (*Scanner)(nil)
|
||||
|
||||
var (
|
||||
client = common.SaneHttpClient()
|
||||
|
||||
//Make sure that your group is surrounded in boundry characters such as below to reduce false positives
|
||||
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"demio"}) + `\b([a-z0-9A-Z]{32})\b`)
|
||||
secretPat = regexp.MustCompile(detectors.PrefixRegex([]string{"demio"}) + `\b([a-z0-9A-Z]{10,20})\b`)
|
||||
|
||||
)
|
||||
|
||||
// Keywords are used for efficiently pre-filtering chunks.
|
||||
// Use identifiers in the secret preferably, or the provider name.
|
||||
func (s Scanner) Keywords() []string {
|
||||
return []string{"demio"}
|
||||
}
|
||||
|
||||
// FromData will find and optionally verify Demio secrets in a given set of bytes.
|
||||
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
|
||||
dataStr := string(data)
|
||||
|
||||
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
|
||||
idMatches := secretPat.FindAllStringSubmatch(dataStr, -1)
|
||||
for _, match := range matches {
|
||||
if len(match) != 2 {
|
||||
continue
|
||||
}
|
||||
resMatch := strings.TrimSpace(match[1])
|
||||
for _, idmatch := range idMatches {
|
||||
if len(match) != 2 {
|
||||
continue
|
||||
}
|
||||
resIdMatch := strings.TrimSpace(idmatch[1])
|
||||
|
||||
s1 := detectors.Result{
|
||||
DetectorType: detectorspb.DetectorType_Demio,
|
||||
Raw: []byte(resMatch),
|
||||
}
|
||||
|
||||
if verify {
|
||||
url := fmt.Sprintf("https://my.demio.com/api/v1/ping/query?api_key=%s&api_secret=%s",resMatch,resIdMatch)
|
||||
req, err := http.NewRequestWithContext(ctx, "GET", url, nil)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
res, err := client.Do(req)
|
||||
if err == nil {
|
||||
defer res.Body.Close()
|
||||
if res.StatusCode >= 200 && res.StatusCode < 300 {
|
||||
s1.Verified = true
|
||||
} else {
|
||||
//This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key
|
||||
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
|
||||
continue
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
results = append(results, s1)
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
return detectors.CleanResults(results), nil
|
||||
}
|
||||
@@ -1,117 +0,0 @@
|
||||
package demio
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/kylelemons/godebug/pretty"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
|
||||
)
|
||||
|
||||
func TestDemio_FromChunk(t *testing.T) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
|
||||
defer cancel()
|
||||
testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors4")
|
||||
if err != nil {
|
||||
t.Fatalf("could not get test secrets from GCP: %s", err)
|
||||
}
|
||||
secret := testSecrets.MustGetField("DEMIO")
|
||||
inactiveSecret := testSecrets.MustGetField("DEMIO_INACTIVE")
|
||||
keySecret := testSecrets.MustGetField("DEMIO_SECRET")
|
||||
|
||||
type args struct {
|
||||
ctx context.Context
|
||||
data []byte
|
||||
verify bool
|
||||
}
|
||||
tests := []struct {
|
||||
name string
|
||||
s Scanner
|
||||
args args
|
||||
want []detectors.Result
|
||||
wantErr bool
|
||||
}{
|
||||
{
|
||||
name: "found, verified",
|
||||
s: Scanner{},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a demio secret %s within demio %s", secret,keySecret)),
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_Demio,
|
||||
Verified: true,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "found, unverified",
|
||||
s: Scanner{},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a demio secret %s within but not valid demio %s", inactiveSecret,keySecret)), // the secret would satisfy the regex but not pass validation
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_Demio,
|
||||
Verified: false,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "not found",
|
||||
s: Scanner{},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte("You cannot find the secret within"),
|
||||
verify: true,
|
||||
},
|
||||
want: nil,
|
||||
wantErr: false,
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
s := Scanner{}
|
||||
got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
|
||||
if (err != nil) != tt.wantErr {
|
||||
t.Errorf("Demio.FromData() error = %v, wantErr %v", err, tt.wantErr)
|
||||
return
|
||||
}
|
||||
for i := range got {
|
||||
if len(got[i].Raw) == 0 {
|
||||
t.Fatalf("no raw secret present: \n %+v", got[i])
|
||||
}
|
||||
got[i].Raw = nil
|
||||
}
|
||||
if diff := pretty.Compare(got, tt.want); diff != "" {
|
||||
t.Errorf("Demio.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkFromData(benchmark *testing.B) {
|
||||
ctx := context.Background()
|
||||
s := Scanner{}
|
||||
for name, data := range detectors.MustGetBenchmarkData() {
|
||||
benchmark.Run(name, func(b *testing.B) {
|
||||
for n := 0; n < b.N; n++ {
|
||||
_, err := s.FromData(ctx, false, data)
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -1,76 +0,0 @@
|
||||
package docparser
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
|
||||
)
|
||||
|
||||
type Scanner struct{}
|
||||
|
||||
// Ensure the Scanner satisfies the interface at compile time
|
||||
var _ detectors.Detector = (*Scanner)(nil)
|
||||
|
||||
var (
|
||||
client = common.SaneHttpClient()
|
||||
|
||||
//Make sure that your group is surrounded in boundry characters such as below to reduce false positives
|
||||
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"docparser"}) + `\b([a-f0-9]{40})\b`)
|
||||
)
|
||||
|
||||
// Keywords are used for efficiently pre-filtering chunks.
|
||||
// Use identifiers in the secret preferably, or the provider name.
|
||||
func (s Scanner) Keywords() []string {
|
||||
return []string{"docparser"}
|
||||
}
|
||||
|
||||
// FromData will find and optionally verify Docparser secrets in a given set of bytes.
|
||||
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
|
||||
dataStr := string(data)
|
||||
|
||||
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
|
||||
|
||||
for _, match := range matches {
|
||||
if len(match) != 2 {
|
||||
continue
|
||||
}
|
||||
resMatch := strings.TrimSpace(match[1])
|
||||
|
||||
s1 := detectors.Result{
|
||||
DetectorType: detectorspb.DetectorType_Docparser,
|
||||
Raw: []byte(resMatch),
|
||||
}
|
||||
|
||||
if verify {
|
||||
url := fmt.Sprintf("https://api.docparser.com/v1/parsers?api_key=%s", resMatch)
|
||||
fmt.Println(url)
|
||||
req, err := http.NewRequestWithContext(ctx, "GET",url, nil)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
res, err := client.Do(req)
|
||||
if err == nil {
|
||||
|
||||
defer res.Body.Close()
|
||||
if res.StatusCode >= 200 && res.StatusCode < 300 {
|
||||
s1.Verified = true
|
||||
} else {
|
||||
//This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key
|
||||
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
|
||||
continue
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
results = append(results, s1)
|
||||
}
|
||||
|
||||
return detectors.CleanResults(results), nil
|
||||
}
|
||||
@@ -1,116 +0,0 @@
|
||||
package docparser
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/kylelemons/godebug/pretty"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
|
||||
)
|
||||
|
||||
func TestDocparser_FromChunk(t *testing.T) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
|
||||
defer cancel()
|
||||
testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
|
||||
if err != nil {
|
||||
t.Fatalf("could not get test secrets from GCP: %s", err)
|
||||
}
|
||||
secret := testSecrets.MustGetField("DOCPARSER")
|
||||
inactiveSecret := testSecrets.MustGetField("DOCPARSER_INACTIVE")
|
||||
|
||||
type args struct {
|
||||
ctx context.Context
|
||||
data []byte
|
||||
verify bool
|
||||
}
|
||||
tests := []struct {
|
||||
name string
|
||||
s Scanner
|
||||
args args
|
||||
want []detectors.Result
|
||||
wantErr bool
|
||||
}{
|
||||
{
|
||||
name: "found, verified",
|
||||
s: Scanner{},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a docparser secret %s within", secret)),
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_Docparser,
|
||||
Verified: true,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "found, unverified",
|
||||
s: Scanner{},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a docparser secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_Docparser,
|
||||
Verified: false,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "not found",
|
||||
s: Scanner{},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte("You cannot find the secret within"),
|
||||
verify: true,
|
||||
},
|
||||
want: nil,
|
||||
wantErr: false,
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
s := Scanner{}
|
||||
got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
|
||||
if (err != nil) != tt.wantErr {
|
||||
t.Errorf("Docparser.FromData() error = %v, wantErr %v", err, tt.wantErr)
|
||||
return
|
||||
}
|
||||
for i := range got {
|
||||
if len(got[i].Raw) == 0 {
|
||||
t.Fatalf("no raw secret present: \n %+v", got[i])
|
||||
}
|
||||
got[i].Raw = nil
|
||||
}
|
||||
if diff := pretty.Compare(got, tt.want); diff != "" {
|
||||
t.Errorf("Docparser.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkFromData(benchmark *testing.B) {
|
||||
ctx := context.Background()
|
||||
s := Scanner{}
|
||||
for name, data := range detectors.MustGetBenchmarkData() {
|
||||
benchmark.Run(name, func(b *testing.B) {
|
||||
for n := 0; n < b.N; n++ {
|
||||
_, err := s.FromData(ctx, false, data)
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -1,87 +0,0 @@
|
||||
package duply
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
|
||||
)
|
||||
|
||||
type Scanner struct{}
|
||||
|
||||
// Ensure the Scanner satisfies the interface at compile time
|
||||
var _ detectors.Detector = (*Scanner)(nil)
|
||||
|
||||
var (
|
||||
client = common.SaneHttpClient()
|
||||
|
||||
//Make sure that your group is surrounded in boundry characters such as below to reduce false positives
|
||||
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"duply"}) + `\b([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})\b`)
|
||||
idPat = regexp.MustCompile(detectors.PrefixRegex([]string{"duply"}) + `\b([0-9A-Z]{7}-[0-9A-Z]{7}-[0-9A-Z]{7}-[0-9A-Z]{7})\b`)
|
||||
)
|
||||
|
||||
// Keywords are used for efficiently pre-filtering chunks.
|
||||
// Use identifiers in the secret preferably, or the provider name.
|
||||
func (s Scanner) Keywords() []string {
|
||||
return []string{"duply"}
|
||||
}
|
||||
|
||||
// FromData will find and optionally verify Duply secrets in a given set of bytes.
|
||||
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
|
||||
dataStr := string(data)
|
||||
|
||||
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
|
||||
idMatches := idPat.FindAllStringSubmatch(dataStr, -1)
|
||||
|
||||
for _, match := range matches {
|
||||
if len(match) != 2 {
|
||||
continue
|
||||
}
|
||||
resMatch := strings.TrimSpace(match[1])
|
||||
|
||||
for _, idMatch := range idMatches {
|
||||
if len(idMatch) != 2 {
|
||||
continue
|
||||
}
|
||||
|
||||
resIdMatch := strings.TrimSpace(idMatch[1])
|
||||
s1 := detectors.Result{
|
||||
DetectorType: detectorspb.DetectorType_Duply,
|
||||
Raw: []byte(resMatch),
|
||||
}
|
||||
|
||||
if verify {
|
||||
timeout := 10 * time.Second
|
||||
client.Timeout = timeout
|
||||
req, err := http.NewRequestWithContext(ctx, "GET", "https://gen.duply.co/v1/usage", nil)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
req.Header.Add("Content-Type", "application/json")
|
||||
req.SetBasicAuth(resIdMatch, resMatch)
|
||||
|
||||
res, err := client.Do(req)
|
||||
if err == nil {
|
||||
defer res.Body.Close()
|
||||
if res.StatusCode >= 200 && res.StatusCode < 300 {
|
||||
s1.Verified = true
|
||||
} else {
|
||||
//This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key
|
||||
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
|
||||
continue
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
results = append(results, s1)
|
||||
}
|
||||
}
|
||||
|
||||
return detectors.CleanResults(results), nil
|
||||
}
|
||||
@@ -1,117 +0,0 @@
|
||||
package duply
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/kylelemons/godebug/pretty"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
|
||||
)
|
||||
|
||||
func TestDuply_FromChunk(t *testing.T) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
|
||||
defer cancel()
|
||||
testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
|
||||
if err != nil {
|
||||
t.Fatalf("could not get test secrets from GCP: %s", err)
|
||||
}
|
||||
secret := testSecrets.MustGetField("DUPLY")
|
||||
id := testSecrets.MustGetField("DUPLY_USER")
|
||||
inactiveSecret := testSecrets.MustGetField("DUPLY_INACTIVE")
|
||||
|
||||
type args struct {
|
||||
ctx context.Context
|
||||
data []byte
|
||||
verify bool
|
||||
}
|
||||
tests := []struct {
|
||||
name string
|
||||
s Scanner
|
||||
args args
|
||||
want []detectors.Result
|
||||
wantErr bool
|
||||
}{
|
||||
{
|
||||
name: "found, verified",
|
||||
s: Scanner{},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a duply secret %s within duply %s", secret, id)),
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_Duply,
|
||||
Verified: true,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "found, unverified",
|
||||
s: Scanner{},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a duply secret %s within duply %s but not valid", inactiveSecret, id)), // the secret would satisfy the regex but not pass validation
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_Duply,
|
||||
Verified: false,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "not found",
|
||||
s: Scanner{},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte("You cannot find the secret within"),
|
||||
verify: true,
|
||||
},
|
||||
want: nil,
|
||||
wantErr: false,
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
s := Scanner{}
|
||||
got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
|
||||
if (err != nil) != tt.wantErr {
|
||||
t.Errorf("Duply.FromData() error = %v, wantErr %v", err, tt.wantErr)
|
||||
return
|
||||
}
|
||||
for i := range got {
|
||||
if len(got[i].Raw) == 0 {
|
||||
t.Fatalf("no raw secret present: \n %+v", got[i])
|
||||
}
|
||||
got[i].Raw = nil
|
||||
}
|
||||
if diff := pretty.Compare(got, tt.want); diff != "" {
|
||||
t.Errorf("Duply.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkFromData(benchmark *testing.B) {
|
||||
ctx := context.Background()
|
||||
s := Scanner{}
|
||||
for name, data := range detectors.MustGetBenchmarkData() {
|
||||
benchmark.Run(name, func(b *testing.B) {
|
||||
for n := 0; n < b.N; n++ {
|
||||
_, err := s.FromData(ctx, false, data)
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -1,75 +0,0 @@
|
||||
package ecostruxureit
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
|
||||
)
|
||||
|
||||
type Scanner struct{}
|
||||
|
||||
// Ensure the Scanner satisfies the interface at compile time
|
||||
var _ detectors.Detector = (*Scanner)(nil)
|
||||
|
||||
var (
|
||||
client = common.SaneHttpClient()
|
||||
|
||||
//Make sure that your group is surrounded in boundry characters such as below to reduce false positives
|
||||
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"ecostruxureit"}) + `\b(AK1[0-9a-zA-Z\/]{50,55})\b`)
|
||||
)
|
||||
|
||||
// Keywords are used for efficiently pre-filtering chunks.
|
||||
// Use identifiers in the secret preferably, or the provider name.
|
||||
func (s Scanner) Keywords() []string {
|
||||
return []string{"ecostruxureit"}
|
||||
}
|
||||
|
||||
// FromData will find and optionally verify EcoStruxureIT secrets in a given set of bytes.
|
||||
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
|
||||
dataStr := string(data)
|
||||
|
||||
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
|
||||
|
||||
for _, match := range matches {
|
||||
if len(match) != 2 {
|
||||
continue
|
||||
}
|
||||
resMatch := strings.TrimSpace(match[1])
|
||||
|
||||
s1 := detectors.Result{
|
||||
DetectorType: detectorspb.DetectorType_EcoStruxureIT,
|
||||
Raw: []byte(resMatch),
|
||||
}
|
||||
|
||||
if verify {
|
||||
req, err := http.NewRequestWithContext(ctx, "GET", "https://api.ecostruxureit.com/rest/v1/organizations", nil)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
req.Header.Add("Accept", "application/json")
|
||||
req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", resMatch))
|
||||
res, err := client.Do(req)
|
||||
if err == nil {
|
||||
defer res.Body.Close()
|
||||
if res.StatusCode >= 200 && res.StatusCode < 300 {
|
||||
s1.Verified = true
|
||||
} else {
|
||||
//This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key
|
||||
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
|
||||
continue
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
results = append(results, s1)
|
||||
}
|
||||
|
||||
return detectors.CleanResults(results), nil
|
||||
}
|
||||
@@ -1,116 +0,0 @@
|
||||
package ecostruxureit
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/kylelemons/godebug/pretty"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
|
||||
)
|
||||
|
||||
func TestEcoStruxureIT_FromChunk(t *testing.T) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
|
||||
defer cancel()
|
||||
testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
|
||||
if err != nil {
|
||||
t.Fatalf("could not get test secrets from GCP: %s", err)
|
||||
}
|
||||
secret := testSecrets.MustGetField("ECOSTRUXUREIT")
|
||||
inactiveSecret := testSecrets.MustGetField("ECOSTRUXUREIT_INACTIVE")
|
||||
|
||||
type args struct {
|
||||
ctx context.Context
|
||||
data []byte
|
||||
verify bool
|
||||
}
|
||||
tests := []struct {
|
||||
name string
|
||||
s Scanner
|
||||
args args
|
||||
want []detectors.Result
|
||||
wantErr bool
|
||||
}{
|
||||
{
|
||||
name: "found, verified",
|
||||
s: Scanner{},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a ecostruxureit secret %s within", secret)),
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_EcoStruxureIT,
|
||||
Verified: true,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "found, unverified",
|
||||
s: Scanner{},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a ecostruxureit secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_EcoStruxureIT,
|
||||
Verified: false,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "not found",
|
||||
s: Scanner{},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte("You cannot find the secret within"),
|
||||
verify: true,
|
||||
},
|
||||
want: nil,
|
||||
wantErr: false,
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
s := Scanner{}
|
||||
got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
|
||||
if (err != nil) != tt.wantErr {
|
||||
t.Errorf("EcoStruxureIT.FromData() error = %v, wantErr %v", err, tt.wantErr)
|
||||
return
|
||||
}
|
||||
for i := range got {
|
||||
if len(got[i].Raw) == 0 {
|
||||
t.Fatalf("no raw secret present: \n %+v", got[i])
|
||||
}
|
||||
got[i].Raw = nil
|
||||
}
|
||||
if diff := pretty.Compare(got, tt.want); diff != "" {
|
||||
t.Errorf("EcoStruxureIT.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkFromData(benchmark *testing.B) {
|
||||
ctx := context.Background()
|
||||
s := Scanner{}
|
||||
for name, data := range detectors.MustGetBenchmarkData() {
|
||||
benchmark.Run(name, func(b *testing.B) {
|
||||
for n := 0; n < b.N; n++ {
|
||||
_, err := s.FromData(ctx, false, data)
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -1,81 +0,0 @@
|
||||
package flightlabs
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
|
||||
)
|
||||
|
||||
type Scanner struct{}
|
||||
|
||||
// Ensure the Scanner satisfies the interface at compile time
|
||||
var _ detectors.Detector = (*Scanner)(nil)
|
||||
|
||||
var (
|
||||
client = common.SaneHttpClient()
|
||||
|
||||
//Make sure that your group is surrounded in boundry characters such as below to reduce false positives
|
||||
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"flightlabs"}) + `\b(ey[a-zA-Z0-9]{34}.ey[a-zA-Z0-9._-]{300,350})\b`)
|
||||
)
|
||||
|
||||
// Keywords are used for efficiently pre-filtering chunks.
|
||||
// Use identifiers in the secret preferably, or the provider name.
|
||||
func (s Scanner) Keywords() []string {
|
||||
return []string{"flightlabs"}
|
||||
}
|
||||
|
||||
// FromData will find and optionally verify FlightLabs secrets in a given set of bytes.
|
||||
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
|
||||
dataStr := string(data)
|
||||
|
||||
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
|
||||
|
||||
for _, match := range matches {
|
||||
if len(match) != 2 {
|
||||
continue
|
||||
}
|
||||
resMatch := strings.TrimSpace(match[1])
|
||||
|
||||
s1 := detectors.Result{
|
||||
DetectorType: detectorspb.DetectorType_FlightLabs,
|
||||
Raw: []byte(resMatch),
|
||||
}
|
||||
|
||||
if verify {
|
||||
req, err := http.NewRequestWithContext(ctx, "GET", fmt.Sprintf("https://app.goflightlabs.com/airports?access_key=%s", resMatch), nil)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
|
||||
res, err := client.Do(req)
|
||||
if err == nil {
|
||||
bodyBytes, err := ioutil.ReadAll(res.Body)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
bodyString := string(bodyBytes)
|
||||
validResponse := strings.Contains(bodyString, `"id"`)
|
||||
defer res.Body.Close()
|
||||
if res.StatusCode >= 200 && res.StatusCode < 300 && validResponse {
|
||||
s1.Verified = true
|
||||
} else {
|
||||
//This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key
|
||||
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
|
||||
continue
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
results = append(results, s1)
|
||||
}
|
||||
|
||||
return detectors.CleanResults(results), nil
|
||||
}
|
||||
@@ -1,116 +0,0 @@
|
||||
package flightlabs
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/kylelemons/godebug/pretty"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
|
||||
)
|
||||
|
||||
func TestFlightLabs_FromChunk(t *testing.T) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
|
||||
defer cancel()
|
||||
testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
|
||||
if err != nil {
|
||||
t.Fatalf("could not get test secrets from GCP: %s", err)
|
||||
}
|
||||
secret := testSecrets.MustGetField("FLIGHTLABS")
|
||||
inactiveSecret := testSecrets.MustGetField("FLIGHTLABS_INACTIVE")
|
||||
|
||||
type args struct {
|
||||
ctx context.Context
|
||||
data []byte
|
||||
verify bool
|
||||
}
|
||||
tests := []struct {
|
||||
name string
|
||||
s Scanner
|
||||
args args
|
||||
want []detectors.Result
|
||||
wantErr bool
|
||||
}{
|
||||
{
|
||||
name: "found, verified",
|
||||
s: Scanner{},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a flightlabs secret %s within", secret)),
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_FlightLabs,
|
||||
Verified: true,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "found, unverified",
|
||||
s: Scanner{},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a flightlabs secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_FlightLabs,
|
||||
Verified: false,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "not found",
|
||||
s: Scanner{},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte("You cannot find the secret within"),
|
||||
verify: true,
|
||||
},
|
||||
want: nil,
|
||||
wantErr: false,
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
s := Scanner{}
|
||||
got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
|
||||
if (err != nil) != tt.wantErr {
|
||||
t.Errorf("FlightLabs.FromData() error = %v, wantErr %v", err, tt.wantErr)
|
||||
return
|
||||
}
|
||||
for i := range got {
|
||||
if len(got[i].Raw) == 0 {
|
||||
t.Fatalf("no raw secret present: \n %+v", got[i])
|
||||
}
|
||||
got[i].Raw = nil
|
||||
}
|
||||
if diff := pretty.Compare(got, tt.want); diff != "" {
|
||||
t.Errorf("FlightLabs.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkFromData(benchmark *testing.B) {
|
||||
ctx := context.Background()
|
||||
s := Scanner{}
|
||||
for name, data := range detectors.MustGetBenchmarkData() {
|
||||
benchmark.Run(name, func(b *testing.B) {
|
||||
for n := 0; n < b.N; n++ {
|
||||
_, err := s.FromData(ctx, false, data)
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -1,90 +0,0 @@
|
||||
package formsite
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
|
||||
)
|
||||
|
||||
type Scanner struct{}
|
||||
|
||||
// Ensure the Scanner satisfies the interface at compile time
|
||||
var _ detectors.Detector = (*Scanner)(nil)
|
||||
|
||||
var (
|
||||
client = common.SaneHttpClient()
|
||||
|
||||
//Make sure that your group is surrounded in boundry characters such as below to reduce false positives
|
||||
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"formsite"}) + `\b([a-zA-Z0-9]{32})\b`)
|
||||
serverPat = regexp.MustCompile(detectors.PrefixRegex([]string{"formsite"}) + `\b(fs[0-9]{1,4})\b`)
|
||||
userPat = regexp.MustCompile(detectors.PrefixRegex([]string{"formsite"}) + `\b([a-zA-Z0-9]{6})\b`)
|
||||
)
|
||||
|
||||
// Keywords are used for efficiently pre-filtering chunks.
|
||||
// Use identifiers in the secret preferably, or the provider name.
|
||||
func (s Scanner) Keywords() []string {
|
||||
return []string{"formsite"}
|
||||
}
|
||||
|
||||
// FromData will find and optionally verify Formsite secrets in a given set of bytes..
|
||||
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
|
||||
dataStr := string(data)
|
||||
|
||||
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
|
||||
serverMatches := serverPat.FindAllStringSubmatch(dataStr, -1)
|
||||
userMatches := userPat.FindAllStringSubmatch(dataStr, -1)
|
||||
|
||||
for _, match := range matches {
|
||||
if len(match) != 2 {
|
||||
continue
|
||||
}
|
||||
resMatch := strings.TrimSpace(match[1])
|
||||
for _, serverMatch := range serverMatches {
|
||||
if len(serverMatch) != 2 {
|
||||
continue
|
||||
}
|
||||
resServerMatch := strings.TrimSpace(serverMatch[1])
|
||||
for _, userMatch := range userMatches {
|
||||
if len(userMatch) != 2 {
|
||||
continue
|
||||
}
|
||||
resUserMatch := strings.TrimSpace(userMatch[1])
|
||||
s1 := detectors.Result{
|
||||
DetectorType: detectorspb.DetectorType_Formsite,
|
||||
Raw: []byte(resMatch),
|
||||
}
|
||||
|
||||
if verify {
|
||||
req, err := http.NewRequestWithContext(ctx, "GET", fmt.Sprintf("https://%s.formsite.com/api/v2/%s/forms",resServerMatch,resUserMatch), nil)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", resMatch))
|
||||
res, err := client.Do(req)
|
||||
if err == nil {
|
||||
defer res.Body.Close()
|
||||
if res.StatusCode >= 200 && res.StatusCode < 300 {
|
||||
s1.Verified = true
|
||||
} else {
|
||||
//This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key
|
||||
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
|
||||
continue
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
results = append(results, s1)
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
return detectors.CleanResults(results), nil
|
||||
}
|
||||
@@ -1,118 +0,0 @@
|
||||
package formsite
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/kylelemons/godebug/pretty"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
|
||||
)
|
||||
|
||||
func TestFormsite_FromChunk(t *testing.T) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
|
||||
defer cancel()
|
||||
testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors4")
|
||||
if err != nil {
|
||||
t.Fatalf("could not get test secrets from GCP: %s", err)
|
||||
}
|
||||
secret := testSecrets.MustGetField("FORMSITE")
|
||||
inactiveSecret := testSecrets.MustGetField("FORMSITE_INACTIVE")
|
||||
server := testSecrets.MustGetField("FORMSITE_SERVER")
|
||||
user := testSecrets.MustGetField("FORMSITE_USER")
|
||||
|
||||
type args struct {
|
||||
ctx context.Context
|
||||
data []byte
|
||||
verify bool
|
||||
}
|
||||
tests := []struct {
|
||||
name string
|
||||
s Scanner
|
||||
args args
|
||||
want []detectors.Result
|
||||
wantErr bool
|
||||
}{
|
||||
{
|
||||
name: "found, verified",
|
||||
s: Scanner{},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a formsite secret %s within formsite server %s formsite user %s", secret,server,user)),
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_Formsite,
|
||||
Verified: true,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "found, unverified",
|
||||
s: Scanner{},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a formsite secret %s within but not valid formsite server %s formsite user %s", inactiveSecret,server,user)), // the secret would satisfy the regex but not pass validation
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_Formsite,
|
||||
Verified: false,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "not found",
|
||||
s: Scanner{},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte("You cannot find the secret within"),
|
||||
verify: true,
|
||||
},
|
||||
want: nil,
|
||||
wantErr: false,
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
s := Scanner{}
|
||||
got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
|
||||
if (err != nil) != tt.wantErr {
|
||||
t.Errorf("Formsite.FromData() error = %v, wantErr %v", err, tt.wantErr)
|
||||
return
|
||||
}
|
||||
for i := range got {
|
||||
if len(got[i].Raw) == 0 {
|
||||
t.Fatalf("no raw secret present: \n %+v", got[i])
|
||||
}
|
||||
got[i].Raw = nil
|
||||
}
|
||||
if diff := pretty.Compare(got, tt.want); diff != "" {
|
||||
t.Errorf("Formsite.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkFromData(benchmark *testing.B) {
|
||||
ctx := context.Background()
|
||||
s := Scanner{}
|
||||
for name, data := range detectors.MustGetBenchmarkData() {
|
||||
benchmark.Run(name, func(b *testing.B) {
|
||||
for n := 0; n < b.N; n++ {
|
||||
_, err := s.FromData(ctx, false, data)
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -1,74 +0,0 @@
|
||||
package getresponse
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
|
||||
)
|
||||
|
||||
type Scanner struct{}
|
||||
|
||||
// Ensure the Scanner satisfies the interface at compile time
|
||||
var _ detectors.Detector = (*Scanner)(nil)
|
||||
|
||||
var (
|
||||
client = common.SaneHttpClient()
|
||||
|
||||
//Make sure that your group is surrounded in boundry characters such as below to reduce false positives
|
||||
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"getresponse"}) + `\b([a-z0-9]{32})\b`)
|
||||
)
|
||||
|
||||
// Keywords are used for efficiently pre-filtering chunks.
|
||||
// Use identifiers in the secret preferably, or the provider name.
|
||||
func (s Scanner) Keywords() []string {
|
||||
return []string{"getresponse"}
|
||||
}
|
||||
|
||||
// FromData will find and optionally verify Getresponse secrets in a given set of bytes.
|
||||
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
|
||||
dataStr := string(data)
|
||||
|
||||
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
|
||||
|
||||
for _, match := range matches {
|
||||
if len(match) != 2 {
|
||||
continue
|
||||
}
|
||||
resMatch := strings.TrimSpace(match[1])
|
||||
|
||||
s1 := detectors.Result{
|
||||
DetectorType: detectorspb.DetectorType_Getresponse,
|
||||
Raw: []byte(resMatch),
|
||||
}
|
||||
|
||||
if verify {
|
||||
req, err := http.NewRequestWithContext(ctx, "GET", "https://api.getresponse.com/v3/accounts", nil)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
req.Header.Add("X-Auth-Token", fmt.Sprintf("api-key %s", resMatch))
|
||||
res, err := client.Do(req)
|
||||
if err == nil {
|
||||
defer res.Body.Close()
|
||||
if res.StatusCode >= 200 && res.StatusCode < 300 {
|
||||
s1.Verified = true
|
||||
} else {
|
||||
//This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key
|
||||
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
|
||||
continue
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
results = append(results, s1)
|
||||
}
|
||||
|
||||
return detectors.CleanResults(results), nil
|
||||
}
|
||||
@@ -1,116 +0,0 @@
|
||||
package getresponse
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/kylelemons/godebug/pretty"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
|
||||
)
|
||||
|
||||
func TestGetresponse_FromChunk(t *testing.T) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
|
||||
defer cancel()
|
||||
testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors4")
|
||||
if err != nil {
|
||||
t.Fatalf("could not get test secrets from GCP: %s", err)
|
||||
}
|
||||
secret := testSecrets.MustGetField("GETRESPONSE")
|
||||
inactiveSecret := testSecrets.MustGetField("GETRESPONSE_INACTIVE")
|
||||
|
||||
type args struct {
|
||||
ctx context.Context
|
||||
data []byte
|
||||
verify bool
|
||||
}
|
||||
tests := []struct {
|
||||
name string
|
||||
s Scanner
|
||||
args args
|
||||
want []detectors.Result
|
||||
wantErr bool
|
||||
}{
|
||||
{
|
||||
name: "found, verified",
|
||||
s: Scanner{},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a getresponse secret %s within", secret)),
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_Getresponse,
|
||||
Verified: true,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "found, unverified",
|
||||
s: Scanner{},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a getresponse secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_Getresponse,
|
||||
Verified: false,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "not found",
|
||||
s: Scanner{},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte("You cannot find the secret within"),
|
||||
verify: true,
|
||||
},
|
||||
want: nil,
|
||||
wantErr: false,
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
s := Scanner{}
|
||||
got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
|
||||
if (err != nil) != tt.wantErr {
|
||||
t.Errorf("Getresponse.FromData() error = %v, wantErr %v", err, tt.wantErr)
|
||||
return
|
||||
}
|
||||
for i := range got {
|
||||
if len(got[i].Raw) == 0 {
|
||||
t.Fatalf("no raw secret present: \n %+v", got[i])
|
||||
}
|
||||
got[i].Raw = nil
|
||||
}
|
||||
if diff := pretty.Compare(got, tt.want); diff != "" {
|
||||
t.Errorf("Getresponse.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkFromData(benchmark *testing.B) {
|
||||
ctx := context.Background()
|
||||
s := Scanner{}
|
||||
for name, data := range detectors.MustGetBenchmarkData() {
|
||||
benchmark.Run(name, func(b *testing.B) {
|
||||
for n := 0; n < b.N; n++ {
|
||||
_, err := s.FromData(ctx, false, data)
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -1,73 +0,0 @@
|
||||
package gtmetrix
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
|
||||
)
|
||||
|
||||
type Scanner struct{}
|
||||
|
||||
// Ensure the Scanner satisfies the interface at compile time
|
||||
var _ detectors.Detector = (*Scanner)(nil)
|
||||
|
||||
var (
|
||||
client = common.SaneHttpClient()
|
||||
|
||||
//Make sure that your group is surrounded in boundry characters such as below to reduce false positives
|
||||
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"gtmetrix"}) + `\b([0-9a-f]{32})\b`)
|
||||
)
|
||||
|
||||
// Keywords are used for efficiently pre-filtering chunks.
|
||||
// Use identifiers in the secret preferably, or the provider name.
|
||||
func (s Scanner) Keywords() []string {
|
||||
return []string{"gtmetrix"}
|
||||
}
|
||||
|
||||
// FromData will find and optionally verify GTMetrix secrets in a given set of bytes.
|
||||
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
|
||||
dataStr := string(data)
|
||||
|
||||
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
|
||||
|
||||
for _, match := range matches {
|
||||
if len(match) != 2 {
|
||||
continue
|
||||
}
|
||||
resMatch := strings.TrimSpace(match[1])
|
||||
|
||||
s1 := detectors.Result{
|
||||
DetectorType: detectorspb.DetectorType_GTMetrix,
|
||||
Raw: []byte(resMatch),
|
||||
}
|
||||
|
||||
if verify {
|
||||
req, err := http.NewRequestWithContext(ctx, "GET", "https://gtmetrix.com/api/2.0/status", nil)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
req.SetBasicAuth(resMatch, "")
|
||||
res, err := client.Do(req)
|
||||
if err == nil {
|
||||
defer res.Body.Close()
|
||||
if res.StatusCode >= 200 && res.StatusCode < 300 {
|
||||
s1.Verified = true
|
||||
} else {
|
||||
//This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key
|
||||
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
|
||||
continue
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
results = append(results, s1)
|
||||
}
|
||||
|
||||
return detectors.CleanResults(results), nil
|
||||
}
|
||||
@@ -1,116 +0,0 @@
|
||||
package gtmetrix
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/kylelemons/godebug/pretty"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
|
||||
)
|
||||
|
||||
func TestGTMetrix_FromChunk(t *testing.T) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
|
||||
defer cancel()
|
||||
testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
|
||||
if err != nil {
|
||||
t.Fatalf("could not get test secrets from GCP: %s", err)
|
||||
}
|
||||
secret := testSecrets.MustGetField("GTMETRIX")
|
||||
inactiveSecret := testSecrets.MustGetField("GTMETRIX_INACTIVE")
|
||||
|
||||
type args struct {
|
||||
ctx context.Context
|
||||
data []byte
|
||||
verify bool
|
||||
}
|
||||
tests := []struct {
|
||||
name string
|
||||
s Scanner
|
||||
args args
|
||||
want []detectors.Result
|
||||
wantErr bool
|
||||
}{
|
||||
{
|
||||
name: "found, verified",
|
||||
s: Scanner{},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a gtmetrix secret %s within", secret)),
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_GTMetrix,
|
||||
Verified: true,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "found, unverified",
|
||||
s: Scanner{},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a gtmetrix secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_GTMetrix,
|
||||
Verified: false,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "not found",
|
||||
s: Scanner{},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte("You cannot find the secret within"),
|
||||
verify: true,
|
||||
},
|
||||
want: nil,
|
||||
wantErr: false,
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
s := Scanner{}
|
||||
got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
|
||||
if (err != nil) != tt.wantErr {
|
||||
t.Errorf("GTMetrix.FromData() error = %v, wantErr %v", err, tt.wantErr)
|
||||
return
|
||||
}
|
||||
for i := range got {
|
||||
if len(got[i].Raw) == 0 {
|
||||
t.Fatalf("no raw secret present: \n %+v", got[i])
|
||||
}
|
||||
got[i].Raw = nil
|
||||
}
|
||||
if diff := pretty.Compare(got, tt.want); diff != "" {
|
||||
t.Errorf("GTMetrix.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkFromData(benchmark *testing.B) {
|
||||
ctx := context.Background()
|
||||
s := Scanner{}
|
||||
for name, data := range detectors.MustGetBenchmarkData() {
|
||||
benchmark.Run(name, func(b *testing.B) {
|
||||
for n := 0; n < b.N; n++ {
|
||||
_, err := s.FromData(ctx, false, data)
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -1,78 +0,0 @@
|
||||
package heatmapapi
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"strings"
|
||||
"io/ioutil"
|
||||
"fmt"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
|
||||
)
|
||||
|
||||
type Scanner struct{}
|
||||
|
||||
// Ensure the Scanner satisfies the interface at compile time
|
||||
var _ detectors.Detector = (*Scanner)(nil)
|
||||
|
||||
var (
|
||||
client = common.SaneHttpClient()
|
||||
|
||||
//Make sure that your group is surrounded in boundry characters such as below to reduce false positives
|
||||
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"heatmapapi"}) + `\b([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})\b`)
|
||||
)
|
||||
|
||||
// Keywords are used for efficiently pre-filtering chunks.
|
||||
// Use identifiers in the secret preferably, or the provider name.
|
||||
func (s Scanner) Keywords() []string {
|
||||
return []string{"heatmapapi"}
|
||||
}
|
||||
|
||||
// FromData will find and optionally verify Heatmapapi secrets in a given set of bytes.
|
||||
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
|
||||
dataStr := string(data)
|
||||
|
||||
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
|
||||
|
||||
for _, match := range matches {
|
||||
if len(match) != 2 {
|
||||
continue
|
||||
}
|
||||
resMatch := strings.TrimSpace(match[1])
|
||||
|
||||
s1 := detectors.Result{
|
||||
DetectorType: detectorspb.DetectorType_Heatmapapi,
|
||||
Raw: []byte(resMatch),
|
||||
}
|
||||
|
||||
if verify {
|
||||
req, err := http.NewRequestWithContext(ctx, "GET", fmt.Sprintf("http://www.heatmapapi.com/javascript/HeatmapAPI2.aspx?k=%s",resMatch), nil)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
res, err := client.Do(req)
|
||||
if err == nil {
|
||||
defer res.Body.Close()
|
||||
bodyBytes, err := ioutil.ReadAll(res.Body)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
body := string(bodyBytes)
|
||||
|
||||
if strings.Contains(body, "poststr") {
|
||||
s1.Verified = true
|
||||
} else {
|
||||
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
|
||||
continue
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
results = append(results, s1)
|
||||
}
|
||||
|
||||
return detectors.CleanResults(results), nil
|
||||
}
|
||||
@@ -1,116 +0,0 @@
|
||||
package heatmapapi
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/kylelemons/godebug/pretty"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
|
||||
)
|
||||
|
||||
func TestHeatmapapi_FromChunk(t *testing.T) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
|
||||
defer cancel()
|
||||
testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors4")
|
||||
if err != nil {
|
||||
t.Fatalf("could not get test secrets from GCP: %s", err)
|
||||
}
|
||||
secret := testSecrets.MustGetField("HEATMAPAPI")
|
||||
inactiveSecret := testSecrets.MustGetField("HEATMAPAPI_INACTIVE")
|
||||
|
||||
type args struct {
|
||||
ctx context.Context
|
||||
data []byte
|
||||
verify bool
|
||||
}
|
||||
tests := []struct {
|
||||
name string
|
||||
s Scanner
|
||||
args args
|
||||
want []detectors.Result
|
||||
wantErr bool
|
||||
}{
|
||||
{
|
||||
name: "found, verified",
|
||||
s: Scanner{},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a heatmapapi secret %s within", secret)),
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_Heatmapapi,
|
||||
Verified: true,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "found, unverified",
|
||||
s: Scanner{},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a heatmapapi secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_Heatmapapi,
|
||||
Verified: false,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "not found",
|
||||
s: Scanner{},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte("You cannot find the secret within"),
|
||||
verify: true,
|
||||
},
|
||||
want: nil,
|
||||
wantErr: false,
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
s := Scanner{}
|
||||
got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
|
||||
if (err != nil) != tt.wantErr {
|
||||
t.Errorf("Heatmapapi.FromData() error = %v, wantErr %v", err, tt.wantErr)
|
||||
return
|
||||
}
|
||||
for i := range got {
|
||||
if len(got[i].Raw) == 0 {
|
||||
t.Fatalf("no raw secret present: \n %+v", got[i])
|
||||
}
|
||||
got[i].Raw = nil
|
||||
}
|
||||
if diff := pretty.Compare(got, tt.want); diff != "" {
|
||||
t.Errorf("Heatmapapi.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkFromData(benchmark *testing.B) {
|
||||
ctx := context.Background()
|
||||
s := Scanner{}
|
||||
for name, data := range detectors.MustGetBenchmarkData() {
|
||||
benchmark.Run(name, func(b *testing.B) {
|
||||
for n := 0; n < b.N; n++ {
|
||||
_, err := s.FromData(ctx, false, data)
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -1,82 +0,0 @@
|
||||
package holistic
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io/ioutil"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
|
||||
)
|
||||
|
||||
type Scanner struct{}
|
||||
|
||||
// Ensure the Scanner satisfies the interface at compile time
|
||||
var _ detectors.Detector = (*Scanner)(nil)
|
||||
|
||||
var (
|
||||
client = common.SaneHttpClient()
|
||||
|
||||
//Make sure that your group is surrounded in boundry characters such as below to reduce false positives
|
||||
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"holistic"}) + `\b([0-9a-f]{64})\b`)
|
||||
)
|
||||
|
||||
// Keywords are used for efficiently pre-filtering chunks.
|
||||
// Use identifiers in the secret preferably, or the provider name.
|
||||
func (s Scanner) Keywords() []string {
|
||||
return []string{"holistic"}
|
||||
}
|
||||
|
||||
// FromData will find and optionally verify Holistic secrets in a given set of bytes.
|
||||
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
|
||||
dataStr := string(data)
|
||||
|
||||
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
|
||||
|
||||
for _, match := range matches {
|
||||
if len(match) != 2 {
|
||||
continue
|
||||
}
|
||||
resMatch := strings.TrimSpace(match[1])
|
||||
|
||||
s1 := detectors.Result{
|
||||
DetectorType: detectorspb.DetectorType_Holistic,
|
||||
Raw: []byte(resMatch),
|
||||
}
|
||||
|
||||
if verify {
|
||||
req, err := http.NewRequestWithContext(ctx, "GET", "https://api.holistic.dev/api/v1/project", nil)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
req.Header.Add("x-api-key", resMatch)
|
||||
res, err := client.Do(req)
|
||||
if err == nil {
|
||||
bodyBytes, err := ioutil.ReadAll(res.Body)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
|
||||
bodyString := string(bodyBytes)
|
||||
errorResponse := strings.Contains(bodyString, `"data":[]`)
|
||||
|
||||
defer res.Body.Close()
|
||||
if res.StatusCode >= 200 && res.StatusCode < 300 && !errorResponse {
|
||||
s1.Verified = true
|
||||
} else {
|
||||
//This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key
|
||||
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
|
||||
continue
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
results = append(results, s1)
|
||||
}
|
||||
|
||||
return detectors.CleanResults(results), nil
|
||||
}
|
||||
@@ -1,116 +0,0 @@
|
||||
package holistic
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/kylelemons/godebug/pretty"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
|
||||
)
|
||||
|
||||
func TestHolistic_FromChunk(t *testing.T) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
|
||||
defer cancel()
|
||||
testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")
|
||||
if err != nil {
|
||||
t.Fatalf("could not get test secrets from GCP: %s", err)
|
||||
}
|
||||
secret := testSecrets.MustGetField("HOLISTIC")
|
||||
inactiveSecret := testSecrets.MustGetField("HOLISTIC_INACTIVE")
|
||||
|
||||
type args struct {
|
||||
ctx context.Context
|
||||
data []byte
|
||||
verify bool
|
||||
}
|
||||
tests := []struct {
|
||||
name string
|
||||
s Scanner
|
||||
args args
|
||||
want []detectors.Result
|
||||
wantErr bool
|
||||
}{
|
||||
{
|
||||
name: "found, verified",
|
||||
s: Scanner{},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a holistic secret %s within", secret)),
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_Holistic,
|
||||
Verified: true,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "found, unverified",
|
||||
s: Scanner{},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a holistic secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_Holistic,
|
||||
Verified: false,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "not found",
|
||||
s: Scanner{},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte("You cannot find the secret within"),
|
||||
verify: true,
|
||||
},
|
||||
want: nil,
|
||||
wantErr: false,
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
s := Scanner{}
|
||||
got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
|
||||
if (err != nil) != tt.wantErr {
|
||||
t.Errorf("Holistic.FromData() error = %v, wantErr %v", err, tt.wantErr)
|
||||
return
|
||||
}
|
||||
for i := range got {
|
||||
if len(got[i].Raw) == 0 {
|
||||
t.Fatalf("no raw secret present: \n %+v", got[i])
|
||||
}
|
||||
got[i].Raw = nil
|
||||
}
|
||||
if diff := pretty.Compare(got, tt.want); diff != "" {
|
||||
t.Errorf("Holistic.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkFromData(benchmark *testing.B) {
|
||||
ctx := context.Background()
|
||||
s := Scanner{}
|
||||
for name, data := range detectors.MustGetBenchmarkData() {
|
||||
benchmark.Run(name, func(b *testing.B) {
|
||||
for n := 0; n < b.N; n++ {
|
||||
_, err := s.FromData(ctx, false, data)
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -1,73 +0,0 @@
|
||||
package honeycomb
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
|
||||
)
|
||||
|
||||
type Scanner struct{}
|
||||
|
||||
// Ensure the Scanner satisfies the interface at compile time.
|
||||
var _ detectors.Detector = (*Scanner)(nil)
|
||||
|
||||
var (
|
||||
client = common.SaneHttpClient()
|
||||
|
||||
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
|
||||
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"Honeycomb"}) + `\b([0-9Aa-z-A-Z]{22})\b`)
|
||||
)
|
||||
|
||||
// Keywords are used for efficiently pre-filtering chunks.
|
||||
// Use identifiers in the secret preferably, or the provider name.
|
||||
func (s Scanner) Keywords() []string {
|
||||
return []string{"Honeycomb"}
|
||||
}
|
||||
|
||||
// FromData will find and optionally verify Honeycomb secrets in a given set of bytes.
|
||||
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
|
||||
dataStr := string(data)
|
||||
|
||||
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
|
||||
|
||||
for _, match := range matches {
|
||||
if len(match) != 2 {
|
||||
continue
|
||||
}
|
||||
resMatch := strings.TrimSpace(match[1])
|
||||
|
||||
s1 := detectors.Result{
|
||||
DetectorType: detectorspb.DetectorType_Honeycomb,
|
||||
Raw: []byte(resMatch),
|
||||
}
|
||||
|
||||
if verify {
|
||||
req, err := http.NewRequestWithContext(ctx, "GET", "https://api.honeycomb.io/1/auth", nil)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
req.Header.Add("X-Honeycomb-Team", resMatch)
|
||||
res, err := client.Do(req)
|
||||
if err == nil {
|
||||
defer res.Body.Close()
|
||||
if res.StatusCode >= 200 && res.StatusCode < 300 {
|
||||
s1.Verified = true
|
||||
} else {
|
||||
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
|
||||
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
|
||||
continue
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
results = append(results, s1)
|
||||
}
|
||||
|
||||
return detectors.CleanResults(results), nil
|
||||
}
|
||||
@@ -1,119 +0,0 @@
|
||||
//go:build detectors
|
||||
// +build detectors
|
||||
|
||||
package honeycomb
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/kylelemons/godebug/pretty"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
|
||||
)
|
||||
|
||||
func TestHoneycomb_FromChunk(t *testing.T) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
|
||||
defer cancel()
|
||||
testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")
|
||||
if err != nil {
|
||||
t.Fatalf("could not get test secrets from GCP: %s", err)
|
||||
}
|
||||
secret := testSecrets.MustGetField("HONEYCOMB")
|
||||
inactiveSecret := testSecrets.MustGetField("HONEYCOMB_INACTIVE")
|
||||
|
||||
type args struct {
|
||||
ctx context.Context
|
||||
data []byte
|
||||
verify bool
|
||||
}
|
||||
tests := []struct {
|
||||
name string
|
||||
s Scanner
|
||||
args args
|
||||
want []detectors.Result
|
||||
wantErr bool
|
||||
}{
|
||||
{
|
||||
name: "found, verified",
|
||||
s: Scanner{},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a honeycomb secret %s within", secret)),
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_Honeycomb,
|
||||
Verified: true,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "found, unverified",
|
||||
s: Scanner{},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a honeycomb secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_Honeycomb,
|
||||
Verified: false,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "not found",
|
||||
s: Scanner{},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte("You cannot find the secret within"),
|
||||
verify: true,
|
||||
},
|
||||
want: nil,
|
||||
wantErr: false,
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
s := Scanner{}
|
||||
got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
|
||||
if (err != nil) != tt.wantErr {
|
||||
t.Errorf("Honeycomb.FromData() error = %v, wantErr %v", err, tt.wantErr)
|
||||
return
|
||||
}
|
||||
for i := range got {
|
||||
if len(got[i].Raw) == 0 {
|
||||
t.Fatalf("no raw secret present: \n %+v", got[i])
|
||||
}
|
||||
got[i].Raw = nil
|
||||
}
|
||||
if diff := pretty.Compare(got, tt.want); diff != "" {
|
||||
t.Errorf("Honeycomb.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkFromData(benchmark *testing.B) {
|
||||
ctx := context.Background()
|
||||
s := Scanner{}
|
||||
for name, data := range detectors.MustGetBenchmarkData() {
|
||||
benchmark.Run(name, func(b *testing.B) {
|
||||
for n := 0; n < b.N; n++ {
|
||||
_, err := s.FromData(ctx, false, data)
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -1,85 +0,0 @@
|
||||
package kanbantool
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
|
||||
)
|
||||
|
||||
type Scanner struct{}
|
||||
|
||||
// Ensure the Scanner satisfies the interface at compile time
|
||||
var _ detectors.Detector = (*Scanner)(nil)
|
||||
|
||||
var (
|
||||
client = common.SaneHttpClient()
|
||||
|
||||
//Make sure that your group is surrounded in boundry characters such as below to reduce false positives
|
||||
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"kanbantool"}) + `\b([0-9A-Z]{12})\b`)
|
||||
domainPat = regexp.MustCompile(detectors.PrefixRegex([]string{"kanbantool"}) + `\b([a-z0-9A-Z]{2,22})\b`)
|
||||
|
||||
)
|
||||
|
||||
// Keywords are used for efficiently pre-filtering chunks.
|
||||
// Use identifiers in the secret preferably, or the provider name.
|
||||
func (s Scanner) Keywords() []string {
|
||||
return []string{"kanbantool"}
|
||||
}
|
||||
|
||||
// FromData will find and optionally verify Kanbantool secrets in a given set of bytes.
|
||||
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
|
||||
dataStr := string(data)
|
||||
|
||||
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
|
||||
idMatches := domainPat.FindAllStringSubmatch(dataStr, -1)
|
||||
fmt.Println(idMatches)
|
||||
for _, match := range matches {
|
||||
if len(match) != 2 {
|
||||
continue
|
||||
}
|
||||
resMatch := strings.TrimSpace(match[1])
|
||||
|
||||
for _, idMatch := range idMatches {
|
||||
if len(idMatch) != 2 {
|
||||
continue
|
||||
}
|
||||
resIdMatch := strings.TrimSpace(idMatch[1])
|
||||
|
||||
s1 := detectors.Result{
|
||||
DetectorType: detectorspb.DetectorType_Kanbantool,
|
||||
Raw: []byte(resMatch),
|
||||
}
|
||||
if verify {
|
||||
url := fmt.Sprintf("https://%s.kanbantool.com/api/v3/users/current.json",resIdMatch)
|
||||
req, err := http.NewRequestWithContext(ctx, "GET", url, nil)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", resMatch))
|
||||
res, err := client.Do(req)
|
||||
if err == nil {
|
||||
defer res.Body.Close()
|
||||
if res.StatusCode >= 200 && res.StatusCode < 300 {
|
||||
s1.Verified = true
|
||||
} else {
|
||||
//This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key
|
||||
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
|
||||
continue
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
results = append(results, s1)
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
return detectors.CleanResults(results), nil
|
||||
}
|
||||
@@ -1,117 +0,0 @@
|
||||
package kanbantool
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/kylelemons/godebug/pretty"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
|
||||
)
|
||||
|
||||
func TestKanbantool_FromChunk(t *testing.T) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
|
||||
defer cancel()
|
||||
testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors4")
|
||||
if err != nil {
|
||||
t.Fatalf("could not get test secrets from GCP: %s", err)
|
||||
}
|
||||
secret := testSecrets.MustGetField("KANBANTOOL")
|
||||
inactiveSecret := testSecrets.MustGetField("KANBANTOOL_INACTIVE")
|
||||
domain := testSecrets.MustGetField("KANBANTOOL_DOMAIN")
|
||||
|
||||
type args struct {
|
||||
ctx context.Context
|
||||
data []byte
|
||||
verify bool
|
||||
}
|
||||
tests := []struct {
|
||||
name string
|
||||
s Scanner
|
||||
args args
|
||||
want []detectors.Result
|
||||
wantErr bool
|
||||
}{
|
||||
{
|
||||
name: "found, verified",
|
||||
s: Scanner{},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a kanbantool secret %s within kanbantool %s", secret,domain)),
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_Kanbantool,
|
||||
Verified: true,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "found, unverified",
|
||||
s: Scanner{},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a kanbantool secret %s within but not valid kanbantool %s", inactiveSecret,domain)), // the secret would satisfy the regex but not pass validation
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_Kanbantool,
|
||||
Verified: false,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "not found",
|
||||
s: Scanner{},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte("You cannot find the secret within"),
|
||||
verify: true,
|
||||
},
|
||||
want: nil,
|
||||
wantErr: false,
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
s := Scanner{}
|
||||
got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
|
||||
if (err != nil) != tt.wantErr {
|
||||
t.Errorf("Kanbantool.FromData() error = %v, wantErr %v", err, tt.wantErr)
|
||||
return
|
||||
}
|
||||
for i := range got {
|
||||
if len(got[i].Raw) == 0 {
|
||||
t.Fatalf("no raw secret present: \n %+v", got[i])
|
||||
}
|
||||
got[i].Raw = nil
|
||||
}
|
||||
if diff := pretty.Compare(got, tt.want); diff != "" {
|
||||
t.Errorf("Kanbantool.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkFromData(benchmark *testing.B) {
|
||||
ctx := context.Background()
|
||||
s := Scanner{}
|
||||
for name, data := range detectors.MustGetBenchmarkData() {
|
||||
benchmark.Run(name, func(b *testing.B) {
|
||||
for n := 0; n < b.N; n++ {
|
||||
_, err := s.FromData(ctx, false, data)
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -1,76 +0,0 @@
|
||||
package lemlist
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"strings"
|
||||
b64 "encoding/base64"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
|
||||
)
|
||||
|
||||
type Scanner struct{}
|
||||
|
||||
// Ensure the Scanner satisfies the interface at compile time
|
||||
var _ detectors.Detector = (*Scanner)(nil)
|
||||
|
||||
var (
|
||||
client = common.SaneHttpClient()
|
||||
|
||||
//Make sure that your group is surrounded in boundry characters such as below to reduce false positives
|
||||
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"lemlist"}) + `\b([a-f0-9]{32})\b`)
|
||||
)
|
||||
|
||||
// Keywords are used for efficiently pre-filtering chunks.
|
||||
// Use identifiers in the secret preferably, or the provider name.
|
||||
func (s Scanner) Keywords() []string {
|
||||
return []string{"lemlist"}
|
||||
}
|
||||
|
||||
// FromData will find and optionally verify Lemlist secrets in a given set of bytes.
|
||||
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
|
||||
dataStr := string(data)
|
||||
|
||||
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
|
||||
|
||||
for _, match := range matches {
|
||||
if len(match) != 2 {
|
||||
continue
|
||||
}
|
||||
resMatch := strings.TrimSpace(match[1])
|
||||
|
||||
s1 := detectors.Result{
|
||||
DetectorType: detectorspb.DetectorType_Lemlist,
|
||||
Raw: []byte(resMatch),
|
||||
}
|
||||
|
||||
if verify {
|
||||
data := fmt.Sprintf(":%s", resMatch)
|
||||
sEnc := b64.StdEncoding.EncodeToString([]byte(data))
|
||||
req, err := http.NewRequestWithContext(ctx, "GET", "https://api.lemlist.com/api/team", nil)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
req.Header.Add("Authorization", fmt.Sprintf("Basic %s", sEnc))
|
||||
res, err := client.Do(req)
|
||||
if err == nil {
|
||||
defer res.Body.Close()
|
||||
if res.StatusCode >= 200 && res.StatusCode < 300 {
|
||||
s1.Verified = true
|
||||
} else {
|
||||
//This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key
|
||||
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
|
||||
continue
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
results = append(results, s1)
|
||||
}
|
||||
|
||||
return detectors.CleanResults(results), nil
|
||||
}
|
||||
@@ -1,116 +0,0 @@
|
||||
package lemlist
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/kylelemons/godebug/pretty"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
|
||||
)
|
||||
|
||||
func TestLemlist_FromChunk(t *testing.T) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
|
||||
defer cancel()
|
||||
testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
|
||||
if err != nil {
|
||||
t.Fatalf("could not get test secrets from GCP: %s", err)
|
||||
}
|
||||
secret := testSecrets.MustGetField("LEMLIST")
|
||||
inactiveSecret := testSecrets.MustGetField("LEMLIST_INACTIVE")
|
||||
|
||||
type args struct {
|
||||
ctx context.Context
|
||||
data []byte
|
||||
verify bool
|
||||
}
|
||||
tests := []struct {
|
||||
name string
|
||||
s Scanner
|
||||
args args
|
||||
want []detectors.Result
|
||||
wantErr bool
|
||||
}{
|
||||
{
|
||||
name: "found, verified",
|
||||
s: Scanner{},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a lemlist secret %s within", secret)),
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_Lemlist,
|
||||
Verified: true,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "found, unverified",
|
||||
s: Scanner{},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a lemlist secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_Lemlist,
|
||||
Verified: false,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "not found",
|
||||
s: Scanner{},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte("You cannot find the secret within"),
|
||||
verify: true,
|
||||
},
|
||||
want: nil,
|
||||
wantErr: false,
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
s := Scanner{}
|
||||
got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
|
||||
if (err != nil) != tt.wantErr {
|
||||
t.Errorf("Lemlist.FromData() error = %v, wantErr %v", err, tt.wantErr)
|
||||
return
|
||||
}
|
||||
for i := range got {
|
||||
if len(got[i].Raw) == 0 {
|
||||
t.Fatalf("no raw secret present: \n %+v", got[i])
|
||||
}
|
||||
got[i].Raw = nil
|
||||
}
|
||||
if diff := pretty.Compare(got, tt.want); diff != "" {
|
||||
t.Errorf("Lemlist.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkFromData(benchmark *testing.B) {
|
||||
ctx := context.Background()
|
||||
s := Scanner{}
|
||||
for name, data := range detectors.MustGetBenchmarkData() {
|
||||
benchmark.Run(name, func(b *testing.B) {
|
||||
for n := 0; n < b.N; n++ {
|
||||
_, err := s.FromData(ctx, false, data)
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -1,75 +0,0 @@
|
||||
package monkeylearn
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
|
||||
)
|
||||
|
||||
type Scanner struct{}
|
||||
|
||||
// Ensure the Scanner satisfies the interface at compile time
|
||||
var _ detectors.Detector = (*Scanner)(nil)
|
||||
|
||||
var (
|
||||
client = common.SaneHttpClient()
|
||||
|
||||
//Make sure that your group is surrounded in boundry characters such as below to reduce false positives
|
||||
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"monkeylearn"}) + `\b([0-9a-f]{40})\b`)
|
||||
)
|
||||
|
||||
// Keywords are used for efficiently pre-filtering chunks.
|
||||
// Use identifiers in the secret preferably, or the provider name.
|
||||
func (s Scanner) Keywords() []string {
|
||||
return []string{"monkeylearn"}
|
||||
}
|
||||
|
||||
// FromData will find and optionally verify MonkeyLearn secrets in a given set of bytes.
|
||||
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
|
||||
dataStr := string(data)
|
||||
|
||||
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
|
||||
|
||||
for _, match := range matches {
|
||||
if len(match) != 2 {
|
||||
continue
|
||||
}
|
||||
resMatch := strings.TrimSpace(match[1])
|
||||
|
||||
s1 := detectors.Result{
|
||||
DetectorType: detectorspb.DetectorType_MonkeyLearn,
|
||||
Raw: []byte(resMatch),
|
||||
}
|
||||
|
||||
if verify {
|
||||
req, err := http.NewRequestWithContext(ctx, "GET", "https://api.monkeylearn.com/v3/classifiers", nil)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
req.Header.Add("Accept", "application/json")
|
||||
req.Header.Add("Authorization", fmt.Sprintf("Token %s", resMatch))
|
||||
res, err := client.Do(req)
|
||||
if err == nil {
|
||||
defer res.Body.Close()
|
||||
if res.StatusCode >= 200 && res.StatusCode < 300 {
|
||||
s1.Verified = true
|
||||
} else {
|
||||
//This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key
|
||||
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
|
||||
continue
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
results = append(results, s1)
|
||||
}
|
||||
|
||||
return detectors.CleanResults(results), nil
|
||||
}
|
||||
@@ -1,116 +0,0 @@
|
||||
package monkeylearn
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/kylelemons/godebug/pretty"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
|
||||
)
|
||||
|
||||
func TestMonkeyLearn_FromChunk(t *testing.T) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
|
||||
defer cancel()
|
||||
testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
|
||||
if err != nil {
|
||||
t.Fatalf("could not get test secrets from GCP: %s", err)
|
||||
}
|
||||
secret := testSecrets.MustGetField("MONKEYLEARN")
|
||||
inactiveSecret := testSecrets.MustGetField("MONKEYLEARN_INACTIVE")
|
||||
|
||||
type args struct {
|
||||
ctx context.Context
|
||||
data []byte
|
||||
verify bool
|
||||
}
|
||||
tests := []struct {
|
||||
name string
|
||||
s Scanner
|
||||
args args
|
||||
want []detectors.Result
|
||||
wantErr bool
|
||||
}{
|
||||
{
|
||||
name: "found, verified",
|
||||
s: Scanner{},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a monkeylearn secret %s within", secret)),
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_MonkeyLearn,
|
||||
Verified: true,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "found, unverified",
|
||||
s: Scanner{},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a monkeylearn secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_MonkeyLearn,
|
||||
Verified: false,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "not found",
|
||||
s: Scanner{},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte("You cannot find the secret within"),
|
||||
verify: true,
|
||||
},
|
||||
want: nil,
|
||||
wantErr: false,
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
s := Scanner{}
|
||||
got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
|
||||
if (err != nil) != tt.wantErr {
|
||||
t.Errorf("MonkeyLearn.FromData() error = %v, wantErr %v", err, tt.wantErr)
|
||||
return
|
||||
}
|
||||
for i := range got {
|
||||
if len(got[i].Raw) == 0 {
|
||||
t.Fatalf("no raw secret present: \n %+v", got[i])
|
||||
}
|
||||
got[i].Raw = nil
|
||||
}
|
||||
if diff := pretty.Compare(got, tt.want); diff != "" {
|
||||
t.Errorf("MonkeyLearn.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkFromData(benchmark *testing.B) {
|
||||
ctx := context.Background()
|
||||
s := Scanner{}
|
||||
for name, data := range detectors.MustGetBenchmarkData() {
|
||||
benchmark.Run(name, func(b *testing.B) {
|
||||
for n := 0; n < b.N; n++ {
|
||||
_, err := s.FromData(ctx, false, data)
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -1,74 +0,0 @@
|
||||
package parseur
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
|
||||
)
|
||||
|
||||
type Scanner struct{}
|
||||
|
||||
// Ensure the Scanner satisfies the interface at compile time
|
||||
var _ detectors.Detector = (*Scanner)(nil)
|
||||
|
||||
var (
|
||||
client = common.SaneHttpClient()
|
||||
|
||||
//Make sure that your group is surrounded in boundry characters such as below to reduce false positives
|
||||
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"parseur"}) + `\b([a-f0-9]{40})\b`)
|
||||
)
|
||||
|
||||
// Keywords are used for efficiently pre-filtering chunks.
|
||||
// Use identifiers in the secret preferably, or the provider name.
|
||||
func (s Scanner) Keywords() []string {
|
||||
return []string{"parseur"}
|
||||
}
|
||||
|
||||
// FromData will find and optionally verify Parseur secrets in a given set of bytes.
|
||||
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
|
||||
dataStr := string(data)
|
||||
|
||||
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
|
||||
|
||||
for _, match := range matches {
|
||||
if len(match) != 2 {
|
||||
continue
|
||||
}
|
||||
resMatch := strings.TrimSpace(match[1])
|
||||
|
||||
s1 := detectors.Result{
|
||||
DetectorType: detectorspb.DetectorType_Parseur,
|
||||
Raw: []byte(resMatch),
|
||||
}
|
||||
|
||||
if verify {
|
||||
req, err := http.NewRequestWithContext(ctx, "GET", "https://api.parseur.com/", nil)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
req.Header.Add("Authorization", fmt.Sprintf("Token %s", resMatch))
|
||||
res, err := client.Do(req)
|
||||
if err == nil {
|
||||
defer res.Body.Close()
|
||||
if res.StatusCode >= 200 && res.StatusCode < 300 {
|
||||
s1.Verified = true
|
||||
} else {
|
||||
//This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key
|
||||
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
|
||||
continue
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
results = append(results, s1)
|
||||
}
|
||||
|
||||
return detectors.CleanResults(results), nil
|
||||
}
|
||||
@@ -1,116 +0,0 @@
|
||||
package parseur
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/kylelemons/godebug/pretty"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
|
||||
)
|
||||
|
||||
func TestParseur_FromChunk(t *testing.T) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
|
||||
defer cancel()
|
||||
testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
|
||||
if err != nil {
|
||||
t.Fatalf("could not get test secrets from GCP: %s", err)
|
||||
}
|
||||
secret := testSecrets.MustGetField("PARSEUR")
|
||||
inactiveSecret := testSecrets.MustGetField("PARSEUR_INACTIVE")
|
||||
|
||||
type args struct {
|
||||
ctx context.Context
|
||||
data []byte
|
||||
verify bool
|
||||
}
|
||||
tests := []struct {
|
||||
name string
|
||||
s Scanner
|
||||
args args
|
||||
want []detectors.Result
|
||||
wantErr bool
|
||||
}{
|
||||
{
|
||||
name: "found, verified",
|
||||
s: Scanner{},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a parseur secret %s within", secret)),
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_Parseur,
|
||||
Verified: true,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "found, unverified",
|
||||
s: Scanner{},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a parseur secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_Parseur,
|
||||
Verified: false,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "not found",
|
||||
s: Scanner{},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte("You cannot find the secret within"),
|
||||
verify: true,
|
||||
},
|
||||
want: nil,
|
||||
wantErr: false,
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
s := Scanner{}
|
||||
got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
|
||||
if (err != nil) != tt.wantErr {
|
||||
t.Errorf("Parseur.FromData() error = %v, wantErr %v", err, tt.wantErr)
|
||||
return
|
||||
}
|
||||
for i := range got {
|
||||
if len(got[i].Raw) == 0 {
|
||||
t.Fatalf("no raw secret present: \n %+v", got[i])
|
||||
}
|
||||
got[i].Raw = nil
|
||||
}
|
||||
if diff := pretty.Compare(got, tt.want); diff != "" {
|
||||
t.Errorf("Parseur.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkFromData(benchmark *testing.B) {
|
||||
ctx := context.Background()
|
||||
s := Scanner{}
|
||||
for name, data := range detectors.MustGetBenchmarkData() {
|
||||
benchmark.Run(name, func(b *testing.B) {
|
||||
for n := 0; n < b.N; n++ {
|
||||
_, err := s.FromData(ctx, false, data)
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -1,75 +0,0 @@
|
||||
package prodpad
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
|
||||
)
|
||||
|
||||
type Scanner struct{}
|
||||
|
||||
// Ensure the Scanner satisfies the interface at compile time
|
||||
var _ detectors.Detector = (*Scanner)(nil)
|
||||
|
||||
var (
|
||||
client = common.SaneHttpClient()
|
||||
|
||||
//Make sure that your group is surrounded in boundry characters such as below to reduce false positives
|
||||
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"prodpad"}) + `\b([a-f0-9]{64})\b`)
|
||||
)
|
||||
|
||||
// Keywords are used for efficiently pre-filtering chunks.
|
||||
// Use identifiers in the secret preferably, or the provider name.
|
||||
func (s Scanner) Keywords() []string {
|
||||
return []string{"prodpad"}
|
||||
}
|
||||
|
||||
// FromData will find and optionally verify Prodpad secrets in a given set of bytes.
|
||||
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
|
||||
dataStr := string(data)
|
||||
|
||||
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
|
||||
|
||||
for _, match := range matches {
|
||||
if len(match) != 2 {
|
||||
continue
|
||||
}
|
||||
resMatch := strings.TrimSpace(match[1])
|
||||
|
||||
s1 := detectors.Result{
|
||||
DetectorType: detectorspb.DetectorType_Prodpad,
|
||||
Raw: []byte(resMatch),
|
||||
}
|
||||
|
||||
if verify {
|
||||
req, err := http.NewRequestWithContext(ctx, "GET", "https://api.prodpad.com/v1/tags", nil)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
req.Header.Add("accept", "application/json")
|
||||
req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", resMatch))
|
||||
res, err := client.Do(req)
|
||||
if err == nil {
|
||||
defer res.Body.Close()
|
||||
if res.StatusCode >= 200 && res.StatusCode < 300 {
|
||||
s1.Verified = true
|
||||
} else {
|
||||
//This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key
|
||||
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
|
||||
continue
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
results = append(results, s1)
|
||||
}
|
||||
|
||||
return detectors.CleanResults(results), nil
|
||||
}
|
||||
@@ -1,116 +0,0 @@
|
||||
package prodpad
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/kylelemons/godebug/pretty"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
|
||||
)
|
||||
|
||||
func TestProdpad_FromChunk(t *testing.T) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
|
||||
defer cancel()
|
||||
testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
|
||||
if err != nil {
|
||||
t.Fatalf("could not get test secrets from GCP: %s", err)
|
||||
}
|
||||
secret := testSecrets.MustGetField("PRODPAD")
|
||||
inactiveSecret := testSecrets.MustGetField("PRODPAD_INACTIVE")
|
||||
|
||||
type args struct {
|
||||
ctx context.Context
|
||||
data []byte
|
||||
verify bool
|
||||
}
|
||||
tests := []struct {
|
||||
name string
|
||||
s Scanner
|
||||
args args
|
||||
want []detectors.Result
|
||||
wantErr bool
|
||||
}{
|
||||
{
|
||||
name: "found, verified",
|
||||
s: Scanner{},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a prodpad secret %s within", secret)),
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_Prodpad,
|
||||
Verified: true,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "found, unverified",
|
||||
s: Scanner{},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a prodpad secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_Prodpad,
|
||||
Verified: false,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "not found",
|
||||
s: Scanner{},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte("You cannot find the secret within"),
|
||||
verify: true,
|
||||
},
|
||||
want: nil,
|
||||
wantErr: false,
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
s := Scanner{}
|
||||
got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
|
||||
if (err != nil) != tt.wantErr {
|
||||
t.Errorf("Prodpad.FromData() error = %v, wantErr %v", err, tt.wantErr)
|
||||
return
|
||||
}
|
||||
for i := range got {
|
||||
if len(got[i].Raw) == 0 {
|
||||
t.Fatalf("no raw secret present: \n %+v", got[i])
|
||||
}
|
||||
got[i].Raw = nil
|
||||
}
|
||||
if diff := pretty.Compare(got, tt.want); diff != "" {
|
||||
t.Errorf("Prodpad.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkFromData(benchmark *testing.B) {
|
||||
ctx := context.Background()
|
||||
s := Scanner{}
|
||||
for name, data := range detectors.MustGetBenchmarkData() {
|
||||
benchmark.Run(name, func(b *testing.B) {
|
||||
for n := 0; n < b.N; n++ {
|
||||
_, err := s.FromData(ctx, false, data)
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -1,85 +0,0 @@
|
||||
package salesmate
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
|
||||
)
|
||||
|
||||
type Scanner struct{}
|
||||
|
||||
// Ensure the Scanner satisfies the interface at compile time
|
||||
var _ detectors.Detector = (*Scanner)(nil)
|
||||
|
||||
var (
|
||||
client = common.SaneHttpClient()
|
||||
|
||||
//Make sure that your group is surrounded in boundry characters such as below to reduce false positives
|
||||
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"salesmate"}) + `\b([0-9Aa-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})\b`)
|
||||
domainPat = regexp.MustCompile(detectors.PrefixRegex([]string{"salesmate"}) + `\b([a-z0-9A-Z]{3,22})\b`)
|
||||
)
|
||||
|
||||
// Keywords are used for efficiently pre-filtering chunks.
|
||||
// Use identifiers in the secret preferably, or the provider name.
|
||||
func (s Scanner) Keywords() []string {
|
||||
return []string{"salesmate"}
|
||||
}
|
||||
|
||||
// FromData will find and optionally verify Salesmate secrets in a given set of bytes.
|
||||
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
|
||||
dataStr := string(data)
|
||||
|
||||
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
|
||||
idmatches := domainPat.FindAllStringSubmatch(dataStr, -1)
|
||||
|
||||
for _, match := range matches {
|
||||
if len(match) != 2 {
|
||||
continue
|
||||
}
|
||||
resMatch := strings.TrimSpace(match[1])
|
||||
for _, idmatch := range idmatches {
|
||||
if len(idmatch) != 2 {
|
||||
continue
|
||||
}
|
||||
resIdMatch := strings.TrimSpace(idmatch[1])
|
||||
s1 := detectors.Result{
|
||||
DetectorType: detectorspb.DetectorType_Salesmate,
|
||||
Raw: []byte(resMatch),
|
||||
}
|
||||
|
||||
if verify {
|
||||
url := fmt.Sprintf("https://%s.salesmate.io/apis/v3/companies/1?trackingRecentSearch=true",resIdMatch)
|
||||
req, err := http.NewRequestWithContext(ctx, "GET", url, nil)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
|
||||
req.Header.Add("Content-Type", "application/json")
|
||||
req.Header.Add("sessionToken", resMatch)
|
||||
res, err := client.Do(req)
|
||||
if err == nil {
|
||||
defer res.Body.Close()
|
||||
if res.StatusCode >= 200 && res.StatusCode < 300 {
|
||||
s1.Verified = true
|
||||
} else {
|
||||
//This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key
|
||||
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
|
||||
continue
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
results = append(results, s1)
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
return detectors.CleanResults(results), nil
|
||||
}
|
||||
@@ -1,117 +0,0 @@
|
||||
package salesmate
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/kylelemons/godebug/pretty"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
|
||||
)
|
||||
|
||||
func TestSalesmate_FromChunk(t *testing.T) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
|
||||
defer cancel()
|
||||
testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors4")
|
||||
if err != nil {
|
||||
t.Fatalf("could not get test secrets from GCP: %s", err)
|
||||
}
|
||||
secret := testSecrets.MustGetField("SALESMATE")
|
||||
inactiveSecret := testSecrets.MustGetField("SALESMATE_INACTIVE")
|
||||
domain := testSecrets.MustGetField("SALESMATE_DOMAIN")
|
||||
|
||||
type args struct {
|
||||
ctx context.Context
|
||||
data []byte
|
||||
verify bool
|
||||
}
|
||||
tests := []struct {
|
||||
name string
|
||||
s Scanner
|
||||
args args
|
||||
want []detectors.Result
|
||||
wantErr bool
|
||||
}{
|
||||
{
|
||||
name: "found, verified",
|
||||
s: Scanner{},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a salesmate secret %s within salesmateDomain %s", secret,domain)),
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_Salesmate,
|
||||
Verified: true,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "found, unverified",
|
||||
s: Scanner{},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a salesmate secret %s within but not valid salesmateDomain %s", inactiveSecret,domain)), // the secret would satisfy the regex but not pass validation
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_Salesmate,
|
||||
Verified: false,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "not found",
|
||||
s: Scanner{},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte("You cannot find the secret within"),
|
||||
verify: true,
|
||||
},
|
||||
want: nil,
|
||||
wantErr: false,
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
s := Scanner{}
|
||||
got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
|
||||
if (err != nil) != tt.wantErr {
|
||||
t.Errorf("Salesmate.FromData() error = %v, wantErr %v", err, tt.wantErr)
|
||||
return
|
||||
}
|
||||
for i := range got {
|
||||
if len(got[i].Raw) == 0 {
|
||||
t.Fatalf("no raw secret present: \n %+v", got[i])
|
||||
}
|
||||
got[i].Raw = nil
|
||||
}
|
||||
if diff := pretty.Compare(got, tt.want); diff != "" {
|
||||
t.Errorf("Salesmate.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkFromData(benchmark *testing.B) {
|
||||
ctx := context.Background()
|
||||
s := Scanner{}
|
||||
for name, data := range detectors.MustGetBenchmarkData() {
|
||||
benchmark.Run(name, func(b *testing.B) {
|
||||
for n := 0; n < b.N; n++ {
|
||||
_, err := s.FromData(ctx, false, data)
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -1,85 +0,0 @@
|
||||
package scalr
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
|
||||
)
|
||||
|
||||
type Scanner struct{}
|
||||
|
||||
// Ensure the Scanner satisfies the interface at compile time
|
||||
var _ detectors.Detector = (*Scanner)(nil)
|
||||
|
||||
var (
|
||||
client = common.SaneHttpClient()
|
||||
|
||||
//Make sure that your group is surrounded in boundry characters such as below to reduce false positives
|
||||
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"scalr"}) + `\b([0-9a-zA-Z._]{136})`)
|
||||
idPat = regexp.MustCompile(detectors.PrefixRegex([]string{"scalr"}) + `\b([0-9a-z]{4,50})\b`)
|
||||
)
|
||||
|
||||
// Keywords are used for efficiently pre-filtering chunks.
|
||||
// Use identifiers in the secret preferably, or the provider name.
|
||||
func (s Scanner) Keywords() []string {
|
||||
return []string{"scalr"}
|
||||
}
|
||||
|
||||
// FromData will find and optionally verify Scalr secrets in a given set of bytes.
|
||||
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
|
||||
dataStr := string(data)
|
||||
|
||||
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
|
||||
idmatches := idPat.FindAllStringSubmatch(dataStr, -1)
|
||||
|
||||
for _, match := range matches {
|
||||
if len(match) != 2 {
|
||||
continue
|
||||
}
|
||||
resMatch := strings.TrimSpace(match[1])
|
||||
for _, idmatch := range idmatches {
|
||||
if len(idmatch) != 2 {
|
||||
continue
|
||||
}
|
||||
resIdMatch := strings.TrimSpace(idmatch[1])
|
||||
s1 := detectors.Result{
|
||||
DetectorType: detectorspb.DetectorType_Scalr,
|
||||
Raw: []byte(resMatch),
|
||||
}
|
||||
|
||||
if verify {
|
||||
url := fmt.Sprintf("https://%s.scalr.io/api/iacp/v3/agents",resIdMatch)
|
||||
req, err := http.NewRequestWithContext(ctx, "GET", url, nil)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
req.Header.Add("Authorization", fmt.Sprintf("Bearer %s",resMatch))
|
||||
req.Header.Add("Content-Type", "application/vnd.api+json")
|
||||
req.Header.Add("Prefer", "profile=preview")
|
||||
res, err := client.Do(req)
|
||||
if err == nil {
|
||||
defer res.Body.Close()
|
||||
if res.StatusCode >= 200 && res.StatusCode < 300 {
|
||||
s1.Verified = true
|
||||
} else {
|
||||
//This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key
|
||||
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
|
||||
continue
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
results = append(results, s1)
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
return detectors.CleanResults(results), nil
|
||||
}
|
||||
@@ -1,118 +0,0 @@
|
||||
package scalr
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/kylelemons/godebug/pretty"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
|
||||
)
|
||||
|
||||
func TestScalr_FromChunk(t *testing.T) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
|
||||
defer cancel()
|
||||
testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors4")
|
||||
if err != nil {
|
||||
t.Fatalf("could not get test secrets from GCP: %s", err)
|
||||
}
|
||||
secret := testSecrets.MustGetField("SCALR")
|
||||
inactiveSecret := testSecrets.MustGetField("SCALR_INACTIVE")
|
||||
id := testSecrets.MustGetField("SCALR_DOMAIN")
|
||||
|
||||
|
||||
type args struct {
|
||||
ctx context.Context
|
||||
data []byte
|
||||
verify bool
|
||||
}
|
||||
tests := []struct {
|
||||
name string
|
||||
s Scanner
|
||||
args args
|
||||
want []detectors.Result
|
||||
wantErr bool
|
||||
}{
|
||||
{
|
||||
name: "found, verified",
|
||||
s: Scanner{},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a scalr secret %s within scalr %s", secret,id)),
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_Scalr,
|
||||
Verified: true,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "found, unverified",
|
||||
s: Scanner{},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a scalr secret %s within but not valid scalr %s", inactiveSecret,id)), // the secret would satisfy the regex but not pass validation
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_Scalr,
|
||||
Verified: false,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "not found",
|
||||
s: Scanner{},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte("You cannot find the secret within"),
|
||||
verify: true,
|
||||
},
|
||||
want: nil,
|
||||
wantErr: false,
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
s := Scanner{}
|
||||
got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
|
||||
if (err != nil) != tt.wantErr {
|
||||
t.Errorf("Scalr.FromData() error = %v, wantErr %v", err, tt.wantErr)
|
||||
return
|
||||
}
|
||||
for i := range got {
|
||||
if len(got[i].Raw) == 0 {
|
||||
t.Fatalf("no raw secret present: \n %+v", got[i])
|
||||
}
|
||||
got[i].Raw = nil
|
||||
}
|
||||
if diff := pretty.Compare(got, tt.want); diff != "" {
|
||||
t.Errorf("Scalr.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkFromData(benchmark *testing.B) {
|
||||
ctx := context.Background()
|
||||
s := Scanner{}
|
||||
for name, data := range detectors.MustGetBenchmarkData() {
|
||||
benchmark.Run(name, func(b *testing.B) {
|
||||
for n := 0; n < b.N; n++ {
|
||||
_, err := s.FromData(ctx, false, data)
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -1,85 +0,0 @@
|
||||
package tokeet
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
|
||||
)
|
||||
|
||||
type Scanner struct{}
|
||||
|
||||
// Ensure the Scanner satisfies the interface at compile time
|
||||
var _ detectors.Detector = (*Scanner)(nil)
|
||||
|
||||
var (
|
||||
client = common.SaneHttpClient()
|
||||
|
||||
//Make sure that your group is surrounded in boundry characters such as below to reduce false positives
|
||||
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"tokeet"}) + `\b([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})\b`)
|
||||
idPat = regexp.MustCompile(detectors.PrefixRegex([]string{"tokeet"}) + `\b([0-9]{10}.[0-9]{4})\b`)
|
||||
|
||||
)
|
||||
|
||||
// Keywords are used for efficiently pre-filtering chunks.
|
||||
// Use identifiers in the secret preferably, or the provider name.
|
||||
func (s Scanner) Keywords() []string {
|
||||
return []string{"tokeet"}
|
||||
}
|
||||
|
||||
// FromData will find and optionally verify Tokeet secrets in a given set of bytes.
|
||||
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
|
||||
dataStr := string(data)
|
||||
|
||||
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
|
||||
idmatches := idPat.FindAllStringSubmatch(dataStr, -1)
|
||||
|
||||
for _, match := range matches {
|
||||
if len(match) != 2 {
|
||||
continue
|
||||
}
|
||||
resMatch := strings.TrimSpace(match[1])
|
||||
|
||||
for _, idmatch := range idmatches {
|
||||
if len(idmatch) != 2 {
|
||||
continue
|
||||
}
|
||||
resIdMatch := strings.TrimSpace(idmatch[1])
|
||||
|
||||
s1 := detectors.Result{
|
||||
DetectorType: detectorspb.DetectorType_Tokeet,
|
||||
Raw: []byte(resMatch),
|
||||
}
|
||||
if verify {
|
||||
req, err := http.NewRequestWithContext(ctx, "GET", fmt.Sprintf("https://capi.tokeet.com/v1/user?account=%s", resIdMatch), nil)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
req.Header.Add("Accept", "application/json")
|
||||
req.Header.Add("Authorization",resMatch)
|
||||
res, err := client.Do(req)
|
||||
if err == nil {
|
||||
defer res.Body.Close()
|
||||
if res.StatusCode >= 200 && res.StatusCode < 300 {
|
||||
s1.Verified = true
|
||||
} else {
|
||||
//This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key
|
||||
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
|
||||
continue
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
results = append(results, s1)
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
return detectors.CleanResults(results), nil
|
||||
}
|
||||
@@ -1,118 +0,0 @@
|
||||
package tokeet
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/kylelemons/godebug/pretty"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
|
||||
)
|
||||
|
||||
func TestTokeet_FromChunk(t *testing.T) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
|
||||
defer cancel()
|
||||
testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors4")
|
||||
if err != nil {
|
||||
t.Fatalf("could not get test secrets from GCP: %s", err)
|
||||
}
|
||||
secret := testSecrets.MustGetField("TOKEET")
|
||||
inactiveSecret := testSecrets.MustGetField("TOKEET_INACTIVE")
|
||||
id := testSecrets.MustGetField("TOKEET_ID")
|
||||
|
||||
|
||||
type args struct {
|
||||
ctx context.Context
|
||||
data []byte
|
||||
verify bool
|
||||
}
|
||||
tests := []struct {
|
||||
name string
|
||||
s Scanner
|
||||
args args
|
||||
want []detectors.Result
|
||||
wantErr bool
|
||||
}{
|
||||
{
|
||||
name: "found, verified",
|
||||
s: Scanner{},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a tokeet secret %s within tokeet %s", secret,id)),
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_Tokeet,
|
||||
Verified: true,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "found, unverified",
|
||||
s: Scanner{},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a tokeet secret %s within but not valid tokeet %s", inactiveSecret,id)), // the secret would satisfy the regex but not pass validation
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_Tokeet,
|
||||
Verified: false,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "not found",
|
||||
s: Scanner{},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte("You cannot find the secret within"),
|
||||
verify: true,
|
||||
},
|
||||
want: nil,
|
||||
wantErr: false,
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
s := Scanner{}
|
||||
got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
|
||||
if (err != nil) != tt.wantErr {
|
||||
t.Errorf("Tokeet.FromData() error = %v, wantErr %v", err, tt.wantErr)
|
||||
return
|
||||
}
|
||||
for i := range got {
|
||||
if len(got[i].Raw) == 0 {
|
||||
t.Fatalf("no raw secret present: \n %+v", got[i])
|
||||
}
|
||||
got[i].Raw = nil
|
||||
}
|
||||
if diff := pretty.Compare(got, tt.want); diff != "" {
|
||||
t.Errorf("Tokeet.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkFromData(benchmark *testing.B) {
|
||||
ctx := context.Background()
|
||||
s := Scanner{}
|
||||
for name, data := range detectors.MustGetBenchmarkData() {
|
||||
benchmark.Run(name, func(b *testing.B) {
|
||||
for n := 0; n < b.N; n++ {
|
||||
_, err := s.FromData(ctx, false, data)
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -1,75 +0,0 @@
|
||||
package transferwise
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
|
||||
)
|
||||
|
||||
type Scanner struct{}
|
||||
|
||||
// Ensure the Scanner satisfies the interface at compile time
|
||||
var _ detectors.Detector = (*Scanner)(nil)
|
||||
|
||||
var (
|
||||
client = common.SaneHttpClient()
|
||||
|
||||
//Make sure that your group is surrounded in boundry characters such as below to reduce false positives
|
||||
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"transferwise"}) + `\b([0-9a-f-]{8}-[0-9a-f-]{4}-[0-9a-f-]{4}-[0-9a-f-]{4}-[0-9a-f-]{12})\b`)
|
||||
)
|
||||
|
||||
// Keywords are used for efficiently pre-filtering chunks.
|
||||
// Use identifiers in the secret preferably, or the provider name.
|
||||
func (s Scanner) Keywords() []string {
|
||||
return []string{"transferwise"}
|
||||
}
|
||||
|
||||
// FromData will find and optionally verify Transferwise secrets in a given set of bytes.
|
||||
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
|
||||
dataStr := string(data)
|
||||
|
||||
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
|
||||
|
||||
for _, match := range matches {
|
||||
if len(match) != 2 {
|
||||
continue
|
||||
}
|
||||
resMatch := strings.TrimSpace(match[1])
|
||||
|
||||
s1 := detectors.Result{
|
||||
DetectorType: detectorspb.DetectorType_Transferwise,
|
||||
Raw: []byte(resMatch),
|
||||
}
|
||||
|
||||
if verify {
|
||||
req, err := http.NewRequestWithContext(ctx, "GET", "https://api.transferwise.com/v2/profiles", nil)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
req.Header.Add("Accept", "application/vnd.transferwise+json; version=3")
|
||||
req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", resMatch))
|
||||
res, err := client.Do(req)
|
||||
if err == nil {
|
||||
defer res.Body.Close()
|
||||
if res.StatusCode >= 200 && res.StatusCode < 300 {
|
||||
s1.Verified = true
|
||||
} else {
|
||||
//This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key
|
||||
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
|
||||
continue
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
results = append(results, s1)
|
||||
}
|
||||
|
||||
return detectors.CleanResults(results), nil
|
||||
}
|
||||
@@ -1,116 +0,0 @@
|
||||
package transferwise
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/kylelemons/godebug/pretty"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
|
||||
)
|
||||
|
||||
func TestTransferwise_FromChunk(t *testing.T) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
|
||||
defer cancel()
|
||||
testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
|
||||
if err != nil {
|
||||
t.Fatalf("could not get test secrets from GCP: %s", err)
|
||||
}
|
||||
secret := testSecrets.MustGetField("TRANSFERWISE")
|
||||
inactiveSecret := testSecrets.MustGetField("TRANSFERWISE_INACTIVE")
|
||||
|
||||
type args struct {
|
||||
ctx context.Context
|
||||
data []byte
|
||||
verify bool
|
||||
}
|
||||
tests := []struct {
|
||||
name string
|
||||
s Scanner
|
||||
args args
|
||||
want []detectors.Result
|
||||
wantErr bool
|
||||
}{
|
||||
{
|
||||
name: "found, verified",
|
||||
s: Scanner{},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a transferwise secret %s within", secret)),
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_Transferwise,
|
||||
Verified: true,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "found, unverified",
|
||||
s: Scanner{},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a transferwise secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_Transferwise,
|
||||
Verified: false,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "not found",
|
||||
s: Scanner{},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte("You cannot find the secret within"),
|
||||
verify: true,
|
||||
},
|
||||
want: nil,
|
||||
wantErr: false,
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
s := Scanner{}
|
||||
got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
|
||||
if (err != nil) != tt.wantErr {
|
||||
t.Errorf("Transferwise.FromData() error = %v, wantErr %v", err, tt.wantErr)
|
||||
return
|
||||
}
|
||||
for i := range got {
|
||||
if len(got[i].Raw) == 0 {
|
||||
t.Fatalf("no raw secret present: \n %+v", got[i])
|
||||
}
|
||||
got[i].Raw = nil
|
||||
}
|
||||
if diff := pretty.Compare(got, tt.want); diff != "" {
|
||||
t.Errorf("Transferwise.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkFromData(benchmark *testing.B) {
|
||||
ctx := context.Background()
|
||||
s := Scanner{}
|
||||
for name, data := range detectors.MustGetBenchmarkData() {
|
||||
benchmark.Run(name, func(b *testing.B) {
|
||||
for n := 0; n < b.N; n++ {
|
||||
_, err := s.FromData(ctx, false, data)
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -1,77 +0,0 @@
|
||||
package twist
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
|
||||
)
|
||||
|
||||
type Scanner struct{}
|
||||
|
||||
// Ensure the Scanner satisfies the interface at compile time
|
||||
var _ detectors.Detector = (*Scanner)(nil)
|
||||
|
||||
var (
|
||||
client = common.SaneHttpClient()
|
||||
|
||||
//Make sure that your group is surrounded in boundry characters such as below to reduce false positives
|
||||
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"twist"}) + `\b([0-9a-f:]{40,47})\b`)
|
||||
)
|
||||
|
||||
// Keywords are used for efficiently pre-filtering chunks.
|
||||
// Use identifiers in the secret preferably, or the provider name.
|
||||
func (s Scanner) Keywords() []string {
|
||||
return []string{"twist"}
|
||||
}
|
||||
|
||||
// FromData will find and optionally verify Twist secrets in a given set of bytes.
|
||||
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
|
||||
dataStr := string(data)
|
||||
|
||||
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
|
||||
|
||||
for _, match := range matches {
|
||||
if len(match) != 2 {
|
||||
continue
|
||||
}
|
||||
resMatch := strings.TrimSpace(match[1])
|
||||
setAuth := resMatch
|
||||
|
||||
if strings.Contains(match[0], "oauth") {
|
||||
setAuth = fmt.Sprintf("oauth2:%s", strings.TrimSpace(match[1]))
|
||||
}
|
||||
|
||||
s1 := detectors.Result{
|
||||
DetectorType: detectorspb.DetectorType_Twist,
|
||||
Raw: []byte(resMatch),
|
||||
}
|
||||
|
||||
if verify {
|
||||
req, err := http.NewRequestWithContext(ctx, "GET", "https://api.twist.com/api/v3/users/get_session_user", nil)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", setAuth))
|
||||
res, err := client.Do(req)
|
||||
if err == nil {
|
||||
defer res.Body.Close()
|
||||
if res.StatusCode >= 200 && res.StatusCode < 300 {
|
||||
s1.Verified = true
|
||||
} else {
|
||||
//This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key
|
||||
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
|
||||
continue
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
results = append(results, s1)
|
||||
}
|
||||
return detectors.CleanResults(results), nil
|
||||
}
|
||||
@@ -1,116 +0,0 @@
|
||||
package twist
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/kylelemons/godebug/pretty"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
|
||||
)
|
||||
|
||||
func TestTwist_FromChunk(t *testing.T) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
|
||||
defer cancel()
|
||||
testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
|
||||
if err != nil {
|
||||
t.Fatalf("could not get test secrets from GCP: %s", err)
|
||||
}
|
||||
secret := testSecrets.MustGetField("TWIST")
|
||||
inactiveSecret := testSecrets.MustGetField("TWIST_INACTIVE")
|
||||
|
||||
type args struct {
|
||||
ctx context.Context
|
||||
data []byte
|
||||
verify bool
|
||||
}
|
||||
tests := []struct {
|
||||
name string
|
||||
s Scanner
|
||||
args args
|
||||
want []detectors.Result
|
||||
wantErr bool
|
||||
}{
|
||||
{
|
||||
name: "found, verified",
|
||||
s: Scanner{},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a twist secret %s within", secret)),
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_Twist,
|
||||
Verified: true,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "found, unverified",
|
||||
s: Scanner{},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a twist secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_Twist,
|
||||
Verified: false,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "not found",
|
||||
s: Scanner{},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte("You cannot find the secret within"),
|
||||
verify: true,
|
||||
},
|
||||
want: nil,
|
||||
wantErr: false,
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
s := Scanner{}
|
||||
got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
|
||||
if (err != nil) != tt.wantErr {
|
||||
t.Errorf("Twist.FromData() error = %v, wantErr %v", err, tt.wantErr)
|
||||
return
|
||||
}
|
||||
for i := range got {
|
||||
if len(got[i].Raw) == 0 {
|
||||
t.Fatalf("no raw secret present: \n %+v", got[i])
|
||||
}
|
||||
got[i].Raw = nil
|
||||
}
|
||||
if diff := pretty.Compare(got, tt.want); diff != "" {
|
||||
t.Errorf("Twist.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkFromData(benchmark *testing.B) {
|
||||
ctx := context.Background()
|
||||
s := Scanner{}
|
||||
for name, data := range detectors.MustGetBenchmarkData() {
|
||||
benchmark.Run(name, func(b *testing.B) {
|
||||
for n := 0; n < b.N; n++ {
|
||||
_, err := s.FromData(ctx, false, data)
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -1,85 +0,0 @@
|
||||
package websitepulse
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"strings"
|
||||
"io/ioutil"
|
||||
"fmt"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
|
||||
)
|
||||
|
||||
type Scanner struct{}
|
||||
|
||||
// Ensure the Scanner satisfies the interface at compile time
|
||||
var _ detectors.Detector = (*Scanner)(nil)
|
||||
|
||||
var (
|
||||
client = common.SaneHttpClient()
|
||||
|
||||
//Make sure that your group is surrounded in boundry characters such as below to reduce false positives
|
||||
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"websitepulse"}) + `\b([0-9a-f]{32})\b`)
|
||||
idPat = regexp.MustCompile(detectors.PrefixRegex([]string{"websitepulse"}) + `\b([0-9a-zA-Z._]{4,22})\b`)
|
||||
)
|
||||
|
||||
// Keywords are used for efficiently pre-filtering chunks.
|
||||
// Use identifiers in the secret preferably, or the provider name.
|
||||
func (s Scanner) Keywords() []string {
|
||||
return []string{"websitepulse"}
|
||||
}
|
||||
|
||||
// FromData will find and optionally verify Websitepulse secrets in a given set of bytes.
|
||||
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
|
||||
dataStr := string(data)
|
||||
|
||||
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
|
||||
idmatches := idPat.FindAllStringSubmatch(dataStr, -1)
|
||||
|
||||
for _, match := range matches {
|
||||
if len(match) != 2 {
|
||||
continue
|
||||
}
|
||||
resMatch := strings.TrimSpace(match[1])
|
||||
|
||||
for _, idmatch := range idmatches {
|
||||
if len(idmatch) != 2 {
|
||||
continue
|
||||
}
|
||||
resIdMatch := strings.TrimSpace(idmatch[1])
|
||||
s1 := detectors.Result{
|
||||
DetectorType: detectorspb.DetectorType_Websitepulse,
|
||||
Raw: []byte(resMatch),
|
||||
}
|
||||
if verify {
|
||||
req, err := http.NewRequestWithContext(ctx, "GET", fmt.Sprintf("https://api.websitepulse.com/textserver.php?method=GetContacts&username=%s&key=%s",resIdMatch,resMatch), nil)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
res, err := client.Do(req)
|
||||
if err == nil {
|
||||
bodyBytes, err := ioutil.ReadAll(res.Body)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
body := string(bodyBytes)
|
||||
|
||||
if strings.Contains(body, "Active") {
|
||||
s1.Verified = true
|
||||
} else {
|
||||
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
|
||||
continue
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
results = append(results, s1)
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
return detectors.CleanResults(results), nil
|
||||
}
|
||||
@@ -1,117 +0,0 @@
|
||||
package websitepulse
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/kylelemons/godebug/pretty"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
|
||||
)
|
||||
|
||||
func TestWebsitepulse_FromChunk(t *testing.T) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
|
||||
defer cancel()
|
||||
testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors4")
|
||||
if err != nil {
|
||||
t.Fatalf("could not get test secrets from GCP: %s", err)
|
||||
}
|
||||
secret := testSecrets.MustGetField("WEBSITEPULSE")
|
||||
inactiveSecret := testSecrets.MustGetField("WEBSITEPULSE_INACTIVE")
|
||||
id := testSecrets.MustGetField("WEBSITEPULSE_ID")
|
||||
|
||||
type args struct {
|
||||
ctx context.Context
|
||||
data []byte
|
||||
verify bool
|
||||
}
|
||||
tests := []struct {
|
||||
name string
|
||||
s Scanner
|
||||
args args
|
||||
want []detectors.Result
|
||||
wantErr bool
|
||||
}{
|
||||
{
|
||||
name: "found, verified",
|
||||
s: Scanner{},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a websitepulse secret %s within websitepulse %s", secret,id)),
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_Websitepulse,
|
||||
Verified: true,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "found, unverified",
|
||||
s: Scanner{},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a websitepulse secret %s within but not valid websitepulse %s", inactiveSecret,id)), // the secret would satisfy the regex but not pass validation
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_Websitepulse,
|
||||
Verified: false,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "not found",
|
||||
s: Scanner{},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte("You cannot find the secret within"),
|
||||
verify: true,
|
||||
},
|
||||
want: nil,
|
||||
wantErr: false,
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
s := Scanner{}
|
||||
got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
|
||||
if (err != nil) != tt.wantErr {
|
||||
t.Errorf("Websitepulse.FromData() error = %v, wantErr %v", err, tt.wantErr)
|
||||
return
|
||||
}
|
||||
for i := range got {
|
||||
if len(got[i].Raw) == 0 {
|
||||
t.Fatalf("no raw secret present: \n %+v", got[i])
|
||||
}
|
||||
got[i].Raw = nil
|
||||
}
|
||||
if diff := pretty.Compare(got, tt.want); diff != "" {
|
||||
t.Errorf("Websitepulse.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkFromData(benchmark *testing.B) {
|
||||
ctx := context.Background()
|
||||
s := Scanner{}
|
||||
for name, data := range detectors.MustGetBenchmarkData() {
|
||||
benchmark.Run(name, func(b *testing.B) {
|
||||
for n := 0; n < b.N; n++ {
|
||||
_, err := s.FromData(ctx, false, data)
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -1,284 +0,0 @@
|
||||
package gitparse
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
"fmt"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
log "github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
// DateFormat is the standard date format for git.
|
||||
const DateFormat = "Mon Jan 02 15:04:05 2006 -0700"
|
||||
|
||||
// Commit contains commit header info and diffs.
|
||||
type Commit struct {
|
||||
Hash string
|
||||
Author string
|
||||
Date time.Time
|
||||
Message strings.Builder
|
||||
Diffs []Diff
|
||||
}
|
||||
|
||||
// Diff contains the info about a file diff in a commit.
|
||||
type Diff struct {
|
||||
PathB string
|
||||
LineStart int
|
||||
Content bytes.Buffer
|
||||
IsBinary bool
|
||||
}
|
||||
|
||||
// RepoPath parses the output of the `git log` command for the `source` path.
|
||||
func RepoPath(source string, head string) (chan Commit, error) {
|
||||
commitChan := make(chan Commit)
|
||||
|
||||
args := []string{"-C", source, "log", "-p", "-U0", "--full-history", "--diff-filter=AM", "--date=format:%a %b %d %H:%M:%S %Y %z"}
|
||||
if head != "" {
|
||||
args = append(args, head)
|
||||
} else {
|
||||
args = append(args, "--all")
|
||||
}
|
||||
|
||||
cmd := exec.Command("git", args...)
|
||||
|
||||
absPath, err := filepath.Abs(source)
|
||||
if err == nil {
|
||||
cmd.Env = append(cmd.Env, fmt.Sprintf("GIT_DIR=%s", filepath.Join(absPath, ".git")))
|
||||
}
|
||||
|
||||
stdOut, err := cmd.StdoutPipe()
|
||||
if err != nil {
|
||||
return commitChan, err
|
||||
}
|
||||
stdErr, err := cmd.StderrPipe()
|
||||
if err != nil {
|
||||
return commitChan, err
|
||||
}
|
||||
|
||||
err = cmd.Start()
|
||||
if err != nil {
|
||||
return commitChan, err
|
||||
}
|
||||
|
||||
outReader := bufio.NewReader(stdOut)
|
||||
var currentCommit *Commit
|
||||
var currentDiff *Diff
|
||||
|
||||
go func() {
|
||||
scanner := bufio.NewScanner(stdErr)
|
||||
for scanner.Scan() {
|
||||
log.Debug(scanner.Text())
|
||||
}
|
||||
}()
|
||||
|
||||
go func() {
|
||||
for {
|
||||
line, err := outReader.ReadBytes([]byte("\n")[0])
|
||||
if err != nil && len(line) == 0 {
|
||||
break
|
||||
}
|
||||
switch {
|
||||
case isCommitLine(line):
|
||||
// If there is a currentDiff, add it to currentCommit.
|
||||
if currentDiff != nil && currentDiff.Content.Len() > 0 {
|
||||
currentCommit.Diffs = append(currentCommit.Diffs, *currentDiff)
|
||||
}
|
||||
// If there is a currentCommit, send it to the channel.
|
||||
if currentCommit != nil {
|
||||
commitChan <- *currentCommit
|
||||
}
|
||||
// Create a new currentDiff and currentCommit
|
||||
currentDiff = &Diff{}
|
||||
currentCommit = &Commit{
|
||||
Message: strings.Builder{},
|
||||
}
|
||||
// Check that the commit line contains a hash and set it.
|
||||
if len(line) >= 47 {
|
||||
currentCommit.Hash = string(line[7:47])
|
||||
}
|
||||
case isAuthorLine(line):
|
||||
currentCommit.Author = string(line[8:])
|
||||
case isDateLine(line):
|
||||
date, err := time.Parse(DateFormat, strings.TrimSpace(string(line[6:])))
|
||||
if err != nil {
|
||||
log.WithError(err).Debug("Could not parse date from git stream.")
|
||||
}
|
||||
currentCommit.Date = date
|
||||
case isDiffLine(line):
|
||||
// This should never be nil, but check in case the stdin stream is messed up.
|
||||
if currentDiff != nil && currentDiff.Content.Len() > 0 {
|
||||
currentCommit.Diffs = append(currentCommit.Diffs, *currentDiff)
|
||||
}
|
||||
currentDiff = &Diff{}
|
||||
case isModeLine(line):
|
||||
// NoOp
|
||||
case isIndexLine(line):
|
||||
// NoOp
|
||||
case isPlusFileLine(line):
|
||||
currentDiff.PathB = strings.TrimRight(string(line[6:]), "\n")
|
||||
case isMinusFileLine(line):
|
||||
// NoOp
|
||||
case isPlusDiffLine(line):
|
||||
currentDiff.Content.Write(line[1:])
|
||||
case isMinusDiffLine(line):
|
||||
// NoOp. We only care about additions.
|
||||
case isMessageLine(line):
|
||||
currentCommit.Message.Write(line[4:])
|
||||
case isBinaryLine(line):
|
||||
currentDiff.IsBinary = true
|
||||
currentDiff.PathB = pathFromBinaryLine(line)
|
||||
case isLineNumberDiffLine(line):
|
||||
if currentDiff != nil && currentDiff.Content.Len() > 0 {
|
||||
currentCommit.Diffs = append(currentCommit.Diffs, *currentDiff)
|
||||
}
|
||||
newDiff := &Diff{
|
||||
PathB: currentDiff.PathB,
|
||||
}
|
||||
|
||||
currentDiff = newDiff
|
||||
|
||||
words := bytes.Split(line, []byte(" "))
|
||||
if len(words) >= 3 {
|
||||
startSlice := bytes.Split(words[2], []byte(","))
|
||||
lineStart, err := strconv.Atoi(string(startSlice[0]))
|
||||
if err == nil {
|
||||
currentDiff.LineStart = lineStart
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
if currentDiff != nil && currentDiff.Content.Len() > 0 {
|
||||
currentCommit.Diffs = append(currentCommit.Diffs, *currentDiff)
|
||||
}
|
||||
if currentCommit != nil {
|
||||
commitChan <- *currentCommit
|
||||
}
|
||||
if err := cmd.Wait(); err != nil {
|
||||
log.WithError(err).Debugf("Error waiting for git command to complete.")
|
||||
}
|
||||
close(commitChan)
|
||||
}()
|
||||
return commitChan, nil
|
||||
}
|
||||
|
||||
// Date: Tue Aug 10 15:20:40 2021 +0100
|
||||
func isDateLine(line []byte) bool {
|
||||
if len(line) > 7 && bytes.Equal(line[:5], []byte("Date:")) {
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// Author: Bill Rich <bill.rich@trufflesec.com>
|
||||
func isAuthorLine(line []byte) bool {
|
||||
if len(line) > 8 && bytes.Equal(line[:7], []byte("Author:")) {
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// commit 7a95bbf0199e280a0e42dbb1d1a3f56cdd0f6e05
|
||||
func isCommitLine(line []byte) bool {
|
||||
if len(line) > 7 && bytes.Equal(line[:6], []byte("commit")) {
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// diff --git a/internal/addrs/move_endpoint_module.go b/internal/addrs/move_endpoint_module.go
|
||||
func isDiffLine(line []byte) bool {
|
||||
if len(line) > 5 && bytes.Equal(line[:4], []byte("diff")) {
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// index 1ed6fbee1..aea1e643a 100644
|
||||
func isIndexLine(line []byte) bool {
|
||||
if len(line) > 6 && bytes.Equal(line[:5], []byte("index")) {
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// new file mode 100644
|
||||
func isModeLine(line []byte) bool {
|
||||
if len(line) > 13 && bytes.Equal(line[:13], []byte("new file mode")) {
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// --- a/internal/addrs/move_endpoint_module.go
|
||||
func isMinusFileLine(line []byte) bool {
|
||||
if len(line) >= 6 && bytes.Equal(line[:3], []byte("---")) {
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// +++ b/internal/addrs/move_endpoint_module.go
|
||||
func isPlusFileLine(line []byte) bool {
|
||||
if len(line) >= 6 && bytes.Equal(line[:3], []byte("+++")) {
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// +fmt.Println("ok")
|
||||
func isPlusDiffLine(line []byte) bool {
|
||||
if len(line) >= 1 && bytes.Equal(line[:1], []byte("+")) {
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// -fmt.Println("ok")
|
||||
func isMinusDiffLine(line []byte) bool {
|
||||
if len(line) >= 1 && bytes.Equal(line[:1], []byte("-")) {
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// Line that starts with 4 spaces
|
||||
func isMessageLine(line []byte) bool {
|
||||
if len(line) > 4 && bytes.Equal(line[:4], []byte(" ")) {
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// Binary files /dev/null and b/plugin.sig differ
|
||||
func isBinaryLine(line []byte) bool {
|
||||
if len(line) > 7 && bytes.Equal(line[:6], []byte("Binary")) {
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// @@ -298 +298 @@ func maxRetryErrorHandler(resp *http.Response, err error, numTries int)
|
||||
func isLineNumberDiffLine(line []byte) bool {
|
||||
if len(line) >= 8 && bytes.Equal(line[:2], []byte("@@")) {
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// Get the b/ file path.
|
||||
func pathFromBinaryLine(line []byte) string {
|
||||
sbytes := bytes.Split(line, []byte(" and "))
|
||||
if len(sbytes) != 2 {
|
||||
log.Debugf("Expected binary line to be in 'Binary files a/filaA and b/fileB differ' format. Got: %s", line)
|
||||
return ""
|
||||
}
|
||||
bRaw := sbytes[1]
|
||||
return string(bRaw[2 : len(bRaw)-7]) // drop the "b/" and " differ"
|
||||
}
|
||||
@@ -1,80 +0,0 @@
|
||||
package gitparse
|
||||
|
||||
import (
|
||||
"testing"
|
||||
)
|
||||
|
||||
type testCase struct {
|
||||
pass []byte
|
||||
fails [][]byte
|
||||
function func([]byte) bool
|
||||
}
|
||||
|
||||
func TestIsIndexLine(t *testing.T) {
|
||||
tests := map[string]testCase{
|
||||
"indexLine": {
|
||||
pass: []byte("index 1ed6fbee1..aea1e643a 100644"),
|
||||
fails: [][]byte{[]byte("notcorrect")},
|
||||
function: isIndexLine,
|
||||
},
|
||||
"modeLine": {
|
||||
pass: []byte("new file mode 100644"),
|
||||
fails: [][]byte{[]byte("notcorrect")},
|
||||
function: isModeLine,
|
||||
},
|
||||
"minusFileLine": {
|
||||
pass: []byte("--- a/internal/addrs/move_endpoint_module.go"),
|
||||
fails: [][]byte{[]byte("notcorrect"), []byte("--- s"), []byte("short")},
|
||||
function: isMinusFileLine,
|
||||
},
|
||||
"plusFileLine": {
|
||||
pass: []byte("+++ b/internal/addrs/move_endpoint_module.go"),
|
||||
fails: [][]byte{[]byte("notcorrect"), []byte("+++ s"), []byte("short")},
|
||||
function: isPlusFileLine,
|
||||
},
|
||||
"plusDiffLine": {
|
||||
pass: []byte("+fmt.Println"),
|
||||
fails: [][]byte{[]byte("notcorrect")},
|
||||
function: isPlusDiffLine,
|
||||
},
|
||||
"minusDiffLine": {
|
||||
pass: []byte("-fmt.Println"),
|
||||
function: isMinusDiffLine,
|
||||
},
|
||||
"messageLine": {
|
||||
pass: []byte(" committed"),
|
||||
fails: [][]byte{[]byte("notcorrect")},
|
||||
function: isMessageLine,
|
||||
},
|
||||
"binaryLine": {
|
||||
pass: []byte("Binary files /dev/null and b/plugin.sig differ"),
|
||||
fails: [][]byte{[]byte("notcorrect")},
|
||||
function: isBinaryLine,
|
||||
},
|
||||
"lineNumberLine": {
|
||||
pass: []byte("@@ -298 +298 @@ func maxRetryErrorHandler(resp *http.Response, err error, numTries int)"),
|
||||
fails: [][]byte{[]byte("notcorrect")},
|
||||
function: isLineNumberDiffLine,
|
||||
},
|
||||
}
|
||||
|
||||
for name, test := range tests {
|
||||
if !test.function(test.pass) {
|
||||
t.Errorf("%s: Parser did not recognize correct line.", name)
|
||||
}
|
||||
for _, fail := range test.fails {
|
||||
if test.function(fail) {
|
||||
t.Errorf("%s: Parser did not recognize incorrect line.", name)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestBinaryPathParse(t *testing.T) {
|
||||
filename := pathFromBinaryLine([]byte("Binary files /dev/null and b/plugin.sig differ"))
|
||||
expected := "plugin.sig"
|
||||
if filename != expected {
|
||||
t.Errorf("Expected: %s, Got: %s", expected, filename)
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,240 @@
|
||||
//package main
|
||||
package gitparse
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
"fmt"
|
||||
"log"
|
||||
"os/exec"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
// DateFormat is the standard date format for git.
|
||||
const DateFormat = "Mon Jan 02 15:04:05 2006 -0700"
|
||||
|
||||
// Commit contains commit header info and diffs.
|
||||
type Commit struct {
|
||||
Hash string
|
||||
Author string
|
||||
Date time.Time
|
||||
Message strings.Builder
|
||||
Diffs []Diff
|
||||
}
|
||||
|
||||
// Diff contains the info about a file diff in a commit.
|
||||
type Diff struct {
|
||||
PathA string
|
||||
PathB string
|
||||
LineStart int
|
||||
Content bytes.Buffer
|
||||
IsBinary bool
|
||||
}
|
||||
|
||||
/*
|
||||
func main() {
|
||||
source := "/home/hrich/go/src/github.com/hashicorp/terraform"
|
||||
start := time.Now()
|
||||
fileChan := BillParser(source)
|
||||
for range fileChan {
|
||||
}
|
||||
newTime := time.Now().Sub(start)
|
||||
|
||||
start = time.Now()
|
||||
errChan := make(chan error)
|
||||
fileChan2, _ := glgo.GitLog(source, glgo.LogOpts{}, errChan)
|
||||
for range fileChan2 {
|
||||
}
|
||||
oldTime := time.Now().Sub(start)
|
||||
|
||||
fmt.Printf("Gitleaks parser: %d seconds\nNew parser: %d seconds", oldTime.Seconds(), newTime.Seconds())
|
||||
|
||||
}
|
||||
*/
|
||||
|
||||
func BillParser(source string) chan Commit {
|
||||
cmd := exec.Command("git", "-C", source, "log", "-p", "-U0", "--full-history", "--all", "--diff-filter=AM", "--date=format:%a %b %d %H:%M:%S %Y %z")
|
||||
|
||||
stdOut, err := cmd.StdoutPipe()
|
||||
if err != nil {
|
||||
errs(err)
|
||||
}
|
||||
// TODO: Handle errors
|
||||
// stdErr, err := cmd.StderrPipe()
|
||||
// if err != nil {
|
||||
// errs(err)
|
||||
// }
|
||||
|
||||
err = cmd.Start()
|
||||
if err != nil {
|
||||
errs(err)
|
||||
}
|
||||
|
||||
outReader := bufio.NewReader(stdOut)
|
||||
var currentCommit *Commit
|
||||
var currentDiff *Diff
|
||||
|
||||
commitChan := make(chan Commit)
|
||||
|
||||
// TODO: Something about nil handling maybe? I don't remember
|
||||
go func() {
|
||||
for {
|
||||
line, _, err := outReader.ReadLine()
|
||||
if err != nil {
|
||||
fmt.Println(err)
|
||||
break
|
||||
}
|
||||
logrus.Debugf("Line: %s", line)
|
||||
switch {
|
||||
case isCommitLine(line):
|
||||
logrus.Debugf("commitline: %s", line)
|
||||
if currentCommit != nil {
|
||||
commitChan <- *currentCommit
|
||||
}
|
||||
currentCommit = &Commit{
|
||||
Message: strings.Builder{},
|
||||
}
|
||||
currentCommit.Hash = string(line[7:])
|
||||
case isAuthorLine(line):
|
||||
currentCommit.Author = string(line[8:])
|
||||
case isDateLine(line):
|
||||
date, err := time.Parse(DateFormat, strings.TrimSpace(string(line[6:])))
|
||||
if err != nil {
|
||||
errs(err)
|
||||
}
|
||||
currentCommit.Date = date
|
||||
case isDiffLine(line):
|
||||
if currentDiff != nil {
|
||||
currentCommit.Diffs = append(currentCommit.Diffs, *currentDiff)
|
||||
}
|
||||
currentDiff = &Diff{}
|
||||
case isModeLine(line):
|
||||
case isIndexLine(line):
|
||||
case isPlusFileLine(line):
|
||||
currentDiff.PathB = string(line[4:])
|
||||
case isMinusFileLine(line):
|
||||
currentDiff.PathA = string(line[4:])
|
||||
case isPlusDiffLine(line):
|
||||
currentDiff.Content.Write(line[1:])
|
||||
currentDiff.Content.Write([]byte("\n"))
|
||||
case isMinusDiffLine(line):
|
||||
case isMessageLine(line):
|
||||
currentCommit.Message.Write(line[4:])
|
||||
case isBinaryLine(line):
|
||||
currentDiff.IsBinary = true
|
||||
currentDiff.PathB = pathFromBinaryLine(line, source)
|
||||
}
|
||||
|
||||
}
|
||||
if currentDiff != nil && currentDiff.Content.Len() > 0 {
|
||||
currentCommit.Diffs = append(currentCommit.Diffs, *currentDiff)
|
||||
}
|
||||
if currentCommit != nil {
|
||||
commitChan <- *currentCommit
|
||||
}
|
||||
cmd.Wait()
|
||||
close(commitChan)
|
||||
}()
|
||||
return commitChan
|
||||
}
|
||||
|
||||
func isDateLine(line []byte) bool {
|
||||
if len(line) > 7 && bytes.Equal(line[:5], []byte("Date:")) {
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func isAuthorLine(line []byte) bool {
|
||||
if len(line) > 8 && bytes.Equal(line[:7], []byte("Author:")) {
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func isCommitLine(line []byte) bool {
|
||||
if len(line) > 7 && bytes.Equal(line[:6], []byte("commit")) {
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func isDiffLine(line []byte) bool {
|
||||
if len(line) > 5 && bytes.Equal(line[:4], []byte("diff")) {
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func isIndexLine(line []byte) bool {
|
||||
if len(line) > 6 && bytes.Equal(line[:5], []byte("index")) {
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func isModeLine(line []byte) bool {
|
||||
if len(line) > 8 && bytes.Equal(line[:8], []byte("new mode")) {
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func isMinusFileLine(line []byte) bool {
|
||||
if len(line) > 3 && bytes.Equal(line[:3], []byte("---")) {
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func isPlusFileLine(line []byte) bool {
|
||||
if len(line) > 3 && bytes.Equal(line[:3], []byte("+++")) {
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func isPlusDiffLine(line []byte) bool {
|
||||
if len(line) > 1 && bytes.Equal(line[:1], []byte("+")) {
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func isMinusDiffLine(line []byte) bool {
|
||||
if len(line) > 1 && bytes.Equal(line[:1], []byte("-")) {
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func isMessageLine(line []byte) bool {
|
||||
if len(line) > 4 && bytes.Equal(line[:4], []byte(" ")) {
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func isBinaryLine(line []byte) bool {
|
||||
if len(line) > 7 && bytes.Equal(line[:6], []byte("Binary")) {
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func pathFromBinaryLine(line []byte, source string) string {
|
||||
sbytes := bytes.Split(line, []byte(" and "))
|
||||
if len(sbytes) != 2 {
|
||||
logrus.Errorf("Expected binary line to be in 'Binary files a/filaA and b/fileB differ' format. Got: %s", line)
|
||||
return ""
|
||||
}
|
||||
bRaw := sbytes[1]
|
||||
return string(bRaw[2 : len(bRaw)-7]) // drop the "b/" and " differ"
|
||||
}
|
||||
|
||||
func errs(err error) {
|
||||
log.Println(err)
|
||||
}
|
||||
@@ -0,0 +1,30 @@
|
||||
package gitparse
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
glgo "github.com/zricethezav/gitleaks/v8/detect/git"
|
||||
)
|
||||
|
||||
func TestMain(t *testing.T) {
|
||||
source := "/home/hrich/go/src/github.com/hashicorp/terraform"
|
||||
start := time.Now()
|
||||
commitChan := BillParser(source)
|
||||
for _ = range commitChan {
|
||||
}
|
||||
fmt.Printf("BillParser took %f seconds.", time.Now().Sub(start).Seconds())
|
||||
start = time.Now()
|
||||
logOpts := glgo.LogOpts{
|
||||
DisableSafeDir: true,
|
||||
}
|
||||
errChan := make(chan error)
|
||||
fileChan, err := glgo.GitLog(source, logOpts, errChan)
|
||||
if err != nil {
|
||||
//errs(err)
|
||||
}
|
||||
for _ = range fileChan {
|
||||
}
|
||||
fmt.Printf("GitLeaksParser took %f seconds.", time.Now().Sub(start).Seconds())
|
||||
}
|
||||
-183
@@ -1,183 +0,0 @@
|
||||
package log
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"io"
|
||||
"time"
|
||||
|
||||
"github.com/TheZeroSlave/zapsentry"
|
||||
"github.com/getsentry/sentry-go"
|
||||
"github.com/go-logr/logr"
|
||||
"github.com/go-logr/zapr"
|
||||
"go.uber.org/zap"
|
||||
"go.uber.org/zap/zapcore"
|
||||
)
|
||||
|
||||
var (
|
||||
// global log level control (see SetLevel)
|
||||
logLevel zap.AtomicLevel = zap.NewAtomicLevel()
|
||||
)
|
||||
|
||||
type logConfig struct {
|
||||
core zapcore.Core
|
||||
cleanup func() error
|
||||
err error
|
||||
}
|
||||
|
||||
// New creates a new log object with the provided configurations. If no sinks
|
||||
// are provided, a no-op sink will be used. Returns the logger and a cleanup
|
||||
// function that should be executed before the program exits.
|
||||
func New(service string, configs ...logConfig) (logr.Logger, func() error) {
|
||||
var cores []zapcore.Core
|
||||
var cleanupFuncs []func() error
|
||||
|
||||
// create cores for the logger
|
||||
for _, config := range configs {
|
||||
if config.err != nil {
|
||||
continue
|
||||
}
|
||||
cores = append(cores, config.core)
|
||||
if config.cleanup != nil {
|
||||
cleanupFuncs = append(cleanupFuncs, config.cleanup)
|
||||
}
|
||||
}
|
||||
// create logger
|
||||
zapLogger := zap.New(zapcore.NewTee(cores...))
|
||||
cleanupFuncs = append(cleanupFuncs, zapLogger.Sync)
|
||||
logger := zapr.NewLogger(zapLogger).WithName(service)
|
||||
|
||||
// report the errors we encountered in the configs
|
||||
for _, config := range configs {
|
||||
if config.err != nil {
|
||||
logger.Error(config.err, "error configuring logger")
|
||||
}
|
||||
}
|
||||
|
||||
return logger, firstErrorFunc(cleanupFuncs...)
|
||||
}
|
||||
|
||||
// SetLevel sets the log level for all loggers created with WithJSONSink and
|
||||
// WithConsoleSink.
|
||||
func SetLevel(level int8) {
|
||||
// Zap's levels get more verbose as the number gets smaller, as explained
|
||||
// by zapr here: https://github.com/go-logr/zapr#increasing-verbosity
|
||||
// For example setting the level to -2 below, means log.V(2) will be enabled.
|
||||
logLevel.SetLevel(zapcore.Level(-level))
|
||||
}
|
||||
|
||||
// WithSentry adds sentry integration to the logger. This configuration may
|
||||
// fail, in which case, sentry will not be added and execution will continue
|
||||
// normally.
|
||||
func WithSentry(opts sentry.ClientOptions, tags map[string]string) logConfig {
|
||||
client, err := sentry.NewClient(opts)
|
||||
if err != nil {
|
||||
return logConfig{err: err}
|
||||
}
|
||||
|
||||
// create sentry core
|
||||
cfg := zapsentry.Configuration{
|
||||
Tags: tags,
|
||||
Level: zapcore.ErrorLevel,
|
||||
EnableBreadcrumbs: true,
|
||||
BreadcrumbLevel: zapcore.InfoLevel,
|
||||
}
|
||||
core, err := zapsentry.NewCore(cfg, zapsentry.NewSentryClientFromClient(client))
|
||||
if err != nil {
|
||||
return logConfig{err: err}
|
||||
}
|
||||
|
||||
return logConfig{
|
||||
core: core,
|
||||
cleanup: func() error {
|
||||
sentry.Flush(5 * time.Second)
|
||||
return nil
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// WithJSONSink adds a JSON encoded output to the logger.
|
||||
func WithJSONSink(sink io.Writer) logConfig {
|
||||
return logConfig{
|
||||
core: zapcore.NewCore(
|
||||
zapcore.NewJSONEncoder(defaultEncoderConfig()),
|
||||
zapcore.Lock(
|
||||
zapcore.AddSync(sink),
|
||||
),
|
||||
logLevel,
|
||||
),
|
||||
}
|
||||
}
|
||||
|
||||
// WithConsoleSink adds a console-style output to the logger.
|
||||
func WithConsoleSink(sink io.Writer) logConfig {
|
||||
return logConfig{
|
||||
core: zapcore.NewCore(
|
||||
zapcore.NewConsoleEncoder(defaultEncoderConfig()),
|
||||
zapcore.Lock(
|
||||
zapcore.AddSync(sink),
|
||||
),
|
||||
logLevel,
|
||||
),
|
||||
}
|
||||
}
|
||||
|
||||
func defaultEncoderConfig() zapcore.EncoderConfig {
|
||||
conf := zap.NewProductionEncoderConfig()
|
||||
// Use more human-readable time format.
|
||||
conf.EncodeTime = zapcore.TimeEncoderOfLayout(time.RFC3339)
|
||||
return conf
|
||||
}
|
||||
|
||||
// WithCore adds any user supplied zap core to the logger.
|
||||
func WithCore(core zapcore.Core) logConfig {
|
||||
return logConfig{core: core}
|
||||
}
|
||||
|
||||
// AddSentry initializes a sentry client and extends an existing
|
||||
// logr.Logger with the hook.
|
||||
func AddSentry(l logr.Logger, opts sentry.ClientOptions, tags map[string]string) (logr.Logger, func() error, error) {
|
||||
conf := WithSentry(opts, tags)
|
||||
if conf.err != nil {
|
||||
return l, nil, conf.err
|
||||
}
|
||||
return AddSink(l, conf)
|
||||
}
|
||||
|
||||
// AddSink extends an existing logr.Logger with a new sink. It returns the new
|
||||
// logr.Logger, a cleanup function, and an error.
|
||||
func AddSink(l logr.Logger, sink logConfig) (logr.Logger, func() error, error) {
|
||||
zapLogger, err := getZapLogger(l)
|
||||
if err != nil {
|
||||
return l, nil, errors.New("unsupported logr implementation")
|
||||
}
|
||||
zapLogger = zapLogger.WithOptions(zap.WrapCore(func(core zapcore.Core) zapcore.Core {
|
||||
return zapcore.NewTee(core, sink.core)
|
||||
}))
|
||||
return zapr.NewLogger(zapLogger), firstErrorFunc(zapLogger.Sync, sink.cleanup), nil
|
||||
}
|
||||
|
||||
// getZapLogger is a helper function that gets the underlying zap logger from a
|
||||
// logr.Logger interface.
|
||||
func getZapLogger(l logr.Logger) (*zap.Logger, error) {
|
||||
if u, ok := l.GetSink().(zapr.Underlier); ok {
|
||||
return u.GetUnderlying(), nil
|
||||
}
|
||||
return nil, errors.New("not a zapr logger")
|
||||
}
|
||||
|
||||
// firstErrorFunc is a helper function that returns a function that executes
|
||||
// all provided args and returns the first error, if any.
|
||||
func firstErrorFunc(fs ...func() error) func() error {
|
||||
return func() error {
|
||||
var firstErr error = nil
|
||||
for _, f := range fs {
|
||||
if f == nil {
|
||||
continue
|
||||
}
|
||||
if err := f(); err != nil && firstErr == nil {
|
||||
firstErr = err
|
||||
}
|
||||
}
|
||||
return firstErr
|
||||
}
|
||||
}
|
||||
@@ -1,152 +0,0 @@
|
||||
package log
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/getsentry/sentry-go"
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
func TestNew(t *testing.T) {
|
||||
var jsonBuffer, consoleBuffer bytes.Buffer
|
||||
logger, sync := New("service-name",
|
||||
WithJSONSink(&jsonBuffer),
|
||||
WithConsoleSink(&consoleBuffer),
|
||||
)
|
||||
logger.Info("yay")
|
||||
assert.Nil(t, sync())
|
||||
|
||||
assert.Contains(t, jsonBuffer.String(), `"logger":"service-name"`)
|
||||
assert.Contains(t, jsonBuffer.String(), `"msg":"yay"`)
|
||||
assert.Contains(t, consoleBuffer.String(), "info\tservice-name\tyay")
|
||||
}
|
||||
|
||||
func TestSetLevel(t *testing.T) {
|
||||
var buffer bytes.Buffer
|
||||
logger, _ := New("service-name",
|
||||
WithConsoleSink(&buffer),
|
||||
)
|
||||
|
||||
assert.Equal(t, true, logger.GetSink().Enabled(0))
|
||||
assert.Equal(t, false, logger.GetSink().Enabled(1))
|
||||
assert.Equal(t, false, logger.GetSink().Enabled(2))
|
||||
|
||||
SetLevel(1)
|
||||
assert.Equal(t, true, logger.GetSink().Enabled(0))
|
||||
assert.Equal(t, true, logger.GetSink().Enabled(1))
|
||||
assert.Equal(t, false, logger.GetSink().Enabled(2))
|
||||
|
||||
SetLevel(2)
|
||||
assert.Equal(t, true, logger.GetSink().Enabled(0))
|
||||
assert.Equal(t, true, logger.GetSink().Enabled(1))
|
||||
assert.Equal(t, true, logger.GetSink().Enabled(2))
|
||||
}
|
||||
|
||||
func TestWithSentryFailure(t *testing.T) {
|
||||
var buffer bytes.Buffer
|
||||
logger, sync := New("service-name",
|
||||
WithSentry(sentry.ClientOptions{Dsn: "fail"}, nil),
|
||||
WithConsoleSink(&buffer),
|
||||
)
|
||||
logger.Info("yay")
|
||||
assert.Nil(t, sync())
|
||||
|
||||
assert.Contains(t, buffer.String(), "error configuring logger")
|
||||
assert.Contains(t, buffer.String(), "yay")
|
||||
}
|
||||
|
||||
func TestAddSentryFailure(t *testing.T) {
|
||||
var buffer bytes.Buffer
|
||||
logger, sync := New("service-name",
|
||||
WithConsoleSink(&buffer),
|
||||
)
|
||||
logger, _, err := AddSentry(logger, sentry.ClientOptions{Dsn: "fail"}, nil)
|
||||
assert.NotNil(t, err)
|
||||
assert.NotContains(t, err.Error(), "unsupported")
|
||||
|
||||
logger.Info("yay")
|
||||
assert.Nil(t, sync())
|
||||
|
||||
assert.Contains(t, buffer.String(), "yay")
|
||||
}
|
||||
|
||||
func TestAddSentry(t *testing.T) {
|
||||
var buffer bytes.Buffer
|
||||
var sentryMessage string
|
||||
logger, _ := New("service-name",
|
||||
WithConsoleSink(&buffer),
|
||||
)
|
||||
logger, sync, err := AddSentry(logger, sentry.ClientOptions{
|
||||
BeforeSend: func(event *sentry.Event, hint *sentry.EventHint) *sentry.Event {
|
||||
sentryMessage = event.Message
|
||||
return nil
|
||||
},
|
||||
}, nil)
|
||||
assert.Nil(t, err)
|
||||
|
||||
logger.Info("yay")
|
||||
logger.Error(nil, "oops")
|
||||
assert.Nil(t, sync())
|
||||
|
||||
assert.Contains(t, buffer.String(), "yay")
|
||||
assert.Contains(t, buffer.String(), "oops")
|
||||
assert.Equal(t, "oops", sentryMessage)
|
||||
}
|
||||
|
||||
func TestWithSentry(t *testing.T) {
|
||||
var buffer bytes.Buffer
|
||||
var sentryMessage string
|
||||
logger, sync := New("service-name",
|
||||
WithConsoleSink(&buffer),
|
||||
WithSentry(sentry.ClientOptions{
|
||||
BeforeSend: func(event *sentry.Event, hint *sentry.EventHint) *sentry.Event {
|
||||
sentryMessage = event.Message
|
||||
return nil
|
||||
},
|
||||
}, nil),
|
||||
)
|
||||
logger.Info("yay")
|
||||
logger.Error(nil, "oops")
|
||||
assert.Nil(t, sync())
|
||||
|
||||
assert.Contains(t, buffer.String(), "yay")
|
||||
assert.Contains(t, buffer.String(), "oops")
|
||||
assert.Equal(t, "oops", sentryMessage)
|
||||
}
|
||||
|
||||
func TestHumanReadableTimestamp(t *testing.T) {
|
||||
var buffer bytes.Buffer
|
||||
logger, sync := New("service-name",
|
||||
WithConsoleSink(&buffer),
|
||||
)
|
||||
logger.Info("yay")
|
||||
assert.Nil(t, sync())
|
||||
|
||||
ts := strings.Split(buffer.String(), "\t")[0]
|
||||
assert.NotContains(t, ts, "e+09")
|
||||
|
||||
parsedTime, err := time.Parse(time.RFC3339, ts)
|
||||
assert.Nil(t, err)
|
||||
assert.Less(t, time.Since(parsedTime), 5*time.Second)
|
||||
}
|
||||
|
||||
func TestAddSink(t *testing.T) {
|
||||
var buf1, buf2 bytes.Buffer
|
||||
logger, _ := New("service-name",
|
||||
WithConsoleSink(&buf1),
|
||||
)
|
||||
logger.Info("line 1")
|
||||
logger, sync, err := AddSink(logger, WithConsoleSink(&buf2))
|
||||
assert.Nil(t, err)
|
||||
logger.Info("line 2")
|
||||
assert.Nil(t, sync())
|
||||
|
||||
assert.Contains(t, buf1.String(), "line 1")
|
||||
assert.Contains(t, buf1.String(), "line 2")
|
||||
// buf2 should only have "line 2"
|
||||
assert.NotContains(t, buf2.String(), "line 1")
|
||||
assert.Contains(t, buf2.String(), "line 2")
|
||||
}
|
||||
@@ -652,9 +652,8 @@ type SlackTokens struct {
|
||||
sizeCache protoimpl.SizeCache
|
||||
unknownFields protoimpl.UnknownFields
|
||||
|
||||
AppToken string `protobuf:"bytes,1,opt,name=app_token,json=appToken,proto3" json:"app_token,omitempty"`
|
||||
BotToken string `protobuf:"bytes,2,opt,name=bot_token,json=botToken,proto3" json:"bot_token,omitempty"`
|
||||
ClientToken string `protobuf:"bytes,3,opt,name=client_token,json=clientToken,proto3" json:"client_token,omitempty"`
|
||||
AppToken string `protobuf:"bytes,1,opt,name=app_token,json=appToken,proto3" json:"app_token,omitempty"`
|
||||
BotToken string `protobuf:"bytes,2,opt,name=bot_token,json=botToken,proto3" json:"bot_token,omitempty"`
|
||||
}
|
||||
|
||||
func (x *SlackTokens) Reset() {
|
||||
@@ -703,13 +702,6 @@ func (x *SlackTokens) GetBotToken() string {
|
||||
return ""
|
||||
}
|
||||
|
||||
func (x *SlackTokens) GetClientToken() string {
|
||||
if x != nil {
|
||||
return x.ClientToken
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
var File_credentials_proto protoreflect.FileDescriptor
|
||||
|
||||
var file_credentials_proto_rawDesc = []byte{
|
||||
@@ -771,18 +763,16 @@ var file_credentials_proto_rawDesc = []byte{
|
||||
0x6c, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52,
|
||||
0x0e, 0x69, 0x6e, 0x73, 0x74, 0x61, 0x6c, 0x6c, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x64, 0x12,
|
||||
0x15, 0x0a, 0x06, 0x61, 0x70, 0x70, 0x5f, 0x69, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52,
|
||||
0x05, 0x61, 0x70, 0x70, 0x49, 0x64, 0x22, 0x6a, 0x0a, 0x0b, 0x53, 0x6c, 0x61, 0x63, 0x6b, 0x54,
|
||||
0x05, 0x61, 0x70, 0x70, 0x49, 0x64, 0x22, 0x47, 0x0a, 0x0b, 0x53, 0x6c, 0x61, 0x63, 0x6b, 0x54,
|
||||
0x6f, 0x6b, 0x65, 0x6e, 0x73, 0x12, 0x1b, 0x0a, 0x09, 0x61, 0x70, 0x70, 0x5f, 0x74, 0x6f, 0x6b,
|
||||
0x65, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x61, 0x70, 0x70, 0x54, 0x6f, 0x6b,
|
||||
0x65, 0x6e, 0x12, 0x1b, 0x0a, 0x09, 0x62, 0x6f, 0x74, 0x5f, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x18,
|
||||
0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x62, 0x6f, 0x74, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x12,
|
||||
0x21, 0x0a, 0x0c, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x5f, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x18,
|
||||
0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x54, 0x6f, 0x6b,
|
||||
0x65, 0x6e, 0x42, 0x3f, 0x5a, 0x3d, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d,
|
||||
0x2f, 0x74, 0x72, 0x75, 0x66, 0x66, 0x6c, 0x65, 0x73, 0x65, 0x63, 0x75, 0x72, 0x69, 0x74, 0x79,
|
||||
0x2f, 0x74, 0x72, 0x75, 0x66, 0x66, 0x6c, 0x65, 0x68, 0x6f, 0x67, 0x2f, 0x76, 0x33, 0x2f, 0x70,
|
||||
0x6b, 0x67, 0x2f, 0x70, 0x62, 0x2f, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c,
|
||||
0x73, 0x70, 0x62, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
|
||||
0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x62, 0x6f, 0x74, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x42,
|
||||
0x3f, 0x5a, 0x3d, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x74, 0x72,
|
||||
0x75, 0x66, 0x66, 0x6c, 0x65, 0x73, 0x65, 0x63, 0x75, 0x72, 0x69, 0x74, 0x79, 0x2f, 0x74, 0x72,
|
||||
0x75, 0x66, 0x66, 0x6c, 0x65, 0x68, 0x6f, 0x67, 0x2f, 0x76, 0x33, 0x2f, 0x70, 0x6b, 0x67, 0x2f,
|
||||
0x70, 0x62, 0x2f, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x73, 0x70, 0x62,
|
||||
0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
|
||||
}
|
||||
|
||||
var (
|
||||
|
||||
@@ -1229,8 +1229,6 @@ func (m *SlackTokens) validate(all bool) error {
|
||||
|
||||
// no validation rules for BotToken
|
||||
|
||||
// no validation rules for ClientToken
|
||||
|
||||
if len(errors) > 0 {
|
||||
return SlackTokensMultiError(errors)
|
||||
}
|
||||
|
||||
@@ -835,7 +835,7 @@ const (
|
||||
DetectorType_Iexapis DetectorType = 813
|
||||
DetectorType_Reachmail DetectorType = 814
|
||||
DetectorType_Chartmogul DetectorType = 815
|
||||
DetectorType_Appointedd DetectorType = 816
|
||||
DetectorType_Appointed DetectorType = 816
|
||||
DetectorType_Wit DetectorType = 817
|
||||
DetectorType_RechargePayments DetectorType = 818
|
||||
DetectorType_Diggernaut DetectorType = 819
|
||||
@@ -896,7 +896,6 @@ const (
|
||||
DetectorType_Api2Convert DetectorType = 874
|
||||
DetectorType_Opsgenie DetectorType = 875
|
||||
DetectorType_Gemini DetectorType = 876
|
||||
DetectorType_Honeycomb DetectorType = 877
|
||||
)
|
||||
|
||||
// Enum value maps for DetectorType.
|
||||
@@ -1714,7 +1713,7 @@ var (
|
||||
813: "Iexapis",
|
||||
814: "Reachmail",
|
||||
815: "Chartmogul",
|
||||
816: "Appointedd",
|
||||
816: "Appointed",
|
||||
817: "Wit",
|
||||
818: "RechargePayments",
|
||||
819: "Diggernaut",
|
||||
@@ -1775,7 +1774,6 @@ var (
|
||||
874: "Api2Convert",
|
||||
875: "Opsgenie",
|
||||
876: "Gemini",
|
||||
877: "Honeycomb",
|
||||
}
|
||||
DetectorType_value = map[string]int32{
|
||||
"Alibaba": 0,
|
||||
@@ -2590,7 +2588,7 @@ var (
|
||||
"Iexapis": 813,
|
||||
"Reachmail": 814,
|
||||
"Chartmogul": 815,
|
||||
"Appointedd": 816,
|
||||
"Appointed": 816,
|
||||
"Wit": 817,
|
||||
"RechargePayments": 818,
|
||||
"Diggernaut": 819,
|
||||
@@ -2651,7 +2649,6 @@ var (
|
||||
"Api2Convert": 874,
|
||||
"Opsgenie": 875,
|
||||
"Gemini": 876,
|
||||
"Honeycomb": 877,
|
||||
}
|
||||
)
|
||||
|
||||
@@ -3001,7 +2998,7 @@ var file_detectors_proto_rawDesc = []byte{
|
||||
0x16, 0x70, 0x75, 0x62, 0x6c, 0x69, 0x63, 0x5f, 0x6b, 0x65, 0x79, 0x5f, 0x66, 0x69, 0x6e, 0x67,
|
||||
0x65, 0x72, 0x70, 0x72, 0x69, 0x6e, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x14, 0x70,
|
||||
0x75, 0x62, 0x6c, 0x69, 0x63, 0x4b, 0x65, 0x79, 0x46, 0x69, 0x6e, 0x67, 0x65, 0x72, 0x70, 0x72,
|
||||
0x69, 0x6e, 0x74, 0x2a, 0xfe, 0x6d, 0x0a, 0x0c, 0x44, 0x65, 0x74, 0x65, 0x63, 0x74, 0x6f, 0x72,
|
||||
0x69, 0x6e, 0x74, 0x2a, 0xed, 0x6d, 0x0a, 0x0c, 0x44, 0x65, 0x74, 0x65, 0x63, 0x74, 0x6f, 0x72,
|
||||
0x54, 0x79, 0x70, 0x65, 0x12, 0x0b, 0x0a, 0x07, 0x41, 0x6c, 0x69, 0x62, 0x61, 0x62, 0x61, 0x10,
|
||||
0x00, 0x12, 0x08, 0x0a, 0x04, 0x41, 0x4d, 0x51, 0x50, 0x10, 0x01, 0x12, 0x07, 0x0a, 0x03, 0x41,
|
||||
0x57, 0x53, 0x10, 0x02, 0x12, 0x09, 0x0a, 0x05, 0x41, 0x7a, 0x75, 0x72, 0x65, 0x10, 0x03, 0x12,
|
||||
@@ -3820,72 +3817,71 @@ var file_detectors_proto_rawDesc = []byte{
|
||||
0x06, 0x12, 0x0c, 0x0a, 0x07, 0x49, 0x65, 0x78, 0x61, 0x70, 0x69, 0x73, 0x10, 0xad, 0x06, 0x12,
|
||||
0x0e, 0x0a, 0x09, 0x52, 0x65, 0x61, 0x63, 0x68, 0x6d, 0x61, 0x69, 0x6c, 0x10, 0xae, 0x06, 0x12,
|
||||
0x0f, 0x0a, 0x0a, 0x43, 0x68, 0x61, 0x72, 0x74, 0x6d, 0x6f, 0x67, 0x75, 0x6c, 0x10, 0xaf, 0x06,
|
||||
0x12, 0x0f, 0x0a, 0x0a, 0x41, 0x70, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x65, 0x64, 0x64, 0x10, 0xb0,
|
||||
0x06, 0x12, 0x08, 0x0a, 0x03, 0x57, 0x69, 0x74, 0x10, 0xb1, 0x06, 0x12, 0x15, 0x0a, 0x10, 0x52,
|
||||
0x65, 0x63, 0x68, 0x61, 0x72, 0x67, 0x65, 0x50, 0x61, 0x79, 0x6d, 0x65, 0x6e, 0x74, 0x73, 0x10,
|
||||
0xb2, 0x06, 0x12, 0x0f, 0x0a, 0x0a, 0x44, 0x69, 0x67, 0x67, 0x65, 0x72, 0x6e, 0x61, 0x75, 0x74,
|
||||
0x10, 0xb3, 0x06, 0x12, 0x10, 0x0a, 0x0b, 0x4d, 0x6f, 0x6e, 0x6b, 0x65, 0x79, 0x4c, 0x65, 0x61,
|
||||
0x72, 0x6e, 0x10, 0xb4, 0x06, 0x12, 0x0a, 0x0a, 0x05, 0x44, 0x75, 0x70, 0x6c, 0x79, 0x10, 0xb5,
|
||||
0x06, 0x12, 0x0e, 0x0a, 0x09, 0x50, 0x6f, 0x73, 0x74, 0x62, 0x61, 0x63, 0x6b, 0x73, 0x10, 0xb6,
|
||||
0x06, 0x12, 0x0d, 0x0a, 0x08, 0x43, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x32, 0x10, 0xb7, 0x06,
|
||||
0x12, 0x0c, 0x0a, 0x07, 0x5a, 0x65, 0x6e, 0x52, 0x6f, 0x77, 0x73, 0x10, 0xb8, 0x06, 0x12, 0x10,
|
||||
0x0a, 0x0b, 0x5a, 0x69, 0x70, 0x63, 0x6f, 0x64, 0x65, 0x62, 0x61, 0x73, 0x65, 0x10, 0xb9, 0x06,
|
||||
0x12, 0x0b, 0x0a, 0x06, 0x54, 0x65, 0x66, 0x74, 0x65, 0x72, 0x10, 0xba, 0x06, 0x12, 0x0a, 0x0a,
|
||||
0x05, 0x54, 0x77, 0x69, 0x73, 0x74, 0x10, 0xbb, 0x06, 0x12, 0x16, 0x0a, 0x11, 0x42, 0x72, 0x61,
|
||||
0x69, 0x6e, 0x74, 0x72, 0x65, 0x65, 0x50, 0x61, 0x79, 0x6d, 0x65, 0x6e, 0x74, 0x73, 0x10, 0xbc,
|
||||
0x06, 0x12, 0x11, 0x0a, 0x0c, 0x43, 0x6c, 0x6f, 0x75, 0x64, 0x43, 0x6f, 0x6e, 0x76, 0x65, 0x72,
|
||||
0x74, 0x10, 0xbd, 0x06, 0x12, 0x0c, 0x0a, 0x07, 0x47, 0x72, 0x61, 0x66, 0x61, 0x6e, 0x61, 0x10,
|
||||
0xbe, 0x06, 0x12, 0x0f, 0x0a, 0x0a, 0x43, 0x6f, 0x6e, 0x76, 0x65, 0x72, 0x74, 0x41, 0x70, 0x69,
|
||||
0x10, 0xbf, 0x06, 0x12, 0x11, 0x0a, 0x0c, 0x54, 0x72, 0x61, 0x6e, 0x73, 0x66, 0x65, 0x72, 0x77,
|
||||
0x69, 0x73, 0x65, 0x10, 0xc0, 0x06, 0x12, 0x0c, 0x0a, 0x07, 0x42, 0x75, 0x6c, 0x6b, 0x73, 0x6d,
|
||||
0x73, 0x10, 0xc1, 0x06, 0x12, 0x0c, 0x0a, 0x07, 0x44, 0x61, 0x74, 0x61, 0x62, 0x6f, 0x78, 0x10,
|
||||
0xc2, 0x06, 0x12, 0x0e, 0x0a, 0x09, 0x4f, 0x6e, 0x65, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x6c, 0x10,
|
||||
0xc3, 0x06, 0x12, 0x0c, 0x0a, 0x07, 0x52, 0x65, 0x6e, 0x74, 0x6d, 0x61, 0x6e, 0x10, 0xc4, 0x06,
|
||||
0x12, 0x0c, 0x0a, 0x07, 0x50, 0x61, 0x72, 0x73, 0x65, 0x75, 0x72, 0x10, 0xc5, 0x06, 0x12, 0x0e,
|
||||
0x0a, 0x09, 0x44, 0x6f, 0x63, 0x70, 0x61, 0x72, 0x73, 0x65, 0x72, 0x10, 0xc6, 0x06, 0x12, 0x0d,
|
||||
0x0a, 0x08, 0x46, 0x6f, 0x72, 0x6d, 0x73, 0x69, 0x74, 0x65, 0x10, 0xc7, 0x06, 0x12, 0x11, 0x0a,
|
||||
0x0c, 0x54, 0x69, 0x63, 0x6b, 0x65, 0x74, 0x74, 0x61, 0x69, 0x6c, 0x6f, 0x72, 0x10, 0xc8, 0x06,
|
||||
0x12, 0x0c, 0x0a, 0x07, 0x4c, 0x65, 0x6d, 0x6c, 0x69, 0x73, 0x74, 0x10, 0xc9, 0x06, 0x12, 0x0c,
|
||||
0x0a, 0x07, 0x50, 0x72, 0x6f, 0x64, 0x70, 0x61, 0x64, 0x10, 0xca, 0x06, 0x12, 0x0e, 0x0a, 0x09,
|
||||
0x46, 0x6f, 0x72, 0x6d, 0x73, 0x74, 0x61, 0x63, 0x6b, 0x10, 0xcb, 0x06, 0x12, 0x10, 0x0a, 0x0b,
|
||||
0x43, 0x6f, 0x64, 0x65, 0x63, 0x6c, 0x69, 0x6d, 0x61, 0x74, 0x65, 0x10, 0xcc, 0x06, 0x12, 0x0e,
|
||||
0x0a, 0x09, 0x43, 0x6f, 0x64, 0x65, 0x6d, 0x61, 0x67, 0x69, 0x63, 0x10, 0xcd, 0x06, 0x12, 0x0a,
|
||||
0x0a, 0x05, 0x56, 0x62, 0x6f, 0x75, 0x74, 0x10, 0xce, 0x06, 0x12, 0x0e, 0x0a, 0x09, 0x4e, 0x69,
|
||||
0x67, 0x68, 0x74, 0x66, 0x61, 0x6c, 0x6c, 0x10, 0xcf, 0x06, 0x12, 0x0f, 0x0a, 0x0a, 0x46, 0x6c,
|
||||
0x69, 0x67, 0x68, 0x74, 0x4c, 0x61, 0x62, 0x73, 0x10, 0xd0, 0x06, 0x12, 0x11, 0x0a, 0x0c, 0x53,
|
||||
0x70, 0x65, 0x65, 0x63, 0x68, 0x54, 0x65, 0x78, 0x74, 0x41, 0x49, 0x10, 0xd1, 0x06, 0x12, 0x0d,
|
||||
0x0a, 0x08, 0x50, 0x6f, 0x6c, 0x6c, 0x73, 0x41, 0x50, 0x49, 0x10, 0xd2, 0x06, 0x12, 0x0b, 0x0a,
|
||||
0x06, 0x53, 0x69, 0x6d, 0x46, 0x69, 0x6e, 0x10, 0xd3, 0x06, 0x12, 0x0a, 0x0a, 0x05, 0x53, 0x63,
|
||||
0x61, 0x6c, 0x72, 0x10, 0xd4, 0x06, 0x12, 0x0f, 0x0a, 0x0a, 0x4b, 0x61, 0x6e, 0x62, 0x61, 0x6e,
|
||||
0x74, 0x6f, 0x6f, 0x6c, 0x10, 0xd5, 0x06, 0x12, 0x10, 0x0a, 0x0b, 0x42, 0x72, 0x69, 0x67, 0x68,
|
||||
0x74, 0x6c, 0x6f, 0x63, 0x61, 0x6c, 0x10, 0xd6, 0x06, 0x12, 0x0c, 0x0a, 0x07, 0x48, 0x6f, 0x74,
|
||||
0x77, 0x69, 0x72, 0x65, 0x10, 0xd7, 0x06, 0x12, 0x0d, 0x0a, 0x08, 0x49, 0x6e, 0x73, 0x74, 0x61,
|
||||
0x62, 0x6f, 0x74, 0x10, 0xd8, 0x06, 0x12, 0x0c, 0x0a, 0x07, 0x54, 0x69, 0x6d, 0x65, 0x6b, 0x69,
|
||||
0x74, 0x10, 0xd9, 0x06, 0x12, 0x10, 0x0a, 0x0b, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x73, 0x65, 0x6c,
|
||||
0x6c, 0x65, 0x72, 0x10, 0xda, 0x06, 0x12, 0x11, 0x0a, 0x0c, 0x4d, 0x6f, 0x6a, 0x6f, 0x68, 0x65,
|
||||
0x6c, 0x70, 0x64, 0x65, 0x73, 0x6b, 0x10, 0xdb, 0x06, 0x12, 0x0f, 0x0a, 0x0a, 0x43, 0x72, 0x65,
|
||||
0x61, 0x74, 0x65, 0x73, 0x65, 0x6e, 0x64, 0x10, 0xdc, 0x06, 0x12, 0x10, 0x0a, 0x0b, 0x47, 0x65,
|
||||
0x74, 0x72, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x10, 0xdd, 0x06, 0x12, 0x0c, 0x0a, 0x07,
|
||||
0x44, 0x79, 0x6e, 0x61, 0x64, 0x6f, 0x74, 0x10, 0xde, 0x06, 0x12, 0x0a, 0x0a, 0x05, 0x44, 0x65,
|
||||
0x6d, 0x69, 0x6f, 0x10, 0xdf, 0x06, 0x12, 0x0b, 0x0a, 0x06, 0x54, 0x6f, 0x6b, 0x65, 0x65, 0x74,
|
||||
0x10, 0xe0, 0x06, 0x12, 0x11, 0x0a, 0x0c, 0x4d, 0x79, 0x65, 0x78, 0x70, 0x65, 0x72, 0x69, 0x6d,
|
||||
0x65, 0x6e, 0x74, 0x10, 0xe1, 0x06, 0x12, 0x0e, 0x0a, 0x09, 0x43, 0x6f, 0x70, 0x79, 0x73, 0x63,
|
||||
0x61, 0x70, 0x65, 0x10, 0xe2, 0x06, 0x12, 0x0d, 0x0a, 0x08, 0x42, 0x65, 0x73, 0x6e, 0x61, 0x70,
|
||||
0x70, 0x79, 0x10, 0xe3, 0x06, 0x12, 0x0e, 0x0a, 0x09, 0x53, 0x61, 0x6c, 0x65, 0x73, 0x6d, 0x61,
|
||||
0x74, 0x65, 0x10, 0xe4, 0x06, 0x12, 0x0f, 0x0a, 0x0a, 0x48, 0x65, 0x61, 0x74, 0x6d, 0x61, 0x70,
|
||||
0x61, 0x70, 0x69, 0x10, 0xe5, 0x06, 0x12, 0x11, 0x0a, 0x0c, 0x57, 0x65, 0x62, 0x73, 0x69, 0x74,
|
||||
0x65, 0x70, 0x75, 0x6c, 0x73, 0x65, 0x10, 0xe6, 0x06, 0x12, 0x0e, 0x0a, 0x09, 0x55, 0x63, 0x6c,
|
||||
0x61, 0x73, 0x73, 0x69, 0x66, 0x79, 0x10, 0xe7, 0x06, 0x12, 0x0c, 0x0a, 0x07, 0x43, 0x6f, 0x6e,
|
||||
0x76, 0x65, 0x72, 0x74, 0x10, 0xe8, 0x06, 0x12, 0x0d, 0x0a, 0x08, 0x50, 0x44, 0x46, 0x6d, 0x79,
|
||||
0x55, 0x52, 0x4c, 0x10, 0xe9, 0x06, 0x12, 0x10, 0x0a, 0x0b, 0x41, 0x70, 0x69, 0x32, 0x43, 0x6f,
|
||||
0x6e, 0x76, 0x65, 0x72, 0x74, 0x10, 0xea, 0x06, 0x12, 0x0d, 0x0a, 0x08, 0x4f, 0x70, 0x73, 0x67,
|
||||
0x65, 0x6e, 0x69, 0x65, 0x10, 0xeb, 0x06, 0x12, 0x0b, 0x0a, 0x06, 0x47, 0x65, 0x6d, 0x69, 0x6e,
|
||||
0x69, 0x10, 0xec, 0x06, 0x12, 0x0e, 0x0a, 0x09, 0x48, 0x6f, 0x6e, 0x65, 0x79, 0x63, 0x6f, 0x6d,
|
||||
0x62, 0x10, 0xed, 0x06, 0x42, 0x3d, 0x5a, 0x3b, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63,
|
||||
0x6f, 0x6d, 0x2f, 0x74, 0x72, 0x75, 0x66, 0x66, 0x6c, 0x65, 0x73, 0x65, 0x63, 0x75, 0x72, 0x69,
|
||||
0x74, 0x79, 0x2f, 0x74, 0x72, 0x75, 0x66, 0x66, 0x6c, 0x65, 0x68, 0x6f, 0x67, 0x2f, 0x76, 0x33,
|
||||
0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x70, 0x62, 0x2f, 0x64, 0x65, 0x74, 0x65, 0x63, 0x74, 0x6f, 0x72,
|
||||
0x73, 0x70, 0x62, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
|
||||
0x12, 0x0e, 0x0a, 0x09, 0x41, 0x70, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x65, 0x64, 0x10, 0xb0, 0x06,
|
||||
0x12, 0x08, 0x0a, 0x03, 0x57, 0x69, 0x74, 0x10, 0xb1, 0x06, 0x12, 0x15, 0x0a, 0x10, 0x52, 0x65,
|
||||
0x63, 0x68, 0x61, 0x72, 0x67, 0x65, 0x50, 0x61, 0x79, 0x6d, 0x65, 0x6e, 0x74, 0x73, 0x10, 0xb2,
|
||||
0x06, 0x12, 0x0f, 0x0a, 0x0a, 0x44, 0x69, 0x67, 0x67, 0x65, 0x72, 0x6e, 0x61, 0x75, 0x74, 0x10,
|
||||
0xb3, 0x06, 0x12, 0x10, 0x0a, 0x0b, 0x4d, 0x6f, 0x6e, 0x6b, 0x65, 0x79, 0x4c, 0x65, 0x61, 0x72,
|
||||
0x6e, 0x10, 0xb4, 0x06, 0x12, 0x0a, 0x0a, 0x05, 0x44, 0x75, 0x70, 0x6c, 0x79, 0x10, 0xb5, 0x06,
|
||||
0x12, 0x0e, 0x0a, 0x09, 0x50, 0x6f, 0x73, 0x74, 0x62, 0x61, 0x63, 0x6b, 0x73, 0x10, 0xb6, 0x06,
|
||||
0x12, 0x0d, 0x0a, 0x08, 0x43, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x32, 0x10, 0xb7, 0x06, 0x12,
|
||||
0x0c, 0x0a, 0x07, 0x5a, 0x65, 0x6e, 0x52, 0x6f, 0x77, 0x73, 0x10, 0xb8, 0x06, 0x12, 0x10, 0x0a,
|
||||
0x0b, 0x5a, 0x69, 0x70, 0x63, 0x6f, 0x64, 0x65, 0x62, 0x61, 0x73, 0x65, 0x10, 0xb9, 0x06, 0x12,
|
||||
0x0b, 0x0a, 0x06, 0x54, 0x65, 0x66, 0x74, 0x65, 0x72, 0x10, 0xba, 0x06, 0x12, 0x0a, 0x0a, 0x05,
|
||||
0x54, 0x77, 0x69, 0x73, 0x74, 0x10, 0xbb, 0x06, 0x12, 0x16, 0x0a, 0x11, 0x42, 0x72, 0x61, 0x69,
|
||||
0x6e, 0x74, 0x72, 0x65, 0x65, 0x50, 0x61, 0x79, 0x6d, 0x65, 0x6e, 0x74, 0x73, 0x10, 0xbc, 0x06,
|
||||
0x12, 0x11, 0x0a, 0x0c, 0x43, 0x6c, 0x6f, 0x75, 0x64, 0x43, 0x6f, 0x6e, 0x76, 0x65, 0x72, 0x74,
|
||||
0x10, 0xbd, 0x06, 0x12, 0x0c, 0x0a, 0x07, 0x47, 0x72, 0x61, 0x66, 0x61, 0x6e, 0x61, 0x10, 0xbe,
|
||||
0x06, 0x12, 0x0f, 0x0a, 0x0a, 0x43, 0x6f, 0x6e, 0x76, 0x65, 0x72, 0x74, 0x41, 0x70, 0x69, 0x10,
|
||||
0xbf, 0x06, 0x12, 0x11, 0x0a, 0x0c, 0x54, 0x72, 0x61, 0x6e, 0x73, 0x66, 0x65, 0x72, 0x77, 0x69,
|
||||
0x73, 0x65, 0x10, 0xc0, 0x06, 0x12, 0x0c, 0x0a, 0x07, 0x42, 0x75, 0x6c, 0x6b, 0x73, 0x6d, 0x73,
|
||||
0x10, 0xc1, 0x06, 0x12, 0x0c, 0x0a, 0x07, 0x44, 0x61, 0x74, 0x61, 0x62, 0x6f, 0x78, 0x10, 0xc2,
|
||||
0x06, 0x12, 0x0e, 0x0a, 0x09, 0x4f, 0x6e, 0x65, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x6c, 0x10, 0xc3,
|
||||
0x06, 0x12, 0x0c, 0x0a, 0x07, 0x52, 0x65, 0x6e, 0x74, 0x6d, 0x61, 0x6e, 0x10, 0xc4, 0x06, 0x12,
|
||||
0x0c, 0x0a, 0x07, 0x50, 0x61, 0x72, 0x73, 0x65, 0x75, 0x72, 0x10, 0xc5, 0x06, 0x12, 0x0e, 0x0a,
|
||||
0x09, 0x44, 0x6f, 0x63, 0x70, 0x61, 0x72, 0x73, 0x65, 0x72, 0x10, 0xc6, 0x06, 0x12, 0x0d, 0x0a,
|
||||
0x08, 0x46, 0x6f, 0x72, 0x6d, 0x73, 0x69, 0x74, 0x65, 0x10, 0xc7, 0x06, 0x12, 0x11, 0x0a, 0x0c,
|
||||
0x54, 0x69, 0x63, 0x6b, 0x65, 0x74, 0x74, 0x61, 0x69, 0x6c, 0x6f, 0x72, 0x10, 0xc8, 0x06, 0x12,
|
||||
0x0c, 0x0a, 0x07, 0x4c, 0x65, 0x6d, 0x6c, 0x69, 0x73, 0x74, 0x10, 0xc9, 0x06, 0x12, 0x0c, 0x0a,
|
||||
0x07, 0x50, 0x72, 0x6f, 0x64, 0x70, 0x61, 0x64, 0x10, 0xca, 0x06, 0x12, 0x0e, 0x0a, 0x09, 0x46,
|
||||
0x6f, 0x72, 0x6d, 0x73, 0x74, 0x61, 0x63, 0x6b, 0x10, 0xcb, 0x06, 0x12, 0x10, 0x0a, 0x0b, 0x43,
|
||||
0x6f, 0x64, 0x65, 0x63, 0x6c, 0x69, 0x6d, 0x61, 0x74, 0x65, 0x10, 0xcc, 0x06, 0x12, 0x0e, 0x0a,
|
||||
0x09, 0x43, 0x6f, 0x64, 0x65, 0x6d, 0x61, 0x67, 0x69, 0x63, 0x10, 0xcd, 0x06, 0x12, 0x0a, 0x0a,
|
||||
0x05, 0x56, 0x62, 0x6f, 0x75, 0x74, 0x10, 0xce, 0x06, 0x12, 0x0e, 0x0a, 0x09, 0x4e, 0x69, 0x67,
|
||||
0x68, 0x74, 0x66, 0x61, 0x6c, 0x6c, 0x10, 0xcf, 0x06, 0x12, 0x0f, 0x0a, 0x0a, 0x46, 0x6c, 0x69,
|
||||
0x67, 0x68, 0x74, 0x4c, 0x61, 0x62, 0x73, 0x10, 0xd0, 0x06, 0x12, 0x11, 0x0a, 0x0c, 0x53, 0x70,
|
||||
0x65, 0x65, 0x63, 0x68, 0x54, 0x65, 0x78, 0x74, 0x41, 0x49, 0x10, 0xd1, 0x06, 0x12, 0x0d, 0x0a,
|
||||
0x08, 0x50, 0x6f, 0x6c, 0x6c, 0x73, 0x41, 0x50, 0x49, 0x10, 0xd2, 0x06, 0x12, 0x0b, 0x0a, 0x06,
|
||||
0x53, 0x69, 0x6d, 0x46, 0x69, 0x6e, 0x10, 0xd3, 0x06, 0x12, 0x0a, 0x0a, 0x05, 0x53, 0x63, 0x61,
|
||||
0x6c, 0x72, 0x10, 0xd4, 0x06, 0x12, 0x0f, 0x0a, 0x0a, 0x4b, 0x61, 0x6e, 0x62, 0x61, 0x6e, 0x74,
|
||||
0x6f, 0x6f, 0x6c, 0x10, 0xd5, 0x06, 0x12, 0x10, 0x0a, 0x0b, 0x42, 0x72, 0x69, 0x67, 0x68, 0x74,
|
||||
0x6c, 0x6f, 0x63, 0x61, 0x6c, 0x10, 0xd6, 0x06, 0x12, 0x0c, 0x0a, 0x07, 0x48, 0x6f, 0x74, 0x77,
|
||||
0x69, 0x72, 0x65, 0x10, 0xd7, 0x06, 0x12, 0x0d, 0x0a, 0x08, 0x49, 0x6e, 0x73, 0x74, 0x61, 0x62,
|
||||
0x6f, 0x74, 0x10, 0xd8, 0x06, 0x12, 0x0c, 0x0a, 0x07, 0x54, 0x69, 0x6d, 0x65, 0x6b, 0x69, 0x74,
|
||||
0x10, 0xd9, 0x06, 0x12, 0x10, 0x0a, 0x0b, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x73, 0x65, 0x6c, 0x6c,
|
||||
0x65, 0x72, 0x10, 0xda, 0x06, 0x12, 0x11, 0x0a, 0x0c, 0x4d, 0x6f, 0x6a, 0x6f, 0x68, 0x65, 0x6c,
|
||||
0x70, 0x64, 0x65, 0x73, 0x6b, 0x10, 0xdb, 0x06, 0x12, 0x0f, 0x0a, 0x0a, 0x43, 0x72, 0x65, 0x61,
|
||||
0x74, 0x65, 0x73, 0x65, 0x6e, 0x64, 0x10, 0xdc, 0x06, 0x12, 0x10, 0x0a, 0x0b, 0x47, 0x65, 0x74,
|
||||
0x72, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x10, 0xdd, 0x06, 0x12, 0x0c, 0x0a, 0x07, 0x44,
|
||||
0x79, 0x6e, 0x61, 0x64, 0x6f, 0x74, 0x10, 0xde, 0x06, 0x12, 0x0a, 0x0a, 0x05, 0x44, 0x65, 0x6d,
|
||||
0x69, 0x6f, 0x10, 0xdf, 0x06, 0x12, 0x0b, 0x0a, 0x06, 0x54, 0x6f, 0x6b, 0x65, 0x65, 0x74, 0x10,
|
||||
0xe0, 0x06, 0x12, 0x11, 0x0a, 0x0c, 0x4d, 0x79, 0x65, 0x78, 0x70, 0x65, 0x72, 0x69, 0x6d, 0x65,
|
||||
0x6e, 0x74, 0x10, 0xe1, 0x06, 0x12, 0x0e, 0x0a, 0x09, 0x43, 0x6f, 0x70, 0x79, 0x73, 0x63, 0x61,
|
||||
0x70, 0x65, 0x10, 0xe2, 0x06, 0x12, 0x0d, 0x0a, 0x08, 0x42, 0x65, 0x73, 0x6e, 0x61, 0x70, 0x70,
|
||||
0x79, 0x10, 0xe3, 0x06, 0x12, 0x0e, 0x0a, 0x09, 0x53, 0x61, 0x6c, 0x65, 0x73, 0x6d, 0x61, 0x74,
|
||||
0x65, 0x10, 0xe4, 0x06, 0x12, 0x0f, 0x0a, 0x0a, 0x48, 0x65, 0x61, 0x74, 0x6d, 0x61, 0x70, 0x61,
|
||||
0x70, 0x69, 0x10, 0xe5, 0x06, 0x12, 0x11, 0x0a, 0x0c, 0x57, 0x65, 0x62, 0x73, 0x69, 0x74, 0x65,
|
||||
0x70, 0x75, 0x6c, 0x73, 0x65, 0x10, 0xe6, 0x06, 0x12, 0x0e, 0x0a, 0x09, 0x55, 0x63, 0x6c, 0x61,
|
||||
0x73, 0x73, 0x69, 0x66, 0x79, 0x10, 0xe7, 0x06, 0x12, 0x0c, 0x0a, 0x07, 0x43, 0x6f, 0x6e, 0x76,
|
||||
0x65, 0x72, 0x74, 0x10, 0xe8, 0x06, 0x12, 0x0d, 0x0a, 0x08, 0x50, 0x44, 0x46, 0x6d, 0x79, 0x55,
|
||||
0x52, 0x4c, 0x10, 0xe9, 0x06, 0x12, 0x10, 0x0a, 0x0b, 0x41, 0x70, 0x69, 0x32, 0x43, 0x6f, 0x6e,
|
||||
0x76, 0x65, 0x72, 0x74, 0x10, 0xea, 0x06, 0x12, 0x0d, 0x0a, 0x08, 0x4f, 0x70, 0x73, 0x67, 0x65,
|
||||
0x6e, 0x69, 0x65, 0x10, 0xeb, 0x06, 0x12, 0x0b, 0x0a, 0x06, 0x47, 0x65, 0x6d, 0x69, 0x6e, 0x69,
|
||||
0x10, 0xec, 0x06, 0x42, 0x3d, 0x5a, 0x3b, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f,
|
||||
0x6d, 0x2f, 0x74, 0x72, 0x75, 0x66, 0x66, 0x6c, 0x65, 0x73, 0x65, 0x63, 0x75, 0x72, 0x69, 0x74,
|
||||
0x79, 0x2f, 0x74, 0x72, 0x75, 0x66, 0x66, 0x6c, 0x65, 0x68, 0x6f, 0x67, 0x2f, 0x76, 0x33, 0x2f,
|
||||
0x70, 0x6b, 0x67, 0x2f, 0x70, 0x62, 0x2f, 0x64, 0x65, 0x74, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x73,
|
||||
0x70, 0x62, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
|
||||
}
|
||||
|
||||
var (
|
||||
|
||||
+35
-30
@@ -15,6 +15,7 @@ import (
|
||||
"time"
|
||||
|
||||
diskbufferreader "github.com/bill-rich/disk-buffer-reader"
|
||||
gitparse "github.com/bill-rich/gitparser"
|
||||
"github.com/go-errors/errors"
|
||||
"github.com/go-git/go-git/v5"
|
||||
"github.com/go-git/go-git/v5/plumbing"
|
||||
@@ -27,7 +28,6 @@ import (
|
||||
"google.golang.org/protobuf/types/known/anypb"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/gitparse"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/handlers"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/source_metadatapb"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
|
||||
@@ -266,12 +266,6 @@ func CloneRepoUsingUnauthenticated(url string, args ...string) (string, *git.Rep
|
||||
return CloneRepo(nil, url, args...)
|
||||
}
|
||||
|
||||
// CloneRepoUsingUnauthenticated clones a repo with no authentication required.
|
||||
func CloneRepoUsingSSH(gitUrl string, args ...string) (string, *git.Repository, error) {
|
||||
userInfo := url.User("git")
|
||||
return CloneRepo(userInfo, gitUrl, args...)
|
||||
}
|
||||
|
||||
func GitCmdCheck() error {
|
||||
if errors.Is(exec.Command("git").Run(), exec.ErrNotFound) {
|
||||
return fmt.Errorf("'git' command not found in $PATH. Make sure git is installed and included in $PATH")
|
||||
@@ -287,22 +281,24 @@ func (s *Git) ScanCommits(repo *git.Repository, path string, scanOptions *ScanOp
|
||||
zerolog.SetGlobalLevel(zerolog.Disabled)
|
||||
}
|
||||
|
||||
commitChan, err := gitparse.RepoPath(path, scanOptions.HeadHash)
|
||||
if err != nil {
|
||||
return err
|
||||
// Errors returned on errChan aren't blocking, so just ignore them.
|
||||
var gitLogArgs []string
|
||||
if scanOptions.HeadHash != "" {
|
||||
gitLogArgs = append(gitLogArgs, scanOptions.HeadHash)
|
||||
}
|
||||
if commitChan == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
commitChan := gitparse.BillParser(path)
|
||||
// parser can return nil chan and nil error
|
||||
// get the URL metadata for reporting (may be empty)
|
||||
urlMetadata := getSafeRemoteURL(repo, "origin")
|
||||
|
||||
var depth int64
|
||||
var reachedBase = false
|
||||
// for file := range fileChan {
|
||||
log.Debugf("Scanning repo")
|
||||
for commit := range commitChan {
|
||||
log.Debugf("Commit: %s", commit.Hash)
|
||||
for _, diff := range commit.Diffs {
|
||||
log.Debugf("Chunk: %s", diff.Content.Bytes())
|
||||
log.WithField("commit", commit.Hash).WithField("file", diff.PathB).Trace("Scanning file from git")
|
||||
if scanOptions.MaxDepth > 0 && depth >= scanOptions.MaxDepth {
|
||||
log.Debugf("reached max depth")
|
||||
@@ -318,10 +314,11 @@ func (s *Git) ScanCommits(repo *git.Repository, path string, scanOptions *ScanOp
|
||||
reachedBase = true
|
||||
}
|
||||
}
|
||||
|
||||
if !scanOptions.Filter.Pass(diff.PathB) {
|
||||
continue
|
||||
}
|
||||
/*
|
||||
if !scanOptions.Filter.Pass(diff.PathB) {
|
||||
continue
|
||||
}
|
||||
*/
|
||||
|
||||
fileName := diff.PathB
|
||||
if fileName == "" {
|
||||
@@ -330,7 +327,7 @@ func (s *Git) ScanCommits(repo *git.Repository, path string, scanOptions *ScanOp
|
||||
var email, hash, when string
|
||||
email = commit.Author
|
||||
hash = commit.Hash
|
||||
when = commit.Date.String()
|
||||
when = commit.Date.GoString()
|
||||
|
||||
// Handle binary files by reading the entire file rather than using the diff.
|
||||
if diff.IsBinary {
|
||||
@@ -344,11 +341,22 @@ func (s *Git) ScanCommits(repo *git.Repository, path string, scanOptions *ScanOp
|
||||
Verify: s.verify,
|
||||
}
|
||||
if err := handleBinary(repo, chunksChan, chunkSkel, commitHash, fileName); err != nil {
|
||||
log.WithError(err).WithField("file", fileName).Debug("Error handling binary file")
|
||||
log.WithError(err).Error("Error handling binary file")
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
/*
|
||||
for _, frag := range file.TextFragments {
|
||||
var sb strings.Builder
|
||||
newLineNumber := frag.NewPosition
|
||||
for _, line := range frag.Lines {
|
||||
if line.Op == gitdiff.OpAdd {
|
||||
sb.WriteString(line.Line)
|
||||
}
|
||||
}
|
||||
*/
|
||||
|
||||
metadata := s.sourceMetadataFunc(fileName, email, hash, when, urlMetadata, int64(diff.LineStart))
|
||||
chunksChan <- &sources.Chunk{
|
||||
SourceName: s.sourceName,
|
||||
@@ -358,6 +366,7 @@ func (s *Git) ScanCommits(repo *git.Repository, path string, scanOptions *ScanOp
|
||||
Data: diff.Content.Bytes(),
|
||||
Verify: s.verify,
|
||||
}
|
||||
//}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
@@ -382,9 +391,11 @@ func (s *Git) ScanUnstaged(repo *git.Repository, scanOptions *ScanOptions, chunk
|
||||
return err
|
||||
}
|
||||
for fh := range status {
|
||||
if !scanOptions.Filter.Pass(fh) {
|
||||
continue
|
||||
}
|
||||
/*
|
||||
if !scanOptions.Filter.Pass(fh) {
|
||||
continue
|
||||
}
|
||||
*/
|
||||
metadata := s.sourceMetadataFunc(
|
||||
fh, "unstaged", "unstaged", time.Now().String(), urlMetadata, 0,
|
||||
)
|
||||
@@ -587,13 +598,6 @@ func PrepareRepo(uriString string) (string, bool, error) {
|
||||
return path, remote, fmt.Errorf("failed to clone unauthenticated Git repo (%s): %s", remotePath, err)
|
||||
}
|
||||
}
|
||||
case "ssh":
|
||||
remotePath := uri.String()
|
||||
remote = true
|
||||
path, _, err = CloneRepoUsingSSH(remotePath)
|
||||
if err != nil {
|
||||
return path, remote, fmt.Errorf("failed to clone unauthenticated Git repo (%s): %s", remotePath, err)
|
||||
}
|
||||
default:
|
||||
return "", remote, fmt.Errorf("unsupported Git URI: %s", uriString)
|
||||
}
|
||||
@@ -631,6 +635,7 @@ func handleBinary(repo *git.Repository, chunksChan chan *sources.Chunk, chunkSke
|
||||
return err
|
||||
}
|
||||
|
||||
log.Error(path)
|
||||
file, err := commit.File(path)
|
||||
if err != nil {
|
||||
return err
|
||||
|
||||
@@ -232,9 +232,9 @@ func TestSource_Chunks_Integration(t *testing.T) {
|
||||
"ce62d79908803153ef6e145e042d3e80488ef747-bump": {B: []byte("\n")},
|
||||
// Normally we might expect to see this commit, and we may in the future.
|
||||
// But at the moment we're ignoring any commit unless it contains at least one non-space character.
|
||||
"27fbead3bf883cdb7de9d7825ed401f28f9398f1-slack": {B: []byte("yup, just did that\n\ngithub_lol: \"ffc7e0f9400fb6300167009e42d2f842cd7956e2\"\n\noh, goodness. there's another one!\n")},
|
||||
"8afb0ecd4998b1179e428db5ebbcdc8221214432-slack": {B: []byte("oops might drop a slack token here\n\ngithub_secret=\"369963c1434c377428ca8531fbc46c0c43d037a0\"\n\nyup, just did that\n"), Multi: true},
|
||||
"8fe6f04ef1839e3fc54b5147e3d0e0b7ab971bd5-aws": {B: []byte("blah blaj\n\nthis is the secret: AKIA2E0A8F3B244C9986\n\nokay thank you bye\n"), Multi: true},
|
||||
"27fbead3bf883cdb7de9d7825ed401f28f9398f1-slack": {B: []byte("yup, just did that\n\ngithub_lol: \"ffc7e0f9400fb6300167009e42d2f842cd7956e2\"\n\noh, goodness. there's another one!")},
|
||||
"8afb0ecd4998b1179e428db5ebbcdc8221214432-slack": {B: []byte("oops might drop a slack token here\n\ngithub_secret=\"369963c1434c377428ca8531fbc46c0c43d037a0\"\n\nyup, just did that"), Multi: true},
|
||||
"8fe6f04ef1839e3fc54b5147e3d0e0b7ab971bd5-aws": {B: []byte("blah blaj\n\nthis is the secret: AKIA2E0A8F3B244C9986\n\nokay thank you bye"), Multi: true},
|
||||
"84e9c75e388ae3e866e121087ea2dd45a71068f2-aws": {B: []byte("this is the secret: [Default]\nAccess key Id: AKIAILE3JG6KMS3HZGCA\nSecret Access Key: 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\n"), Multi: true},
|
||||
},
|
||||
},
|
||||
@@ -262,10 +262,11 @@ func TestSource_Chunks_Integration(t *testing.T) {
|
||||
}()
|
||||
|
||||
for chunk := range chunksCh {
|
||||
|
||||
key := ""
|
||||
switch meta := chunk.SourceMetadata.GetData().(type) {
|
||||
case *source_metadatapb.MetaData_Git:
|
||||
key = strings.TrimRight(meta.Git.Commit+"-"+meta.Git.File, "\n")
|
||||
key = meta.Git.Commit + "-" + meta.Git.File
|
||||
}
|
||||
|
||||
if expectedData, exists := tt.expectedChunkData[key]; !exists {
|
||||
|
||||
+26
-21
@@ -3,6 +3,7 @@ package s3
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
@@ -12,11 +13,9 @@ import (
|
||||
"github.com/aws/aws-sdk-go/aws/session"
|
||||
"github.com/aws/aws-sdk-go/service/s3"
|
||||
"github.com/aws/aws-sdk-go/service/s3/s3manager"
|
||||
diskbufferreader "github.com/bill-rich/disk-buffer-reader"
|
||||
"github.com/go-errors/errors"
|
||||
log "github.com/sirupsen/logrus"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/handlers"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/source_metadatapb"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/sanitizer"
|
||||
@@ -254,11 +253,29 @@ func (s *Source) pageChunker(ctx context.Context, client *s3.S3, chunksChan chan
|
||||
log.Debugf("Error Counts: %s:%d", prefix, nErr)
|
||||
return
|
||||
}
|
||||
|
||||
defer res.Body.Close()
|
||||
reader, err := diskbufferreader.New(res.Body)
|
||||
body, err := ioutil.ReadAll(res.Body)
|
||||
if err != nil {
|
||||
log.WithError(err).Error("Could not create reader.")
|
||||
s.log.WithError(err).Error("could not read S3 object body")
|
||||
nErr, ok := errorCount.Load(prefix)
|
||||
if !ok {
|
||||
nErr = 0
|
||||
}
|
||||
//too many consective errors on this page
|
||||
if nErr.(int) > 3 {
|
||||
log.Debugf("Skipped: %s", *obj.Key)
|
||||
return
|
||||
}
|
||||
nErr = nErr.(int) + 1
|
||||
errorCount.Store(prefix, nErr)
|
||||
|
||||
if nErr.(int) > 3 {
|
||||
s.log.Warnf("Too many consecutive errors. Excluding %s", prefix)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// ignore files that don't have secrets
|
||||
if common.SkipFile(*obj.Key, body) {
|
||||
return
|
||||
}
|
||||
|
||||
@@ -267,10 +284,11 @@ func (s *Source) pageChunker(ctx context.Context, client *s3.S3, chunksChan chan
|
||||
email = *obj.Owner.DisplayName
|
||||
}
|
||||
modified := obj.LastModified.String()
|
||||
chunkSkel := &sources.Chunk{
|
||||
chunk := sources.Chunk{
|
||||
SourceType: s.Type(),
|
||||
SourceName: s.name,
|
||||
SourceID: s.SourceID(),
|
||||
Data: body,
|
||||
SourceMetadata: &source_metadatapb.MetaData{
|
||||
Data: &source_metadatapb.MetaData_S3{
|
||||
S3: &source_metadatapb.S3{
|
||||
@@ -284,20 +302,6 @@ func (s *Source) pageChunker(ctx context.Context, client *s3.S3, chunksChan chan
|
||||
},
|
||||
Verify: s.verify,
|
||||
}
|
||||
if handlers.HandleFile(reader, chunkSkel, chunksChan) {
|
||||
return
|
||||
}
|
||||
|
||||
if err := reader.Reset(); err != nil {
|
||||
log.WithError(err).Error("Error resetting reader to start.")
|
||||
}
|
||||
reader.Stop()
|
||||
|
||||
for chunkData := range common.ChunkReader(reader) {
|
||||
chunk := *chunkSkel
|
||||
chunk.Data = chunkData
|
||||
chunksChan <- &chunk
|
||||
}
|
||||
nErr, ok = errorCount.Load(prefix)
|
||||
if !ok {
|
||||
nErr = 0
|
||||
@@ -305,6 +309,7 @@ func (s *Source) pageChunker(ctx context.Context, client *s3.S3, chunksChan chan
|
||||
if nErr.(int) > 0 {
|
||||
errorCount.Store(prefix, 0)
|
||||
}
|
||||
chunksChan <- &chunk
|
||||
}(ctx, &wg, sem, obj)
|
||||
}
|
||||
wg.Wait()
|
||||
|
||||
@@ -96,12 +96,12 @@ type Config struct {
|
||||
}
|
||||
|
||||
// NewConfig returns a new Config with optional values.
|
||||
func NewConfig(opts ...func(*Config)) Config {
|
||||
c := &Config{}
|
||||
func NewConfig(opts ...func(Config)) Config {
|
||||
c := Config{}
|
||||
for _, opt := range opts {
|
||||
opt(c)
|
||||
}
|
||||
return *c
|
||||
return c
|
||||
}
|
||||
|
||||
// PercentComplete is used to update job completion percentages across sources
|
||||
|
||||
@@ -63,5 +63,4 @@ message GitHubApp {
|
||||
message SlackTokens {
|
||||
string app_token = 1;
|
||||
string bot_token = 2;
|
||||
string client_token = 3;
|
||||
}
|
||||
|
||||
@@ -817,7 +817,7 @@ enum DetectorType {
|
||||
Iexapis = 813;
|
||||
Reachmail = 814;
|
||||
Chartmogul = 815;
|
||||
Appointedd = 816;
|
||||
Appointed = 816;
|
||||
Wit = 817;
|
||||
RechargePayments = 818;
|
||||
Diggernaut = 819;
|
||||
@@ -878,7 +878,6 @@ enum DetectorType {
|
||||
Api2Convert = 874;
|
||||
Opsgenie = 875;
|
||||
Gemini = 876;
|
||||
Honeycomb = 877;
|
||||
}
|
||||
|
||||
message Result {
|
||||
|
||||
Reference in New Issue
Block a user