Compare commits
27 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 30008b46a5 | |||
| cf2e56e483 | |||
| a076410b53 | |||
| 98edac9dfb | |||
| 9beee3cbba | |||
| 4a3c8e852c | |||
| 53813884b5 | |||
| caeef5c5ab | |||
| b55013cad3 | |||
| 9dfd307ff4 | |||
| 5c1aa826b1 | |||
| e7313cabf9 | |||
| 948f3ef375 | |||
| 0d755d82eb | |||
| 185a7635f3 | |||
| d0d521123a | |||
| bcab69b8ee | |||
| c84cf53ed6 | |||
| d6d6940cf9 | |||
| e0e29f5b5f | |||
| aa94718ba4 | |||
| 7e84313366 | |||
| f1a9dcff3f | |||
| c5662d76a2 | |||
| 9e8b24ae6b | |||
| 8ed8d6ab04 | |||
| d9f296e7d3 |
@@ -31,66 +31,6 @@ https://join.slack.com/t/trufflehog-community/shared_invite/zt-pw2qbi43-Aa86hkii
|
||||
docker run -it -v "$PWD:/pwd" trufflesecurity/trufflehog:latest github --org=trufflesecurity
|
||||
```
|
||||
|
||||
## Examples
|
||||
|
||||
### Example 1: Scan a repo for only verified secrets
|
||||
|
||||
Command:
|
||||
```
|
||||
trufflehog git https://github.com/trufflesecurity/test_keys --only-verified
|
||||
```
|
||||
|
||||
Expected output:
|
||||
```
|
||||
🐷🔑🐷 TruffleHog. Unearth your secrets. 🐷🔑🐷
|
||||
|
||||
Found verified result 🐷🔑
|
||||
Detector Type: AWS
|
||||
Decoder Type: PLAIN
|
||||
Raw result: AKIAYVP4CIPPERUVIFXG
|
||||
Line: 4
|
||||
Commit: fbc14303ffbf8fb1c2c1914e8dda7d0121633aca
|
||||
File: keys
|
||||
Email: counter <counter@counters-MacBook-Air.local>
|
||||
Repository: https://github.com/trufflesecurity/test_keys
|
||||
Timestamp: 2022-06-16 10:17:40 -0700 PDT
|
||||
...
|
||||
```
|
||||
|
||||
### Example 2: Scan a GitHub Org for only verified secrets
|
||||
|
||||
```
|
||||
trufflehog github --org=trufflesecurity --only-verified
|
||||
```
|
||||
|
||||
### Example 3: Scan a GitHub Repo for only verified keys and get JSON output
|
||||
|
||||
Command:
|
||||
```
|
||||
trufflehog git https://github.com/trufflesecurity/test_keys --only-verified --json
|
||||
```
|
||||
|
||||
Expected output:
|
||||
```
|
||||
{"SourceMetadata":{"Data":{"Git":{"commit":"fbc14303ffbf8fb1c2c1914e8dda7d0121633aca","file":"keys","email":"counter \u003ccounter@counters-MacBook-Air.local\u003e","repository":"https://github.com/trufflesecurity/test_keys","timestamp":"2022-06-16 10:17:40 -0700 PDT","line":4}}},"SourceID":0,"SourceType":16,"SourceName":"trufflehog - git","DetectorType":2,"DetectorName":"AWS","DecoderName":"PLAIN","Verified":true,"Raw":"AKIAYVP4CIPPERUVIFXG","Redacted":"AKIAYVP4CIPPERUVIFXG","ExtraData":{"account":"595918472158","arn":"arn:aws:iam::595918472158:user/canarytokens.com@@mirux23ppyky6hx3l6vclmhnj","user_id":"AIDAYVP4CIPPJ5M54LRCY"},"StructuredData":null}
|
||||
...
|
||||
```
|
||||
|
||||
### Example 4: Scan an S3 bucket for verified keys
|
||||
```
|
||||
trufflehog s3 --bucket=<bucket name> --only-verified
|
||||
```
|
||||
|
||||
### FAQ
|
||||
|
||||
+ All I see is `🐷🔑🐷 TruffleHog. Unearth your secrets. 🐷🔑🐷` and the program exits, what gives?
|
||||
+ That means no secrets were detected
|
||||
+ Why is the scan is taking a long time when I scan a GitHub org
|
||||
+ Unathenticated GitHub scans have rate limits. To improve your rate limits, include the `--token` flag with a personal access token
|
||||
+ It says a private key was verified, what does that mean?
|
||||
+ Check out our Driftwood blog post to learn how to do this, in short we've confirmed the key can be used live for SSH or SSL [Blog post](https://trufflesecurity.com/blog/driftwood-know-if-private-keys-are-sensitive/)
|
||||
|
||||
|
||||
# What's new in v3?
|
||||
|
||||
TruffleHog v3 is a complete rewrite in Go with many new powerful features.
|
||||
|
||||
@@ -18,11 +18,11 @@ require (
|
||||
github.com/bitfinexcom/bitfinex-api-go v0.0.0-20210608095005-9e0b26f200fb
|
||||
github.com/bradleyfalzon/ghinstallation/v2 v2.1.0
|
||||
github.com/crewjam/rfc5424 v0.1.0
|
||||
github.com/denisenkom/go-mssqldb v0.12.3
|
||||
github.com/envoyproxy/protoc-gen-validate v0.6.13
|
||||
github.com/denisenkom/go-mssqldb v0.12.2
|
||||
github.com/envoyproxy/protoc-gen-validate v0.6.8
|
||||
github.com/fatih/color v1.13.0
|
||||
github.com/felixge/fgprof v0.9.3
|
||||
github.com/getsentry/sentry-go v0.14.0
|
||||
github.com/getsentry/sentry-go v0.13.0
|
||||
github.com/go-errors/errors v1.4.2
|
||||
github.com/go-git/go-git/v5 v5.4.2
|
||||
github.com/go-logr/logr v1.2.3
|
||||
@@ -43,15 +43,16 @@ require (
|
||||
github.com/mholt/archiver/v4 v4.0.0-alpha.7
|
||||
github.com/paulbellamy/ratecounter v0.2.0
|
||||
github.com/pkg/errors v0.9.1
|
||||
github.com/razorpay/razorpay-go v0.0.0-20210728161131-0341409a6ab2
|
||||
github.com/rs/zerolog v1.28.0
|
||||
github.com/sergi/go-diff v1.2.0
|
||||
github.com/sirupsen/logrus v1.9.0
|
||||
github.com/stretchr/testify v1.8.1
|
||||
github.com/stretchr/testify v1.8.0
|
||||
github.com/tailscale/depaware v0.0.0-20210622194025-720c4b409502
|
||||
github.com/xanzy/go-gitlab v0.73.1
|
||||
go.mongodb.org/mongo-driver v1.10.3
|
||||
go.mongodb.org/mongo-driver v1.10.2
|
||||
go.uber.org/zap v1.23.0
|
||||
golang.org/x/crypto v0.0.0-20220926161630-eccd6366d1be
|
||||
golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d
|
||||
golang.org/x/exp v0.0.0-20221018205818-5c77f4b2bbd7
|
||||
golang.org/x/oauth2 v0.0.0-20220909003341-f21342109be1
|
||||
golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4
|
||||
@@ -104,7 +105,7 @@ require (
|
||||
github.com/jmespath/go-jmespath v0.4.0 // indirect
|
||||
github.com/jpillora/s3 v1.1.4 // indirect
|
||||
github.com/kevinburke/ssh_config v0.0.0-20201106050909-4977a11b4351 // indirect
|
||||
github.com/klauspost/compress v1.15.11 // indirect
|
||||
github.com/klauspost/compress v1.15.5 // indirect
|
||||
github.com/klauspost/pgzip v1.2.5 // indirect
|
||||
github.com/mattn/go-isatty v0.0.16 // indirect
|
||||
github.com/mitchellh/go-homedir v1.1.0 // indirect
|
||||
@@ -125,10 +126,10 @@ require (
|
||||
go.uber.org/atomic v1.7.0 // indirect
|
||||
go.uber.org/multierr v1.6.0 // indirect
|
||||
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4 // indirect
|
||||
golang.org/x/net v0.0.0-20221002022538-bcab6841153b // indirect
|
||||
golang.org/x/sys v0.0.0-20220928140112-f11e5e49a4ec // indirect
|
||||
golang.org/x/net v0.0.0-20220909164309-bea034e7d591 // indirect
|
||||
golang.org/x/sys v0.0.0-20220908150016-7ac13a9a928d // indirect
|
||||
golang.org/x/text v0.3.7 // indirect
|
||||
golang.org/x/time v0.0.0-20220922220347-f3bd1da661af // indirect
|
||||
golang.org/x/time v0.0.0-20220722155302-e5dcc9cfc0b9 // indirect
|
||||
golang.org/x/tools v0.1.12 // indirect
|
||||
google.golang.org/api v0.96.0 // indirect
|
||||
google.golang.org/appengine v1.6.7 // indirect
|
||||
|
||||
@@ -142,8 +142,8 @@ github.com/crewjam/rfc5424 v0.1.0/go.mod h1:RCi9M3xHVOeerf6ULZzqv2xOGRO/zYaVUeRy
|
||||
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
||||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/denisenkom/go-mssqldb v0.12.3 h1:pBSGx9Tq67pBOTLmxNuirNTeB8Vjmf886Kx+8Y+8shw=
|
||||
github.com/denisenkom/go-mssqldb v0.12.3/go.mod h1:k0mtMFOnU+AihqFxPMiF05rtiDrorD1Vrm1KEz5hxDo=
|
||||
github.com/denisenkom/go-mssqldb v0.12.2 h1:1OcPn5GBIobjWNd+8yjfHNIaFX14B1pWI3F9HZy5KXw=
|
||||
github.com/denisenkom/go-mssqldb v0.12.2/go.mod h1:lnIw1mZukFRZDJYQ0Pb833QS2IaC3l5HkEfra2LJ+sk=
|
||||
github.com/dimchansky/utfbom v1.1.1 h1:vV6w1AhK4VMnhBno/TPVCoK9U/LP0PkLCS9tbxHdi/U=
|
||||
github.com/dimchansky/utfbom v1.1.1/go.mod h1:SxdoEBH5qIqFocHMyGOXVAybYJdr71b1Q/j0mACtrfE=
|
||||
github.com/dnaeon/go-vcr v1.2.0/go.mod h1:R4UdLID7HZT3taECzJs4YgbbH6PIGXB6W/sc5OLb6RQ=
|
||||
@@ -162,15 +162,15 @@ github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.m
|
||||
github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go.mod h1:AFq3mo9L8Lqqiid3OhADV3RfLJnjiw63cSpi+fDTRC0=
|
||||
github.com/envoyproxy/go-control-plane v0.10.2-0.20220325020618-49ff273808a1/go.mod h1:KJwIaB5Mv44NWtYuAOFCVOjcI94vtpEz2JU/D2v6IjE=
|
||||
github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
|
||||
github.com/envoyproxy/protoc-gen-validate v0.6.13 h1:TvDcILLkjuZV3ER58VkBmncKsLUBqBDxra/XctCzuMM=
|
||||
github.com/envoyproxy/protoc-gen-validate v0.6.13/go.mod h1:qEySVqXrEugbHKvmhI8ZqtQi75/RHSSRNpffvB4I6Bw=
|
||||
github.com/envoyproxy/protoc-gen-validate v0.6.8 h1:B2cR/FAaiMtYDHv5BQpaqtkjGuWQIgr2KQZtHQ7f6i8=
|
||||
github.com/envoyproxy/protoc-gen-validate v0.6.8/go.mod h1:0ZMblUx0cxNoWRswEEXoj9kHBmqX8pxGweMiyIAfR6A=
|
||||
github.com/fatih/color v1.13.0 h1:8LOYc1KYPPmyKMuN8QV2DNRWNbLo6LZ0iLs8+mlH53w=
|
||||
github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk=
|
||||
github.com/felixge/fgprof v0.9.3 h1:VvyZxILNuCiUCSXtPtYmmtGvb65nqXh2QFWc0Wpf2/g=
|
||||
github.com/felixge/fgprof v0.9.3/go.mod h1:RdbpDgzqYVh/T9fPELJyV7EYJuHB55UTEULNun8eiPw=
|
||||
github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc=
|
||||
github.com/getsentry/sentry-go v0.14.0 h1:rlOBkuFZRKKdUnKO+0U3JclRDQKlRu5vVQtkWSQvC70=
|
||||
github.com/getsentry/sentry-go v0.14.0/go.mod h1:RZPJKSw+adu8PBNygiri/A98FqVr2HtRckJk9XVxJ9I=
|
||||
github.com/getsentry/sentry-go v0.13.0 h1:20dgTiUSfxRB/EhMPtxcL9ZEbM1ZdR+W/7f7NWD+xWo=
|
||||
github.com/getsentry/sentry-go v0.13.0/go.mod h1:EOsfu5ZdvKPfeHYV6pTVQnsjfp30+XA7//UooKNumH0=
|
||||
github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
|
||||
github.com/gliderlabs/ssh v0.2.2 h1:6zsha5zo/TWhRhwqCD3+EarCAgZ2yN28ipRnGPnwkI0=
|
||||
github.com/gliderlabs/ssh v0.2.2/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aevW3Awn0=
|
||||
@@ -350,8 +350,8 @@ github.com/kevinburke/ssh_config v0.0.0-20201106050909-4977a11b4351/go.mod h1:CT
|
||||
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
|
||||
github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=
|
||||
github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk=
|
||||
github.com/klauspost/compress v1.15.11 h1:Lcadnb3RKGin4FYM/orgq0qde+nc15E5Cbqg4B9Sx9c=
|
||||
github.com/klauspost/compress v1.15.11/go.mod h1:QPwzmACJjUTFsnSHH934V6woptycfrDDJnH7hvFVbGM=
|
||||
github.com/klauspost/compress v1.15.5 h1:qyCLMz2JCrKADihKOh9FxnW3houKeNsp2h5OEz0QSEA=
|
||||
github.com/klauspost/compress v1.15.5/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU=
|
||||
github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek=
|
||||
github.com/klauspost/pgzip v1.2.5 h1:qnWYvvKqedOF2ulHpMG72XQol4ILEJ8k2wwRl/Km8oE=
|
||||
github.com/klauspost/pgzip v1.2.5/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs=
|
||||
@@ -410,6 +410,8 @@ github.com/pkg/sftp v1.13.1/go.mod h1:3HaPG6Dq1ILlpPZRO0HVMrsydcdLt6HRDccSgb87qR
|
||||
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
|
||||
github.com/razorpay/razorpay-go v0.0.0-20210728161131-0341409a6ab2 h1:8XGvK6qfvE4l749HHWSdmkrXczWJPQLKNDFosFYDbOE=
|
||||
github.com/razorpay/razorpay-go v0.0.0-20210728161131-0341409a6ab2/go.mod h1:VcljkUylUJAUEvFfGVv/d5ht1to1dUgF4H1+3nv7i+Q=
|
||||
github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ=
|
||||
github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
|
||||
github.com/rs/xid v1.4.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg=
|
||||
@@ -432,7 +434,6 @@ github.com/spf13/afero v1.9.2/go.mod h1:iUV7ddyEEZPO5gA3zD4fJt6iStLlL+Lg4m2cihcD
|
||||
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||
github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||
github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
|
||||
github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
|
||||
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
|
||||
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
|
||||
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
|
||||
@@ -440,9 +441,8 @@ github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5
|
||||
github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
||||
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
||||
github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
||||
github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk=
|
||||
github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
|
||||
github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk=
|
||||
github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
|
||||
github.com/tailscale/depaware v0.0.0-20210622194025-720c4b409502 h1:34icjjmqJ2HPjrSuJYEkdZ+0ItmGQAQ75cRHIiftIyE=
|
||||
github.com/tailscale/depaware v0.0.0-20210622194025-720c4b409502/go.mod h1:p9lPsd+cx33L3H9nNoecRRxPssFKUwwI50I3pZ0yT+8=
|
||||
github.com/therootcompany/xz v1.0.1 h1:CmOtsn1CbtmyYiusbfmhmkpAAETj0wBIH6kCYaX+xzw=
|
||||
@@ -474,8 +474,8 @@ github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1
|
||||
github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
|
||||
github.com/yusufpapurcu/wmi v1.2.2 h1:KBNDSne4vP5mbSWnJbO+51IMOXJB67QiYCSBrubbPRg=
|
||||
github.com/yusufpapurcu/wmi v1.2.2/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0=
|
||||
go.mongodb.org/mongo-driver v1.10.3 h1:XDQEvmh6z1EUsXuIkXE9TaVeqHw6SwS1uf93jFs0HBA=
|
||||
go.mongodb.org/mongo-driver v1.10.3/go.mod h1:z4XpeoU6w+9Vht+jAFyLgVrD+jGSQQe0+CBWFHNiHt8=
|
||||
go.mongodb.org/mongo-driver v1.10.2 h1:4Wk3cnqOrQCn0P92L3/mmurMxzdvWWs5J9jinAVKD+k=
|
||||
go.mongodb.org/mongo-driver v1.10.2/go.mod h1:z4XpeoU6w+9Vht+jAFyLgVrD+jGSQQe0+CBWFHNiHt8=
|
||||
go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
|
||||
go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
|
||||
go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
|
||||
@@ -508,9 +508,8 @@ golang.org/x/crypto v0.0.0-20210817164053-32db794688a5/go.mod h1:GvvjBRRGRdwPK5y
|
||||
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
|
||||
golang.org/x/crypto v0.0.0-20211108221036-ceb1ce70b4fa/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
|
||||
golang.org/x/crypto v0.0.0-20211215153901-e495a2d5b3d3/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
|
||||
golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d h1:sK3txAijHtOK88l68nt020reeT1ZdKLIYetKl95FzVY=
|
||||
golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
|
||||
golang.org/x/crypto v0.0.0-20220926161630-eccd6366d1be h1:fmw3UbQh+nxngCAHrDCCztao/kbYFnWjoqop8dHx05A=
|
||||
golang.org/x/crypto v0.0.0-20220926161630-eccd6366d1be/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
|
||||
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
|
||||
golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
|
||||
golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
|
||||
@@ -598,9 +597,8 @@ golang.org/x/net v0.0.0-20220607020251-c690dde0001d/go.mod h1:XRhObCWvk6IyKnWLug
|
||||
golang.org/x/net v0.0.0-20220624214902-1bab6f366d9e/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
|
||||
golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
|
||||
golang.org/x/net v0.0.0-20220907135653-1e95f45603a7/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk=
|
||||
golang.org/x/net v0.0.0-20220909164309-bea034e7d591 h1:D0B/7al0LLrVC8aWF4+oxpv/m8bc7ViFfVS8/gXGdqI=
|
||||
golang.org/x/net v0.0.0-20220909164309-bea034e7d591/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk=
|
||||
golang.org/x/net v0.0.0-20221002022538-bcab6841153b h1:6e93nYa3hNqAvLr0pD4PN1fFS+gKzp2zAXqrnTCstqU=
|
||||
golang.org/x/net v0.0.0-20221002022538-bcab6841153b/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk=
|
||||
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
|
||||
golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
|
||||
golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
|
||||
@@ -709,9 +707,8 @@ golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBc
|
||||
golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220908150016-7ac13a9a928d h1:RoyzQTK76Rktm3p4xyZslc8T8I1tBz4UEjZCzeh57mM=
|
||||
golang.org/x/sys v0.0.0-20220908150016-7ac13a9a928d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220928140112-f11e5e49a4ec h1:BkDtF2Ih9xZ7le9ndzTA7KJow28VbQW3odyk/8drmuI=
|
||||
golang.org/x/sys v0.0.0-20220928140112-f11e5e49a4ec/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
|
||||
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 h1:JGgROgKl9N8DuW20oFS5gxc+lE67/N3FcwmBPMe7ArY=
|
||||
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
|
||||
@@ -728,8 +725,8 @@ golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
|
||||
golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
|
||||
golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
|
||||
golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
|
||||
golang.org/x/time v0.0.0-20220922220347-f3bd1da661af h1:Yx9k8YCG3dvF87UAn2tu2HQLf2dt/eR1bXxpLMWeH+Y=
|
||||
golang.org/x/time v0.0.0-20220922220347-f3bd1da661af/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
|
||||
golang.org/x/time v0.0.0-20220722155302-e5dcc9cfc0b9 h1:ftMN5LMiBFjbzleLqtoBZk7KdJwhuybIU+FckUHgoyQ=
|
||||
golang.org/x/time v0.0.0-20220722155302-e5dcc9cfc0b9/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
|
||||
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
|
||||
|
||||
@@ -156,7 +156,9 @@ func main() {
|
||||
}
|
||||
|
||||
for _, repo := range strings.Split(*scanCmdRepo, ",") {
|
||||
sem.Acquire(ctx, 1)
|
||||
if err := sem.Acquire(ctx, 1); err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
wgChunkers.Add(1)
|
||||
go func(r string) {
|
||||
defer sem.Release(1)
|
||||
|
||||
@@ -0,0 +1,36 @@
|
||||
package common
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"errors"
|
||||
"io"
|
||||
|
||||
log "github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
const (
|
||||
ChunkSize = 10 * 1024
|
||||
PeekSize = 3 * 1024
|
||||
)
|
||||
|
||||
func ChunkReader(r io.Reader) chan []byte {
|
||||
chunkChan := make(chan []byte)
|
||||
go func() {
|
||||
defer close(chunkChan)
|
||||
reader := bufio.NewReaderSize(bufio.NewReader(r), ChunkSize)
|
||||
for {
|
||||
chunk := make([]byte, ChunkSize)
|
||||
n, err := reader.Read(chunk)
|
||||
if err != nil && !errors.Is(err, io.EOF) {
|
||||
log.WithError(err).Error("Error chunking reader.")
|
||||
break
|
||||
}
|
||||
peekData, _ := reader.Peek(PeekSize)
|
||||
chunkChan <- append(chunk[:n], peekData...)
|
||||
if errors.Is(err, io.EOF) {
|
||||
break
|
||||
}
|
||||
}
|
||||
}()
|
||||
return chunkChan
|
||||
}
|
||||
@@ -1,4 +1,4 @@
|
||||
package sources
|
||||
package common
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
@@ -55,20 +55,16 @@ func TestChunker(t *testing.T) {
|
||||
_ = reReader.Reset()
|
||||
|
||||
testChunkCount := 0
|
||||
chunkData, _ := io.ReadAll(reReader)
|
||||
originalChunk := &Chunk{
|
||||
Data: chunkData,
|
||||
}
|
||||
for chunk := range Chunker(originalChunk) {
|
||||
for chunk := range ChunkReader(reReader) {
|
||||
testChunkCount++
|
||||
switch testChunkCount {
|
||||
case 1:
|
||||
if !bytes.Equal(baseChunkOne, chunk.Data) {
|
||||
t.Errorf("First chunk did not match expected. Got: %d bytes, expected: %d bytes", len(chunk.Data), len(baseChunkOne))
|
||||
if !bytes.Equal(baseChunkOne, chunk) {
|
||||
t.Errorf("First chunk did not match expected. Got: %d bytes, expected: %d bytes", len(chunk), len(baseChunkOne))
|
||||
}
|
||||
case 2:
|
||||
if !bytes.Equal(baseChunkTwo, chunk.Data) {
|
||||
t.Errorf("Second chunk did not match expected. Got: %d bytes, expected: %d bytes", len(chunk.Data), len(baseChunkTwo))
|
||||
if !bytes.Equal(baseChunkTwo, chunk) {
|
||||
t.Errorf("Second chunk did not match expected. Got: %d bytes, expected: %d bytes", len(chunk), len(baseChunkTwo))
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -20,8 +20,8 @@ var (
|
||||
client = common.SaneHttpClient()
|
||||
|
||||
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
|
||||
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"datadog", "dd"}) + `\b([a-zA-Z-0-9]{40})\b`)
|
||||
apiPat = regexp.MustCompile(detectors.PrefixRegex([]string{"datadog", "dd"}) + `\b([a-zA-Z-0-9]{32})\b`)
|
||||
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"datadog"}) + `\b([a-zA-Z-0-9]{40})\b`)
|
||||
apiPat = regexp.MustCompile(detectors.PrefixRegex([]string{"datadog"}) + `\b([a-zA-Z-0-9]{32})\b`)
|
||||
)
|
||||
|
||||
// Keywords are used for efficiently pre-filtering chunks.
|
||||
|
||||
@@ -20,7 +20,7 @@ var _ detectors.Detector = (*Scanner)(nil)
|
||||
var (
|
||||
client = common.SaneHttpClient()
|
||||
|
||||
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"ocean", "do"}) + `\b([A-Za-z0-9_-]{64})\b`)
|
||||
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"digitalocean"}) + `\b([A-Za-z0-9_-]{64})\b`)
|
||||
)
|
||||
|
||||
// Keywords are used for efficiently pre-filtering chunks.
|
||||
|
||||
@@ -2,9 +2,7 @@ package fastlypersonaltoken
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"strings"
|
||||
@@ -31,14 +29,6 @@ func (s Scanner) Keywords() []string {
|
||||
return []string{"fastly"}
|
||||
}
|
||||
|
||||
type fastlyUserRes struct {
|
||||
Login string `json:"login"`
|
||||
Name string `json:"name"`
|
||||
Role string `json:"role"`
|
||||
TwoFactorAuthEnabled bool `json:"two_factor_auth_enabled"`
|
||||
Locked bool `json:"locked"`
|
||||
}
|
||||
|
||||
// FromData will find and optionally verify FastlyPersonalToken secrets in a given set of bytes.
|
||||
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
|
||||
dataStr := string(data)
|
||||
@@ -57,32 +47,16 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
}
|
||||
|
||||
if verify {
|
||||
req, err := http.NewRequestWithContext(ctx, "GET", "https://api.fastly.com/current_user", nil)
|
||||
req, err := http.NewRequestWithContext(ctx, "GET", "https://api.fastly.com/services", nil)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
req.Header.Add("Fastly-Key", fmt.Sprintf("%s", resMatch))
|
||||
res, err := client.Do(req)
|
||||
if err == nil {
|
||||
bodyBytes, err := io.ReadAll(res.Body)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
defer res.Body.Close()
|
||||
if res.StatusCode >= 200 && res.StatusCode < 300 {
|
||||
var userRes fastlyUserRes
|
||||
err = json.Unmarshal(bodyBytes, &userRes)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
s1.Verified = true
|
||||
s1.ExtraData = map[string]string{
|
||||
"username": userRes.Login,
|
||||
"name": userRes.Name,
|
||||
"role": userRes.Role,
|
||||
"locked": fmt.Sprintf("%t", userRes.Locked),
|
||||
"two_factor_auth_enabled": fmt.Sprintf("%t", userRes.TwoFactorAuthEnabled),
|
||||
}
|
||||
} else {
|
||||
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
|
||||
continue
|
||||
|
||||
@@ -23,8 +23,7 @@ var (
|
||||
// https://developer.github.com/v3/#oauth2-token-sent-in-a-header
|
||||
// Token type list:
|
||||
// https://github.blog/2021-04-05-behind-githubs-new-authentication-token-formats/
|
||||
// https://github.blog/changelog/2022-10-18-introducing-fine-grained-personal-access-tokens/
|
||||
keyPat = regexp.MustCompile(`\b((?:ghp|gho|ghu|ghs|ghr|github_pat)_[a-zA-Z0-9_]{36,255})\b`)
|
||||
keyPat = regexp.MustCompile(`\b((?:ghp|gho|ghu|ghs|ghr)_[a-zA-Z0-9]{36,255})\b`)
|
||||
|
||||
//TODO: Oauth2 client_id and client_secret
|
||||
// https://developer.github.com/v3/#oauth2-keysecret
|
||||
@@ -37,13 +36,12 @@ type userRes struct {
|
||||
SiteAdmin bool `json:"site_admin"`
|
||||
Name string `json:"name"`
|
||||
Company string `json:"company"`
|
||||
UserURL string `json:"html_url"`
|
||||
}
|
||||
|
||||
// Keywords are used for efficiently pre-filtering chunks.
|
||||
// Use identifiers in the secret preferably, or the provider name.
|
||||
func (s Scanner) Keywords() []string {
|
||||
return []string{"ghp_", "gho_", "ghu_", "ghs_", "ghr_", "github_pat_"}
|
||||
return []string{"ghp_", "gho_", "ghu_", "ghs_", "ghr_"}
|
||||
}
|
||||
|
||||
// FromData will find and optionally verify GitHub secrets in a given set of bytes.
|
||||
@@ -60,7 +58,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
|
||||
token := match[1]
|
||||
|
||||
s1 := detectors.Result{
|
||||
s := detectors.Result{
|
||||
DetectorType: detectorspb.DetectorType_Github,
|
||||
Raw: []byte(token),
|
||||
}
|
||||
@@ -81,25 +79,17 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
err = json.NewDecoder(res.Body).Decode(&userResponse)
|
||||
res.Body.Close()
|
||||
if err == nil {
|
||||
s1.Verified = true
|
||||
s1.ExtraData = map[string]string{
|
||||
"username": userResponse.Login,
|
||||
"url": userResponse.UserURL,
|
||||
"account_type": userResponse.Type,
|
||||
"site_admin": fmt.Sprintf("%t", userResponse.SiteAdmin),
|
||||
"name": userResponse.Name,
|
||||
"company": userResponse.Company,
|
||||
}
|
||||
s.Verified = true
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if !s1.Verified && detectors.IsKnownFalsePositive(string(s1.Raw), detectors.DefaultFalsePositives, true) {
|
||||
if !s.Verified && detectors.IsKnownFalsePositive(string(s.Raw), detectors.DefaultFalsePositives, true) {
|
||||
continue
|
||||
}
|
||||
|
||||
results = append(results, s1)
|
||||
results = append(results, s)
|
||||
}
|
||||
|
||||
return
|
||||
|
||||
@@ -2,14 +2,13 @@ package razorpay
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"io"
|
||||
"net/http"
|
||||
"regexp"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
log "github.com/sirupsen/logrus"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
|
||||
|
||||
"github.com/razorpay/razorpay-go"
|
||||
)
|
||||
|
||||
type Scanner struct{}
|
||||
@@ -19,10 +18,8 @@ var _ detectors.Detector = (*Scanner)(nil)
|
||||
|
||||
// The (`) character adds secondary encoding to parsed strings by Golang which also allows for escape sequences
|
||||
var (
|
||||
client = common.SaneHttpClient()
|
||||
|
||||
keyPat = regexp.MustCompile(`(?i)\brzp_live_\w{10,20}\b`)
|
||||
secretPat = regexp.MustCompile(detectors.PrefixRegex([]string{"razor|secret|rzp|key"}) + `([A-Za-z0-9]{20,50})`)
|
||||
keyPat = regexp.MustCompile(`(?i)\brzp_\w{2,6}_\w{10,20}\b`)
|
||||
secretPat = regexp.MustCompile(`(?:razor|secret|rzp|key)[-\w]*[\" :=']*([A-Za-z0-9]{20,50})`)
|
||||
)
|
||||
|
||||
// Keywords are used for efficiently pre-filtering chunks.
|
||||
@@ -35,54 +32,54 @@ func (s Scanner) Keywords() []string {
|
||||
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
|
||||
dataStr := string(data)
|
||||
|
||||
keyMatches := keyPat.FindAllString(dataStr, -1)
|
||||
matches := keyPat.FindAllString(dataStr, -1)
|
||||
|
||||
for _, key := range keyMatches {
|
||||
for _, match := range matches {
|
||||
token := match
|
||||
|
||||
s := detectors.Result{
|
||||
DetectorType: detectorspb.DetectorType_RazorPay,
|
||||
Raw: []byte(token),
|
||||
Redacted: token,
|
||||
}
|
||||
|
||||
if verify {
|
||||
secMatches := secretPat.FindAllString(dataStr, -1)
|
||||
|
||||
for _, secret := range secMatches {
|
||||
|
||||
s1 := detectors.Result{
|
||||
DetectorType: detectorspb.DetectorType_RazorPay,
|
||||
Raw: []byte(key),
|
||||
RawV2: []byte(key + secret),
|
||||
Redacted: key,
|
||||
}
|
||||
|
||||
req, err := http.NewRequest("GET", "https://api.razorpay.com/v1/items?count=1", nil)
|
||||
if err != nil {
|
||||
//https://dashboard.razorpay.com/#/access/signin
|
||||
//https://gitlab.com/trufflesec/trufflehog/-/blob/master/webapi/secrets/razorpay.py
|
||||
secMatches := secretPat.FindAllStringSubmatch(dataStr, -1)
|
||||
if len(secMatches) == 0 {
|
||||
//no secret keys were found. Declare unverified (This is how AWS secret handles the same logic)
|
||||
//TODO determine if key alone without secret is reportable
|
||||
s.Verified = false
|
||||
return
|
||||
}
|
||||
//we only want the secret, not its surrounding info - grabbing capture groups
|
||||
for _, secMatch := range secMatches {
|
||||
client := razorpay.NewClient(token, secMatch[1])
|
||||
resp, err := client.Order.All(nil, nil)
|
||||
//TODO Error handling is broken in SDK, fixed by https://github.com/razorpay/razorpay-go/pull/23
|
||||
//waiting to be reviewed and merged
|
||||
if resp == nil {
|
||||
continue
|
||||
}
|
||||
req.SetBasicAuth(key, secret)
|
||||
res, err := client.Do(req)
|
||||
if err == nil {
|
||||
bodyBytes, err := io.ReadAll(res.Body)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
defer res.Body.Close()
|
||||
if res.StatusCode >= 200 && res.StatusCode < 300 {
|
||||
if json.Valid(bodyBytes) {
|
||||
s1.Verified = true
|
||||
} else {
|
||||
s1.Verified = false
|
||||
}
|
||||
} else {
|
||||
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
|
||||
if detectors.IsKnownFalsePositive(key, detectors.DefaultFalsePositives, true) {
|
||||
continue
|
||||
}
|
||||
}
|
||||
if err != nil {
|
||||
log.Debugf("Error verifying likely razorpay key/secret combo: %v", err)
|
||||
continue
|
||||
}
|
||||
//TODO debug with responses. could still be invalid at this stage
|
||||
|
||||
results = append(results, s1)
|
||||
s.Verified = true
|
||||
}
|
||||
}
|
||||
|
||||
if !s.Verified {
|
||||
if detectors.IsKnownFalsePositive(string(s.Raw), detectors.DefaultFalsePositives, false) {
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
results = append(results, s)
|
||||
}
|
||||
|
||||
results = detectors.CleanResults(results)
|
||||
return
|
||||
}
|
||||
|
||||
@@ -42,13 +42,13 @@ func TestRazorPay_FromChunk(t *testing.T) {
|
||||
s: Scanner{},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("25:RAZORPAY_KEY='rzp_live_SnDTaP1ncfliDt'\n\"rzp_secret\" : \" %s\", ", secretInactive)),
|
||||
data: []byte(fmt.Sprintf("25:RAZORPAY_KEY='rzp_test_SnDTaP1ncfliDt'\n\"rzp_secret\" : \" %s\", ", secretInactive)),
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_RazorPay,
|
||||
Redacted: "rzp_live_SnDTaP1ncfliDt",
|
||||
Redacted: "rzp_test_SnDTaP1ncfliDt",
|
||||
Verified: false,
|
||||
},
|
||||
},
|
||||
@@ -82,7 +82,6 @@ func TestRazorPay_FromChunk(t *testing.T) {
|
||||
t.Fatal("no raw secret present")
|
||||
}
|
||||
got[i].Raw = nil
|
||||
got[i].RawV2 = nil
|
||||
}
|
||||
if diff := pretty.Compare(got, tt.want); diff != "" {
|
||||
t.Errorf("RazorPay.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
|
||||
|
||||
@@ -1,89 +0,0 @@
|
||||
package sqlserver
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"regexp"
|
||||
|
||||
"github.com/denisenkom/go-mssqldb/msdsn"
|
||||
log "github.com/sirupsen/logrus"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
)
|
||||
|
||||
type Scanner struct{}
|
||||
|
||||
// Ensure the Scanner satisfies the interface at compile time.
|
||||
var _ detectors.Detector = (*Scanner)(nil)
|
||||
|
||||
var (
|
||||
// SQLServer connection string is a semicolon delimited set of case-insensitive parameters which may go in any order.
|
||||
pattern = regexp.MustCompile("(?:\n|`|'|\"| )?((?:[A-Za-z0-9_ ]+=[^;$'`\"$]+;?){3,})(?:'|`|\"|\r\n|\n)?")
|
||||
)
|
||||
|
||||
// Keywords are used for efficiently pre-filtering chunks.
|
||||
// Use identifiers in the secret preferably, or the provider name.
|
||||
func (s Scanner) Keywords() []string {
|
||||
return []string{"sqlserver"}
|
||||
}
|
||||
|
||||
// FromData will find and optionally verify SpotifyKey secrets in a given set of bytes.
|
||||
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
|
||||
matches := pattern.FindAllStringSubmatch(string(data), -1)
|
||||
for _, match := range matches {
|
||||
params, _, err := msdsn.Parse(match[1])
|
||||
if err != nil {
|
||||
log.Debugf("sqlserver: unable to parse connection string '%s' because '%s'", match[1], err.Error())
|
||||
continue
|
||||
}
|
||||
|
||||
if params.Password == "" {
|
||||
log.Debugf("sqlserver: skip connection string '%s' because it does not contain password", match[1])
|
||||
continue
|
||||
}
|
||||
|
||||
detected := detectors.Result{
|
||||
DetectorType: detectorspb.DetectorType_SQLServer,
|
||||
Raw: []byte(params.Password),
|
||||
}
|
||||
|
||||
if verify {
|
||||
verified, err := ping(params)
|
||||
if err != nil {
|
||||
log.Debugf("sqlserver: unable to verify '%s' because '%s'", params.URL(), err.Error())
|
||||
} else {
|
||||
detected.Verified = verified
|
||||
}
|
||||
}
|
||||
|
||||
results = append(results, detected)
|
||||
}
|
||||
|
||||
return detectors.CleanResults(results), nil
|
||||
}
|
||||
|
||||
var ping = func(config msdsn.Config) (bool, error) {
|
||||
url := config.URL()
|
||||
query := url.Query()
|
||||
query.Set("dial timeout", "3")
|
||||
query.Set("connection timeout", "3")
|
||||
url.RawQuery = query.Encode()
|
||||
|
||||
conn, err := sql.Open("mssql", url.String())
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
|
||||
err = conn.Ping()
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
|
||||
err = conn.Close()
|
||||
if err != nil {
|
||||
log.Debugf("sqlserver: unable to close connection '%s' because '%s'", url.String(), err.Error())
|
||||
}
|
||||
|
||||
return true, nil
|
||||
}
|
||||
@@ -1,145 +0,0 @@
|
||||
//go:build detectors
|
||||
// +build detectors
|
||||
|
||||
package sqlserver
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"github.com/denisenkom/go-mssqldb/msdsn"
|
||||
"github.com/google/go-cmp/cmp"
|
||||
"github.com/google/go-cmp/cmp/cmpopts"
|
||||
"testing"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
|
||||
)
|
||||
|
||||
func TestSQLServer_FromChunk(t *testing.T) {
|
||||
secret := "Server=localhost;Initial Catalog=Demo;User ID=sa;Password=P@ssw0rd!;Persist Security Info=true;MultipleActiveResultSets=true;"
|
||||
inactiveSecret := "Server=localhost;User ID=sa;Password=123"
|
||||
|
||||
type args struct {
|
||||
ctx context.Context
|
||||
data []byte
|
||||
verify bool
|
||||
}
|
||||
tests := []struct {
|
||||
name string
|
||||
s Scanner
|
||||
args args
|
||||
want []detectors.Result
|
||||
wantErr bool
|
||||
mockFunc func()
|
||||
}{
|
||||
{
|
||||
name: "found, verified",
|
||||
s: Scanner{},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a sqlserver secret %s within", secret)),
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_SQLServer,
|
||||
Verified: true,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
mockFunc: func() {
|
||||
ping = func(config msdsn.Config) (bool, error) {
|
||||
return true, nil
|
||||
}
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "found, unverified",
|
||||
s: Scanner{},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a sqlserver secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_SQLServer,
|
||||
Verified: false,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
mockFunc: func() {
|
||||
ping = func(config msdsn.Config) (bool, error) {
|
||||
return false, nil
|
||||
}
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "not found",
|
||||
s: Scanner{},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte("You cannot find the secret within"),
|
||||
verify: true,
|
||||
},
|
||||
want: nil,
|
||||
wantErr: false,
|
||||
mockFunc: func() {},
|
||||
},
|
||||
}
|
||||
|
||||
// preserve the original function
|
||||
originalPing := ping
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
tt.mockFunc()
|
||||
s := Scanner{}
|
||||
got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
|
||||
if (err != nil) != tt.wantErr {
|
||||
t.Errorf("SQLServer.FromData() error = %v, wantErr %v", err, tt.wantErr)
|
||||
return
|
||||
}
|
||||
for i := range got {
|
||||
if len(got[i].Raw) == 0 {
|
||||
t.Fatalf("no raw secret present: \n %+v", got[i])
|
||||
}
|
||||
got[i].Raw = nil
|
||||
}
|
||||
ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "RawV2")
|
||||
if diff := cmp.Diff(tt.want, got, ignoreOpts); diff != "" {
|
||||
t.Errorf("SQLServer.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
ping = originalPing
|
||||
}
|
||||
|
||||
func TestSQLServer_pattern(t *testing.T) {
|
||||
if !pattern.Match([]byte(`builder.Services.AddDbContext<Database>(optionsBuilder => optionsBuilder.UseSqlServer("Server=localhost;Initial Catalog=master;User ID=sa;Password=P@ssw0rd!;Persist Security Info=true;MultipleActiveResultSets=true;"));`)) {
|
||||
t.Errorf("SQLServer.pattern: did not catched connection string from Program.cs")
|
||||
}
|
||||
if !pattern.Match([]byte(`{"ConnectionStrings": {"Demo": "Server=localhost;Initial Catalog=master;User ID=sa;Password=P@ssw0rd!;Persist Security Info=true;MultipleActiveResultSets=true;"}}`)) {
|
||||
t.Errorf("SQLServer.pattern: did not catched connection string from appsettings.json")
|
||||
}
|
||||
if !pattern.Match([]byte(`CONNECTION_STRING: Server=localhost;Initial Catalog=master;User ID=sa;Password=P@ssw0rd!;Persist Security Info=true;MultipleActiveResultSets=true`)) {
|
||||
t.Errorf("SQLServer.pattern: did not catched connection string from .env")
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkFromData(benchmark *testing.B) {
|
||||
ctx := context.Background()
|
||||
s := Scanner{}
|
||||
for name, data := range detectors.MustGetBenchmarkData() {
|
||||
benchmark.Run(name, func(b *testing.B) {
|
||||
for n := 0; n < b.N; n++ {
|
||||
_, err := s.FromData(ctx, false, data)
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -426,8 +426,8 @@ import (
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/noticeable"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/notion"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/nozbeteams"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/npmtoken"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/npmtokenv2"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/npmtoken"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/npmtokenv2"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/numverify"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/nutritionix"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/nylas"
|
||||
@@ -593,7 +593,6 @@ import (
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/spoonacular"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/sportradar"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/sportsmonk"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/sqlserver"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/square"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/squareapp"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/squarespace"
|
||||
@@ -1483,6 +1482,5 @@ func DefaultDetectors() []detectors.Detector {
|
||||
digitaloceanv2.Scanner{},
|
||||
npmtoken.Scanner{},
|
||||
npmtokenv2.Scanner{},
|
||||
sqlserver.Scanner{},
|
||||
}
|
||||
}
|
||||
|
||||
+66
-68
@@ -161,76 +161,74 @@ func (e *Engine) DetectorAvgTime() map[string][]time.Duration {
|
||||
}
|
||||
|
||||
func (e *Engine) detectorWorker(ctx context.Context) {
|
||||
for originalChunk := range e.chunks {
|
||||
for chunk := range sources.Chunker(originalChunk) {
|
||||
fragStart, mdLine := fragmentFirstLine(chunk)
|
||||
for _, decoder := range e.decoders {
|
||||
var decoderType detectorspb.DecoderType
|
||||
switch decoder.(type) {
|
||||
case *decoders.Plain:
|
||||
decoderType = detectorspb.DecoderType_PLAIN
|
||||
case *decoders.Base64:
|
||||
decoderType = detectorspb.DecoderType_BASE64
|
||||
default:
|
||||
logrus.Warnf("unknown decoder type: %T", decoder)
|
||||
decoderType = detectorspb.DecoderType_UNKNOWN
|
||||
}
|
||||
decoded := decoder.FromChunk(chunk)
|
||||
if decoded == nil {
|
||||
continue
|
||||
}
|
||||
dataLower := strings.ToLower(string(decoded.Data))
|
||||
for verify, detectorsSet := range e.detectors {
|
||||
for _, detector := range detectorsSet {
|
||||
start := time.Now()
|
||||
foundKeyword := false
|
||||
for _, kw := range detector.Keywords() {
|
||||
if strings.Contains(dataLower, strings.ToLower(kw)) {
|
||||
foundKeyword = true
|
||||
break
|
||||
for chunk := range e.chunks {
|
||||
fragStart, mdLine := fragmentFirstLine(chunk)
|
||||
for _, decoder := range e.decoders {
|
||||
var decoderType detectorspb.DecoderType
|
||||
switch decoder.(type) {
|
||||
case *decoders.Plain:
|
||||
decoderType = detectorspb.DecoderType_PLAIN
|
||||
case *decoders.Base64:
|
||||
decoderType = detectorspb.DecoderType_BASE64
|
||||
default:
|
||||
logrus.Warnf("unknown decoder type: %T", decoder)
|
||||
decoderType = detectorspb.DecoderType_UNKNOWN
|
||||
}
|
||||
decoded := decoder.FromChunk(chunk)
|
||||
if decoded == nil {
|
||||
continue
|
||||
}
|
||||
dataLower := strings.ToLower(string(decoded.Data))
|
||||
for verify, detectorsSet := range e.detectors {
|
||||
for _, detector := range detectorsSet {
|
||||
start := time.Now()
|
||||
foundKeyword := false
|
||||
for _, kw := range detector.Keywords() {
|
||||
if strings.Contains(dataLower, strings.ToLower(kw)) {
|
||||
foundKeyword = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !foundKeyword {
|
||||
continue
|
||||
}
|
||||
|
||||
results, err := func() ([]detectors.Result, error) {
|
||||
ctx, cancel := context.WithTimeout(ctx, time.Second*10)
|
||||
defer cancel()
|
||||
defer common.Recover(ctx)
|
||||
return detector.FromData(ctx, verify, decoded.Data)
|
||||
}()
|
||||
if err != nil {
|
||||
logrus.WithFields(logrus.Fields{
|
||||
"source_type": decoded.SourceType.String(),
|
||||
"metadata": decoded.SourceMetadata,
|
||||
}).WithError(err).Error("could not scan chunk")
|
||||
continue
|
||||
}
|
||||
|
||||
for _, result := range results {
|
||||
if isGitSource(chunk.SourceType) {
|
||||
offset := FragmentLineOffset(chunk, &result)
|
||||
*mdLine = fragStart + offset
|
||||
}
|
||||
result.DecoderType = decoderType
|
||||
e.results <- detectors.CopyMetadata(chunk, result)
|
||||
|
||||
}
|
||||
if len(results) > 0 {
|
||||
elapsed := time.Since(start)
|
||||
detectorName := results[0].DetectorType.String()
|
||||
avgTimeI, ok := e.detectorAvgTime.Load(detectorName)
|
||||
var avgTime []time.Duration
|
||||
if ok {
|
||||
avgTime, ok = avgTimeI.([]time.Duration)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
}
|
||||
if !foundKeyword {
|
||||
continue
|
||||
}
|
||||
|
||||
results, err := func() ([]detectors.Result, error) {
|
||||
ctx, cancel := context.WithTimeout(ctx, time.Second*10)
|
||||
defer cancel()
|
||||
defer common.Recover(ctx)
|
||||
return detector.FromData(ctx, verify, decoded.Data)
|
||||
}()
|
||||
if err != nil {
|
||||
logrus.WithFields(logrus.Fields{
|
||||
"source_type": decoded.SourceType.String(),
|
||||
"metadata": decoded.SourceMetadata,
|
||||
}).WithError(err).Error("could not scan chunk")
|
||||
continue
|
||||
}
|
||||
|
||||
for _, result := range results {
|
||||
if isGitSource(chunk.SourceType) {
|
||||
offset := FragmentLineOffset(chunk, &result)
|
||||
*mdLine = fragStart + offset
|
||||
}
|
||||
result.DecoderType = decoderType
|
||||
e.results <- detectors.CopyMetadata(chunk, result)
|
||||
|
||||
}
|
||||
if len(results) > 0 {
|
||||
elapsed := time.Since(start)
|
||||
detectorName := results[0].DetectorType.String()
|
||||
avgTimeI, ok := e.detectorAvgTime.Load(detectorName)
|
||||
var avgTime []time.Duration
|
||||
if ok {
|
||||
avgTime, ok = avgTimeI.([]time.Duration)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
}
|
||||
avgTime = append(avgTime, elapsed)
|
||||
e.detectorAvgTime.Store(detectorName, avgTime)
|
||||
}
|
||||
avgTime = append(avgTime, elapsed)
|
||||
e.detectorAvgTime.Store(detectorName, avgTime)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -966,7 +966,6 @@ const (
|
||||
DetectorType_MongoDB DetectorType = 895
|
||||
DetectorType_NGC DetectorType = 896
|
||||
DetectorType_DigitalOceanV2 DetectorType = 897
|
||||
DetectorType_SQLServer DetectorType = 898
|
||||
)
|
||||
|
||||
// Enum value maps for DetectorType.
|
||||
@@ -1866,7 +1865,6 @@ var (
|
||||
895: "MongoDB",
|
||||
896: "NGC",
|
||||
897: "DigitalOceanV2",
|
||||
898: "SQLServer",
|
||||
}
|
||||
DetectorType_value = map[string]int32{
|
||||
"Alibaba": 0,
|
||||
@@ -2763,7 +2761,6 @@ var (
|
||||
"MongoDB": 895,
|
||||
"NGC": 896,
|
||||
"DigitalOceanV2": 897,
|
||||
"SQLServer": 898,
|
||||
}
|
||||
)
|
||||
|
||||
@@ -3128,7 +3125,7 @@ var file_detectors_proto_rawDesc = []byte{
|
||||
0x0a, 0x0b, 0x44, 0x65, 0x63, 0x6f, 0x64, 0x65, 0x72, 0x54, 0x79, 0x70, 0x65, 0x12, 0x0b, 0x0a,
|
||||
0x07, 0x55, 0x4e, 0x4b, 0x4e, 0x4f, 0x57, 0x4e, 0x10, 0x00, 0x12, 0x09, 0x0a, 0x05, 0x50, 0x4c,
|
||||
0x41, 0x49, 0x4e, 0x10, 0x01, 0x12, 0x0a, 0x0a, 0x06, 0x42, 0x41, 0x53, 0x45, 0x36, 0x34, 0x10,
|
||||
0x02, 0x2a, 0xc1, 0x70, 0x0a, 0x0c, 0x44, 0x65, 0x74, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x54, 0x79,
|
||||
0x02, 0x2a, 0xb1, 0x70, 0x0a, 0x0c, 0x44, 0x65, 0x74, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x54, 0x79,
|
||||
0x70, 0x65, 0x12, 0x0b, 0x0a, 0x07, 0x41, 0x6c, 0x69, 0x62, 0x61, 0x62, 0x61, 0x10, 0x00, 0x12,
|
||||
0x08, 0x0a, 0x04, 0x41, 0x4d, 0x51, 0x50, 0x10, 0x01, 0x12, 0x07, 0x0a, 0x03, 0x41, 0x57, 0x53,
|
||||
0x10, 0x02, 0x12, 0x09, 0x0a, 0x05, 0x41, 0x7a, 0x75, 0x72, 0x65, 0x10, 0x03, 0x12, 0x0a, 0x0a,
|
||||
@@ -4027,8 +4024,7 @@ var file_detectors_proto_rawDesc = []byte{
|
||||
0x72, 0x6b, 0x64, 0x61, 0x79, 0x10, 0xfe, 0x06, 0x12, 0x0c, 0x0a, 0x07, 0x4d, 0x6f, 0x6e, 0x67,
|
||||
0x6f, 0x44, 0x42, 0x10, 0xff, 0x06, 0x12, 0x08, 0x0a, 0x03, 0x4e, 0x47, 0x43, 0x10, 0x80, 0x07,
|
||||
0x12, 0x13, 0x0a, 0x0e, 0x44, 0x69, 0x67, 0x69, 0x74, 0x61, 0x6c, 0x4f, 0x63, 0x65, 0x61, 0x6e,
|
||||
0x56, 0x32, 0x10, 0x81, 0x07, 0x12, 0x0e, 0x0a, 0x09, 0x53, 0x51, 0x4c, 0x53, 0x65, 0x72, 0x76,
|
||||
0x65, 0x72, 0x10, 0x82, 0x07, 0x42, 0x3d, 0x5a, 0x3b, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e,
|
||||
0x56, 0x32, 0x10, 0x81, 0x07, 0x42, 0x3d, 0x5a, 0x3b, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e,
|
||||
0x63, 0x6f, 0x6d, 0x2f, 0x74, 0x72, 0x75, 0x66, 0x66, 0x6c, 0x65, 0x73, 0x65, 0x63, 0x75, 0x72,
|
||||
0x69, 0x74, 0x79, 0x2f, 0x74, 0x72, 0x75, 0x66, 0x66, 0x6c, 0x65, 0x68, 0x6f, 0x67, 0x2f, 0x76,
|
||||
0x33, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x70, 0x62, 0x2f, 0x64, 0x65, 0x74, 0x65, 0x63, 0x74, 0x6f,
|
||||
|
||||
@@ -1,47 +0,0 @@
|
||||
package sources
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
"errors"
|
||||
"io"
|
||||
|
||||
"github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
const (
|
||||
// ChunkSize is the maximum size of a chunk.
|
||||
ChunkSize = 10 * 1024
|
||||
// PeekSize is the size of the peek into the previous chunk.
|
||||
PeekSize = 3 * 1024
|
||||
)
|
||||
|
||||
// Chunker takes a chunk and splits it into chunks of ChunkSize.
|
||||
func Chunker(originalChunk *Chunk) chan *Chunk {
|
||||
chunkChan := make(chan *Chunk)
|
||||
go func() {
|
||||
defer close(chunkChan)
|
||||
if len(originalChunk.Data) <= ChunkSize+PeekSize {
|
||||
chunkChan <- originalChunk
|
||||
return
|
||||
}
|
||||
r := bytes.NewReader(originalChunk.Data)
|
||||
reader := bufio.NewReaderSize(bufio.NewReader(r), ChunkSize)
|
||||
for {
|
||||
chunkBytes := make([]byte, ChunkSize)
|
||||
chunk := *originalChunk
|
||||
n, err := reader.Read(chunkBytes)
|
||||
if err != nil && !errors.Is(err, io.EOF) {
|
||||
logrus.WithError(err).Error("Error chunking reader.")
|
||||
break
|
||||
}
|
||||
peekData, _ := reader.Peek(PeekSize)
|
||||
chunk.Data = append(chunkBytes[:n], peekData...)
|
||||
chunkChan <- &chunk
|
||||
if errors.Is(err, io.EOF) {
|
||||
break
|
||||
}
|
||||
}
|
||||
}()
|
||||
return chunkChan
|
||||
}
|
||||
@@ -13,6 +13,7 @@ import (
|
||||
"google.golang.org/protobuf/proto"
|
||||
"google.golang.org/protobuf/types/known/anypb"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/context"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/handlers"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/source_metadatapb"
|
||||
@@ -78,8 +79,8 @@ func (s *Source) Init(aCtx context.Context, name string, jobId, sourceId int64,
|
||||
|
||||
// Chunks emits chunks of bytes over a channel.
|
||||
func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) error {
|
||||
for i, path := range s.paths {
|
||||
s.SetProgressComplete(i, len(s.paths), fmt.Sprintf("Path: %s", path), "")
|
||||
for _, path := range s.paths {
|
||||
s.Update(len(s.paths), fmt.Sprintf("Path: %s", path), "")
|
||||
|
||||
cleanPath := filepath.Clean(path)
|
||||
done := false
|
||||
@@ -139,23 +140,22 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) err
|
||||
return err
|
||||
}
|
||||
reReader.Stop()
|
||||
data, err := io.ReadAll(reReader)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
chunksChan <- &sources.Chunk{
|
||||
SourceType: s.Type(),
|
||||
SourceName: s.name,
|
||||
SourceID: s.SourceID(),
|
||||
Data: data,
|
||||
SourceMetadata: &source_metadatapb.MetaData{
|
||||
Data: &source_metadatapb.MetaData_Filesystem{
|
||||
Filesystem: &source_metadatapb.Filesystem{
|
||||
File: sanitizer.UTF8(path),
|
||||
|
||||
for chunkData := range common.ChunkReader(reReader) {
|
||||
chunksChan <- &sources.Chunk{
|
||||
SourceType: s.Type(),
|
||||
SourceName: s.name,
|
||||
SourceID: s.SourceID(),
|
||||
Data: chunkData,
|
||||
SourceMetadata: &source_metadatapb.MetaData{
|
||||
Data: &source_metadatapb.MetaData_Filesystem{
|
||||
Filesystem: &source_metadatapb.Filesystem{
|
||||
File: sanitizer.UTF8(path),
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
Verify: s.verify,
|
||||
Verify: s.verify,
|
||||
}
|
||||
}
|
||||
return nil
|
||||
})
|
||||
|
||||
+19
-80
@@ -1,10 +1,7 @@
|
||||
package git
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"net/url"
|
||||
"os"
|
||||
@@ -27,6 +24,7 @@ import (
|
||||
"google.golang.org/protobuf/proto"
|
||||
"google.golang.org/protobuf/types/known/anypb"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/context"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/gitparse"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/handlers"
|
||||
@@ -134,8 +132,8 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) err
|
||||
user := cred.BasicAuth.Username
|
||||
token := cred.BasicAuth.Password
|
||||
|
||||
for i, repoURI := range s.conn.Repositories {
|
||||
s.SetProgressComplete(i, len(s.conn.Repositories), fmt.Sprintf("Repo: %s", repoURI), "")
|
||||
for _, repoURI := range s.conn.Repositories {
|
||||
s.Update(len(s.conn.Repositories), fmt.Sprintf("Repo: %s", repoURI), "")
|
||||
if len(repoURI) == 0 {
|
||||
continue
|
||||
}
|
||||
@@ -152,8 +150,8 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) err
|
||||
}
|
||||
}
|
||||
case *sourcespb.Git_Unauthenticated:
|
||||
for i, repoURI := range s.conn.Repositories {
|
||||
s.SetProgressComplete(i, len(s.conn.Repositories), fmt.Sprintf("Repo: %s", repoURI), "")
|
||||
for _, repoURI := range s.conn.Repositories {
|
||||
s.Update(len(s.conn.Repositories), fmt.Sprintf("Repo: %s", repoURI), "")
|
||||
if len(repoURI) == 0 {
|
||||
continue
|
||||
}
|
||||
@@ -170,8 +168,8 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) err
|
||||
}
|
||||
}
|
||||
case *sourcespb.Git_SshAuth:
|
||||
for i, repoURI := range s.conn.Repositories {
|
||||
s.SetProgressComplete(i, len(s.conn.Repositories), fmt.Sprintf("Repo: %s", repoURI), "")
|
||||
for _, repoURI := range s.conn.Repositories {
|
||||
s.Update(len(s.conn.Repositories), fmt.Sprintf("Repo: %s", repoURI), "")
|
||||
if len(repoURI) == 0 {
|
||||
continue
|
||||
}
|
||||
@@ -191,8 +189,12 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) err
|
||||
return errors.New("invalid connection type for git source")
|
||||
}
|
||||
|
||||
for i, u := range s.conn.Directories {
|
||||
s.SetProgressComplete(i, len(s.conn.Repositories), fmt.Sprintf("Repo: %s", u), "")
|
||||
for _, u := range s.conn.Directories {
|
||||
s.Counter.IncTotal()
|
||||
go func() {
|
||||
defer s.Counter.IncSuccess()
|
||||
}()
|
||||
s.Update(len(s.conn.Repositories), fmt.Sprintf("Repo: %s", u), "")
|
||||
|
||||
if len(u) == 0 {
|
||||
continue
|
||||
@@ -217,7 +219,7 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) err
|
||||
}
|
||||
|
||||
}
|
||||
s.SetProgressComplete(len(s.conn.Repositories), len(s.conn.Repositories), fmt.Sprintf("Completed scanning source %s", s.name), "")
|
||||
s.Complete(fmt.Sprintf("Completed scanning source %s", s.name))
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -324,7 +326,7 @@ func (s *Git) ScanCommits(ctx context.Context, repo *git.Repository, path string
|
||||
var reachedBase = false
|
||||
log.WithField("repo", urlMetadata).Debugf("Scanning repo")
|
||||
for commit := range commitChan {
|
||||
log.Tracef("Scanning commit %s", commit.Hash)
|
||||
log.Debugf("Scanning commit %s", commit.Hash)
|
||||
if scanOptions.MaxDepth > 0 && depth >= scanOptions.MaxDepth {
|
||||
log.Debugf("reached max depth")
|
||||
break
|
||||
@@ -372,10 +374,6 @@ func (s *Git) ScanCommits(ctx context.Context, repo *git.Repository, path string
|
||||
continue
|
||||
}
|
||||
|
||||
if diff.Content.Len() > sources.ChunkSize+sources.PeekSize {
|
||||
s.gitChunk(diff, fileName, email, hash, when, urlMetadata, chunksChan)
|
||||
continue
|
||||
}
|
||||
metadata := s.sourceMetadataFunc(fileName, email, hash, when, urlMetadata, int64(diff.LineStart))
|
||||
chunksChan <- &sources.Chunk{
|
||||
SourceName: s.sourceName,
|
||||
@@ -390,62 +388,6 @@ func (s *Git) ScanCommits(ctx context.Context, repo *git.Repository, path string
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Git) gitChunk(diff gitparse.Diff, fileName, email, hash, when, urlMetadata string, chunksChan chan *sources.Chunk) {
|
||||
originalChunk := bufio.NewScanner(&diff.Content)
|
||||
newChunkBuffer := bytes.Buffer{}
|
||||
lastOffset := 0
|
||||
for offset := 0; originalChunk.Scan(); offset++ {
|
||||
line := originalChunk.Bytes()
|
||||
if len(line) > sources.ChunkSize || len(line)+newChunkBuffer.Len() > sources.ChunkSize {
|
||||
// Add oversize chunk info
|
||||
if newChunkBuffer.Len() > 0 {
|
||||
// Send the existing fragment.
|
||||
metadata := s.sourceMetadataFunc(fileName, email, hash, when, urlMetadata, int64(diff.LineStart+lastOffset))
|
||||
chunksChan <- &sources.Chunk{
|
||||
SourceName: s.sourceName,
|
||||
SourceID: s.sourceID,
|
||||
SourceType: s.sourceType,
|
||||
SourceMetadata: metadata,
|
||||
Data: append([]byte{}, newChunkBuffer.Bytes()...),
|
||||
Verify: s.verify,
|
||||
}
|
||||
newChunkBuffer.Reset()
|
||||
lastOffset = offset
|
||||
}
|
||||
if len(line) > sources.ChunkSize {
|
||||
// Send the oversize line.
|
||||
metadata := s.sourceMetadataFunc(fileName, email, hash, when, urlMetadata, int64(diff.LineStart+offset))
|
||||
chunksChan <- &sources.Chunk{
|
||||
SourceName: s.sourceName,
|
||||
SourceID: s.sourceID,
|
||||
SourceType: s.sourceType,
|
||||
SourceMetadata: metadata,
|
||||
Data: line,
|
||||
Verify: s.verify,
|
||||
}
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
_, err := newChunkBuffer.Write(line)
|
||||
if err != nil {
|
||||
log.WithError(err).Error("Could not write line to git diff buffer.")
|
||||
}
|
||||
}
|
||||
// Send anything still in the new chunk buffer
|
||||
if newChunkBuffer.Len() > 0 {
|
||||
metadata := s.sourceMetadataFunc(fileName, email, hash, when, urlMetadata, int64(diff.LineStart+lastOffset))
|
||||
chunksChan <- &sources.Chunk{
|
||||
SourceName: s.sourceName,
|
||||
SourceID: s.sourceID,
|
||||
SourceType: s.sourceType,
|
||||
SourceMetadata: metadata,
|
||||
Data: append([]byte{}, newChunkBuffer.Bytes()...),
|
||||
Verify: s.verify,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ScanUnstaged chunks unstaged changes.
|
||||
func (s *Git) ScanUnstaged(ctx context.Context, repo *git.Repository, path string, scanOptions *ScanOptions, chunksChan chan *sources.Chunk) error {
|
||||
// get the URL metadata for reporting (may be empty)
|
||||
@@ -815,14 +757,11 @@ func handleBinary(repo *git.Repository, chunksChan chan *sources.Chunk, chunkSke
|
||||
}
|
||||
reader.Stop()
|
||||
|
||||
chunkData, err := io.ReadAll(reader)
|
||||
if err != nil {
|
||||
return err
|
||||
for chunkData := range common.ChunkReader(reader) {
|
||||
chunk := *chunkSkel
|
||||
chunk.Data = chunkData
|
||||
chunksChan <- &chunk
|
||||
}
|
||||
|
||||
chunk := *chunkSkel
|
||||
chunk.Data = chunkData
|
||||
chunksChan <- &chunk
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -496,15 +496,17 @@ func (s *Source) scan(ctx context.Context, installationClient *github.Client, ch
|
||||
s.repos = reposToScan
|
||||
|
||||
var scanErrs []error
|
||||
for i, repoURL := range s.repos {
|
||||
i, repoURL := i, repoURL
|
||||
repoCnt := len(s.repos)
|
||||
for _, repoURL := range s.repos {
|
||||
repoURL := repoURL
|
||||
s.jobPool.Go(func() error {
|
||||
if common.IsDone(ctx) {
|
||||
return nil
|
||||
}
|
||||
cnt := s.Counter.IncTotal()
|
||||
|
||||
// TODO: set progress complete is being called concurrently with i
|
||||
s.setProgressCompleteWithRepo(i, progressIndexOffset, repoURL)
|
||||
defer s.Counter.IncSuccess()
|
||||
s.setProgressCompleteWithRepo(progressIndexOffset, repoURL)
|
||||
// Ensure the repo is removed from the resume info after being scanned.
|
||||
defer func(s *Source, repoURL string) {
|
||||
s.resumeInfoMutex.Lock()
|
||||
@@ -512,15 +514,15 @@ func (s *Source) scan(ctx context.Context, installationClient *github.Client, ch
|
||||
s.resumeInfoSlice = sources.RemoveRepoFromResumeInfo(s.resumeInfoSlice, repoURL)
|
||||
}(s, repoURL)
|
||||
|
||||
var err error
|
||||
if !strings.HasSuffix(repoURL, ".git") {
|
||||
scanErrs = append(scanErrs, fmt.Errorf("repo %s does not end in .git", repoURL))
|
||||
return nil
|
||||
}
|
||||
|
||||
s.log.WithField("repo", repoURL).Debugf("attempting to clone repo %d/%d", i+1, len(s.repos))
|
||||
s.log.WithField("repo", repoURL).Debugf("attempting to clone repo #%d of %d", cnt, repoCnt)
|
||||
var path string
|
||||
var repo *gogit.Repository
|
||||
var err error
|
||||
|
||||
path, repo, err = s.cloneRepo(ctx, repoURL, installationClient)
|
||||
if err != nil {
|
||||
@@ -537,21 +539,20 @@ func (s *Source) scan(ctx context.Context, installationClient *github.Client, ch
|
||||
git.ScanOptionHeadCommit(s.conn.Head),
|
||||
)
|
||||
|
||||
log.Debugf("Starting to scan repo #%d of %d: %s", cnt, repoCnt, repo)
|
||||
if err = s.git.ScanRepo(ctx, repo, path, scanOptions, chunksChan); err != nil {
|
||||
log.WithError(err).Errorf("unable to scan repo, continuing")
|
||||
return nil
|
||||
}
|
||||
atomic.AddUint64(&scanned, 1)
|
||||
log.Debugf("scanned %d/%d repos", scanned, len(s.repos))
|
||||
log.Debugf("scanned %d/%d repos", scanned, repoCnt)
|
||||
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
_ = s.jobPool.Wait()
|
||||
if len(scanErrs) == 0 {
|
||||
s.SetProgressComplete(len(s.repos), len(s.repos), "Completed Github scan", "")
|
||||
}
|
||||
s.Complete(fmt.Sprintf("Completed scanning source %s", s.name))
|
||||
|
||||
return scanErrs
|
||||
}
|
||||
@@ -1006,8 +1007,8 @@ func (s *Source) normalizeRepos(ctx context.Context) {
|
||||
}
|
||||
}
|
||||
|
||||
// setProgressCompleteWithRepo calls the s.SetProgressComplete after safely setting up the encoded resume info string.
|
||||
func (s *Source) setProgressCompleteWithRepo(index int, offset int, repoURL string) {
|
||||
// setProgressCompleteWithRepo calls the s.Update after safely setting up the encoded resume info string.
|
||||
func (s *Source) setProgressCompleteWithRepo(offset int, repoURL string) {
|
||||
s.resumeInfoMutex.Lock()
|
||||
defer s.resumeInfoMutex.Unlock()
|
||||
|
||||
@@ -1018,5 +1019,5 @@ func (s *Source) setProgressCompleteWithRepo(index int, offset int, repoURL stri
|
||||
// Make the resume info string from the slice.
|
||||
encodedResumeInfo := sources.EncodeResumeInfo(s.resumeInfoSlice)
|
||||
|
||||
s.SetProgressComplete(index+offset, len(s.repos)+offset, fmt.Sprintf("Repo: %s", repoURL), encodedResumeInfo)
|
||||
s.Update(len(s.repos)+offset, fmt.Sprintf("Repo: %s", repoURL), encodedResumeInfo)
|
||||
}
|
||||
|
||||
@@ -308,29 +308,6 @@ func TestEnumerateWithToken(t *testing.T) {
|
||||
assert.True(t, gock.IsDone())
|
||||
}
|
||||
|
||||
func TestEnumerateWithToken_IncludeRepos(t *testing.T) {
|
||||
defer gock.Off()
|
||||
|
||||
gock.New("https://api.github.com").
|
||||
Get("/user").
|
||||
Reply(200).
|
||||
JSON(map[string]string{"login": "super-secret-user"})
|
||||
|
||||
gock.New("https://api.github.com").
|
||||
Get("/users/super-secret-user/gists").
|
||||
Reply(200).
|
||||
JSON([]map[string]string{{"clone_url": ""}})
|
||||
|
||||
s := initTestSource(nil)
|
||||
s.repos = []string{"some-special-repo"}
|
||||
|
||||
err := s.enumerateWithToken(context.TODO(), "https://api.github.com", "token")
|
||||
assert.Nil(t, err)
|
||||
assert.Equal(t, 2, len(s.repos))
|
||||
assert.Equal(t, []string{"some-special-repo", ""}, s.repos)
|
||||
assert.True(t, gock.IsDone())
|
||||
}
|
||||
|
||||
func TestEnumerateWithApp(t *testing.T) {
|
||||
defer gock.Off()
|
||||
|
||||
@@ -406,7 +383,7 @@ func Test_setProgressCompleteWithRepo_resumeInfo(t *testing.T) {
|
||||
|
||||
for _, tt := range tests {
|
||||
s.resumeInfoSlice = tt.startingResumeInfoSlice
|
||||
s.setProgressCompleteWithRepo(0, 0, tt.repoURL)
|
||||
s.setProgressCompleteWithRepo(0, tt.repoURL)
|
||||
if !reflect.DeepEqual(s.resumeInfoSlice, tt.wantResumeInfoSlice) {
|
||||
t.Errorf("s.setProgressCompleteWithRepo() got: %v, want: %v", s.resumeInfoSlice, tt.wantResumeInfoSlice)
|
||||
}
|
||||
@@ -417,7 +394,6 @@ func Test_setProgressCompleteWithRepo_Progress(t *testing.T) {
|
||||
repos := []string{"a", "b", "c", "d", "e"}
|
||||
tests := map[string]struct {
|
||||
repos []string
|
||||
index int
|
||||
offset int
|
||||
wantPercentComplete int64
|
||||
wantSectionsCompleted int32
|
||||
@@ -425,7 +401,6 @@ func Test_setProgressCompleteWithRepo_Progress(t *testing.T) {
|
||||
}{
|
||||
"starting from the beginning, no offset": {
|
||||
repos: repos,
|
||||
index: 0,
|
||||
offset: 0,
|
||||
wantPercentComplete: 0,
|
||||
wantSectionsCompleted: 0,
|
||||
@@ -433,18 +408,16 @@ func Test_setProgressCompleteWithRepo_Progress(t *testing.T) {
|
||||
},
|
||||
"resume from the third, offset 2": {
|
||||
repos: repos[2:],
|
||||
index: 0,
|
||||
offset: 2,
|
||||
wantPercentComplete: 40,
|
||||
wantSectionsCompleted: 2,
|
||||
wantSectionsRemaining: 5,
|
||||
},
|
||||
"resume from the third, on last repo, offset 2": {
|
||||
repos: repos[2:],
|
||||
index: 2,
|
||||
offset: 2,
|
||||
wantPercentComplete: 80,
|
||||
wantSectionsCompleted: 4,
|
||||
"resume from the third, on last repo, offset 3": {
|
||||
repos: repos[3:],
|
||||
offset: 3,
|
||||
wantPercentComplete: 60,
|
||||
wantSectionsCompleted: 3,
|
||||
wantSectionsRemaining: 5,
|
||||
},
|
||||
}
|
||||
@@ -453,22 +426,29 @@ func Test_setProgressCompleteWithRepo_Progress(t *testing.T) {
|
||||
logger.Out = io.Discard
|
||||
|
||||
for _, tt := range tests {
|
||||
s := &Source{
|
||||
repos: tt.repos,
|
||||
log: logger.WithField("no", "output"),
|
||||
}
|
||||
t.Run("", func(t *testing.T) {
|
||||
s := &Source{
|
||||
repos: tt.repos,
|
||||
log: logger.WithField("no", "output"),
|
||||
}
|
||||
|
||||
s.setProgressCompleteWithRepo(tt.index, tt.offset, "")
|
||||
gotProgress := s.GetProgress()
|
||||
if gotProgress.PercentComplete != tt.wantPercentComplete {
|
||||
t.Errorf("s.setProgressCompleteWithRepo() PercentComplete got: %v want: %v", gotProgress.PercentComplete, tt.wantPercentComplete)
|
||||
}
|
||||
if gotProgress.SectionsCompleted != tt.wantSectionsCompleted {
|
||||
t.Errorf("s.setProgressCompleteWithRepo() PercentComplete got: %v want: %v", gotProgress.SectionsCompleted, tt.wantSectionsCompleted)
|
||||
}
|
||||
if gotProgress.SectionsRemaining != tt.wantSectionsRemaining {
|
||||
t.Errorf("s.setProgressCompleteWithRepo() PercentComplete got: %v want: %v", gotProgress.SectionsRemaining, tt.wantSectionsRemaining)
|
||||
}
|
||||
// Increment the progress counter by number of repos already processed.
|
||||
for i := 1; i <= tt.offset; i++ {
|
||||
s.Counter.IncSuccess()
|
||||
}
|
||||
|
||||
s.setProgressCompleteWithRepo(tt.offset, "")
|
||||
gotProgress := s.GetProgress()
|
||||
if gotProgress.PercentComplete != tt.wantPercentComplete {
|
||||
t.Errorf("s.setProgressCompleteWithRepo() PercentComplete got: %v want: %v", gotProgress.PercentComplete, tt.wantPercentComplete)
|
||||
}
|
||||
if gotProgress.SectionsCompleted != tt.wantSectionsCompleted {
|
||||
t.Errorf("s.setProgressCompleteWithRepo() SectionsCompleted got: %v want: %v", gotProgress.SectionsCompleted, tt.wantSectionsCompleted)
|
||||
}
|
||||
if gotProgress.SectionsRemaining != tt.wantSectionsRemaining {
|
||||
t.Errorf("s.setProgressCompleteWithRepo() SectionsRemaining got: %v want: %v", gotProgress.SectionsRemaining, tt.wantSectionsRemaining)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -477,16 +457,15 @@ func Test_scan_SetProgressComplete(t *testing.T) {
|
||||
name string
|
||||
repos []string
|
||||
wantComplete bool
|
||||
wantErr bool
|
||||
}{
|
||||
{
|
||||
name: "no repos",
|
||||
wantComplete: true,
|
||||
},
|
||||
{
|
||||
name: "one valid repo",
|
||||
repos: []string{"a"},
|
||||
wantErr: true,
|
||||
name: "one valid repo",
|
||||
repos: []string{"a"},
|
||||
wantComplete: true,
|
||||
},
|
||||
}
|
||||
|
||||
@@ -498,9 +477,6 @@ func Test_scan_SetProgressComplete(t *testing.T) {
|
||||
src.jobPool = &errgroup.Group{}
|
||||
|
||||
_ = src.scan(context.Background(), nil, nil)
|
||||
if !tc.wantErr {
|
||||
assert.Equal(t, "", src.GetProgress().EncodedResumeInfo)
|
||||
}
|
||||
|
||||
gotComplete := src.GetProgress().PercentComplete == 100
|
||||
if gotComplete != tc.wantComplete {
|
||||
|
||||
@@ -8,6 +8,9 @@ import (
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
|
||||
"golang.org/x/sync/errgroup"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/context"
|
||||
@@ -46,6 +49,7 @@ type Source struct {
|
||||
resumeInfoMutex sync.Mutex
|
||||
jobSem *semaphore.Weighted
|
||||
sources.Progress
|
||||
jobPool *errgroup.Group
|
||||
}
|
||||
|
||||
// Ensure the Source satisfies the interface at compile time.
|
||||
@@ -73,7 +77,8 @@ func (s *Source) Init(aCtx context.Context, name string, jobId, sourceId int64,
|
||||
s.sourceId = sourceId
|
||||
s.jobId = jobId
|
||||
s.verify = verify
|
||||
s.jobSem = semaphore.NewWeighted(int64(concurrency))
|
||||
s.jobPool = &errgroup.Group{}
|
||||
s.jobPool.SetLimit(concurrency)
|
||||
|
||||
var conn sourcespb.GitLab
|
||||
err := anypb.UnmarshalTo(connection, &conn, proto.UnmarshalOptions{})
|
||||
@@ -269,34 +274,32 @@ func (s *Source) getRepos() ([]string, []error) {
|
||||
}
|
||||
|
||||
func (s *Source) scanRepos(ctx context.Context, chunksChan chan *sources.Chunk) []error {
|
||||
wg := sync.WaitGroup{}
|
||||
var errs []error
|
||||
var errsMut sync.Mutex
|
||||
var scanned uint64
|
||||
|
||||
// If there is resume information available, limit this scan to only the repos that still need scanning.
|
||||
reposToScan, progressIndexOffset := sources.FilterReposToResume(s.repos, s.GetProgress().EncodedResumeInfo)
|
||||
s.repos = reposToScan
|
||||
log.Debugf("Found %v total repos to scan", len(s.repos))
|
||||
|
||||
for i, repo := range s.repos {
|
||||
if common.IsDone(ctx) {
|
||||
// We are returning nil instead of the scanErrors slice here because
|
||||
// we don't want to mark this scan as errored if we cancelled it.
|
||||
return nil
|
||||
}
|
||||
if err := s.jobSem.Acquire(ctx, 1); err != nil {
|
||||
log.WithError(err).Debug("could not acquire semaphore")
|
||||
continue
|
||||
}
|
||||
wg.Add(1)
|
||||
go func(ctx context.Context, repoURL string, i int) {
|
||||
defer s.jobSem.Release(1)
|
||||
defer wg.Done()
|
||||
var errMut sync.Mutex
|
||||
var scanErrs []error
|
||||
repoCnt := len(s.repos)
|
||||
for _, repoURL := range s.repos {
|
||||
repoURL := repoURL
|
||||
s.jobPool.Go(func() error {
|
||||
if common.IsDone(ctx) {
|
||||
// We are returning nil instead of the scanErrors slice here because
|
||||
// we don't want to mark this scan as errored if we cancelled it.
|
||||
return nil
|
||||
}
|
||||
if len(repoURL) == 0 {
|
||||
return
|
||||
return nil
|
||||
}
|
||||
|
||||
s.setProgressCompleteWithRepo(i, progressIndexOffset, repoURL)
|
||||
// Ensure the repo is removed from the resume info after being scanned.
|
||||
cnt := s.Counter.IncTotal()
|
||||
defer s.Counter.IncSuccess()
|
||||
s.setProgressCompleteWithRepo(progressIndexOffset, repoURL)
|
||||
// Ensure the repoURL is removed from the resume info after being scanned.
|
||||
defer func(s *Source) {
|
||||
s.resumeInfoMutex.Lock()
|
||||
defer s.resumeInfoMutex.Unlock()
|
||||
@@ -309,7 +312,7 @@ func (s *Source) scanRepos(ctx context.Context, chunksChan chan *sources.Chunk)
|
||||
if s.authMethod == "UNAUTHENTICATED" {
|
||||
path, repo, err = git.CloneRepoUsingUnauthenticated(repoURL)
|
||||
} else {
|
||||
// If a username is not provided we need to use a default one in order to clone a private repo.
|
||||
// If a username is not provided we need to use a default one in order to clone a private repoURL.
|
||||
// Not setting "placeholder" as s.user on purpose in case any downstream services rely on a "" value for s.user.
|
||||
user := s.user
|
||||
if user == "" {
|
||||
@@ -319,28 +322,32 @@ func (s *Source) scanRepos(ctx context.Context, chunksChan chan *sources.Chunk)
|
||||
}
|
||||
defer os.RemoveAll(path)
|
||||
if err != nil {
|
||||
errsMut.Lock()
|
||||
errs = append(errs, err)
|
||||
errsMut.Unlock()
|
||||
return
|
||||
errMut.Lock()
|
||||
scanErrs = append(scanErrs, err)
|
||||
errMut.Unlock()
|
||||
return nil
|
||||
}
|
||||
log.Debugf("Starting to scan repo %d/%d: %s", i+1, len(s.repos), repoURL)
|
||||
|
||||
log.Debugf("Starting to scan repoURL #%d of %d: %s", cnt, repoCnt, repo)
|
||||
err = s.git.ScanRepo(ctx, repo, path, git.NewScanOptions(), chunksChan)
|
||||
if err != nil {
|
||||
errsMut.Lock()
|
||||
errs = append(errs, err)
|
||||
errsMut.Unlock()
|
||||
return
|
||||
errMut.Lock()
|
||||
scanErrs = append(scanErrs, err)
|
||||
errMut.Unlock()
|
||||
return nil
|
||||
}
|
||||
log.Debugf("Completed scanning repo %d/%d: %s", i+1, len(s.repos), repoURL)
|
||||
}(ctx, repo, i)
|
||||
}
|
||||
wg.Wait()
|
||||
if len(errs) == 0 {
|
||||
s.SetProgressComplete(len(s.repos), len(s.repos), fmt.Sprintf("Completed scanning source %s", s.name), "")
|
||||
atomic.AddUint64(&scanned, 1)
|
||||
log.Debugf("scanned %d/%d repos", scanned, repoCnt)
|
||||
log.Debugf("Completed scanning repoURL # %d of %d: %s", cnt, repoCnt, repo)
|
||||
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
return errs
|
||||
_ = s.jobPool.Wait()
|
||||
s.Complete(fmt.Sprintf("Completed scanning source %s", s.name))
|
||||
|
||||
return scanErrs
|
||||
}
|
||||
|
||||
// Chunks emits chunks of bytes over a channel.
|
||||
@@ -417,8 +424,8 @@ func (s *Source) basicAuthSuccessful(apiClient *gitlab.Client) bool {
|
||||
return false
|
||||
}
|
||||
|
||||
// setProgressCompleteWithRepo calls the s.SetProgressComplete after safely setting up the encoded resume info string.
|
||||
func (s *Source) setProgressCompleteWithRepo(index int, offset int, repoURL string) {
|
||||
// setProgressCompleteWithRepo calls the s.Update after safely setting up the encoded resume info string.
|
||||
func (s *Source) setProgressCompleteWithRepo(offset int, repoURL string) {
|
||||
s.resumeInfoMutex.Lock()
|
||||
defer s.resumeInfoMutex.Unlock()
|
||||
|
||||
@@ -430,5 +437,5 @@ func (s *Source) setProgressCompleteWithRepo(index int, offset int, repoURL stri
|
||||
encodedResumeInfo := sources.EncodeResumeInfo(s.resumeInfoSlice)
|
||||
|
||||
// Add the offset to both the index and the repos to give the proper place and proper repo count.
|
||||
s.SetProgressComplete(index+offset, len(s.repos)+offset, fmt.Sprintf("Repo: %s", repoURL), encodedResumeInfo)
|
||||
s.Update(len(s.repos)+offset, fmt.Sprintf("Repo: %s", repoURL), encodedResumeInfo)
|
||||
}
|
||||
|
||||
@@ -9,7 +9,7 @@ import (
|
||||
"github.com/kylelemons/godebug/pretty"
|
||||
log "github.com/sirupsen/logrus"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"golang.org/x/sync/semaphore"
|
||||
"golang.org/x/sync/errgroup"
|
||||
"google.golang.org/protobuf/types/known/anypb"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
@@ -187,7 +187,7 @@ func Test_setProgressCompleteWithRepo_resumeInfo(t *testing.T) {
|
||||
|
||||
for _, tt := range tests {
|
||||
s.resumeInfoSlice = tt.startingResumeInfoSlice
|
||||
s.setProgressCompleteWithRepo(0, 0, tt.repoURL)
|
||||
s.setProgressCompleteWithRepo(0, tt.repoURL)
|
||||
if !reflect.DeepEqual(s.resumeInfoSlice, tt.wantResumeInfoSlice) {
|
||||
t.Errorf("s.setProgressCompleteWithRepo() got: %v, want: %v", s.resumeInfoSlice, tt.wantResumeInfoSlice)
|
||||
}
|
||||
@@ -198,7 +198,6 @@ func Test_setProgressCompleteWithRepo_Progress(t *testing.T) {
|
||||
repos := []string{"a", "b", "c", "d", "e"}
|
||||
tests := map[string]struct {
|
||||
repos []string
|
||||
index int
|
||||
offset int
|
||||
wantPercentComplete int64
|
||||
wantSectionsCompleted int32
|
||||
@@ -206,7 +205,6 @@ func Test_setProgressCompleteWithRepo_Progress(t *testing.T) {
|
||||
}{
|
||||
"starting from the beginning, no offset": {
|
||||
repos: repos,
|
||||
index: 0,
|
||||
offset: 0,
|
||||
wantPercentComplete: 0,
|
||||
wantSectionsCompleted: 0,
|
||||
@@ -214,18 +212,16 @@ func Test_setProgressCompleteWithRepo_Progress(t *testing.T) {
|
||||
},
|
||||
"resume from the third, offset 2": {
|
||||
repos: repos[2:],
|
||||
index: 0,
|
||||
offset: 2,
|
||||
wantPercentComplete: 40,
|
||||
wantSectionsCompleted: 2,
|
||||
wantSectionsRemaining: 5,
|
||||
},
|
||||
"resume from the third, on last repo, offset 2": {
|
||||
repos: repos[2:],
|
||||
index: 2,
|
||||
offset: 2,
|
||||
wantPercentComplete: 80,
|
||||
wantSectionsCompleted: 4,
|
||||
repos: repos[3:],
|
||||
offset: 3,
|
||||
wantPercentComplete: 60,
|
||||
wantSectionsCompleted: 3,
|
||||
wantSectionsRemaining: 5,
|
||||
},
|
||||
}
|
||||
@@ -233,21 +229,28 @@ func Test_setProgressCompleteWithRepo_Progress(t *testing.T) {
|
||||
log.SetOutput(io.Discard)
|
||||
|
||||
for _, tt := range tests {
|
||||
s := &Source{
|
||||
repos: tt.repos,
|
||||
}
|
||||
t.Run("", func(t *testing.T) {
|
||||
s := &Source{
|
||||
repos: tt.repos,
|
||||
}
|
||||
|
||||
s.setProgressCompleteWithRepo(tt.index, tt.offset, "")
|
||||
gotProgress := s.GetProgress()
|
||||
if gotProgress.PercentComplete != tt.wantPercentComplete {
|
||||
t.Errorf("s.setProgressCompleteWithRepo() PercentComplete got: %v want: %v", gotProgress.PercentComplete, tt.wantPercentComplete)
|
||||
}
|
||||
if gotProgress.SectionsCompleted != tt.wantSectionsCompleted {
|
||||
t.Errorf("s.setProgressCompleteWithRepo() PercentComplete got: %v want: %v", gotProgress.SectionsCompleted, tt.wantSectionsCompleted)
|
||||
}
|
||||
if gotProgress.SectionsRemaining != tt.wantSectionsRemaining {
|
||||
t.Errorf("s.setProgressCompleteWithRepo() PercentComplete got: %v want: %v", gotProgress.SectionsRemaining, tt.wantSectionsRemaining)
|
||||
}
|
||||
// Increment the progress counter by number of repos already processed.
|
||||
for i := 1; i <= tt.offset; i++ {
|
||||
s.Counter.IncSuccess()
|
||||
}
|
||||
|
||||
s.setProgressCompleteWithRepo(tt.offset, "")
|
||||
gotProgress := s.GetProgress()
|
||||
if gotProgress.PercentComplete != tt.wantPercentComplete {
|
||||
t.Errorf("s.setProgressCompleteWithRepo() PercentComplete got: %v want: %v", gotProgress.PercentComplete, tt.wantPercentComplete)
|
||||
}
|
||||
if gotProgress.SectionsCompleted != tt.wantSectionsCompleted {
|
||||
t.Errorf("s.setProgressCompleteWithRepo() SectionsCompleted got: %v want: %v", gotProgress.SectionsCompleted, tt.wantSectionsCompleted)
|
||||
}
|
||||
if gotProgress.SectionsRemaining != tt.wantSectionsRemaining {
|
||||
t.Errorf("s.setProgressCompleteWithRepo() SectionsRemaining got: %v want: %v", gotProgress.SectionsRemaining, tt.wantSectionsRemaining)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -256,16 +259,15 @@ func Test_scanRepos_SetProgressComplete(t *testing.T) {
|
||||
name string
|
||||
repos []string
|
||||
wantComplete bool
|
||||
wantErr bool
|
||||
}{
|
||||
{
|
||||
name: "no repos",
|
||||
wantComplete: true,
|
||||
},
|
||||
{
|
||||
name: "one valid repo",
|
||||
repos: []string{"a"},
|
||||
wantErr: true,
|
||||
name: "one valid repo",
|
||||
repos: []string{"a"},
|
||||
wantComplete: true,
|
||||
},
|
||||
}
|
||||
|
||||
@@ -274,12 +276,9 @@ func Test_scanRepos_SetProgressComplete(t *testing.T) {
|
||||
src := &Source{
|
||||
repos: tc.repos,
|
||||
}
|
||||
src.jobSem = semaphore.NewWeighted(1)
|
||||
src.jobPool = &errgroup.Group{}
|
||||
|
||||
_ = src.scanRepos(context.Background(), nil)
|
||||
if !tc.wantErr {
|
||||
assert.Equal(t, "", src.GetProgress().EncodedResumeInfo)
|
||||
}
|
||||
|
||||
gotComplete := src.GetProgress().PercentComplete == 100
|
||||
if gotComplete != tc.wantComplete {
|
||||
|
||||
+12
-12
@@ -2,7 +2,6 @@ package s3
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
@@ -136,12 +135,15 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) err
|
||||
return errors.Errorf("invalid configuration given for %s source", s.name)
|
||||
}
|
||||
|
||||
for i, bucket := range bucketsToScan {
|
||||
numBuckets := len(bucketsToScan)
|
||||
log.Debugf("Found %v total buckets to scan", numBuckets)
|
||||
for _, bucket := range bucketsToScan {
|
||||
if common.IsDone(ctx) {
|
||||
return nil
|
||||
}
|
||||
|
||||
s.SetProgressComplete(i, len(bucketsToScan), fmt.Sprintf("Bucket: %s", bucket), "")
|
||||
cnt := s.Counter.IncSuccess()
|
||||
s.Update(len(bucketsToScan), fmt.Sprintf("Bucket: %s", bucket), "")
|
||||
|
||||
s.log.Debugf("Scanning bucket: %s", bucket)
|
||||
region, err := s3manager.GetBucketRegionWithClient(context.Background(), client, bucket)
|
||||
@@ -162,6 +164,7 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) err
|
||||
// pf := "public"
|
||||
errorCount := sync.Map{}
|
||||
|
||||
log.Debugf("Starting to scan bucket #%d of %d: %s", cnt, numBuckets, bucket)
|
||||
err = regionalClient.ListObjectsV2PagesWithContext(
|
||||
ctx, &s3.ListObjectsV2Input{Bucket: &bucket},
|
||||
func(page *s3.ListObjectsV2Output, last bool) bool {
|
||||
@@ -173,8 +176,9 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) err
|
||||
s.log.WithError(err).Errorf("could not list objects in s3 bucket: %s", bucket)
|
||||
return errors.WrapPrefix(err, fmt.Sprintf("could not list objects in s3 bucket: %s", bucket), 0)
|
||||
}
|
||||
log.Debugf("scanned %d/%d buckets", cnt, numBuckets)
|
||||
}
|
||||
s.SetProgressComplete(len(bucketsToScan), len(bucketsToScan), fmt.Sprintf("Completed scanning source %s", s.name), "")
|
||||
s.Complete(fmt.Sprintf("Completed scanning source %s", s.name))
|
||||
|
||||
return nil
|
||||
}
|
||||
@@ -296,15 +300,11 @@ func (s *Source) pageChunker(ctx context.Context, client *s3.S3, chunksChan chan
|
||||
}
|
||||
reader.Stop()
|
||||
|
||||
chunk := *chunkSkel
|
||||
chunkData, err := io.ReadAll(reader)
|
||||
if err != nil {
|
||||
log.WithError(err).Error("Could not read file data.")
|
||||
return
|
||||
for chunkData := range common.ChunkReader(reader) {
|
||||
chunk := *chunkSkel
|
||||
chunk.Data = chunkData
|
||||
chunksChan <- &chunk
|
||||
}
|
||||
chunk.Data = chunkData
|
||||
chunksChan <- &chunk
|
||||
|
||||
nErr, ok = errorCount.Load(prefix)
|
||||
if !ok {
|
||||
nErr = 0
|
||||
|
||||
+45
-12
@@ -2,6 +2,7 @@ package sources
|
||||
|
||||
import (
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
|
||||
"google.golang.org/protobuf/types/known/anypb"
|
||||
|
||||
@@ -104,39 +105,71 @@ func NewConfig(opts ...func(*Config)) Config {
|
||||
return *c
|
||||
}
|
||||
|
||||
// Counter is a simple monotonically-increasing counter for tracking progress.
|
||||
// It includes a successCnt that records the number of successful scans.
|
||||
// It also includes a totalCnt that records the total number of scans.
|
||||
// This is necessary in order to display the number of concurrent scans
|
||||
// regardless of whether they are successful or not.
|
||||
type Counter struct {
|
||||
successCnt, totalCnt uint32
|
||||
}
|
||||
|
||||
// IncTotal increments the total field of counter and returns the new value.
|
||||
func (c *Counter) IncTotal() int {
|
||||
return int(atomic.AddUint32(&c.totalCnt, 1))
|
||||
}
|
||||
|
||||
// IncSuccess increments the success field of counter and returns the new value.
|
||||
func (c *Counter) IncSuccess() int {
|
||||
return int(atomic.AddUint32(&c.successCnt, 1))
|
||||
}
|
||||
|
||||
// Get returns the current successCnt.
|
||||
func (c *Counter) Get() uint32 {
|
||||
return atomic.LoadUint32(&c.successCnt)
|
||||
}
|
||||
|
||||
// Progress is used to update job completion progress across sources.
|
||||
type Progress struct {
|
||||
mut sync.Mutex
|
||||
SectionsCompleted int32
|
||||
SectionsRemaining int32
|
||||
PercentComplete int64
|
||||
Message string
|
||||
EncodedResumeInfo string
|
||||
SectionsCompleted int32
|
||||
SectionsRemaining int32
|
||||
mut sync.Mutex
|
||||
Counter Counter
|
||||
}
|
||||
|
||||
// SetProgressComplete sets job progress information for a running job based on the highest level objects in the source.
|
||||
// i is the current iteration in the loop of target scope
|
||||
// Update sets job progress information for a running job based on the highest level objects in the source.
|
||||
// scope should be the len(scopedItems)
|
||||
// message is the public facing user information about the current progress
|
||||
// encodedResumeInfo is an optional string representing any information necessary to resume the job if interrupted
|
||||
func (p *Progress) SetProgressComplete(i, scope int, message, encodedResumeInfo string) {
|
||||
func (p *Progress) Update(scope int, message, encodedResumeInfo string) {
|
||||
p.mut.Lock()
|
||||
defer p.mut.Unlock()
|
||||
|
||||
i := p.Counter.Get()
|
||||
|
||||
p.Message = message
|
||||
p.EncodedResumeInfo = encodedResumeInfo
|
||||
p.SectionsCompleted = int32(i)
|
||||
p.SectionsRemaining = int32(scope)
|
||||
|
||||
// If the iteration and scope are both 0, completion is 100%.
|
||||
if i == 0 && scope == 0 {
|
||||
p.PercentComplete = 100
|
||||
return
|
||||
}
|
||||
|
||||
p.PercentComplete = int64((float64(i) / float64(scope)) * 100)
|
||||
}
|
||||
|
||||
// Complete sets job progress information for a completed job.
|
||||
func (p *Progress) Complete(msg string) {
|
||||
p.mut.Lock()
|
||||
defer p.mut.Unlock()
|
||||
|
||||
p.SectionsRemaining = 0
|
||||
p.SectionsCompleted = int32(p.Counter.Get())
|
||||
p.Message = msg
|
||||
p.EncodedResumeInfo = ""
|
||||
p.PercentComplete = 100
|
||||
}
|
||||
|
||||
// GetProgress gets job completion percentage for metrics reporting.
|
||||
func (p *Progress) GetProgress() *Progress {
|
||||
p.mut.Lock()
|
||||
|
||||
@@ -0,0 +1,192 @@
|
||||
package sources
|
||||
|
||||
import (
|
||||
"sync"
|
||||
"testing"
|
||||
|
||||
"github.com/google/go-cmp/cmp"
|
||||
"github.com/google/go-cmp/cmp/cmpopts"
|
||||
"google.golang.org/protobuf/testing/protocmp"
|
||||
)
|
||||
|
||||
func TestCounter_IncTotal(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
want int
|
||||
}{
|
||||
{
|
||||
name: "increment total 50 times",
|
||||
want: 50,
|
||||
},
|
||||
{
|
||||
name: "increment total 0 times",
|
||||
},
|
||||
{
|
||||
name: "increment total 1 times",
|
||||
want: 1,
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
c := &Counter{}
|
||||
|
||||
var wg sync.WaitGroup
|
||||
wg.Add(tt.want)
|
||||
for i := 0; i < tt.want; i++ {
|
||||
go func(c *Counter) {
|
||||
defer wg.Done()
|
||||
c.IncTotal()
|
||||
}(c)
|
||||
}
|
||||
|
||||
wg.Wait()
|
||||
if got := c.totalCnt; int(got) != tt.want {
|
||||
t.Errorf("Counter.IncTotal() = %v, want %v", got, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestCounter_IncSuccess(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
want int
|
||||
}{
|
||||
{
|
||||
name: "increment total 50 times",
|
||||
want: 50,
|
||||
},
|
||||
{
|
||||
name: "increment total 0 times",
|
||||
},
|
||||
{
|
||||
name: "increment total 1 times",
|
||||
want: 1,
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
c := &Counter{}
|
||||
|
||||
var wg sync.WaitGroup
|
||||
wg.Add(tt.want)
|
||||
for i := 0; i < tt.want; i++ {
|
||||
go func(c *Counter) {
|
||||
defer wg.Done()
|
||||
c.IncSuccess()
|
||||
}(c)
|
||||
}
|
||||
|
||||
wg.Wait()
|
||||
if got := c.Get(); int(got) != tt.want {
|
||||
t.Errorf("Counter.IncTotal() = %v, want %v", got, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestProgress_Update(t *testing.T) {
|
||||
tests := []struct {
|
||||
name, msg, encodeResumeInfo string
|
||||
counter uint32
|
||||
scope, routines, want int
|
||||
}{
|
||||
{
|
||||
name: "update progress, no counter",
|
||||
scope: 100,
|
||||
routines: 10,
|
||||
want: 10,
|
||||
},
|
||||
{
|
||||
name: "update progress, counter 5",
|
||||
scope: 100,
|
||||
routines: 10,
|
||||
counter: 5,
|
||||
want: 15,
|
||||
},
|
||||
{
|
||||
name: "update progress, counter 50",
|
||||
scope: 100,
|
||||
routines: 10,
|
||||
counter: 50,
|
||||
want: 60,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
p := &Progress{}
|
||||
p.Counter.successCnt = tt.counter
|
||||
|
||||
var wg sync.WaitGroup
|
||||
wg.Add(tt.routines)
|
||||
for i := 0; i < tt.routines; i++ {
|
||||
p.Counter.IncSuccess()
|
||||
go func(p *Progress) {
|
||||
defer wg.Done()
|
||||
p.Update(tt.scope, tt.msg, tt.encodeResumeInfo)
|
||||
}(p)
|
||||
}
|
||||
|
||||
wg.Wait()
|
||||
if got := p.PercentComplete; int(got) != tt.want {
|
||||
t.Errorf("Progress.Update(%v, %v, %v): got %v, want %v", tt.scope, tt.msg, tt.encodeResumeInfo, got, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestProgress_Complete(t *testing.T) {
|
||||
tests := []struct {
|
||||
name,
|
||||
msg string
|
||||
curProgress, wantProgress *Progress
|
||||
}{
|
||||
{
|
||||
name: "complete progress",
|
||||
msg: "completed scanning thinger",
|
||||
curProgress: &Progress{
|
||||
PercentComplete: 42,
|
||||
Message: "scanning thinger",
|
||||
EncodedResumeInfo: "resume info",
|
||||
SectionsRemaining: 12,
|
||||
SectionsCompleted: 2,
|
||||
},
|
||||
wantProgress: &Progress{
|
||||
PercentComplete: 100,
|
||||
Message: "completed scanning thinger",
|
||||
EncodedResumeInfo: "",
|
||||
SectionsRemaining: 0,
|
||||
SectionsCompleted: 14,
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "complete progress with no message",
|
||||
curProgress: &Progress{
|
||||
PercentComplete: 80,
|
||||
Message: "scanning thinger2",
|
||||
EncodedResumeInfo: "resume info2",
|
||||
SectionsRemaining: 2,
|
||||
SectionsCompleted: 42,
|
||||
},
|
||||
wantProgress: &Progress{
|
||||
PercentComplete: 100,
|
||||
EncodedResumeInfo: "",
|
||||
SectionsRemaining: 0,
|
||||
SectionsCompleted: 44,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
tt.curProgress.Counter.successCnt = uint32(tt.wantProgress.SectionsCompleted)
|
||||
tt.curProgress.Complete(tt.msg)
|
||||
|
||||
ignoreOpts := cmpopts.IgnoreFields(Progress{}, "mut", "Counter")
|
||||
if !cmp.Equal(tt.curProgress, tt.wantProgress, protocmp.Transform(), ignoreOpts) {
|
||||
t.Errorf("Progress.Complete(%v): got %v, want %v", tt.msg, tt.curProgress, tt.wantProgress)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -905,7 +905,6 @@ enum DetectorType {
|
||||
MongoDB = 895;
|
||||
NGC = 896;
|
||||
DigitalOceanV2 = 897;
|
||||
SQLServer = 898;
|
||||
}
|
||||
|
||||
message Result {
|
||||
|
||||
Reference in New Issue
Block a user