Compare commits
17 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 Ahrav Dutta
|
7798005fd5 | ||
|
Ahrav Dutta
|
32555f5e87 | ||
|
ahrav
|
f3071aa46a | ||
|
Ahrav Dutta
|
ea2de122be | ||
|
Miccah
|
e5ede17c77 | ||
|
Miccah
|
74831f63d5 | ||
|
Ahrav Dutta
|
83e22d8b16 | ||
|
ahrav
|
09d4422cdb | ||
|
dependabot[bot]
|
7de0178842 | ||
|
dependabot[bot]
|
6d384ce3e8 | ||
|
dependabot[bot]
|
0e24d406d5 | ||
|
dependabot[bot]
|
d72c31b8b6 | ||
|
Yassine Ilmi
|
d720c0c0f3 | ||
|
dependabot[bot]
|
705c01e5f3 | ||
|
Pulkit Aggarwal
|
fc6fd29f3f | ||
|
ahrav
|
ee6817ad85 | ||
|
Dustin Decker
|
5f6143f09a |
@@ -44,7 +44,7 @@ jobs:
|
||||
with:
|
||||
go-version: '1.18'
|
||||
- name: Run GoReleaser
|
||||
uses: goreleaser/goreleaser-action@v3
|
||||
uses: goreleaser/goreleaser-action@v4
|
||||
with:
|
||||
distribution: goreleaser
|
||||
version: latest
|
||||
|
||||
@@ -96,7 +96,7 @@ trufflehog s3 --bucket=<bucket name> --only-verified
|
||||
TruffleHog v3 is a complete rewrite in Go with many new powerful features.
|
||||
|
||||
- We've **added over 700 credential detectors that support active verification against their respective APIs**.
|
||||
- We've also added native **support for scanning GitHub, GitLab, filesystems, and S3**.
|
||||
- We've also added native **support for scanning GitHub, GitLab, filesystems, S3, and Circle CI**.
|
||||
- **Instantly verify private keys** against millions of github users and **billions** of TLS certificates using our [Driftwood](https://trufflesecurity.com/blog/driftwood) technology.
|
||||
|
||||
|
||||
@@ -160,6 +160,7 @@ TruffleHog has a sub-command for each source of data that you may want to scan:
|
||||
- S3
|
||||
- filesystem
|
||||
- syslog
|
||||
- circleci
|
||||
- file and stdin (coming soon)
|
||||
|
||||
Each subcommand can have options that you can see with the `--help` flag provided to the sub command:
|
||||
@@ -172,27 +173,19 @@ Find credentials in git repositories.
|
||||
|
||||
Flags:
|
||||
--help Show context-sensitive help (also try --help-long and --help-man).
|
||||
--debug Run in debug mode
|
||||
--version Prints trufflehog version.
|
||||
--debug Run in debug mode.
|
||||
--trace Run in trace mode.
|
||||
-j, --json Output in JSON format.
|
||||
--json-legacy Use the pre-v3.0 JSON format. Only works with git, gitlab, and github sources.
|
||||
--concurrency=1 Number of concurrent workers.
|
||||
--concurrency=10 Number of concurrent workers.
|
||||
--no-verification Don't verify the results.
|
||||
--only-verified Only output verified results.
|
||||
--filter-unverified Only output first unverified result per chunk per detector if there are more than one results.
|
||||
--config=CONFIG Path to configuration file.
|
||||
--print-avg-detector-time Print the average time spent on each detector.
|
||||
--no-update Don't check for updates.
|
||||
-i, --include-paths=INCLUDE-PATHS
|
||||
Path to file with newline separated regexes for files to include in scan.
|
||||
-x, --exclude-paths=EXCLUDE-PATHS
|
||||
Path to file with newline separated regexes for files to exclude in scan.
|
||||
--since-commit=SINCE-COMMIT
|
||||
Commit to start scan from.
|
||||
--branch=BRANCH Branch to scan.
|
||||
--max-depth=MAX-DEPTH Maximum depth of commits to scan.
|
||||
--allow No-op flag for backwards compat.
|
||||
--entropy No-op flag for backwards compat.
|
||||
--regex No-op flag for backwards compat.
|
||||
--fail Exit with code 183 if results are found.
|
||||
--version Show application version.
|
||||
|
||||
Args:
|
||||
<uri> Git repository URL. https://, file://, or ssh:// schema expected.
|
||||
|
||||
+3
-2
@@ -27,9 +27,10 @@ runs:
|
||||
- git
|
||||
- file://${{ inputs.path }}
|
||||
- --since-commit
|
||||
- ${{ inputs.base }}
|
||||
- ${{ inputs.base.default }}
|
||||
- --branch
|
||||
- ${{ inputs.head }}
|
||||
- --fail
|
||||
- --no-update
|
||||
- ${{ inputs.extra_args }}
|
||||
- ${{arg in inputs.extra_args.split(' ')}}${{arg}}${{end}}
|
||||
- ${{ arg }}
|
||||
+2
-3
@@ -1,5 +1,4 @@
|
||||
#!/usr/bin/env ash
|
||||
|
||||
# `$*` expands the `args` supplied in an `array` individually
|
||||
# or splits `args` in a string separated by whitespace.
|
||||
/usr/bin/trufflehog $*
|
||||
# "$@" should expand all args in an 'array' as they were passed in
|
||||
/usr/bin/trufflehog "$@"
|
||||
@@ -9,7 +9,7 @@ replace github.com/zricethezav/gitleaks/v8 => github.com/trufflesecurity/gitleak
|
||||
replace github.com/gitleaks/go-gitdiff => github.com/trufflesecurity/go-gitdiff v0.7.6-zombies2
|
||||
|
||||
require (
|
||||
cloud.google.com/go/secretmanager v1.9.0
|
||||
cloud.google.com/go/secretmanager v1.10.0
|
||||
github.com/Azure/go-autorest/autorest/azure/auth v0.5.11
|
||||
github.com/TheZeroSlave/zapsentry v1.12.0
|
||||
github.com/aws/aws-sdk-go v1.44.83
|
||||
@@ -24,7 +24,7 @@ require (
|
||||
github.com/felixge/fgprof v0.9.3
|
||||
github.com/getsentry/sentry-go v0.16.0
|
||||
github.com/go-errors/errors v1.4.2
|
||||
github.com/go-git/go-git/v5 v5.5.1
|
||||
github.com/go-git/go-git/v5 v5.5.2
|
||||
github.com/go-ldap/ldap/v3 v3.4.4
|
||||
github.com/go-logr/logr v1.2.3
|
||||
github.com/go-logr/zapr v1.2.3
|
||||
@@ -35,7 +35,7 @@ require (
|
||||
github.com/google/go-cmp v0.5.9
|
||||
github.com/google/go-github/v42 v42.0.0
|
||||
github.com/gorilla/mux v1.8.0
|
||||
github.com/hashicorp/go-retryablehttp v0.7.1
|
||||
github.com/hashicorp/go-retryablehttp v0.7.2
|
||||
github.com/jlaffaye/ftp v0.1.0
|
||||
github.com/joho/godotenv v1.4.0
|
||||
github.com/jpillora/overseer v1.1.6
|
||||
@@ -54,11 +54,11 @@ require (
|
||||
github.com/xanzy/go-gitlab v0.77.0
|
||||
go.mongodb.org/mongo-driver v1.11.1
|
||||
go.uber.org/zap v1.24.0
|
||||
golang.org/x/crypto v0.4.0
|
||||
golang.org/x/crypto v0.5.0
|
||||
golang.org/x/exp v0.0.0-20221018205818-5c77f4b2bbd7
|
||||
golang.org/x/oauth2 v0.0.0-20221014153046-6fdb5e3db783
|
||||
golang.org/x/sync v0.1.0
|
||||
google.golang.org/genproto v0.0.0-20221027153422-115e99e71e1c
|
||||
google.golang.org/genproto v0.0.0-20221201164419-0e50fba7f41c
|
||||
google.golang.org/protobuf v1.28.1
|
||||
gopkg.in/alecthomas/kingpin.v2 v2.2.6
|
||||
gopkg.in/h2non/gock.v1 v1.1.2
|
||||
@@ -68,7 +68,7 @@ require (
|
||||
require (
|
||||
cloud.google.com/go/compute v1.12.1 // indirect
|
||||
cloud.google.com/go/compute/metadata v0.2.1 // indirect
|
||||
cloud.google.com/go/iam v0.6.0 // indirect
|
||||
cloud.google.com/go/iam v0.8.0 // indirect
|
||||
github.com/Azure/go-autorest v14.2.0+incompatible // indirect
|
||||
github.com/Azure/go-autorest/autorest v0.11.24 // indirect
|
||||
github.com/Azure/go-autorest/autorest/adal v0.9.18 // indirect
|
||||
@@ -91,7 +91,7 @@ require (
|
||||
github.com/emirpasic/gods v1.18.1 // indirect
|
||||
github.com/go-asn1-ber/asn1-ber v1.5.4 // indirect
|
||||
github.com/go-git/gcfg v1.5.0 // indirect
|
||||
github.com/go-git/go-billy/v5 v5.3.1 // indirect
|
||||
github.com/go-git/go-billy/v5 v5.4.0 // indirect
|
||||
github.com/go-ole/go-ole v1.2.6 // indirect
|
||||
github.com/golang-jwt/jwt/v4 v4.4.1 // indirect
|
||||
github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe // indirect
|
||||
@@ -103,7 +103,7 @@ require (
|
||||
github.com/google/go-querystring v1.1.0 // indirect
|
||||
github.com/google/pprof v0.0.0-20211214055906-6f57359322fd // indirect
|
||||
github.com/googleapis/enterprise-certificate-proxy v0.2.0 // indirect
|
||||
github.com/googleapis/gax-go/v2 v2.6.0 // indirect
|
||||
github.com/googleapis/gax-go/v2 v2.7.0 // indirect
|
||||
github.com/h2non/parth v0.0.0-20190131123155-b4df798d6542 // indirect
|
||||
github.com/hashicorp/errwrap v1.0.0 // indirect
|
||||
github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
|
||||
@@ -134,18 +134,18 @@ require (
|
||||
github.com/xdg-go/stringprep v1.0.3 // indirect
|
||||
github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d // indirect
|
||||
github.com/yusufpapurcu/wmi v1.2.2 // indirect
|
||||
go.opencensus.io v0.23.0 // indirect
|
||||
go.opencensus.io v0.24.0 // indirect
|
||||
go.uber.org/atomic v1.7.0 // indirect
|
||||
go.uber.org/multierr v1.6.0 // indirect
|
||||
golang.org/x/mod v0.7.0 // indirect
|
||||
golang.org/x/net v0.3.0 // indirect
|
||||
golang.org/x/sys v0.3.0 // indirect
|
||||
golang.org/x/text v0.5.0 // indirect
|
||||
golang.org/x/net v0.5.0 // indirect
|
||||
golang.org/x/sys v0.4.0 // indirect
|
||||
golang.org/x/text v0.6.0 // indirect
|
||||
golang.org/x/time v0.0.0-20220922220347-f3bd1da661af // indirect
|
||||
golang.org/x/tools v0.3.0 // indirect
|
||||
google.golang.org/api v0.102.0 // indirect
|
||||
google.golang.org/api v0.103.0 // indirect
|
||||
google.golang.org/appengine v1.6.7 // indirect
|
||||
google.golang.org/grpc v1.50.1 // indirect
|
||||
google.golang.org/grpc v1.51.0 // indirect
|
||||
gopkg.in/warnings.v0 v0.1.2 // indirect
|
||||
gopkg.in/yaml.v2 v2.4.0 // indirect
|
||||
gopkg.in/yaml.v3 v3.0.1 // indirect
|
||||
|
||||
@@ -4,11 +4,11 @@ cloud.google.com/go/compute v1.12.1 h1:gKVJMEyqV5c/UnpzjjQbo3Rjvvqpr9B1DFSbJC4OX
|
||||
cloud.google.com/go/compute v1.12.1/go.mod h1:e8yNOBcBONZU1vJKCvCoDw/4JQsA0dpM4x/6PIIOocU=
|
||||
cloud.google.com/go/compute/metadata v0.2.1 h1:efOwf5ymceDhK6PKMnnrTHP4pppY5L22mle96M1yP48=
|
||||
cloud.google.com/go/compute/metadata v0.2.1/go.mod h1:jgHgmJd2RKBGzXqF5LR2EZMGxBkeanZ9wwa75XHJgOM=
|
||||
cloud.google.com/go/iam v0.6.0 h1:nsqQC88kT5Iwlm4MeNGTpfMWddp6NB/UOLFTH6m1QfQ=
|
||||
cloud.google.com/go/iam v0.6.0/go.mod h1:+1AH33ueBne5MzYccyMHtEKqLE4/kJOibtffMHDMFMc=
|
||||
cloud.google.com/go/longrunning v0.1.1 h1:y50CXG4j0+qvEukslYFBCrzaXX0qpFbBzc3PchSu/LE=
|
||||
cloud.google.com/go/secretmanager v1.9.0 h1:xE6uXljAC1kCR8iadt9+/blg1fvSbmenlsDN4fT9gqw=
|
||||
cloud.google.com/go/secretmanager v1.9.0/go.mod h1:b71qH2l1yHmWQHt9LC80akm86mX8AL6X1MA01dW8ht4=
|
||||
cloud.google.com/go/iam v0.8.0 h1:E2osAkZzxI/+8pZcxVLcDtAQx/u+hZXVryUaYQ5O0Kk=
|
||||
cloud.google.com/go/iam v0.8.0/go.mod h1:lga0/y3iH6CX7sYqypWJ33hf7kkfXJag67naqGESjkE=
|
||||
cloud.google.com/go/longrunning v0.3.0 h1:NjljC+FYPV3uh5/OwWT6pVU+doBqMg2x/rZlE+CamDs=
|
||||
cloud.google.com/go/secretmanager v1.10.0 h1:pu03bha7ukxF8otyPKTFdDz+rr9sE3YauS5PliDXK60=
|
||||
cloud.google.com/go/secretmanager v1.10.0/go.mod h1:MfnrdvKMPNra9aZtQFvBcvRU54hbPD8/HayQdlUgJpU=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/azcore v0.19.0/go.mod h1:h6H6c8enJmmocHUbLiiGY6sx7f9i+X3m1CHdd5c6Rdw=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/azidentity v0.11.0/go.mod h1:HcM1YX14R7CJcghJGOYCgdezslRSVzqwLf/q+4Y2r/0=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/internal v0.7.0/go.mod h1:yqy467j36fJxcRV2TzfVZ1pCb5vxm4BtZPUdYWe/Xo8=
|
||||
@@ -111,12 +111,13 @@ github.com/go-errors/errors v1.4.2 h1:J6MZopCL4uSllY1OfXM374weqZFFItUbrImctkmUxI
|
||||
github.com/go-errors/errors v1.4.2/go.mod h1:sIVyrIiJhuEF+Pj9Ebtd6P/rEYROXFi3BopGUQ5a5Og=
|
||||
github.com/go-git/gcfg v1.5.0 h1:Q5ViNfGF8zFgyJWPqYwA7qGFoMTEiBmdlkcfRmpIMa4=
|
||||
github.com/go-git/gcfg v1.5.0/go.mod h1:5m20vg6GwYabIxaOonVkTdrILxQMpEShl1xiMF4ua+E=
|
||||
github.com/go-git/go-billy/v5 v5.3.1 h1:CPiOUAzKtMRvolEKw+bG1PLRpT7D3LIs3/3ey4Aiu34=
|
||||
github.com/go-git/go-billy/v5 v5.3.1/go.mod h1:pmpqyWchKfYfrkb/UVH4otLvyi/5gJlGI4Hb3ZqZ3W0=
|
||||
github.com/go-git/go-billy/v5 v5.4.0 h1:Vaw7LaSTRJOUric7pe4vnzBSgyuf2KrLsu2Y4ZpQBDE=
|
||||
github.com/go-git/go-billy/v5 v5.4.0/go.mod h1:vjbugF6Fz7JIflbVpl1hJsGjSHNltrSw45YK/ukIvQg=
|
||||
github.com/go-git/go-git-fixtures/v4 v4.3.1 h1:y5z6dd3qi8Hl+stezc8p3JxDkoTRqMAlKnXHuzrfjTQ=
|
||||
github.com/go-git/go-git-fixtures/v4 v4.3.1/go.mod h1:8LHG1a3SRW71ettAD/jW13h8c6AqjVSeL11RAdgaqpo=
|
||||
github.com/go-git/go-git/v5 v5.5.1 h1:5vtv2TB5PM/gPM+EvsHJ16hJh4uAkdGcKilcwY7FYwo=
|
||||
github.com/go-git/go-git/v5 v5.5.1/go.mod h1:uz5PQ3d0gz7mSgzZhSJToM6ALPaKCdSnl58/Xb5hzr8=
|
||||
github.com/go-git/go-git/v5 v5.5.2 h1:v8lgZa5k9ylUw+OR/roJHTxR4QItsNFI5nKtAXFuynw=
|
||||
github.com/go-git/go-git/v5 v5.5.2/go.mod h1:BE5hUJ5yaV2YMxhmaP4l6RBQ08kMxKSPD4BlxtH7OjI=
|
||||
github.com/go-ldap/ldap/v3 v3.4.4 h1:qPjipEpt+qDa6SI/h1fzuGWoRUY+qqQ9sOZq67/PYUs=
|
||||
github.com/go-ldap/ldap/v3 v3.4.4/go.mod h1:fe1MsuN5eJJ1FeLT/LEBVdWfNWKh459R7aXgXtJC+aI=
|
||||
github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
|
||||
@@ -189,8 +190,8 @@ github.com/google/pprof v0.0.0-20211214055906-6f57359322fd/go.mod h1:KgnwoLYCZ8I
|
||||
github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
||||
github.com/googleapis/enterprise-certificate-proxy v0.2.0 h1:y8Yozv7SZtlU//QXbezB6QkpuE6jMD2/gfzk4AftXjs=
|
||||
github.com/googleapis/enterprise-certificate-proxy v0.2.0/go.mod h1:8C0jb7/mgJe/9KK8Lm7X9ctZC2t60YyIpYEI16jx0Qg=
|
||||
github.com/googleapis/gax-go/v2 v2.6.0 h1:SXk3ABtQYDT/OH8jAyvEOQ58mgawq5C4o/4/89qN2ZU=
|
||||
github.com/googleapis/gax-go/v2 v2.6.0/go.mod h1:1mjbznJAPHFpesgE5ucqfYEscaz5kMdcIDwU/6+DDoY=
|
||||
github.com/googleapis/gax-go/v2 v2.7.0 h1:IcsPKeInNvYi7eqSaDjiZqDDKu5rsmunY0Y1YupQSSQ=
|
||||
github.com/googleapis/gax-go/v2 v2.7.0/go.mod h1:TEop28CZZQ2y+c0VxMUmu1lV+fQx57QpBWsYpwqHJx8=
|
||||
github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI=
|
||||
github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So=
|
||||
github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
|
||||
@@ -198,15 +199,14 @@ github.com/h2non/parth v0.0.0-20190131123155-b4df798d6542 h1:2VTzZjLZBgl62/EtslC
|
||||
github.com/h2non/parth v0.0.0-20190131123155-b4df798d6542/go.mod h1:Ow0tF8D4Kplbc8s8sSb3V2oUCygFHVp8gC3Dn6U4MNI=
|
||||
github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA=
|
||||
github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
|
||||
github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80=
|
||||
github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ=
|
||||
github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48=
|
||||
github.com/hashicorp/go-hclog v0.9.2 h1:CG6TE5H9/JXsFWJCfoIVpKFIkFe6ysEuHirp4DxCsHI=
|
||||
github.com/hashicorp/go-hclog v0.9.2/go.mod h1:5CU+agLiy3J7N7QjHK5d05KxGsuXiQLrjA0H7acj2lQ=
|
||||
github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo=
|
||||
github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM=
|
||||
github.com/hashicorp/go-retryablehttp v0.7.1 h1:sUiuQAnLlbvmExtFQs72iFW/HXeUn8Z1aJLQ4LJJbTQ=
|
||||
github.com/hashicorp/go-retryablehttp v0.7.1/go.mod h1:vAew36LZh98gCBJNLH42IQ1ER/9wtLZZ8meHqQvEYWY=
|
||||
github.com/hashicorp/go-retryablehttp v0.7.2 h1:AcYqCvkpalPnPF2pn0KamgwamS42TqUDDYFRKq/RAd0=
|
||||
github.com/hashicorp/go-retryablehttp v0.7.2/go.mod h1:Jy/gPYAdjqffZ/yFGCFV2doI5wjtH1ewM9u8iYVjtX8=
|
||||
github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
|
||||
github.com/ianlancetaylor/demangle v0.0.0-20210905161508-09a460cdf81d/go.mod h1:aYm2/VgdVmcIU8iMfdMvDMsRAQjcfZSKFby6HOFvi/w=
|
||||
github.com/imdario/mergo v0.3.13 h1:lFzP57bqS/wsqKssCGmtLAb8A0wKjLGrve2q3PPVcBk=
|
||||
@@ -354,8 +354,8 @@ github.com/yusufpapurcu/wmi v1.2.2 h1:KBNDSne4vP5mbSWnJbO+51IMOXJB67QiYCSBrubbPR
|
||||
github.com/yusufpapurcu/wmi v1.2.2/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0=
|
||||
go.mongodb.org/mongo-driver v1.11.1 h1:QP0znIRTuL0jf1oBQoAoM0C6ZJfBK4kx0Uumtv1A7w8=
|
||||
go.mongodb.org/mongo-driver v1.11.1/go.mod h1:s7p5vEtfbeR1gYi6pnj3c3/urpbLv2T5Sfd6Rp2HBB8=
|
||||
go.opencensus.io v0.23.0 h1:gqCw0LfLxScz8irSi8exQc7fyQ0fKQU/qnC/X8+V/1M=
|
||||
go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E=
|
||||
go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0=
|
||||
go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo=
|
||||
go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw=
|
||||
go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
|
||||
go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A=
|
||||
@@ -377,8 +377,8 @@ golang.org/x/crypto v0.0.0-20220525230936-793ad666bf5e/go.mod h1:IxCIyHEi3zRg3s0
|
||||
golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
|
||||
golang.org/x/crypto v0.0.0-20220826181053-bd7e27e6170d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
|
||||
golang.org/x/crypto v0.3.0/go.mod h1:hebNnKkNXi2UzZN1eVRvBB7co0a+JxK6XbPiWVs/3J4=
|
||||
golang.org/x/crypto v0.4.0 h1:UVQgzMY87xqpKNgb+kDsll2Igd33HszWHFLmpaRMq/8=
|
||||
golang.org/x/crypto v0.4.0/go.mod h1:3quD/ATkf6oY+rnes5c3ExXTbLc8mueNue5/DoinL80=
|
||||
golang.org/x/crypto v0.5.0 h1:U/0M97KRkSFvyD/3FSmdP5W5swImpNgle/EHFhOsQPE=
|
||||
golang.org/x/crypto v0.5.0/go.mod h1:NK/OQwhpMQP3MwtdjgLlYHnH9ebylxKWv3e0fK+mkQU=
|
||||
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
|
||||
golang.org/x/exp v0.0.0-20221018205818-5c77f4b2bbd7 h1:lhh9BuEmjIBOdOhMIGSQzdW5LbTtFbUV1k9O/Rlq3SE=
|
||||
golang.org/x/exp v0.0.0-20221018205818-5c77f4b2bbd7/go.mod h1:cyybsKvd6eL0RnXn6p/Grxp8F5bW7iYuBgsNCOHpMYE=
|
||||
@@ -411,8 +411,8 @@ golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su
|
||||
golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
|
||||
golang.org/x/net v0.0.0-20220826154423-83b083e8dc8b/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk=
|
||||
golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY=
|
||||
golang.org/x/net v0.3.0 h1:VWL6FNY2bEEmsGVKabSlHu5Irp34xmMRoqb/9lF9lxk=
|
||||
golang.org/x/net v0.3.0/go.mod h1:MBQ8lrhLObU/6UmLb4fmbmk5OcyYmqtbGd/9yIeKjEE=
|
||||
golang.org/x/net v0.5.0 h1:GyT4nK/YDHSqa1c4753ouYCDajOYKTja9Xb/OHtgvSw=
|
||||
golang.org/x/net v0.5.0/go.mod h1:DivGGAXEgPSlEBzxGzZI+ZLohi+xUj054jfeKui00ws=
|
||||
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
|
||||
golang.org/x/oauth2 v0.0.0-20221014153046-6fdb5e3db783 h1:nt+Q6cXKz4MosCSpnbMtqiQ8Oz0pxTef2B4Vca2lvfk=
|
||||
golang.org/x/oauth2 v0.0.0-20221014153046-6fdb5e3db783/go.mod h1:h4gKUeWbJ4rQPri7E0u6Gs4e9Ri2zaLxzw5DI5XGrYg=
|
||||
@@ -457,21 +457,21 @@ golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBc
|
||||
golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220825204002-c680a09ffe64/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.3.0 h1:w8ZOecv6NaNa/zC8944JTU3vz4u6Lagfk4RPQxv92NQ=
|
||||
golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.4.0 h1:Zr2JFtRQNX3BCZ8YtxRE9hNJYC8J6I1MVbMg6owUp18=
|
||||
golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
|
||||
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
|
||||
golang.org/x/term v0.0.0-20220722155259-a9ba230a4035/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
|
||||
golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc=
|
||||
golang.org/x/term v0.3.0 h1:qoo4akIqOcDME5bhc/NgxUdovd6BSS2uMsVjB56q1xI=
|
||||
golang.org/x/term v0.4.0 h1:O7UWfv5+A2qiuulQk30kVinPoMtoIPeVaKLEgLpVkvg=
|
||||
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
|
||||
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
|
||||
golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
|
||||
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
|
||||
golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
|
||||
golang.org/x/text v0.5.0 h1:OLmvp0KP+FVG99Ct/qFiL/Fhk4zp4QQnZ7b2U+5piUM=
|
||||
golang.org/x/text v0.5.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
|
||||
golang.org/x/text v0.6.0 h1:3XmdazWV+ubf7QgHSTWeykHOci5oeekaGJBLkrkaw4k=
|
||||
golang.org/x/text v0.6.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
|
||||
golang.org/x/time v0.0.0-20220922220347-f3bd1da661af h1:Yx9k8YCG3dvF87UAn2tu2HQLf2dt/eR1bXxpLMWeH+Y=
|
||||
golang.org/x/time v0.0.0-20220922220347-f3bd1da661af/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
|
||||
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
@@ -491,8 +491,8 @@ golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8T
|
||||
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
google.golang.org/api v0.102.0 h1:JxJl2qQ85fRMPNvlZY/enexbxpCjLwGhZUtgfGeQ51I=
|
||||
google.golang.org/api v0.102.0/go.mod h1:3VFl6/fzoA+qNuS1N1/VfXY4LjoXN/wzeIp7TweWwGo=
|
||||
google.golang.org/api v0.103.0 h1:9yuVqlu2JCvcLg9p8S3fcFLZij8EPSyvODIY1rkMizQ=
|
||||
google.golang.org/api v0.103.0/go.mod h1:hGtW6nK1AC+d9si/UBhw8Xli+QMOf6xyNAyJw4qU9w0=
|
||||
google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
|
||||
google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
|
||||
google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c=
|
||||
@@ -500,15 +500,15 @@ google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCID
|
||||
google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
|
||||
google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
|
||||
google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=
|
||||
google.golang.org/genproto v0.0.0-20221027153422-115e99e71e1c h1:QgY/XxIAIeccR+Ca/rDdKubLIU9rcJ3xfy1DC/Wd2Oo=
|
||||
google.golang.org/genproto v0.0.0-20221027153422-115e99e71e1c/go.mod h1:CGI5F/G+E5bKwmfYo09AXuVN4dD894kIKUFmVbP2/Fo=
|
||||
google.golang.org/genproto v0.0.0-20221201164419-0e50fba7f41c h1:S34D59DS2GWOEwWNt4fYmTcFrtlOgukG2k9WsomZ7tg=
|
||||
google.golang.org/genproto v0.0.0-20221201164419-0e50fba7f41c/go.mod h1:rZS5c/ZVYMaOGBfO68GWtjOw/eLaZM1X6iVtgjZ+EWg=
|
||||
google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
|
||||
google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
|
||||
google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY=
|
||||
google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
|
||||
google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc=
|
||||
google.golang.org/grpc v1.50.1 h1:DS/BukOZWp8s6p4Dt/tOaJaTQyPyOoCcrjroHuCeLzY=
|
||||
google.golang.org/grpc v1.50.1/go.mod h1:ZgQEeidpAuNRZ8iRrlBKXZQP1ghovWIVhdJRyCDK+GI=
|
||||
google.golang.org/grpc v1.51.0 h1:E1eGv1FTqoLIdnBCZufiSHgKjlqG6fKFf6pPWtMTh8U=
|
||||
google.golang.org/grpc v1.51.0/go.mod h1:wgNDFcnuBGmxLKI/qn4T+m5BtEBYXJPvibbUPsAIPww=
|
||||
google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
|
||||
google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
|
||||
google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
|
||||
|
||||
@@ -95,6 +95,9 @@ var (
|
||||
syslogTLSCert = syslogScan.Flag("cert", "Path to TLS cert.").String()
|
||||
syslogTLSKey = syslogScan.Flag("key", "Path to TLS key.").String()
|
||||
syslogFormat = syslogScan.Flag("format", "Log format. Can be rfc3164 or rfc5424").String()
|
||||
|
||||
circleCiScan = cli.Command("circleci", "Scan CircleCI")
|
||||
circleCiScanToken = circleCiScan.Flag("token", "CircleCI token. Can also be provided with environment variable").Envar("CIRCLECI_TOKEN").Required().String()
|
||||
)
|
||||
|
||||
func init() {
|
||||
@@ -293,6 +296,10 @@ func run(state overseer.State) {
|
||||
if err = e.ScanSyslog(ctx, sources.NewConfig(syslog)); err != nil {
|
||||
logrus.WithError(err).Fatal("Failed to scan syslog.")
|
||||
}
|
||||
case circleCiScan.FullCommand():
|
||||
if err = e.ScanCircleCI(ctx, *circleCiScanToken); err != nil {
|
||||
logrus.WithError(err).Fatal("Failed to scan CircleCI.")
|
||||
}
|
||||
}
|
||||
// asynchronously wait for scanning to finish and cleanup
|
||||
go e.Finish(ctx)
|
||||
|
||||
+4
-3
@@ -9,6 +9,7 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/hashicorp/go-retryablehttp"
|
||||
"github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
var caCerts = []string{
|
||||
@@ -93,7 +94,7 @@ func NewCustomTransport(T http.RoundTripper) *CustomTransport {
|
||||
|
||||
func PinnedRetryableHttpClient() *http.Client {
|
||||
httpClient := retryablehttp.NewClient()
|
||||
httpClient.Logger = nil
|
||||
httpClient.Logger = logrus.WithField("name", "PinnedRetryableHttpClient").WithField("level", logrus.DebugLevel)
|
||||
httpClient.HTTPClient.Transport = NewCustomTransport(&http.Transport{
|
||||
TLSClientConfig: &tls.Config{
|
||||
RootCAs: PinnedCertPool(),
|
||||
@@ -115,7 +116,7 @@ func PinnedRetryableHttpClient() *http.Client {
|
||||
func RetryableHttpClient() *http.Client {
|
||||
httpClient := retryablehttp.NewClient()
|
||||
httpClient.RetryMax = 3
|
||||
httpClient.Logger = nil
|
||||
httpClient.Logger = logrus.WithField("name", "RetryableHttpClient").WithField("level", logrus.DebugLevel)
|
||||
httpClient.HTTPClient.Timeout = 3 * time.Second
|
||||
httpClient.HTTPClient.Transport = NewCustomTransport(nil)
|
||||
return httpClient.StandardClient()
|
||||
@@ -124,7 +125,7 @@ func RetryableHttpClient() *http.Client {
|
||||
func RetryableHttpClientTimeout(timeOutSeconds int64) *http.Client {
|
||||
httpClient := retryablehttp.NewClient()
|
||||
httpClient.RetryMax = 3
|
||||
httpClient.Logger = nil
|
||||
httpClient.Logger = logrus.WithField("name", "RetryableHttpClientTimeout").WithField("level", logrus.DebugLevel)
|
||||
httpClient.HTTPClient.Timeout = time.Duration(timeOutSeconds) * time.Second
|
||||
httpClient.HTTPClient.Transport = NewCustomTransport(nil)
|
||||
return httpClient.StandardClient()
|
||||
|
||||
+39
-6
@@ -2,6 +2,8 @@ package context
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"runtime/debug"
|
||||
"time"
|
||||
|
||||
"github.com/go-logr/logr"
|
||||
@@ -38,6 +40,7 @@ type logCtx struct {
|
||||
// Embed context.Context to get all methods for free.
|
||||
context.Context
|
||||
log logr.Logger
|
||||
err *error
|
||||
}
|
||||
|
||||
// Logger returns a structured logger.
|
||||
@@ -45,6 +48,13 @@ func (l logCtx) Logger() logr.Logger {
|
||||
return l.log
|
||||
}
|
||||
|
||||
func (l logCtx) Err() error {
|
||||
if l.err != nil && *l.err != nil {
|
||||
return *l.err
|
||||
}
|
||||
return l.Context.Err()
|
||||
}
|
||||
|
||||
// Background returns context.Background with a default logger.
|
||||
func Background() Context {
|
||||
return logCtx{
|
||||
@@ -64,30 +74,33 @@ func TODO() Context {
|
||||
// WithCancel returns context.WithCancel with the log object propagated.
|
||||
func WithCancel(parent Context) (Context, context.CancelFunc) {
|
||||
ctx, cancel := context.WithCancel(parent)
|
||||
return logCtx{
|
||||
lCtx := logCtx{
|
||||
log: parent.Logger(),
|
||||
Context: ctx,
|
||||
}, cancel
|
||||
}
|
||||
return captureCancelCallstack(lCtx, cancel)
|
||||
}
|
||||
|
||||
// WithDeadline returns context.WithDeadline with the log object propagated and
|
||||
// the deadline added to the structured log values.
|
||||
func WithDeadline(parent Context, d time.Time) (Context, context.CancelFunc) {
|
||||
ctx, cancel := context.WithDeadline(parent, d)
|
||||
return logCtx{
|
||||
lCtx := logCtx{
|
||||
log: parent.Logger().WithValues("deadline", d),
|
||||
Context: ctx,
|
||||
}, cancel
|
||||
}
|
||||
return captureCancelCallstack(lCtx, cancel)
|
||||
}
|
||||
|
||||
// WithTimeout returns context.WithTimeout with the log object propagated and
|
||||
// the timeout added to the structured log values.
|
||||
func WithTimeout(parent Context, timeout time.Duration) (Context, context.CancelFunc) {
|
||||
ctx, cancel := context.WithTimeout(parent, timeout)
|
||||
return logCtx{
|
||||
lCtx := logCtx{
|
||||
log: parent.Logger().WithValues("timeout", timeout),
|
||||
Context: ctx,
|
||||
}, cancel
|
||||
}
|
||||
return captureCancelCallstack(lCtx, cancel)
|
||||
}
|
||||
|
||||
// WithValue returns context.WithValue with the log object propagated and
|
||||
@@ -136,3 +149,23 @@ func AddLogger(parent context.Context) Context {
|
||||
func SetDefaultLogger(l logr.Logger) {
|
||||
defaultLogger = l
|
||||
}
|
||||
|
||||
// captureCancelCallstack is a helper function to capture the callstack where
|
||||
// the cancel function was first called.
|
||||
func captureCancelCallstack(ctx logCtx, f context.CancelFunc) (Context, context.CancelFunc) {
|
||||
if ctx.err == nil {
|
||||
var err error
|
||||
ctx.err = &err
|
||||
}
|
||||
return ctx, func() {
|
||||
// We must check Err() before calling f() since f() sets the error.
|
||||
// If there's already an error, do nothing special.
|
||||
if ctx.Err() != nil {
|
||||
f()
|
||||
return
|
||||
}
|
||||
f()
|
||||
// Set the error with the stacktrace if the err pointer is non-nil.
|
||||
*ctx.err = fmt.Errorf("%w (%s)", ctx.Err(), string(debug.Stack()))
|
||||
}
|
||||
}
|
||||
|
||||
@@ -169,3 +169,46 @@ func TestDiscardLogger(t *testing.T) {
|
||||
ctx := Background()
|
||||
ctx.Logger().Info("this shouldn't panic")
|
||||
}
|
||||
|
||||
func TestErrCallstack(t *testing.T) {
|
||||
c, cancel := WithCancel(Background())
|
||||
ctx := c.(logCtx)
|
||||
cancel()
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
assert.Contains(t, ctx.Err().Error(), "TestErrCallstack")
|
||||
case <-time.After(1 * time.Second):
|
||||
assert.Fail(t, "context should be done")
|
||||
}
|
||||
}
|
||||
|
||||
func TestErrCallstackTimeout(t *testing.T) {
|
||||
ctx, cancel := WithTimeout(Background(), 10*time.Millisecond)
|
||||
defer cancel()
|
||||
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
// Deadline exceeded errors will not have a callstack from the cancel
|
||||
// function.
|
||||
assert.NotContains(t, ctx.Err().Error(), "TestErrCallstackTimeout")
|
||||
case <-time.After(1 * time.Second):
|
||||
assert.Fail(t, "context should be done")
|
||||
}
|
||||
}
|
||||
|
||||
func TestErrCallstackTimeoutCancel(t *testing.T) {
|
||||
ctx, cancel := WithTimeout(Background(), 10*time.Millisecond)
|
||||
|
||||
var err error
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
err = ctx.Err()
|
||||
case <-time.After(1 * time.Second):
|
||||
assert.Fail(t, "context should be done")
|
||||
}
|
||||
|
||||
// Calling cancel after deadline exceeded should not overwrite the original
|
||||
// error.
|
||||
cancel()
|
||||
assert.Equal(t, err, ctx.Err())
|
||||
}
|
||||
|
||||
@@ -64,9 +64,8 @@ func (c *customRegexWebhook) FromData(ctx context.Context, verify bool, data []b
|
||||
for name, regex := range c.GetRegex() {
|
||||
regex, err := regexp.Compile(regex)
|
||||
if err != nil {
|
||||
// TODO: Log error.
|
||||
// This should never happen due to validation.
|
||||
continue
|
||||
// This will only happen if the regex is invalid.
|
||||
return nil, err
|
||||
}
|
||||
regexMatches[name] = regex.FindAllStringSubmatch(dataStr, -1)
|
||||
}
|
||||
@@ -183,7 +182,8 @@ func productIndices(lengths ...int) [][]int {
|
||||
// permutateMatches converts the list of all regex matches into all possible
|
||||
// permutations selecting one from each named entry in the map. For example:
|
||||
// {"foo": [matchA, matchB], "bar": [matchC]} becomes
|
||||
// [{"foo": matchA, "bar": matchC}, {"foo": matchB, "bar": matchC}]
|
||||
//
|
||||
// [{"foo": matchA, "bar": matchC}, {"foo": matchB, "bar": matchC}]
|
||||
func permutateMatches(regexMatches map[string][][]string) []map[string][]string {
|
||||
// Get a consistent order for names and their matching lengths.
|
||||
// The lengths are used in calculating the permutation so order matters.
|
||||
|
||||
@@ -1,9 +1,11 @@
|
||||
package custom_detectors
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/custom_detectorspb"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/protoyaml"
|
||||
)
|
||||
@@ -102,6 +104,21 @@ func TestCustomDetectorsParsing(t *testing.T) {
|
||||
assert.Equal(t, []string{"Authorization: Bearer token"}, got.Verify[0].Headers)
|
||||
}
|
||||
|
||||
func TestFromData_InvalidRegEx(t *testing.T) {
|
||||
c := &customRegexWebhook{
|
||||
&custom_detectorspb.CustomRegex{
|
||||
Name: "Internal bi tool",
|
||||
Keywords: []string{"secret_v1_", "pat_v2_"},
|
||||
Regex: map[string]string{
|
||||
"test": "!!?(?:?)[a-zA-Z0-9]{32}", // invalid regex
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
_, err := c.FromData(context.Background(), false, []byte("test"))
|
||||
assert.Error(t, err)
|
||||
}
|
||||
|
||||
func TestProductIndices(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
|
||||
@@ -2,6 +2,7 @@ package custom_detectors
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
)
|
||||
@@ -23,6 +24,11 @@ func ValidateRegex(regex map[string]string) error {
|
||||
if len(regex) == 0 {
|
||||
return fmt.Errorf("no regex")
|
||||
}
|
||||
for name, reg := range regex {
|
||||
if _, err := regexp.Compile(reg); err != nil {
|
||||
return fmt.Errorf("regex '%s': %w", name, err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
|
||||
@@ -53,6 +53,13 @@ func TestCustomDetectorsRegexValidation(t *testing.T) {
|
||||
input: map[string]string{},
|
||||
wantErr: true,
|
||||
},
|
||||
{
|
||||
name: "Test invalid regex",
|
||||
input: map[string]string{
|
||||
"test": "!!?(?:?)[a-zA-Z0-9]{32}",
|
||||
},
|
||||
wantErr: true,
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
|
||||
@@ -0,0 +1,48 @@
|
||||
package engine
|
||||
|
||||
import (
|
||||
"runtime"
|
||||
|
||||
"github.com/go-errors/errors"
|
||||
"github.com/sirupsen/logrus"
|
||||
"google.golang.org/protobuf/proto"
|
||||
"google.golang.org/protobuf/types/known/anypb"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/context"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/sources/circleci"
|
||||
)
|
||||
|
||||
// ScanS3 scans CircleCI logs.
|
||||
func (e *Engine) ScanCircleCI(ctx context.Context, token string) error {
|
||||
connection := &sourcespb.CircleCI{
|
||||
Credential: &sourcespb.CircleCI_Token{
|
||||
Token: token,
|
||||
},
|
||||
}
|
||||
|
||||
var conn anypb.Any
|
||||
err := anypb.MarshalFrom(&conn, connection, proto.MarshalOptions{})
|
||||
if err != nil {
|
||||
logrus.WithError(err).Error("failed to marshal Circle CI connection")
|
||||
return err
|
||||
}
|
||||
|
||||
circleSource := circleci.Source{}
|
||||
err = circleSource.Init(ctx, "trufflehog - Circle CI", 0, int64(sourcespb.SourceType_SOURCE_TYPE_CIRCLECI), true, &conn, runtime.NumCPU())
|
||||
if err != nil {
|
||||
return errors.WrapPrefix(err, "failed to init Circle CI source", 0)
|
||||
}
|
||||
|
||||
e.sourcesWg.Add(1)
|
||||
go func() {
|
||||
defer common.RecoverWithExit(ctx)
|
||||
defer e.sourcesWg.Done()
|
||||
err := circleSource.Chunks(ctx, e.ChunksChan())
|
||||
if err != nil {
|
||||
logrus.WithError(err).Error("error scanning Circle CI")
|
||||
}
|
||||
}()
|
||||
return nil
|
||||
}
|
||||
+2
-2
@@ -42,7 +42,7 @@ func (e *Engine) ScanS3(ctx context.Context, c sources.Config) error {
|
||||
var conn anypb.Any
|
||||
err := anypb.MarshalFrom(&conn, connection, proto.MarshalOptions{})
|
||||
if err != nil {
|
||||
logrus.WithError(err).Error("failed to marshal github connection")
|
||||
logrus.WithError(err).Error("failed to marshal S3 connection")
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -58,7 +58,7 @@ func (e *Engine) ScanS3(ctx context.Context, c sources.Config) error {
|
||||
defer e.sourcesWg.Done()
|
||||
err := s3Source.Chunks(ctx, e.ChunksChan())
|
||||
if err != nil {
|
||||
logrus.WithError(err).Error("error scanning s3")
|
||||
logrus.WithError(err).Error("error scanning S3")
|
||||
}
|
||||
}()
|
||||
return nil
|
||||
|
||||
@@ -12,7 +12,7 @@ func NormalizeBitbucketRepo(repoURL string) (string, error) {
|
||||
return "", errors.New("Bitbucket requires https repo urls: e.g. https://bitbucket.org/org/repo.git")
|
||||
}
|
||||
|
||||
return NormalizeOrgRepoURL("Gitlab", repoURL)
|
||||
return NormalizeOrgRepoURL("Bitbucket", repoURL)
|
||||
}
|
||||
|
||||
func NormalizeGerritProject(project string) (string, error) {
|
||||
|
||||
@@ -0,0 +1,256 @@
|
||||
package circleci
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"net/http"
|
||||
|
||||
"github.com/go-errors/errors"
|
||||
"google.golang.org/protobuf/proto"
|
||||
"google.golang.org/protobuf/types/known/anypb"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/context"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/source_metadatapb"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
|
||||
)
|
||||
|
||||
const (
|
||||
baseURL = "https://circleci.com/api/v1.1/"
|
||||
)
|
||||
|
||||
type Source struct {
|
||||
name string
|
||||
sourceId int64
|
||||
jobId int64
|
||||
verify bool
|
||||
sources.Progress
|
||||
token string
|
||||
client *http.Client
|
||||
}
|
||||
|
||||
// Ensure the Source satisfies the interface at compile time
|
||||
var _ sources.Source = (*Source)(nil)
|
||||
|
||||
// Type returns the type of source.
|
||||
// It is used for matching source types in configuration and job input.
|
||||
func (s *Source) Type() sourcespb.SourceType {
|
||||
return sourcespb.SourceType_SOURCE_TYPE_CIRCLECI
|
||||
}
|
||||
|
||||
func (s *Source) SourceID() int64 {
|
||||
return s.sourceId
|
||||
}
|
||||
|
||||
func (s *Source) JobID() int64 {
|
||||
return s.jobId
|
||||
}
|
||||
|
||||
// Init returns an initialized Filesystem source.
|
||||
func (s *Source) Init(aCtx context.Context, name string, jobId, sourceId int64, verify bool, connection *anypb.Any, _ int) error {
|
||||
s.name = name
|
||||
s.sourceId = sourceId
|
||||
s.jobId = jobId
|
||||
s.verify = verify
|
||||
s.client = common.RetryableHttpClientTimeout(3)
|
||||
|
||||
var conn sourcespb.CircleCI
|
||||
if err := anypb.UnmarshalTo(connection, &conn, proto.UnmarshalOptions{}); err != nil {
|
||||
return errors.WrapPrefix(err, "error unmarshalling connection", 0)
|
||||
}
|
||||
|
||||
switch conn.Credential.(type) {
|
||||
case *sourcespb.CircleCI_Token:
|
||||
s.token = conn.GetToken()
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Chunks emits chunks of bytes over a channel.
|
||||
func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) error {
|
||||
projects, err := s.projects(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
for _, proj := range projects {
|
||||
builds, err := s.buildsForProject(ctx, proj)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
for _, bld := range builds {
|
||||
buildSteps, err := s.stepsForBuild(ctx, proj, bld)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
for _, step := range buildSteps {
|
||||
for _, action := range step.Actions {
|
||||
err = s.chunkAction(ctx, proj, bld, action, step.Name, chunksChan)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
type project struct {
|
||||
VCS string `json:"vcs_type"`
|
||||
Username string `json:"username"`
|
||||
RepoName string `json:"reponame"`
|
||||
}
|
||||
|
||||
func (s *Source) projects(ctx context.Context) ([]project, error) {
|
||||
reqURL := fmt.Sprintf("%sprojects", baseURL)
|
||||
req, err := http.NewRequest("GET", reqURL, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
req.Header.Set("Circle-Token", s.token)
|
||||
req.Header.Set("Accept", "application/json")
|
||||
res, err := s.client.Do(req)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer res.Body.Close()
|
||||
|
||||
if res.StatusCode > 399 && res.StatusCode < 500 {
|
||||
return nil, fmt.Errorf("invalid credentials, status %d", res.StatusCode)
|
||||
}
|
||||
|
||||
var projects []project
|
||||
if err := json.NewDecoder(res.Body).Decode(&projects); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return projects, nil
|
||||
}
|
||||
|
||||
type build struct {
|
||||
BuildNum int `json:"build_num"`
|
||||
}
|
||||
|
||||
func (s *Source) buildsForProject(ctx context.Context, proj project) ([]build, error) {
|
||||
reqURL := fmt.Sprintf("%sproject/%s/%s/%s", baseURL, proj.VCS, proj.Username, proj.RepoName)
|
||||
req, err := http.NewRequest("GET", reqURL, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
req.Header.Set("Circle-Token", s.token)
|
||||
req.Header.Set("Accept", "application/json")
|
||||
res, err := s.client.Do(req)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer res.Body.Close()
|
||||
|
||||
var builds []build
|
||||
if err := json.NewDecoder(res.Body).Decode(&builds); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return builds, nil
|
||||
}
|
||||
|
||||
type action struct {
|
||||
Index int `json:"index"`
|
||||
OutputURL string `json:"output_url"`
|
||||
}
|
||||
|
||||
type buildStep struct {
|
||||
Name string `json:"name"`
|
||||
Actions []action `json:"actions"`
|
||||
}
|
||||
|
||||
func (s *Source) stepsForBuild(ctx context.Context, proj project, bld build) ([]buildStep, error) {
|
||||
reqURL := fmt.Sprintf("%sproject/%s/%s/%s/%d", baseURL, proj.VCS, proj.Username, proj.RepoName, bld.BuildNum)
|
||||
req, err := http.NewRequest("GET", reqURL, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
req.Header.Set("Circle-Token", s.token)
|
||||
req.Header.Set("Accept", "application/json")
|
||||
res, err := s.client.Do(req)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer res.Body.Close()
|
||||
|
||||
type buildRes struct {
|
||||
Steps []buildStep `json:"steps"`
|
||||
}
|
||||
|
||||
var bldRes buildRes
|
||||
if err := json.NewDecoder(res.Body).Decode(&bldRes); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return bldRes.Steps, nil
|
||||
}
|
||||
|
||||
func (s *Source) chunkAction(ctx context.Context, proj project, bld build, act action, stepName string, chunksChan chan *sources.Chunk) error {
|
||||
req, err := http.NewRequest("GET", act.OutputURL, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
res, err := s.client.Do(req)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer res.Body.Close()
|
||||
logOutput, err := ioutil.ReadAll(res.Body)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
linkURL := fmt.Sprintf("https://app.circleci.com/pipelines/%s/%s/%s/%d", proj.VCS, proj.Username, proj.RepoName, bld.BuildNum)
|
||||
|
||||
chunk := &sources.Chunk{
|
||||
SourceType: s.Type(),
|
||||
SourceName: s.name,
|
||||
SourceID: s.SourceID(),
|
||||
Data: removeCircleSha1Line(logOutput),
|
||||
SourceMetadata: &source_metadatapb.MetaData{
|
||||
Data: &source_metadatapb.MetaData_Circleci{
|
||||
Circleci: &source_metadatapb.CircleCI{
|
||||
VcsType: proj.VCS,
|
||||
Username: proj.Username,
|
||||
Repository: proj.RepoName,
|
||||
BuildNumber: int64(bld.BuildNum),
|
||||
BuildStep: stepName,
|
||||
Link: linkURL,
|
||||
},
|
||||
},
|
||||
},
|
||||
Verify: s.verify,
|
||||
}
|
||||
|
||||
chunksChan <- chunk
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func removeCircleSha1Line(input []byte) []byte {
|
||||
// Split the input slice into a slice of lines
|
||||
lines := bytes.Split(input, []byte("\n"))
|
||||
|
||||
// Iterate over the lines and add the ones that don't contain "CIRCLE_SHA1=" to the result slice
|
||||
result := make([][]byte, 0, len(lines))
|
||||
for _, line := range lines {
|
||||
if !bytes.Contains(line, []byte("CIRCLE_SHA1=")) {
|
||||
result = append(result, line)
|
||||
}
|
||||
}
|
||||
|
||||
// Join the lines in the result slice and return the resulting slice
|
||||
return bytes.Join(result, []byte("\n"))
|
||||
}
|
||||
@@ -0,0 +1,93 @@
|
||||
package circleci
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/kylelemons/godebug/pretty"
|
||||
"google.golang.org/protobuf/types/known/anypb"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/context"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/source_metadatapb"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
|
||||
)
|
||||
|
||||
func TestSource_Scan(t *testing.T) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), time.Second*3)
|
||||
defer cancel()
|
||||
|
||||
secret, err := common.GetTestSecret(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(fmt.Errorf("failed to access secret: %v", err))
|
||||
}
|
||||
token := secret.MustGetField("CIRCLECI_TOKEN")
|
||||
|
||||
type init struct {
|
||||
name string
|
||||
verify bool
|
||||
connection *sourcespb.CircleCI
|
||||
}
|
||||
tests := []struct {
|
||||
name string
|
||||
init init
|
||||
wantSourceMetadata *source_metadatapb.MetaData
|
||||
wantErr bool
|
||||
}{
|
||||
{
|
||||
name: "get a chunk",
|
||||
init: init{
|
||||
name: "example repo",
|
||||
connection: &sourcespb.CircleCI{
|
||||
Credential: &sourcespb.CircleCI_Token{
|
||||
Token: token,
|
||||
},
|
||||
},
|
||||
verify: true,
|
||||
},
|
||||
wantSourceMetadata: &source_metadatapb.MetaData{
|
||||
Data: &source_metadatapb.MetaData_Circleci{
|
||||
Circleci: &source_metadatapb.CircleCI{
|
||||
VcsType: "github",
|
||||
Username: "dustin-decker",
|
||||
Repository: "circle-ci",
|
||||
BuildNumber: 2,
|
||||
BuildStep: "Spin up environment",
|
||||
Link: "https://app.circleci.com/pipelines/github/dustin-decker/circle-ci/2",
|
||||
},
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
s := Source{}
|
||||
|
||||
conn, err := anypb.New(tt.init.connection)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
err = s.Init(ctx, tt.init.name, 0, 0, tt.init.verify, conn, 5)
|
||||
if (err != nil) != tt.wantErr {
|
||||
t.Errorf("Source.Init() error = %v, wantErr %v", err, tt.wantErr)
|
||||
return
|
||||
}
|
||||
chunksCh := make(chan *sources.Chunk, 1)
|
||||
go func() {
|
||||
err = s.Chunks(ctx, chunksCh)
|
||||
if (err != nil) != tt.wantErr {
|
||||
t.Errorf("Source.Chunks() error = %v, wantErr %v", err, tt.wantErr)
|
||||
return
|
||||
}
|
||||
}()
|
||||
gotChunk := <-chunksCh
|
||||
if diff := pretty.Compare(gotChunk.SourceMetadata, tt.wantSourceMetadata); diff != "" {
|
||||
t.Errorf("Source.Chunks() %s diff: (-got +want)\n%s", tt.name, diff)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -560,8 +560,16 @@ func (s *Git) ScanRepo(ctx context.Context, repo *git.Repository, repoPath strin
|
||||
ctx.Logger().V(1).Info("error scanning unstaged changes", "error", err)
|
||||
}
|
||||
|
||||
// We're logging time, but the repoPath is usally a dynamically generated folder in /tmp
|
||||
// To make this duration logging useful, we need to log the remote as well
|
||||
remotes, _ := repo.Remotes()
|
||||
repoUrl := "Could not get remote for repo"
|
||||
if len(remotes) != 0 {
|
||||
repoUrl = getSafeRemoteURL(repo, remotes[0].Config().Name)
|
||||
}
|
||||
|
||||
scanTime := time.Now().UnixNano() - start
|
||||
ctx.Logger().V(1).Info("scanning git repo complete", "path", repoPath, "time", scanTime)
|
||||
ctx.Logger().V(1).Info("scanning git repo complete", "Repo", repoUrl, "path", repoPath, "time", scanTime)
|
||||
return nil
|
||||
}
|
||||
|
||||
|
||||
@@ -134,7 +134,7 @@ func (s *Source) Init(aCtx context.Context, name string, jobID, sourceID int64,
|
||||
s.jobPool = &errgroup.Group{}
|
||||
s.jobPool.SetLimit(concurrency)
|
||||
|
||||
s.httpClient = common.SaneHttpClient()
|
||||
s.httpClient = common.RetryableHttpClientTimeout(60)
|
||||
s.apiClient = github.NewClient(s.httpClient)
|
||||
|
||||
var conn sourcespb.GitHub
|
||||
@@ -280,17 +280,20 @@ func (s *Source) enumerateWithToken(ctx context.Context, apiEndpoint, token stri
|
||||
ts := oauth2.StaticTokenSource(
|
||||
&oauth2.Token{AccessToken: token},
|
||||
)
|
||||
tc := oauth2.NewClient(context.TODO(), ts)
|
||||
s.httpClient.Transport = &oauth2.Transport{
|
||||
Base: s.httpClient.Transport,
|
||||
Source: oauth2.ReuseTokenSource(nil, ts),
|
||||
}
|
||||
|
||||
var err error
|
||||
// If we're using public Github, make a regular client.
|
||||
// Otherwise, make an enterprise client.
|
||||
var isGHE bool
|
||||
if apiEndpoint == "https://api.github.com" {
|
||||
s.apiClient = github.NewClient(tc)
|
||||
s.apiClient = github.NewClient(s.httpClient)
|
||||
} else {
|
||||
isGHE = true
|
||||
s.apiClient, err = github.NewEnterpriseClient(apiEndpoint, apiEndpoint, tc)
|
||||
s.apiClient, err = github.NewEnterpriseClient(apiEndpoint, apiEndpoint, s.httpClient)
|
||||
if err != nil {
|
||||
return errors.New(err)
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user