Compare commits
105 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 7c9952d2ef | |||
| ea8c21051d | |||
| eee01e0361 | |||
| 2de5c0b217 | |||
| ea6e8b6bb5 | |||
| eae66ccf7e | |||
| 09795c3591 | |||
| fa9469cfc7 | |||
| e894540632 | |||
| c9f58b3780 | |||
| 62cbef5292 | |||
| 18f854d142 | |||
| 2570fdc244 | |||
| 47c2b6bed9 | |||
| e5aeb219de | |||
| 2d2595a2e3 | |||
| 13999227b9 | |||
| 18b3d3dd01 | |||
| c2b49b060b | |||
| cda88ebdf0 | |||
| 1cd600f70f | |||
| d763097fdf | |||
| e322c4b29d | |||
| d062834997 | |||
| 5a5e8a607e | |||
| c995e93dcc | |||
| 06d2eab204 | |||
| b1947246d9 | |||
| 0c7ed19270 | |||
| 1cf419e478 | |||
| 160fd830dd | |||
| 0ad46381d9 | |||
| 91aa75679b | |||
| b906a51d5c | |||
| 3c79b6b133 | |||
| 78d06658ca | |||
| 69021f59c5 | |||
| b8c43ea58f | |||
| a4b1fb7752 | |||
| 7d2f126411 | |||
| 5043fc8756 | |||
| 661c6b47b7 | |||
| eb00d0d4e1 | |||
| 406ce7bc55 | |||
| 5e7a6ca11c | |||
| b54683acb9 | |||
| 32e3f1f015 | |||
| a07b6664f8 | |||
| 61bee6c8b1 | |||
| ad57de50cd | |||
| e0faac8d1c | |||
| 070014f380 | |||
| 10b6e2898d | |||
| 6bd48583ae | |||
| 431d26f5fa | |||
| 9f3809f19e | |||
| e391e89f3e | |||
| 3897454dbb | |||
| ade5d91d5c | |||
| 84d4bf0d51 | |||
| f925da7cea | |||
| e0011ac08e | |||
| 7a29f8a514 | |||
| 46823f77c9 | |||
| 10f0963bc9 | |||
| 35b332c498 | |||
| c814ae4d83 | |||
| 2290954b02 | |||
| f48a635c34 | |||
| 91cbca941a | |||
| 1a1977f7e6 | |||
| f39303495a | |||
| b5b01d3eba | |||
| 85f363f093 | |||
| 302c3ce8d1 | |||
| fd6c281589 | |||
| 93c561f324 | |||
| fab80445d1 | |||
| 9e0a2e9ddd | |||
| 69515bb7ac | |||
| 91c5472876 | |||
| a613bbb979 | |||
| ebf1038392 | |||
| be68eb044a | |||
| 06a562688d | |||
| af428d5e88 | |||
| a3cc756ec6 | |||
| 3e077d874f | |||
| e8b5e3cea3 | |||
| 8ec5e4916c | |||
| 20b7793828 | |||
| 8fad5fff79 | |||
| cb1a63a4e2 | |||
| f6aaa4924f | |||
| be549a7287 | |||
| 3b00c0062c | |||
| cab416b533 | |||
| ee814a67bd | |||
| 17b90b32d0 | |||
| 9af31f00a9 | |||
| d03a74776e | |||
| aab8fddc67 | |||
| 4e774d1f01 | |||
| 4b7f94dea1 | |||
| a9213a1103 |
@@ -6,11 +6,7 @@ labels: bug, needs triage
|
||||
assignees: trufflesecurity/product-eng
|
||||
---
|
||||
|
||||
### Community Note
|
||||
|
||||
* Please vote on this issue by adding a 👍 [reaction](https://blog.github.com/2016-03-10-add-reactions-to-pull-requests-issues-and-comments/) to the original issue to help the community and maintainers prioritize this request
|
||||
* Please do not leave "+1" or other comments that do not add relevant new information or questions, they generate extra noise for issue followers and do not help prioritize the request
|
||||
* If you are interested in working on this issue or have submitted a pull request, please leave a comment
|
||||
Please review the [Community Note](https://github.com/trufflesecurity/trufflehog/blob/main/.github/community_note.md) before submitting
|
||||
|
||||
### TruffleHog Version
|
||||
<!--- Please run `trufflehog --version` to show the version. If you are not running the latest version, please upgrade because your issue may have already been fixed. --->
|
||||
|
||||
@@ -6,33 +6,27 @@ labels: enhancement, needs triage
|
||||
assignees: trufflesecurity/product-eng
|
||||
---
|
||||
|
||||
### Community Note
|
||||
|
||||
* Please vote on this issue by adding a 👍 [reaction](https://blog.github.com/2016-03-10-add-reactions-to-pull-requests-issues-and-comments/) to the original issue to help the community and maintainers prioritize this request
|
||||
* Please do not leave "+1" or other comments that do not add relevant new information or questions, they generate extra noise for issue followers and do not help prioritize the request
|
||||
* If you are interested in working on this issue or have submitted a pull request, please leave a comment
|
||||
|
||||
### Description
|
||||
Please review the [Community Note](https://github.com/trufflesecurity/trufflehog/blob/main/.github/community_note.md) before submitting
|
||||
|
||||
## Description
|
||||
<!--- Please leave a helpful description of the feature request here. --->
|
||||
|
||||
## Problem to be Addressed
|
||||
<!--- A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] --->
|
||||
|
||||
## Description of the Preferred Solution
|
||||
<!--- A clear and concise description of what you want to happen. What
|
||||
information may be required and what would be the preferred way to provide it?
|
||||
### Preferred Solution
|
||||
<!--- A clear and concise description of what you want to happen. What
|
||||
information may be required and what would be the preferred way to provide it?
|
||||
What should the output include? --->
|
||||
|
||||
## Additional Context
|
||||
### Additional Context
|
||||
<!--- Add any other context or screenshots about the feature request here. --->
|
||||
|
||||
### References
|
||||
#### References
|
||||
|
||||
<!---
|
||||
Information about referencing Github Issues: https://help.github.com/articles/basic-writing-and-formatting-syntax/#referencing-issues-and-pull-requests
|
||||
|
||||
Are there any other GitHub issues (open or closed) or pull requests that should be linked here? Vendor blog posts or documentation? For example:
|
||||
--->
|
||||
|
||||
* #0000
|
||||
--->
|
||||
|
||||
|
||||
|
||||
@@ -2,3 +2,11 @@
|
||||
Please create an issue to collect feedback prior to feature additions. Please also reference that issue in any PRs.
|
||||
If possible try to keep PRs scoped to one feature, and add tests for new features.
|
||||
-->
|
||||
|
||||
### Description:
|
||||
Explain the purpose of the PR.
|
||||
|
||||
### Checklist:
|
||||
* [ ] Tests passing (`make test-community`)?
|
||||
* [ ] Lint passing (`make lint` this requires [golangci-lint](https://golangci-lint.run/usage/install/#local-installation))?
|
||||
|
||||
|
||||
@@ -0,0 +1,7 @@
|
||||
## Community Note
|
||||
|
||||
Please vote on this issue by adding a 👍 reaction to the original issue to help the community and maintainers prioritize this request.
|
||||
|
||||
Please do not leave "+1" or other comments that do not add relevant new information or questions, they generate extra noise for issue followers and do not help prioritize the request.
|
||||
|
||||
If you are interested in working on this issue or have submitted a pull request, please leave a comment.
|
||||
@@ -8,9 +8,8 @@ updates:
|
||||
- package-ecosystem: "gomod" # See documentation for possible values
|
||||
directory: "/" # Location of package manifests
|
||||
schedule:
|
||||
interval: "weekly"
|
||||
interval: "monthly"
|
||||
- package-ecosystem: "github-actions"
|
||||
directory: "/"
|
||||
schedule:
|
||||
# Check for updates to GitHub Actions every weekday
|
||||
interval: "weekly"
|
||||
interval: "monthly"
|
||||
|
||||
@@ -17,7 +17,7 @@ jobs:
|
||||
steps:
|
||||
- uses: actions/setup-go@v4
|
||||
with:
|
||||
go-version: '1.20'
|
||||
go-version: '1.21'
|
||||
- uses: actions/checkout@v3
|
||||
- name: golangci-lint
|
||||
uses: golangci/golangci-lint-action@v3
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
name: Performance Test
|
||||
name: Performance Test
|
||||
|
||||
on: [pull_request]
|
||||
|
||||
@@ -7,13 +7,13 @@ jobs:
|
||||
# skip if PR is from a fork.
|
||||
# TODO: this could probabaly be refactored a bit so that it runs on forks
|
||||
if: ${{ ! github.event.pull_request.head.repo.fork }}
|
||||
|
||||
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Install Go
|
||||
uses: actions/setup-go@v4
|
||||
with:
|
||||
go-version: '1.18'
|
||||
go-version: '1.21'
|
||||
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v3
|
||||
|
||||
@@ -35,7 +35,7 @@ jobs:
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v4
|
||||
with:
|
||||
go-version: '1.20'
|
||||
go-version: '1.21'
|
||||
- name: Run GoReleaser
|
||||
uses: goreleaser/goreleaser-action@v4
|
||||
with:
|
||||
|
||||
@@ -16,7 +16,7 @@ jobs:
|
||||
- name: Install Go
|
||||
uses: actions/setup-go@v4
|
||||
with:
|
||||
go-version: '1.20'
|
||||
go-version: '1.21'
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v3
|
||||
with:
|
||||
|
||||
@@ -11,7 +11,7 @@ jobs:
|
||||
- name: Install Go
|
||||
uses: actions/setup-go@v4
|
||||
with:
|
||||
go-version: '1.18'
|
||||
go-version: '1.21'
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v3
|
||||
- name: Smoke
|
||||
|
||||
@@ -18,7 +18,7 @@ jobs:
|
||||
steps:
|
||||
- uses: actions/setup-go@v4
|
||||
with:
|
||||
go-version: '1.20'
|
||||
go-version: '1.21'
|
||||
- uses: actions/checkout@v3
|
||||
- name: Run Snifftest
|
||||
run: make snifftest
|
||||
run: make snifftest
|
||||
|
||||
@@ -21,7 +21,7 @@ jobs:
|
||||
- name: Install Go
|
||||
uses: actions/setup-go@v4
|
||||
with:
|
||||
go-version: '1.20'
|
||||
go-version: '1.21'
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v3
|
||||
- id: 'auth'
|
||||
@@ -42,7 +42,7 @@ jobs:
|
||||
- name: Install Go
|
||||
uses: actions/setup-go@v4
|
||||
with:
|
||||
go-version: '1.20'
|
||||
go-version: '1.21'
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v3
|
||||
- id: 'auth'
|
||||
@@ -52,8 +52,8 @@ jobs:
|
||||
service_account: 'github-ci-external@trufflehog-testing.iam.gserviceaccount.com'
|
||||
- name: Test
|
||||
run: make test-detectors
|
||||
continue-on-error: true
|
||||
test-fork:
|
||||
continue-on-error: true
|
||||
test-community:
|
||||
if: ${{ github.event.pull_request.head.repo.fork }}
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
@@ -63,8 +63,8 @@ jobs:
|
||||
- name: Install Go
|
||||
uses: actions/setup-go@v4
|
||||
with:
|
||||
go-version: '1.20'
|
||||
go-version: '1.21'
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v3
|
||||
- name: Test
|
||||
run: make test-forks
|
||||
run: make test-community
|
||||
|
||||
@@ -39,7 +39,7 @@ test-race:
|
||||
test-detectors:
|
||||
CGO_ENABLED=0 go test -tags=detectors -timeout=5m $(shell go list ./... | grep pkg/detectors)
|
||||
|
||||
test-forks:
|
||||
test-community:
|
||||
CGO_ENABLED=0 go test -timeout=5m $(shell go list ./... | grep -v /vendor/ | grep -v pkg/detectors | grep -v pkg/sources)
|
||||
|
||||
bench:
|
||||
|
||||
@@ -8,7 +8,6 @@
|
||||
|
||||
<div align="center">
|
||||
|
||||
[](https://github.com/trufflesecurity/trufflehog/actions)
|
||||
[](https://goreportcard.com/report/github.com/trufflesecurity/trufflehog/v3)
|
||||
[](/LICENSE)
|
||||
[](/pkg/detectors)
|
||||
@@ -264,9 +263,9 @@ jobs:
|
||||
extra_args: --debug --only-verified
|
||||
```
|
||||
|
||||
# Precommit Hook
|
||||
# Pre-commit Hook
|
||||
|
||||
Trufflehog can be used in a precommit hook to prevent credentials from leaking before they ever leave your computer.
|
||||
Trufflehog can be used in a pre-commit hook to prevent credentials from leaking before they ever leave your computer.
|
||||
An example `.pre-commit-config.yaml` is provided (see [pre-commit.com](https://pre-commit.com/) for installation).
|
||||
|
||||
```yaml
|
||||
|
||||
@@ -4,16 +4,25 @@ go 1.20
|
||||
|
||||
replace github.com/jpillora/overseer => github.com/trufflesecurity/overseer v1.1.7-custom5
|
||||
|
||||
// go wants to pull in the latest but after v10.2.5 has a breaking API change
|
||||
replace github.com/couchbase/gocbcore/v10 => github.com/couchbase/gocbcore/v10 v10.2.4
|
||||
|
||||
require (
|
||||
cloud.google.com/go/secretmanager v1.11.1
|
||||
cloud.google.com/go/storage v1.31.0
|
||||
github.com/Azure/go-autorest/autorest/azure/auth v0.5.11
|
||||
github.com/BobuSumisu/aho-corasick v1.0.3
|
||||
github.com/TheZeroSlave/zapsentry v1.17.0
|
||||
github.com/aws/aws-sdk-go v1.44.83
|
||||
github.com/aymanbagabas/go-osc52 v1.2.1
|
||||
github.com/bill-rich/disk-buffer-reader v0.1.7
|
||||
github.com/bill-rich/go-syslog v0.0.0-20220413021637-49edb52a574c
|
||||
github.com/bitfinexcom/bitfinex-api-go v0.0.0-20210608095005-9e0b26f200fb
|
||||
github.com/bradleyfalzon/ghinstallation/v2 v2.4.0
|
||||
github.com/bradleyfalzon/ghinstallation/v2 v2.6.0
|
||||
github.com/charmbracelet/bubbles v0.16.1
|
||||
github.com/charmbracelet/bubbletea v0.24.1
|
||||
github.com/charmbracelet/glamour v0.6.0
|
||||
github.com/charmbracelet/lipgloss v0.7.1
|
||||
github.com/couchbase/gocb/v2 v2.6.3
|
||||
github.com/crewjam/rfc5424 v0.1.0
|
||||
github.com/denisenkom/go-mssqldb v0.12.3
|
||||
@@ -22,7 +31,7 @@ require (
|
||||
github.com/felixge/fgprof v0.9.3
|
||||
github.com/getsentry/sentry-go v0.22.0
|
||||
github.com/go-errors/errors v1.4.2
|
||||
github.com/go-git/go-git/v5 v5.7.0
|
||||
github.com/go-git/go-git/v5 v5.8.1
|
||||
github.com/go-ldap/ldap/v3 v3.4.5
|
||||
github.com/go-logr/logr v1.2.4
|
||||
github.com/go-logr/zapr v1.2.4
|
||||
@@ -31,35 +40,39 @@ require (
|
||||
github.com/gobwas/glob v0.2.3
|
||||
github.com/golang-jwt/jwt v3.2.2+incompatible
|
||||
github.com/google/go-cmp v0.5.9
|
||||
github.com/google/go-containerregistry v0.14.0
|
||||
github.com/google/go-containerregistry v0.15.2
|
||||
github.com/google/go-github/v42 v42.0.0
|
||||
github.com/googleapis/gax-go/v2 v2.11.0
|
||||
github.com/googleapis/gax-go/v2 v2.12.0
|
||||
github.com/hashicorp/go-retryablehttp v0.7.4
|
||||
github.com/hashicorp/golang-lru v0.5.1
|
||||
github.com/jlaffaye/ftp v0.2.0
|
||||
github.com/joho/godotenv v1.5.1
|
||||
github.com/jpillora/overseer v1.1.6
|
||||
github.com/kylelemons/godebug v1.1.0
|
||||
github.com/lib/pq v1.10.9
|
||||
github.com/lrstanley/bubblezone v0.0.0-20221222153816-e95291e2243e
|
||||
github.com/mattn/go-isatty v0.0.18
|
||||
github.com/mattn/go-sqlite3 v1.14.17
|
||||
github.com/mholt/archiver/v4 v4.0.0-alpha.8
|
||||
github.com/muesli/reflow v0.3.0
|
||||
github.com/patrickmn/go-cache v2.1.0+incompatible
|
||||
github.com/paulbellamy/ratecounter v0.2.0
|
||||
github.com/petar-dambovaliev/aho-corasick v0.0.0-20211021192214-5ab2d9280aa9
|
||||
github.com/pkg/errors v0.9.1
|
||||
github.com/prometheus/client_golang v1.16.0
|
||||
github.com/rabbitmq/amqp091-go v1.8.1
|
||||
github.com/sergi/go-diff v1.3.1
|
||||
github.com/stretchr/testify v1.8.4
|
||||
github.com/tailscale/depaware v0.0.0-20210622194025-720c4b409502
|
||||
github.com/xanzy/go-gitlab v0.86.0
|
||||
github.com/xanzy/go-gitlab v0.88.0
|
||||
go.mongodb.org/mongo-driver v1.12.0
|
||||
go.uber.org/mock v0.2.0
|
||||
go.uber.org/zap v1.24.0
|
||||
golang.org/x/crypto v0.11.0
|
||||
golang.org/x/exp v0.0.0-20221018205818-5c77f4b2bbd7
|
||||
golang.org/x/oauth2 v0.9.0
|
||||
golang.org/x/oauth2 v0.10.0
|
||||
golang.org/x/sync v0.3.0
|
||||
golang.org/x/text v0.11.0
|
||||
google.golang.org/api v0.130.0
|
||||
google.golang.org/api v0.131.0
|
||||
google.golang.org/protobuf v1.31.0
|
||||
gopkg.in/alecthomas/kingpin.v2 v2.2.6
|
||||
gopkg.in/h2non/gock.v1 v1.1.2
|
||||
@@ -68,9 +81,10 @@ require (
|
||||
|
||||
require (
|
||||
cloud.google.com/go v0.110.2 // indirect
|
||||
cloud.google.com/go/compute v1.19.3 // indirect
|
||||
cloud.google.com/go/compute v1.20.1 // indirect
|
||||
cloud.google.com/go/compute/metadata v0.2.3 // indirect
|
||||
cloud.google.com/go/iam v1.1.0 // indirect
|
||||
dario.cat/mergo v1.0.0 // indirect
|
||||
github.com/Azure/go-autorest v14.2.0+incompatible // indirect
|
||||
github.com/Azure/go-autorest/autorest v0.11.24 // indirect
|
||||
github.com/Azure/go-autorest/autorest/adal v0.9.18 // indirect
|
||||
@@ -80,11 +94,15 @@ require (
|
||||
github.com/Azure/go-autorest/tracing v0.6.0 // indirect
|
||||
github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 // indirect
|
||||
github.com/Microsoft/go-winio v0.6.1 // indirect
|
||||
github.com/ProtonMail/go-crypto v0.0.0-20230518184743-7afd39499903 // indirect
|
||||
github.com/ProtonMail/go-crypto v0.0.0-20230717121422-5aa5874ade95 // indirect
|
||||
github.com/acomagu/bufpipe v1.0.4 // indirect
|
||||
github.com/alecthomas/chroma v0.10.0 // indirect
|
||||
github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 // indirect
|
||||
github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 // indirect
|
||||
github.com/andybalholm/brotli v1.0.5 // indirect
|
||||
github.com/atotto/clipboard v0.1.4 // indirect
|
||||
github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect
|
||||
github.com/aymerick/douceur v0.2.0 // indirect
|
||||
github.com/benbjohnson/clock v1.1.0 // indirect
|
||||
github.com/beorn7/perks v1.0.1 // indirect
|
||||
github.com/bodgit/plumbing v1.2.0 // indirect
|
||||
@@ -93,10 +111,12 @@ require (
|
||||
github.com/cespare/xxhash/v2 v2.2.0 // indirect
|
||||
github.com/cloudflare/circl v1.3.3 // indirect
|
||||
github.com/connesc/cipherio v0.2.1 // indirect
|
||||
github.com/containerd/console v1.0.4-0.20230313162750-1ae8d489ac81 // indirect
|
||||
github.com/containerd/stargz-snapshotter/estargz v0.14.3 // indirect
|
||||
github.com/couchbase/gocbcore/v10 v10.2.3 // indirect
|
||||
github.com/couchbase/gocbcore/v10 v10.2.4 // indirect
|
||||
github.com/davecgh/go-spew v1.1.1 // indirect
|
||||
github.com/dimchansky/utfbom v1.1.1 // indirect
|
||||
github.com/dlclark/regexp2 v1.4.0 // indirect
|
||||
github.com/docker/cli v23.0.5+incompatible // indirect
|
||||
github.com/docker/distribution v2.8.2+incompatible // indirect
|
||||
github.com/docker/docker v23.0.5+incompatible // indirect
|
||||
@@ -113,29 +133,36 @@ require (
|
||||
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
|
||||
github.com/golang/protobuf v1.5.3 // indirect
|
||||
github.com/golang/snappy v0.0.4 // indirect
|
||||
github.com/google/go-github/v52 v52.0.0 // indirect
|
||||
github.com/google/go-github/v53 v53.2.0 // indirect
|
||||
github.com/google/go-querystring v1.1.0 // indirect
|
||||
github.com/google/pprof v0.0.0-20211214055906-6f57359322fd // indirect
|
||||
github.com/google/s2a-go v0.1.4 // indirect
|
||||
github.com/google/uuid v1.3.0 // indirect
|
||||
github.com/googleapis/enterprise-certificate-proxy v0.2.5 // indirect
|
||||
github.com/gorilla/css v1.0.0 // indirect
|
||||
github.com/h2non/parth v0.0.0-20190131123155-b4df798d6542 // indirect
|
||||
github.com/hashicorp/errwrap v1.0.0 // indirect
|
||||
github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
|
||||
github.com/hashicorp/go-multierror v1.1.1 // indirect
|
||||
github.com/imdario/mergo v0.3.15 // indirect
|
||||
github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect
|
||||
github.com/jmespath/go-jmespath v0.4.0 // indirect
|
||||
github.com/jpillora/s3 v1.1.4 // indirect
|
||||
github.com/kevinburke/ssh_config v1.2.0 // indirect
|
||||
github.com/klauspost/compress v1.16.5 // indirect
|
||||
github.com/klauspost/pgzip v1.2.5 // indirect
|
||||
github.com/lucasb-eyer/go-colorful v1.2.0 // indirect
|
||||
github.com/mattn/go-colorable v0.1.13 // indirect
|
||||
github.com/mattn/go-isatty v0.0.17 // indirect
|
||||
github.com/mattn/go-localereader v0.0.1 // indirect
|
||||
github.com/mattn/go-runewidth v0.0.14 // indirect
|
||||
github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
|
||||
github.com/microcosm-cc/bluemonday v1.0.23 // indirect
|
||||
github.com/mitchellh/go-homedir v1.1.0 // indirect
|
||||
github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe // indirect
|
||||
github.com/muesli/ansi v0.0.0-20211031195517-c9f0611b6c70 // indirect
|
||||
github.com/muesli/cancelreader v0.2.2 // indirect
|
||||
github.com/muesli/termenv v0.15.1 // indirect
|
||||
github.com/nwaples/rardecode/v2 v2.0.0-beta.2 // indirect
|
||||
github.com/olekukonko/tablewriter v0.0.5 // indirect
|
||||
github.com/onsi/ginkgo v1.16.5 // indirect
|
||||
github.com/onsi/gomega v1.23.0 // indirect
|
||||
github.com/opencontainers/go-digest v1.0.0 // indirect
|
||||
@@ -147,9 +174,11 @@ require (
|
||||
github.com/prometheus/client_model v0.3.0 // indirect
|
||||
github.com/prometheus/common v0.42.0 // indirect
|
||||
github.com/prometheus/procfs v0.10.1 // indirect
|
||||
github.com/rivo/uniseg v0.4.2 // indirect
|
||||
github.com/rogpeppe/go-internal v1.10.0 // indirect
|
||||
github.com/sahilm/fuzzy v0.1.0 // indirect
|
||||
github.com/sirupsen/logrus v1.9.0 // indirect
|
||||
github.com/skeema/knownhosts v1.1.1 // indirect
|
||||
github.com/skeema/knownhosts v1.2.0 // indirect
|
||||
github.com/therootcompany/xz v1.0.1 // indirect
|
||||
github.com/ulikunitz/xz v0.5.10 // indirect
|
||||
github.com/vbatts/tar-split v0.11.3 // indirect
|
||||
@@ -158,6 +187,8 @@ require (
|
||||
github.com/xdg-go/scram v1.1.2 // indirect
|
||||
github.com/xdg-go/stringprep v1.0.4 // indirect
|
||||
github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d // indirect
|
||||
github.com/yuin/goldmark v1.5.2 // indirect
|
||||
github.com/yuin/goldmark-emoji v1.0.1 // indirect
|
||||
github.com/yusufpapurcu/wmi v1.2.2 // indirect
|
||||
go.opencensus.io v0.24.0 // indirect
|
||||
go.uber.org/atomic v1.7.0 // indirect
|
||||
@@ -166,14 +197,15 @@ require (
|
||||
golang.org/x/mod v0.11.0 // indirect
|
||||
golang.org/x/net v0.12.0 // indirect
|
||||
golang.org/x/sys v0.10.0 // indirect
|
||||
golang.org/x/term v0.10.0 // indirect
|
||||
golang.org/x/time v0.3.0 // indirect
|
||||
golang.org/x/tools v0.10.0 // indirect
|
||||
golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect
|
||||
google.golang.org/appengine v1.6.7 // indirect
|
||||
google.golang.org/genproto v0.0.0-20230530153820-e85fd2cbaebc // indirect
|
||||
google.golang.org/genproto/googleapis/api v0.0.0-20230530153820-e85fd2cbaebc // indirect
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20230629202037-9506855d4529 // indirect
|
||||
google.golang.org/grpc v1.56.1 // indirect
|
||||
google.golang.org/genproto v0.0.0-20230629202037-9506855d4529 // indirect
|
||||
google.golang.org/genproto/googleapis/api v0.0.0-20230629202037-9506855d4529 // indirect
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20230706204954-ccb25ca9f130 // indirect
|
||||
google.golang.org/grpc v1.56.2 // indirect
|
||||
gopkg.in/warnings.v0 v0.1.2 // indirect
|
||||
gopkg.in/yaml.v2 v2.4.0 // indirect
|
||||
gopkg.in/yaml.v3 v3.0.1 // indirect
|
||||
|
||||
@@ -11,8 +11,8 @@ cloud.google.com/go v0.110.2 h1:sdFPBr6xG9/wkBbfhmUz/JmZC7X6LavQgcrVINrKiVA=
|
||||
cloud.google.com/go v0.110.2/go.mod h1:k04UEeEtb6ZBRTv3dZz4CeJC3jKGxyhl0sAiVVquxiw=
|
||||
cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o=
|
||||
cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE=
|
||||
cloud.google.com/go/compute v1.19.3 h1:DcTwsFgGev/wV5+q8o2fzgcHOaac+DKGC91ZlvpsQds=
|
||||
cloud.google.com/go/compute v1.19.3/go.mod h1:qxvISKp/gYnXkSAD1ppcSOveRAmzxicEv/JlizULFrI=
|
||||
cloud.google.com/go/compute v1.20.1 h1:6aKEtlUiwEpJzM001l0yFkpXmUVXaN8W+fbkb2AZNbg=
|
||||
cloud.google.com/go/compute v1.20.1/go.mod h1:4tCnrn48xsqlwSAiLf1HXMQk8CONslYbdiEZc9FEIbM=
|
||||
cloud.google.com/go/compute/metadata v0.2.0/go.mod h1:zFmK7XCadkQkj6TtorcaGlCW1hT1fIilQDwofLpJ20k=
|
||||
cloud.google.com/go/compute/metadata v0.2.3 h1:mg4jlk7mCAj6xXp9UJ4fjI9VUI5rubuGBW5aJ7UnBMY=
|
||||
cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA=
|
||||
@@ -27,6 +27,8 @@ cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiy
|
||||
cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos=
|
||||
cloud.google.com/go/storage v1.31.0 h1:+S3LjjEN2zZ+L5hOwj4+1OkGCsLVe0NzpXKQ1pSdTCI=
|
||||
cloud.google.com/go/storage v1.31.0/go.mod h1:81ams1PrhW16L4kF7qg+4mTq7SRs5HsbDTM0bWvrwJ0=
|
||||
dario.cat/mergo v1.0.0 h1:AGCNq9Evsj31mOgNPcLyXc+4PNABt905YmuqPYYpBWk=
|
||||
dario.cat/mergo v1.0.0/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk=
|
||||
dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/azcore v0.19.0/go.mod h1:h6H6c8enJmmocHUbLiiGY6sx7f9i+X3m1CHdd5c6Rdw=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/azidentity v0.11.0/go.mod h1:HcM1YX14R7CJcghJGOYCgdezslRSVzqwLf/q+4Y2r/0=
|
||||
@@ -51,6 +53,8 @@ github.com/Azure/go-autorest/tracing v0.6.0 h1:TYi4+3m5t6K48TGI9AUdb+IzbnSxvnvUM
|
||||
github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU=
|
||||
github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 h1:mFRzDkZVAjdal+s7s0MwaRv9igoPqLRdzOLzw/8Xvq8=
|
||||
github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358/go.mod h1:chxPXzSsl7ZWRAuOIE23GDNzjWuZquvFlgA8xmpunjU=
|
||||
github.com/BobuSumisu/aho-corasick v1.0.3 h1:uuf+JHwU9CHP2Vx+wAy6jcksJThhJS9ehR8a+4nPE9g=
|
||||
github.com/BobuSumisu/aho-corasick v1.0.3/go.mod h1:hm4jLcvZKI2vRF2WDU1N4p/jpWtpOzp3nLmi9AzX/XE=
|
||||
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
|
||||
github.com/BurntSushi/toml v1.2.1/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ=
|
||||
github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
|
||||
@@ -58,12 +62,14 @@ github.com/Microsoft/go-winio v0.5.2/go.mod h1:WpS1mjBmmwHBEWmogvA2mj8546UReBk4v
|
||||
github.com/Microsoft/go-winio v0.6.1 h1:9/kr64B9VUZrLm5YYwbGtUJnMgqWVOdUAXu6Migciow=
|
||||
github.com/Microsoft/go-winio v0.6.1/go.mod h1:LRdKpFKfdobln8UmuiYcKPot9D2v6svN5+sAH+4kjUM=
|
||||
github.com/ProtonMail/go-crypto v0.0.0-20230217124315-7d5c6f04bbb8/go.mod h1:I0gYDMZ6Z5GRU7l58bNFSkPTFN6Yl12dsUlAZ8xy98g=
|
||||
github.com/ProtonMail/go-crypto v0.0.0-20230518184743-7afd39499903 h1:ZK3C5DtzV2nVAQTx5S5jQvMeDqWtD1By5mOoyY/xJek=
|
||||
github.com/ProtonMail/go-crypto v0.0.0-20230518184743-7afd39499903/go.mod h1:8TI4H3IbrackdNgv+92dI+rhpCaLqM0IfpgCgenFvRE=
|
||||
github.com/ProtonMail/go-crypto v0.0.0-20230717121422-5aa5874ade95 h1:KLq8BE0KwCL+mmXnjLWEAOYO+2l2AE4YMmqG1ZpZHBs=
|
||||
github.com/ProtonMail/go-crypto v0.0.0-20230717121422-5aa5874ade95/go.mod h1:EjAoLdwvbIOoOQr3ihjnSoLZRtE8azugULFRteWMNc0=
|
||||
github.com/TheZeroSlave/zapsentry v1.17.0 h1:RIQCG89U7vWWZVmmCxeUz/g32WEcAYXUrXHoMlbSDmc=
|
||||
github.com/TheZeroSlave/zapsentry v1.17.0/go.mod h1:D1YMfSuu6xnkhwFXxrronesmsiyDhIqo+86I3Ok+r64=
|
||||
github.com/acomagu/bufpipe v1.0.4 h1:e3H4WUzM3npvo5uv95QuJM3cQspFNtFBzvJ2oNjKIDQ=
|
||||
github.com/acomagu/bufpipe v1.0.4/go.mod h1:mxdxdup/WdsKVreO5GpW4+M/1CE2sMG4jeGJ2sYmHc4=
|
||||
github.com/alecthomas/chroma v0.10.0 h1:7XDcGkCQopCNKjZHfYrNLraA+M7e0fMiJ/Mfikbfjek=
|
||||
github.com/alecthomas/chroma v0.10.0/go.mod h1:jtJATyUxlIORhUOFNA9NZDWGAQ8wpxQQqNSB4rjA/1s=
|
||||
github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 h1:JYp7IbQjafoB+tBA3gMyHYHrpOtNuDiK/uB5uXxq5wM=
|
||||
github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
|
||||
github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 h1:s6gZFSlWYmbqAuRjVTiNNhvNRfY2Wxp9nhfyel4rklc=
|
||||
@@ -75,8 +81,17 @@ github.com/andybalholm/brotli v1.0.5/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHG
|
||||
github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be h1:9AeTilPcZAjCFIImctFaOjnTIavg87rW78vTPkQqLI8=
|
||||
github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY=
|
||||
github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio=
|
||||
github.com/atotto/clipboard v0.1.4 h1:EH0zSVneZPSuFR11BlR9YppQTVDbh5+16AmcJi4g1z4=
|
||||
github.com/atotto/clipboard v0.1.4/go.mod h1:ZY9tmq7sm5xIbd9bOK4onWV4S6X0u6GY7Vn0Yu86PYI=
|
||||
github.com/aws/aws-sdk-go v1.44.83 h1:7+Rtc2Eio6EKUNoZeMV/IVxzVrY5oBQcNPtCcgIHYJA=
|
||||
github.com/aws/aws-sdk-go v1.44.83/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo=
|
||||
github.com/aymanbagabas/go-osc52 v1.0.3/go.mod h1:zT8H+Rk4VSabYN90pWyugflM3ZhpTZNC7cASDfUCdT4=
|
||||
github.com/aymanbagabas/go-osc52 v1.2.1 h1:q2sWUyDcozPLcLabEMd+a+7Ea2DitxZVN9hTxab9L4E=
|
||||
github.com/aymanbagabas/go-osc52 v1.2.1/go.mod h1:zT8H+Rk4VSabYN90pWyugflM3ZhpTZNC7cASDfUCdT4=
|
||||
github.com/aymanbagabas/go-osc52/v2 v2.0.1 h1:HwpRHbFMcZLEVr42D4p7XBqjyuxQH5SMiErDT4WkJ2k=
|
||||
github.com/aymanbagabas/go-osc52/v2 v2.0.1/go.mod h1:uYgXzlJ7ZpABp8OJ+exZzJJhRNQ2ASbcXHWsFqH8hp8=
|
||||
github.com/aymerick/douceur v0.2.0 h1:Mv+mAeH1Q+n9Fr+oyamOlAkUNPWPlA8PPGR0QAaYuPk=
|
||||
github.com/aymerick/douceur v0.2.0/go.mod h1:wlT5vV2O3h55X9m7iVYN0TBM0NH/MmbLnd30/FjWUq4=
|
||||
github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8=
|
||||
github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA=
|
||||
github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
|
||||
@@ -93,13 +108,22 @@ github.com/bodgit/sevenzip v1.3.0 h1:1ljgELgtHqvgIp8W8kgeEGHIWP4ch3xGI8uOBZgLVKY
|
||||
github.com/bodgit/sevenzip v1.3.0/go.mod h1:omwNcgZTEooWM8gA/IJ2Nk/+ZQ94+GsytRzOJJ8FBlM=
|
||||
github.com/bodgit/windows v1.0.0 h1:rLQ/XjsleZvx4fR1tB/UxQrK+SJ2OFHzfPjLWWOhDIA=
|
||||
github.com/bodgit/windows v1.0.0/go.mod h1:a6JLwrB4KrTR5hBpp8FI9/9W9jJfeQ2h4XDXU74ZCdM=
|
||||
github.com/bradleyfalzon/ghinstallation/v2 v2.4.0 h1:zYSzkoIwekCQAr6GT6KxISLt4YRS6kd4/ixfzMN+7yc=
|
||||
github.com/bradleyfalzon/ghinstallation/v2 v2.4.0/go.mod h1:4MwZLSgBJJgg4i3nJwZJ95AMooSqN8fJDmegLVn9Q2U=
|
||||
github.com/bradleyfalzon/ghinstallation/v2 v2.6.0 h1:IRY7Xy588KylkoycsUhFpW7cdGpy5Y5BPsz4IfuJtGk=
|
||||
github.com/bradleyfalzon/ghinstallation/v2 v2.6.0/go.mod h1:oQ3etOwN3TRH4EwgW5/7MxSVMGlMlzG/O8TU7eYdoSk=
|
||||
github.com/bwesterb/go-ristretto v1.2.0/go.mod h1:fUIoIZaG73pV5biE2Blr2xEzDoMj7NFEuV9ekS419A0=
|
||||
github.com/bwesterb/go-ristretto v1.2.3/go.mod h1:fUIoIZaG73pV5biE2Blr2xEzDoMj7NFEuV9ekS419A0=
|
||||
github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
|
||||
github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
|
||||
github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44=
|
||||
github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
|
||||
github.com/charmbracelet/bubbles v0.16.1 h1:6uzpAAaT9ZqKssntbvZMlksWHruQLNxg49H5WdeuYSY=
|
||||
github.com/charmbracelet/bubbles v0.16.1/go.mod h1:2QCp9LFlEsBQMvIYERr7Ww2H2bA7xen1idUDIzm/+Xc=
|
||||
github.com/charmbracelet/bubbletea v0.24.1 h1:LpdYfnu+Qc6XtvMz6d/6rRY71yttHTP5HtrjMgWvixc=
|
||||
github.com/charmbracelet/bubbletea v0.24.1/go.mod h1:rK3g/2+T8vOSEkNHvtq40umJpeVYDn6bLaqbgzhL/hg=
|
||||
github.com/charmbracelet/glamour v0.6.0 h1:wi8fse3Y7nfcabbbDuwolqTqMQPMnVPeZhDM273bISc=
|
||||
github.com/charmbracelet/glamour v0.6.0/go.mod h1:taqWV4swIMMbWALc0m7AfE9JkPSU8om2538k9ITBxOc=
|
||||
github.com/charmbracelet/lipgloss v0.7.1 h1:17WMwi7N1b1rVWOjMT+rCh7sQkvDU75B2hbZpc5Kc1E=
|
||||
github.com/charmbracelet/lipgloss v0.7.1/go.mod h1:yG0k3giv8Qj8edTCbbg6AlQ5e8KNWpFujkNawKNhE2c=
|
||||
github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
|
||||
github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
|
||||
github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
|
||||
@@ -115,12 +139,14 @@ github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWH
|
||||
github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
|
||||
github.com/connesc/cipherio v0.2.1 h1:FGtpTPMbKNNWByNrr9aEBtaJtXjqOzkIXNYJp6OEycw=
|
||||
github.com/connesc/cipherio v0.2.1/go.mod h1:ukY0MWJDFnJEbXMQtOcn2VmTpRfzcTz4OoVrWGGJZcA=
|
||||
github.com/containerd/console v1.0.4-0.20230313162750-1ae8d489ac81 h1:q2hJAaP1k2wIvVRd/hEHD7lacgqrCPS+k8g1MndzfWY=
|
||||
github.com/containerd/console v1.0.4-0.20230313162750-1ae8d489ac81/go.mod h1:YynlIjWYF8myEu6sdkwKIvGQq+cOckRm6So2avqoYAk=
|
||||
github.com/containerd/stargz-snapshotter/estargz v0.14.3 h1:OqlDCK3ZVUO6C3B/5FSkDwbkEETK84kQgEeFwDC+62k=
|
||||
github.com/containerd/stargz-snapshotter/estargz v0.14.3/go.mod h1:KY//uOCIkSuNAHhJogcZtrNHdKrA99/FCCRjE3HD36o=
|
||||
github.com/couchbase/gocb/v2 v2.6.3 h1:5RsMo+RRfK0mVxHLAfpBz3/tHlgXZb1WBNItLk9Ab+c=
|
||||
github.com/couchbase/gocb/v2 v2.6.3/go.mod h1:yF5F6BHTZ/ZowhEuZbySbXrlI4rHd1TIhm5azOaMbJU=
|
||||
github.com/couchbase/gocbcore/v10 v10.2.3 h1:PEkRSNSkKjUBXx82Ucr094+anoiCG5GleOOQZOHo6D4=
|
||||
github.com/couchbase/gocbcore/v10 v10.2.3/go.mod h1:lYQIIk+tzoMcwtwU5GzPbDdqEkwkH3isI2rkSpfL0oM=
|
||||
github.com/couchbase/gocbcore/v10 v10.2.4 h1:TDTQ1mSBUw9eajuV71ZDJcLRbmfGcsvZV9SMeUDAVGc=
|
||||
github.com/couchbase/gocbcore/v10 v10.2.4/go.mod h1:lYQIIk+tzoMcwtwU5GzPbDdqEkwkH3isI2rkSpfL0oM=
|
||||
github.com/couchbaselabs/gocaves/client v0.0.0-20230307083111-cc3960c624b1/go.mod h1:AVekAZwIY2stsJOMWLAS/0uA/+qdp7pjO8EHnl61QkY=
|
||||
github.com/couchbaselabs/gocaves/client v0.0.0-20230404095311-05e3ba4f0259 h1:2TXy68EGEzIMHOx9UvczR5ApVecwCfQZ0LjkmwMI6g4=
|
||||
github.com/couchbaselabs/gocaves/client v0.0.0-20230404095311-05e3ba4f0259/go.mod h1:AVekAZwIY2stsJOMWLAS/0uA/+qdp7pjO8EHnl61QkY=
|
||||
@@ -135,6 +161,8 @@ github.com/denisenkom/go-mssqldb v0.12.3 h1:pBSGx9Tq67pBOTLmxNuirNTeB8Vjmf886Kx+
|
||||
github.com/denisenkom/go-mssqldb v0.12.3/go.mod h1:k0mtMFOnU+AihqFxPMiF05rtiDrorD1Vrm1KEz5hxDo=
|
||||
github.com/dimchansky/utfbom v1.1.1 h1:vV6w1AhK4VMnhBno/TPVCoK9U/LP0PkLCS9tbxHdi/U=
|
||||
github.com/dimchansky/utfbom v1.1.1/go.mod h1:SxdoEBH5qIqFocHMyGOXVAybYJdr71b1Q/j0mACtrfE=
|
||||
github.com/dlclark/regexp2 v1.4.0 h1:F1rxgk7p4uKjwIQxBs9oAXe5CqrXlCduYEJvrF4u93E=
|
||||
github.com/dlclark/regexp2 v1.4.0/go.mod h1:2pZnwuY/m+8K6iRw6wQdMtk+rH5tNGR1i55kozfMjCc=
|
||||
github.com/dnaeon/go-vcr v1.2.0/go.mod h1:R4UdLID7HZT3taECzJs4YgbbH6PIGXB6W/sc5OLb6RQ=
|
||||
github.com/docker/cli v23.0.5+incompatible h1:ufWmAOuD3Vmr7JP2G5K3cyuNC4YZWiAsuDEvFVVDafE=
|
||||
github.com/docker/cli v23.0.5+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8=
|
||||
@@ -178,8 +206,8 @@ github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376/go.mod h1:an3vInlBmS
|
||||
github.com/go-git/go-billy/v5 v5.4.1 h1:Uwp5tDRkPr+l/TnbHOQzp+tmJfLceOlbVucgpTz8ix4=
|
||||
github.com/go-git/go-billy/v5 v5.4.1/go.mod h1:vjbugF6Fz7JIflbVpl1hJsGjSHNltrSw45YK/ukIvQg=
|
||||
github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20230305113008-0c11038e723f h1:Pz0DHeFij3XFhoBRGUDPzSJ+w2UcK5/0JvF8DRI58r8=
|
||||
github.com/go-git/go-git/v5 v5.7.0 h1:t9AudWVLmqzlo+4bqdf7GY+46SUuRsx59SboFxkq2aE=
|
||||
github.com/go-git/go-git/v5 v5.7.0/go.mod h1:coJHKEOk5kUClpsNlXrUvPrDxY3w3gjHvhcZd8Fodw8=
|
||||
github.com/go-git/go-git/v5 v5.8.1 h1:Zo79E4p7TRk0xoRgMq0RShiTHGKcKI4+DI6BfJc/Q+A=
|
||||
github.com/go-git/go-git/v5 v5.8.1/go.mod h1:FHFuoD6yGz5OSKEBK+aWN9Oah0q54Jxl0abmj6GnqAo=
|
||||
github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
|
||||
github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
|
||||
github.com/go-ldap/ldap/v3 v3.4.5 h1:ekEKmaDrpvR2yf5Nc/DClsGG9lAmdDixe44mLzlW5r8=
|
||||
@@ -255,12 +283,12 @@ github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/
|
||||
github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
|
||||
github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
|
||||
github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
|
||||
github.com/google/go-containerregistry v0.14.0 h1:z58vMqHxuwvAsVwvKEkmVBz2TlgBgH5k6koEXBtlYkw=
|
||||
github.com/google/go-containerregistry v0.14.0/go.mod h1:aiJ2fp/SXvkWgmYHioXnbMdlgB8eXiiYOY55gfN91Wk=
|
||||
github.com/google/go-containerregistry v0.15.2 h1:MMkSh+tjSdnmJZO7ljvEqV1DjfekB6VUEAZgy3a+TQE=
|
||||
github.com/google/go-containerregistry v0.15.2/go.mod h1:wWK+LnOv4jXMM23IT/F1wdYftGWGr47Is8CG+pmHK1Q=
|
||||
github.com/google/go-github/v42 v42.0.0 h1:YNT0FwjPrEysRkLIiKuEfSvBPCGKphW5aS5PxwaoLec=
|
||||
github.com/google/go-github/v42 v42.0.0/go.mod h1:jgg/jvyI0YlDOM1/ps6XYh04HNQ3vKf0CVko62/EhRg=
|
||||
github.com/google/go-github/v52 v52.0.0 h1:uyGWOY+jMQ8GVGSX8dkSwCzlehU3WfdxQ7GweO/JP7M=
|
||||
github.com/google/go-github/v52 v52.0.0/go.mod h1:WJV6VEEUPuMo5pXqqa2ZCZEdbQqua4zAk2MZTIo+m+4=
|
||||
github.com/google/go-github/v53 v53.2.0 h1:wvz3FyF53v4BK+AsnvCmeNhf8AkTaeh2SoYu/XUvTtI=
|
||||
github.com/google/go-github/v53 v53.2.0/go.mod h1:XhFRObz+m/l+UCm9b7KSIC3lT3NWSXGt7mOsAWEloao=
|
||||
github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8=
|
||||
github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU=
|
||||
github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPgecwXBIDzw5no=
|
||||
@@ -281,8 +309,10 @@ github.com/googleapis/enterprise-certificate-proxy v0.2.5 h1:UR4rDjcgpgEnqpIEvki
|
||||
github.com/googleapis/enterprise-certificate-proxy v0.2.5/go.mod h1:RxW0N9901Cko1VOCW3SXCpWP+mlIEkk2tP7jnHy9a3w=
|
||||
github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
|
||||
github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
|
||||
github.com/googleapis/gax-go/v2 v2.11.0 h1:9V9PWXEsWnPpQhu/PeQIkS4eGzMlTLGgt80cUUI8Ki4=
|
||||
github.com/googleapis/gax-go/v2 v2.11.0/go.mod h1:DxmR61SGKkGLa2xigwuZIQpkCI2S5iydzRfb3peWZJI=
|
||||
github.com/googleapis/gax-go/v2 v2.12.0 h1:A+gCJKdRfqXkr+BIRGtZLibNXf0m1f9E4HG56etFpas=
|
||||
github.com/googleapis/gax-go/v2 v2.12.0/go.mod h1:y+aIqrI5eb1YGMVJfuV3185Ts/D7qKpsEkdD5+I6QGU=
|
||||
github.com/gorilla/css v1.0.0 h1:BQqNyPTi50JCFMTw/b67hByjMVXZRwGha6wxVGkeihY=
|
||||
github.com/gorilla/css v1.0.0/go.mod h1:Dn721qIggHpt4+EFCcTLTU/vk5ySda2ReITrtgBl60c=
|
||||
github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
|
||||
github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw=
|
||||
github.com/h2non/parth v0.0.0-20190131123155-b4df798d6542 h1:2VTzZjLZBgl62/EtslCrtky5vbi9dd7HrQPQIx6wqiw=
|
||||
@@ -298,12 +328,11 @@ github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9
|
||||
github.com/hashicorp/go-retryablehttp v0.7.4 h1:ZQgVdpTdAL7WpMIwLzCfbalOcSUdkDZnpUv3/+BxzFA=
|
||||
github.com/hashicorp/go-retryablehttp v0.7.4/go.mod h1:Jy/gPYAdjqffZ/yFGCFV2doI5wjtH1ewM9u8iYVjtX8=
|
||||
github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
|
||||
github.com/hashicorp/golang-lru v0.5.1 h1:0hERBMJE1eitiLkihrMvRVBYAkpHzc/J3QdDN+dAcgU=
|
||||
github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
|
||||
github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
|
||||
github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
|
||||
github.com/ianlancetaylor/demangle v0.0.0-20210905161508-09a460cdf81d/go.mod h1:aYm2/VgdVmcIU8iMfdMvDMsRAQjcfZSKFby6HOFvi/w=
|
||||
github.com/imdario/mergo v0.3.15 h1:M8XP7IuFNsqUx6VPK2P9OSmsYsI/YFaGil0uD21V3dM=
|
||||
github.com/imdario/mergo v0.3.15/go.mod h1:WBLT9ZmE3lPoWsEzCh9LPo3TiwVN+ZKEjmz+hD27ysY=
|
||||
github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOlocH6Fxy8MmwDt+yVQYULKfN0RoTN8A=
|
||||
github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i6rXxKeerYnT8Nvf0QmHCRC1n8sfWVwXF2Frvo=
|
||||
github.com/jlaffaye/ftp v0.2.0 h1:lXNvW7cBu7R/68bknOX3MrRIIqZ61zELs1P2RAiA3lg=
|
||||
@@ -339,24 +368,46 @@ github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0
|
||||
github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
|
||||
github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw=
|
||||
github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
|
||||
github.com/lrstanley/bubblezone v0.0.0-20221222153816-e95291e2243e h1:XoxHx8K6ZKoMtjzWOMDuM69LCdjDDsTOtTfWGrT/fns=
|
||||
github.com/lrstanley/bubblezone v0.0.0-20221222153816-e95291e2243e/go.mod h1:v5lEwWaguF1o2MW/ucO0ZIA/IZymdBYJJ+2cMRLE7LU=
|
||||
github.com/lucasb-eyer/go-colorful v1.2.0 h1:1nnpGOrhyZZuNyfu1QjKiUICQ74+3FNCN69Aj6K7nkY=
|
||||
github.com/lucasb-eyer/go-colorful v1.2.0/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i7ruzyGqttikkLy0=
|
||||
github.com/matryer/is v1.2.0 h1:92UTHpy8CDwaJ08GqLDzhhuixiBUUD1p3AU6PHddz4A=
|
||||
github.com/matryer/is v1.2.0/go.mod h1:2fLPjFQM9rhQ15aVEtbuwhJinnOqrmgXPNdZsdwlWXA=
|
||||
github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=
|
||||
github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg=
|
||||
github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
|
||||
github.com/mattn/go-isatty v0.0.17 h1:BTarxUcIeDqL27Mc+vyvdWYSL28zpIhv3RoTdsLMPng=
|
||||
github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
|
||||
github.com/mattn/go-isatty v0.0.18 h1:DOKFKCQ7FNG2L1rbrmstDN4QVRdS89Nkh85u68Uwp98=
|
||||
github.com/mattn/go-isatty v0.0.18/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
|
||||
github.com/mattn/go-localereader v0.0.1 h1:ygSAOl7ZXTx4RdPYinUpg6W99U8jWvWi9Ye2JC/oIi4=
|
||||
github.com/mattn/go-localereader v0.0.1/go.mod h1:8fBrzywKY7BI3czFoHkuzRoWE9C+EiG4R1k4Cjx5p88=
|
||||
github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI=
|
||||
github.com/mattn/go-runewidth v0.0.12/go.mod h1:RAqKPSqVFrSLVXbA8x7dzmKdmGzieGRCM46jaSJTDAk=
|
||||
github.com/mattn/go-runewidth v0.0.14 h1:+xnbZSEeDbOIg5/mE6JF0w6n9duR1l3/WmbinWVwUuU=
|
||||
github.com/mattn/go-runewidth v0.0.14/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
|
||||
github.com/mattn/go-sqlite3 v1.14.17 h1:mCRHCLDUBXgpKAqIKsaAaAsrAlbkeomtRFKXh2L6YIM=
|
||||
github.com/mattn/go-sqlite3 v1.14.17/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg=
|
||||
github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo=
|
||||
github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4=
|
||||
github.com/mholt/archiver/v4 v4.0.0-alpha.8 h1:tRGQuDVPh66WCOelqe6LIGh0gwmfwxUrSSDunscGsRM=
|
||||
github.com/mholt/archiver/v4 v4.0.0-alpha.8/go.mod h1:5f7FUYGXdJWUjESffJaYR4R60VhnHxb2X3T1teMyv5A=
|
||||
github.com/microcosm-cc/bluemonday v1.0.21/go.mod h1:ytNkv4RrDrLJ2pqlsSI46O6IVXmZOBBD4SaJyDwwTkM=
|
||||
github.com/microcosm-cc/bluemonday v1.0.23 h1:SMZe2IGa0NuHvnVNAZ+6B38gsTbi5e4sViiWJyDDqFY=
|
||||
github.com/microcosm-cc/bluemonday v1.0.23/go.mod h1:mN70sk7UkkF8TUr2IGBpNN0jAgStuPzlK76QuruE/z4=
|
||||
github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y=
|
||||
github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
|
||||
github.com/modocache/gover v0.0.0-20171022184752-b58185e213c5/go.mod h1:caMODM3PzxT8aQXRPkAt8xlV/e7d7w8GM5g0fa5F0D8=
|
||||
github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe h1:iruDEfMl2E6fbMZ9s0scYfZQ84/6SPL6zC8ACM2oIL0=
|
||||
github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc=
|
||||
github.com/muesli/ansi v0.0.0-20211031195517-c9f0611b6c70 h1:kMlmsLSbjkikxQJ1IPwaM+7LJ9ltFu/fi8CRzvSnQmA=
|
||||
github.com/muesli/ansi v0.0.0-20211031195517-c9f0611b6c70/go.mod h1:fQuZ0gauxyBcmsdE3ZT4NasjaRdxmbCS0jRHsrWu3Ho=
|
||||
github.com/muesli/cancelreader v0.2.2 h1:3I4Kt4BQjOR54NavqnDogx/MIoWBFa0StPA8ELUXHmA=
|
||||
github.com/muesli/cancelreader v0.2.2/go.mod h1:3XuTXfFS2VjM+HTLZY9Ak0l6eUKfijIfMUZ4EgX0QYo=
|
||||
github.com/muesli/reflow v0.3.0 h1:IFsN6K9NfGtjeggFP+68I4chLZV2yIKsXJFNZ+eWh6s=
|
||||
github.com/muesli/reflow v0.3.0/go.mod h1:pbwTDkVPibjO2kyvBQRBxTWEEGDGq0FlB1BIKtnHY/8=
|
||||
github.com/muesli/termenv v0.13.0/go.mod h1:sP1+uffeLaEYpyOTb8pLCUctGcGLnoFjSn4YJK5e2bc=
|
||||
github.com/muesli/termenv v0.15.1 h1:UzuTb/+hhlBugQz28rpzey4ZuKcZ03MeKsoG7IJZIxs=
|
||||
github.com/muesli/termenv v0.15.1/go.mod h1:HeAQPTzpfs016yGtA4g00CsdYnVLJvxsS4ANqrZs2sQ=
|
||||
github.com/nbio/st v0.0.0-20140626010706-e9e8d9816f32 h1:W6apQkHrMkS0Muv8G/TipAy/FJl/rCYT0+EuS8+Z0z4=
|
||||
github.com/nbio/st v0.0.0-20140626010706-e9e8d9816f32/go.mod h1:9wM+0iRr9ahx58uYLpLIr5fm8diHn0JbqRycJi6w0Ms=
|
||||
github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
|
||||
@@ -365,6 +416,8 @@ github.com/nwaples/rardecode/v2 v2.0.0-beta.2/go.mod h1:yntwv/HfMc/Hbvtq9I19D1n5
|
||||
github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A=
|
||||
github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE=
|
||||
github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU=
|
||||
github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec=
|
||||
github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY=
|
||||
github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
|
||||
github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk=
|
||||
github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE=
|
||||
@@ -382,8 +435,6 @@ github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaR
|
||||
github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ=
|
||||
github.com/paulbellamy/ratecounter v0.2.0 h1:2L/RhJq+HA8gBQImDXtLPrDXK5qAj6ozWVK/zFXVJGs=
|
||||
github.com/paulbellamy/ratecounter v0.2.0/go.mod h1:Hfx1hDpSGoqxkVVpBi/IlYD7kChlfo5C6hzIHwPqfFE=
|
||||
github.com/petar-dambovaliev/aho-corasick v0.0.0-20211021192214-5ab2d9280aa9 h1:lL+y4Xv20pVlCGyLzNHRC0I0rIHhIL1lTvHizoS/dU8=
|
||||
github.com/petar-dambovaliev/aho-corasick v0.0.0-20211021192214-5ab2d9280aa9/go.mod h1:EHPiTAKtiFmrMldLUNswFwfZ2eJIYBHktdaUTZxYWRw=
|
||||
github.com/pierrec/lz4/v4 v4.1.15 h1:MO0/ucJhngq7299dKLwIMtgTfbkoSPF6AoMYDd8Q4q0=
|
||||
github.com/pierrec/lz4/v4 v4.1.15/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
|
||||
github.com/pingcap/errors v0.11.4 h1:lFuQV/oaUMGcD2tqt+01ROSmJs75VG1ToEOkZIZ4nE4=
|
||||
@@ -408,20 +459,26 @@ github.com/prometheus/procfs v0.10.1 h1:kYK1Va/YMlutzCGazswoHKo//tZVlFpKYh+Pymzi
|
||||
github.com/prometheus/procfs v0.10.1/go.mod h1:nwNm2aOCAYw8uTR/9bWRREkZFxAUcWzPHWJq+XBB/FM=
|
||||
github.com/rabbitmq/amqp091-go v1.8.1 h1:RejT1SBUim5doqcL6s7iN6SBmsQqyTgXb1xMlH0h1hA=
|
||||
github.com/rabbitmq/amqp091-go v1.8.1/go.mod h1:+jPrT9iY2eLjRaMSRHUhc3z14E/l85kv/f+6luSD3pc=
|
||||
github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
|
||||
github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
|
||||
github.com/rivo/uniseg v0.4.2 h1:YwD0ulJSJytLpiaWua0sBDusfsCZohxjxzVTYjwxfV8=
|
||||
github.com/rivo/uniseg v0.4.2/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
|
||||
github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ=
|
||||
github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
|
||||
github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ=
|
||||
github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog=
|
||||
github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
|
||||
github.com/rwcarlsen/goexif v0.0.0-20190401172101-9e8deecbddbd/go.mod h1:hPqNNc0+uJM6H+SuU8sEs5K5IQeKccPqeSjfgcKGgPk=
|
||||
github.com/sahilm/fuzzy v0.1.0 h1:FzWGaw2Opqyu+794ZQ9SYifWv2EIXpwP4q8dY1kDAwI=
|
||||
github.com/sahilm/fuzzy v0.1.0/go.mod h1:VFvziUEIMCrT6A6tw2RFIXPXXmzXbOsSHF0DOI8ZK9Y=
|
||||
github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
|
||||
github.com/sergi/go-diff v1.3.1 h1:xkr+Oxo4BOQKmkn/B9eMK0g5Kg/983T9DqqPHwYqD+8=
|
||||
github.com/sergi/go-diff v1.3.1/go.mod h1:aMJSSKb2lpPvRNec0+w3fl7LP9IOFzdc9Pa4NFbPK1I=
|
||||
github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
|
||||
github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0=
|
||||
github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
|
||||
github.com/skeema/knownhosts v1.1.1 h1:MTk78x9FPgDFVFkDLTrsnnfCJl7g1C/nnKvePgrIngE=
|
||||
github.com/skeema/knownhosts v1.1.1/go.mod h1:g4fPeYpque7P0xefxtGzV81ihjC8sX2IqpAoNkjxbMo=
|
||||
github.com/skeema/knownhosts v1.2.0 h1:h9r9cf0+u7wSE+M183ZtMGgOJKiL96brpaz5ekfJCpM=
|
||||
github.com/skeema/knownhosts v1.2.0/go.mod h1:g4fPeYpque7P0xefxtGzV81ihjC8sX2IqpAoNkjxbMo=
|
||||
github.com/smartystreets/assertions v1.0.1 h1:voD4ITNjPL5jjBfgR/r8fPIIBrliWrWHeiJApdr3r4w=
|
||||
github.com/smartystreets/assertions v1.0.1/go.mod h1:kHHU4qYBaI3q23Pp3VPrmWhuIUrLW/7eUrw0BU5VaoM=
|
||||
github.com/smartystreets/gunit v1.1.3 h1:32x+htJCu3aMswhPw3teoJ+PnWPONqdNgaGs6Qt8ZaU=
|
||||
@@ -454,8 +511,8 @@ github.com/ulikunitz/xz v0.5.10/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0o
|
||||
github.com/urfave/cli v1.22.12/go.mod h1:sSBEIC79qR6OvcmsD4U3KABeOTxDqQtdDnaFuUN30b8=
|
||||
github.com/vbatts/tar-split v0.11.3 h1:hLFqsOLQ1SsppQNTMpkpPXClLDfC2A3Zgy9OUU+RVck=
|
||||
github.com/vbatts/tar-split v0.11.3/go.mod h1:9QlHN18E+fEH7RdG+QAJJcuya3rqT7eXSTY7wGrAokY=
|
||||
github.com/xanzy/go-gitlab v0.86.0 h1:jR8V9cK9jXRQDb46KOB20NCF3ksY09luaG0IfXE6p7w=
|
||||
github.com/xanzy/go-gitlab v0.86.0/go.mod h1:5ryv+MnpZStBH8I/77HuQBsMbBGANtVpLWC15qOjWAw=
|
||||
github.com/xanzy/go-gitlab v0.88.0 h1:9GHBrxyCUNZZNuAsbJ1NbEH6XAYsKyTn6NfE0wYO5SY=
|
||||
github.com/xanzy/go-gitlab v0.88.0/go.mod h1:5ryv+MnpZStBH8I/77HuQBsMbBGANtVpLWC15qOjWAw=
|
||||
github.com/xanzy/ssh-agent v0.3.3 h1:+/15pJfg/RsTxqYcX6fHqOXZwwMP+2VyYWJeWM2qQFM=
|
||||
github.com/xanzy/ssh-agent v0.3.3/go.mod h1:6dzNDKs0J9rVPHPhaGCukekBHKqfl+L3KghI1Bc68Uw=
|
||||
github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c=
|
||||
@@ -469,6 +526,10 @@ github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7Jul
|
||||
github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
|
||||
github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
|
||||
github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
|
||||
github.com/yuin/goldmark v1.5.2 h1:ALmeCk/px5FSm1MAcFBAsVKZjDuMVj8Tm7FFIlMJnqU=
|
||||
github.com/yuin/goldmark v1.5.2/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
|
||||
github.com/yuin/goldmark-emoji v1.0.1 h1:ctuWEyzGBwiucEqxzwe0SOYDXPAucOrE9NQC18Wa1os=
|
||||
github.com/yuin/goldmark-emoji v1.0.1/go.mod h1:2w1E6FEWLcDQkoTE+7HU6QF1F6SLlNGjRIBbIZQFqkQ=
|
||||
github.com/yusufpapurcu/wmi v1.2.2 h1:KBNDSne4vP5mbSWnJbO+51IMOXJB67QiYCSBrubbPRg=
|
||||
github.com/yusufpapurcu/wmi v1.2.2/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0=
|
||||
go.mongodb.org/mongo-driver v1.12.0 h1:aPx33jmn/rQuJXPQLZQ8NtfPQG8CaqgLThFtqRb0PiE=
|
||||
@@ -485,6 +546,8 @@ go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
|
||||
go.uber.org/goleak v1.1.11/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ=
|
||||
go.uber.org/goleak v1.2.1 h1:NBol2c7O1ZokfZ0LEU9K6Whx/KnwvepVetCUhtKja4A=
|
||||
go.uber.org/goleak v1.2.1/go.mod h1:qlT2yGI9QafXHhZZLxlSuNsMw3FFLxBr+tBRlmO1xH4=
|
||||
go.uber.org/mock v0.2.0 h1:TaP3xedm7JaAgScZO7tlvlKrqT0p7I6OsdGB5YNSMDU=
|
||||
go.uber.org/mock v0.2.0/go.mod h1:J0y0rp9L3xiff1+ZBfKxlC1fz2+aO16tw0tsDOixfuM=
|
||||
go.uber.org/multierr v1.6.0 h1:y6IPFStTAIT5Ytl7/XYmHvzXQ7S3g/IeZW9hyZ5thw4=
|
||||
go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU=
|
||||
go.uber.org/zap v1.24.0 h1:FiJd5l1UOLj0wCgbSE0rwwXHzEdAZS6hiiSnxJN/D60=
|
||||
@@ -501,6 +564,7 @@ golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5y
|
||||
golang.org/x/crypto v0.0.0-20211215153901-e495a2d5b3d3/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
|
||||
golang.org/x/crypto v0.0.0-20220314234659-1baeb1ce4c0b/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
|
||||
golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
|
||||
golang.org/x/crypto v0.3.1-0.20221117191849-2c476679df9a/go.mod h1:hebNnKkNXi2UzZN1eVRvBB7co0a+JxK6XbPiWVs/3J4=
|
||||
golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU=
|
||||
golang.org/x/crypto v0.11.0 h1:6Ewdq3tDic1mg5xRO4milcWCfMVQhI4NkqWWvqejpuA=
|
||||
golang.org/x/crypto v0.11.0/go.mod h1:xgJhtzW8F9jGdVFWZESrid1U1bjeNy4zgy5cRr/CIio=
|
||||
@@ -563,9 +627,11 @@ golang.org/x/net v0.0.0-20210610132358-84b48f89b13b/go.mod h1:9nx3DQGgdP8bBQD5qx
|
||||
golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
|
||||
golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
|
||||
golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
|
||||
golang.org/x/net v0.0.0-20221002022538-bcab6841153b/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk=
|
||||
golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY=
|
||||
golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
|
||||
golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc=
|
||||
golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns=
|
||||
golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg=
|
||||
golang.org/x/net v0.12.0 h1:cfawfvKITfUsFCeJIHJrbSxpeu/E81khclypR0GVT50=
|
||||
golang.org/x/net v0.12.0/go.mod h1:zEVYFnQC7m/vmpQFELhcD1EWkZlX69l4oqgmer6hfKA=
|
||||
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
|
||||
@@ -573,9 +639,9 @@ golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4Iltr
|
||||
golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
|
||||
golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
|
||||
golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
|
||||
golang.org/x/oauth2 v0.7.0/go.mod h1:hPLQkd9LyjfXTiRohC/41GhcFqxisoUQ99sCUOHO9x4=
|
||||
golang.org/x/oauth2 v0.9.0 h1:BPpt2kU7oMRq3kCHAA1tbSEshXRw1LpG2ztgDwrzuAs=
|
||||
golang.org/x/oauth2 v0.9.0/go.mod h1:qYgFZaFiu6Wg24azG8bdV52QJXJGbZzIIsRCdVKzbLw=
|
||||
golang.org/x/oauth2 v0.8.0/go.mod h1:yr7u4HXZRm1R1kBWqr/xKNqewf0plRYoB7sla+BCIXE=
|
||||
golang.org/x/oauth2 v0.10.0 h1:zHCpF2Khkwy4mMB4bv0U37YtJdTGW8jI0glAApi0Kh8=
|
||||
golang.org/x/oauth2 v0.10.0/go.mod h1:kTpgurOux7LqtuxjuyZa4Gj2gdezIt/jQtGnNFfypQI=
|
||||
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
@@ -622,20 +688,25 @@ golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBc
|
||||
golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220906165534-d0df966e6959/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.10.0 h1:SqMFp9UcQJZa+pmYuAKjd9xq1f0j5rLcDIk0mj4qAsA=
|
||||
golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
|
||||
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
|
||||
golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc=
|
||||
golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
|
||||
golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U=
|
||||
golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY=
|
||||
golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo=
|
||||
golang.org/x/term v0.10.0 h1:3R7pNqamzBraeqj/Tj8qt1aQ2HpmlC+Cx/qL/7hn4/c=
|
||||
golang.org/x/term v0.10.0/go.mod h1:lpqdcUyK/oCiQxvxVrppt5ggO2KCZ5QblwqPnfZ6d5o=
|
||||
golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
@@ -644,6 +715,7 @@ golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
|
||||
golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
|
||||
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
|
||||
golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ=
|
||||
golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
|
||||
golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
|
||||
golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
|
||||
golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
|
||||
@@ -698,8 +770,8 @@ google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsb
|
||||
google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI=
|
||||
google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI=
|
||||
google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
|
||||
google.golang.org/api v0.130.0 h1:A50ujooa1h9iizvfzA4rrJr2B7uRmWexwbekQ2+5FPQ=
|
||||
google.golang.org/api v0.130.0/go.mod h1:J/LCJMYSDFvAVREGCbrESb53n4++NMBDetSHGL5I5RY=
|
||||
google.golang.org/api v0.131.0 h1:AcgWS2edQ4chVEt/SxgDKubVu/9/idCJy00tBGuGB4M=
|
||||
google.golang.org/api v0.131.0/go.mod h1:7vtkbKv2REjJbxmHSkBTBQ5LUGvPdAqjjvt84XAfhpA=
|
||||
google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
|
||||
google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
|
||||
google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
|
||||
@@ -722,12 +794,12 @@ google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvx
|
||||
google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
|
||||
google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
|
||||
google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=
|
||||
google.golang.org/genproto v0.0.0-20230530153820-e85fd2cbaebc h1:8DyZCyvI8mE1IdLy/60bS+52xfymkE72wv1asokgtao=
|
||||
google.golang.org/genproto v0.0.0-20230530153820-e85fd2cbaebc/go.mod h1:xZnkP7mREFX5MORlOPEzLMr+90PPZQ2QWzrVTWfAq64=
|
||||
google.golang.org/genproto/googleapis/api v0.0.0-20230530153820-e85fd2cbaebc h1:kVKPf/IiYSBWEWtkIn6wZXwWGCnLKcC8oWfZvXjsGnM=
|
||||
google.golang.org/genproto/googleapis/api v0.0.0-20230530153820-e85fd2cbaebc/go.mod h1:vHYtlOoi6TsQ3Uk2yxR7NI5z8uoV+3pZtR4jmHIkRig=
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20230629202037-9506855d4529 h1:DEH99RbiLZhMxrpEJCZ0A+wdTe0EOgou/poSLx9vWf4=
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20230629202037-9506855d4529/go.mod h1:66JfowdXAEgad5O9NnYcsNPLCPZJD++2L9X0PCMODrA=
|
||||
google.golang.org/genproto v0.0.0-20230629202037-9506855d4529 h1:9JucMWR7sPvCxUFd6UsOUNmA5kCcWOfORaT3tpAsKQs=
|
||||
google.golang.org/genproto v0.0.0-20230629202037-9506855d4529/go.mod h1:xZnkP7mREFX5MORlOPEzLMr+90PPZQ2QWzrVTWfAq64=
|
||||
google.golang.org/genproto/googleapis/api v0.0.0-20230629202037-9506855d4529 h1:s5YSX+ZH5b5vS9rnpGymvIyMpLRJizowqDlOuyjXnTk=
|
||||
google.golang.org/genproto/googleapis/api v0.0.0-20230629202037-9506855d4529/go.mod h1:vHYtlOoi6TsQ3Uk2yxR7NI5z8uoV+3pZtR4jmHIkRig=
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20230706204954-ccb25ca9f130 h1:2FZP5XuJY9zQyGM5N0rtovnoXjiMUEIUMvw0m9wlpLc=
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20230706204954-ccb25ca9f130/go.mod h1:8mL13HKkDa+IuJ8yruA3ci0q+0vsUz4m//+ottjwS5o=
|
||||
google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
|
||||
google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
|
||||
google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
|
||||
@@ -740,8 +812,8 @@ google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTp
|
||||
google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc=
|
||||
google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU=
|
||||
google.golang.org/grpc v1.45.0/go.mod h1:lN7owxKUQEqMfSyQikvvk5tf/6zMPsrK+ONuO11+0rQ=
|
||||
google.golang.org/grpc v1.56.1 h1:z0dNfjIl0VpaZ9iSVjA6daGatAYwPGstTjt5vkRMFkQ=
|
||||
google.golang.org/grpc v1.56.1/go.mod h1:I9bI3vqKfayGqPUAwGdOSu7kt6oIJLixfffKrpXqQ9s=
|
||||
google.golang.org/grpc v1.56.2 h1:fVRFRnXvU+x6C4IlHZewvJOVHoOv1TUuQyoRsYnB4bI=
|
||||
google.golang.org/grpc v1.56.2/go.mod h1:I9bI3vqKfayGqPUAwGdOSu7kt6oIJLixfffKrpXqQ9s=
|
||||
google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
|
||||
google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
|
||||
google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
|
||||
|
||||
@@ -62,15 +62,31 @@ If you think that something should be included outside of these guidelines, plea
|
||||
1. Add the test secret to GCP Secrets. See [managing test secrets](#managing-test-secrets)
|
||||
2. Update the pattern regex and keywords. Try iterating with [regex101.com](http://regex101.com/).
|
||||
3. Update the verifier code to use a non-destructive API call that can determine whether the secret is valid or not.
|
||||
* Make sure you understand [verification indeterminacy](#verification-indeterminacy).
|
||||
4. Update the tests with these test cases at minimum:
|
||||
1. Found and verified (using a credential loaded from GCP Secrets)
|
||||
2. Found and unverified
|
||||
3. Not found
|
||||
4. Any false positive cases that you come across
|
||||
2. Found and unverified (determinately, i.e. the secret is invalid)
|
||||
3. Found and unverified (indeterminately due to timeout)
|
||||
4. Found and unverified (indeterminately due to an unexpected API response)
|
||||
5. Not found
|
||||
6. Any false positive cases that you come across
|
||||
5. Create a merge request for review. CI tests must be passing.
|
||||
|
||||
## Addendum
|
||||
|
||||
### Verification indeterminacy
|
||||
|
||||
There are two types of reasons that secret verification can fail:
|
||||
* The candidate secret is not actually a valid secret.
|
||||
* Something went wrong in the process unrelated to the candidate secret, such as a transient network error or an unexpected API response.
|
||||
|
||||
In Trufflehog parlance, the first type of verification response is called _determinate_ and the second type is called _indeterminate_. Verification code should distinguish between the two by returning an error object in the result struct **only** for indeterminate failures. In general, a verifier should return an error (indicating an indeterminate failure) in all cases that haven't been explicitly identified as determinate failure states.
|
||||
|
||||
For example, consider a hypothetical authentication endpoint that returns `200 OK` for valid credentials and `403 Forbidden` for invalid credentials. The verifier for this endpoint could make an HTTP request and use the response status code to decide what to return:
|
||||
* A `200` response would indicate that verification succeeded. (Or maybe any `2xx` response.)
|
||||
* A `403` response would indicate that verification failed **determinately** and no error object should be returned.
|
||||
* Any other response would indicate that verification failed **indeterminately** and an error object should be returned.
|
||||
|
||||
### Managing Test Secrets
|
||||
|
||||
Do not embed test credentials in the test code. Instead, use GCP Secrets Manager.
|
||||
|
||||
@@ -62,15 +62,31 @@ If you think that something should be included outside of these guidelines, plea
|
||||
1. Create a [test secrets file, and export the variable](#using-a-test-secret-file)
|
||||
2. Update the pattern regex and keywords. Try iterating with [regex101.com](http://regex101.com/).
|
||||
3. Update the verifier code to use a non-destructive API call that can determine whether the secret is valid or not.
|
||||
* Make sure you understand [verification indeterminacy](#verification-indeterminacy).
|
||||
4. Update the tests with these test cases at minimum:
|
||||
1. Found and verified (using a credential loaded from GCP Secrets)
|
||||
2. Found and unverified
|
||||
3. Not found
|
||||
4. Any false positive cases that you come across
|
||||
2. Found and unverified (determinately, i.e. the secret is invalid)
|
||||
3. Found and unverified (indeterminately due to timeout)
|
||||
4. Found and unverified (indeterminately due to an unexpected API response)
|
||||
5. Not found
|
||||
6. Any false positive cases that you come across
|
||||
5. Create a pull request for review.
|
||||
|
||||
## Addendum
|
||||
|
||||
### Verification indeterminacy
|
||||
|
||||
There are two types of reasons that secret verification can fail:
|
||||
* The candidate secret is not actually a valid secret.
|
||||
* Something went wrong in the process unrelated to the candidate secret, such as a transient network error or an unexpected API response.
|
||||
|
||||
In Trufflehog parlance, the first type of verification response is called _determinate_ and the second type is called _indeterminate_. Verification code should distinguish between the two by returning an error object in the result struct **only** for indeterminate failures. In general, a verifier should return an error (indicating an indeterminate failure) in all cases that haven't been explicitly identified as determinate failure states.
|
||||
|
||||
For example, consider a hypothetical authentication endpoint that returns `200 OK` for valid credentials and `403 Forbidden` for invalid credentials. The verifier for this endpoint could make an HTTP request and use the response status code to decide what to return:
|
||||
* A `200` response would indicate that verification succeeded. (Or maybe any `2xx` response.)
|
||||
* A `403` response would indicate that verification failed **determinately** and no error object should be returned.
|
||||
* Any other response would indicate that verification failed **indeterminately** and an error object should be returned.
|
||||
|
||||
### Using a test secret file
|
||||
|
||||
1. Create a file called `.env` with this env file format:
|
||||
|
||||
@@ -34,14 +34,14 @@ func main() {
|
||||
case "detector":
|
||||
mustWriteTemplates([]templateJob{
|
||||
{
|
||||
TemplatePath: "pkg/detectors/heroku/heroku.go",
|
||||
TemplatePath: "pkg/detectors/alchemy/alchemy.go",
|
||||
WritePath: filepath.Join(folderPath(), nameLower+".go"),
|
||||
ReplaceString: []string{"heroku"},
|
||||
ReplaceString: []string{"alchemy"},
|
||||
},
|
||||
{
|
||||
TemplatePath: "pkg/detectors/heroku/heroku_test.go",
|
||||
TemplatePath: "pkg/detectors/alchemy/alchemy_test.go",
|
||||
WritePath: filepath.Join(folderPath(), nameLower+"_test.go"),
|
||||
ReplaceString: []string{"heroku"},
|
||||
ReplaceString: []string{"alchemy"},
|
||||
},
|
||||
})
|
||||
// case "source":
|
||||
|
||||
@@ -9,11 +9,11 @@ import (
|
||||
"strconv"
|
||||
"strings"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"github.com/felixge/fgprof"
|
||||
"github.com/go-logr/logr"
|
||||
"github.com/jpillora/overseer"
|
||||
"github.com/mattn/go-isatty"
|
||||
"google.golang.org/protobuf/types/known/anypb"
|
||||
"gopkg.in/alecthomas/kingpin.v2"
|
||||
|
||||
@@ -29,6 +29,7 @@ import (
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/sources/git"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/tui"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/updater"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/version"
|
||||
)
|
||||
@@ -39,6 +40,7 @@ var (
|
||||
debug = cli.Flag("debug", "Run in debug mode.").Bool()
|
||||
trace = cli.Flag("trace", "Run in trace mode.").Bool()
|
||||
profile = cli.Flag("profile", "Enables profiling and sets a pprof and fgprof server on :18066.").Bool()
|
||||
localDev = cli.Flag("local-dev", "Hidden feature to disable overseer for local dev.").Hidden().Bool()
|
||||
jsonOut = cli.Flag("json", "Output in JSON format.").Short('j').Bool()
|
||||
jsonLegacy = cli.Flag("json-legacy", "Use the pre-v3.0 JSON format. Only works with git, gitlab, and github sources.").Bool()
|
||||
gitHubActionsFormat = cli.Flag("github-actions", "Output in GitHub Actions format.").Bool()
|
||||
@@ -66,6 +68,7 @@ var (
|
||||
gitScanSinceCommit = gitScan.Flag("since-commit", "Commit to start scan from.").String()
|
||||
gitScanBranch = gitScan.Flag("branch", "Branch to scan.").String()
|
||||
gitScanMaxDepth = gitScan.Flag("max-depth", "Maximum depth of commits to scan.").Int()
|
||||
gitScanBare = gitScan.Flag("bare", "Scan bare repository (e.g. useful while using in pre-receive hooks)").Bool()
|
||||
_ = gitScan.Flag("allow", "No-op flag for backwards compat.").Bool()
|
||||
_ = gitScan.Flag("entropy", "No-op flag for backwards compat.").Bool()
|
||||
_ = gitScan.Flag("regex", "No-op flag for backwards compat.").Bool()
|
||||
@@ -143,6 +146,18 @@ func init() {
|
||||
}
|
||||
|
||||
cli.Version("trufflehog " + version.BuildVersion)
|
||||
|
||||
if len(os.Args) <= 1 && isatty.IsTerminal(os.Stdout.Fd()) {
|
||||
args := tui.Run()
|
||||
if len(args) == 0 {
|
||||
os.Exit(0)
|
||||
}
|
||||
|
||||
// Overwrite the Args slice so overseer works properly.
|
||||
os.Args = os.Args[:1]
|
||||
os.Args = append(os.Args, args...)
|
||||
}
|
||||
|
||||
cmd = kingpin.MustParse(cli.Parse(os.Args[1:]))
|
||||
|
||||
switch {
|
||||
@@ -162,6 +177,12 @@ func main() {
|
||||
logger, sync := log.New("trufflehog", logFormat(os.Stderr))
|
||||
// make it the default logger for contexts
|
||||
context.SetDefaultLogger(logger)
|
||||
|
||||
if *localDev {
|
||||
run(overseer.State{})
|
||||
os.Exit(0)
|
||||
}
|
||||
|
||||
defer func() { _ = sync() }()
|
||||
logFatal := logFatalFunc(logger)
|
||||
|
||||
@@ -207,7 +228,7 @@ func run(state overseer.State) {
|
||||
if *profile {
|
||||
go func() {
|
||||
router := http.NewServeMux()
|
||||
router.Handle("/debug/pprof", http.DefaultServeMux)
|
||||
router.Handle("/debug/pprof/", http.DefaultServeMux)
|
||||
router.Handle("/debug/fgprof", fgprof.Handler())
|
||||
logger.Info("starting pprof and fgprof server on :18066 /debug/pprof and /debug/fgprof")
|
||||
if err := http.ListenAndServe(":18066", router); err != nil {
|
||||
@@ -316,8 +337,21 @@ func run(state overseer.State) {
|
||||
return true
|
||||
}
|
||||
|
||||
e := engine.Start(ctx,
|
||||
engine.WithConcurrency(*concurrency),
|
||||
// Set how the engine will print its results.
|
||||
var printer engine.Printer
|
||||
switch {
|
||||
case *jsonLegacy:
|
||||
printer = new(output.LegacyJSONPrinter)
|
||||
case *jsonOut:
|
||||
printer = new(output.JSONPrinter)
|
||||
case *gitHubActionsFormat:
|
||||
printer = new(output.GitHubActionsPrinter)
|
||||
default:
|
||||
printer = new(output.PlainPrinter)
|
||||
}
|
||||
|
||||
e, err := engine.Start(ctx,
|
||||
engine.WithConcurrency(uint8(*concurrency)),
|
||||
engine.WithDecoders(decoders.DefaultDecoders()...),
|
||||
engine.WithDetectors(!*noVerification, engine.DefaultDetectors()...),
|
||||
engine.WithDetectors(!*noVerification, conf.Detectors...),
|
||||
@@ -325,7 +359,13 @@ func run(state overseer.State) {
|
||||
engine.WithFilterDetectors(excludeFilter),
|
||||
engine.WithFilterDetectors(endpointCustomizer),
|
||||
engine.WithFilterUnverified(*filterUnverified),
|
||||
engine.WithOnlyVerified(*onlyVerified),
|
||||
engine.WithPrintAvgDetectorTime(*printAvgDetectorTime),
|
||||
engine.WithPrinter(printer),
|
||||
)
|
||||
if err != nil {
|
||||
logFatal(err, "error initializing engine")
|
||||
}
|
||||
|
||||
var repoPath string
|
||||
var remote bool
|
||||
@@ -352,6 +392,7 @@ func run(state overseer.State) {
|
||||
HeadRef: *gitScanBranch,
|
||||
BaseRef: *gitScanSinceCommit,
|
||||
MaxDepth: *gitScanMaxDepth,
|
||||
Bare: *gitScanBare,
|
||||
Filter: filter,
|
||||
ExcludeGlobs: excludedGlobs,
|
||||
}
|
||||
@@ -475,52 +516,49 @@ func run(state overseer.State) {
|
||||
logFatal(err, "Failed to scan Docker.")
|
||||
}
|
||||
}
|
||||
// asynchronously wait for scanning to finish and cleanup
|
||||
go e.Finish(ctx, logFatal)
|
||||
|
||||
if !*jsonLegacy && !*jsonOut {
|
||||
fmt.Fprintf(os.Stderr, "🐷🔑🐷 TruffleHog. Unearth your secrets. 🐷🔑🐷\n\n")
|
||||
}
|
||||
|
||||
// NOTE: this loop will terminate when the results channel is closed in
|
||||
// e.Finish()
|
||||
foundResults := false
|
||||
for r := range e.ResultsChan() {
|
||||
if *onlyVerified && !r.Verified {
|
||||
continue
|
||||
}
|
||||
foundResults = true
|
||||
|
||||
var err error
|
||||
switch {
|
||||
case *jsonLegacy:
|
||||
err = output.PrintLegacyJSON(ctx, &r)
|
||||
case *jsonOut:
|
||||
err = output.PrintJSON(&r)
|
||||
case *gitHubActionsFormat:
|
||||
err = output.PrintGitHubActionsOutput(&r)
|
||||
default:
|
||||
err = output.PrintPlainOutput(&r)
|
||||
}
|
||||
if err != nil {
|
||||
logFatal(err, "error printing results")
|
||||
}
|
||||
// Wait for all workers to finish.
|
||||
if err = e.Finish(ctx); err != nil {
|
||||
logFatal(err, "engine failed to finish execution")
|
||||
}
|
||||
logger.V(2).Info("finished scanning",
|
||||
"chunks", e.ChunksScanned(),
|
||||
"bytes", e.BytesScanned(),
|
||||
|
||||
metrics := e.GetMetrics()
|
||||
// Print results.
|
||||
logger.Info("finished scanning",
|
||||
"chunks", metrics.ChunksScanned,
|
||||
"bytes", metrics.BytesScanned,
|
||||
"verified_secrets", metrics.VerifiedSecretsFound,
|
||||
"unverified_secrets", metrics.UnverifiedSecretsFound,
|
||||
"scan_duration", metrics.ScanDuration.String(),
|
||||
)
|
||||
|
||||
if *printAvgDetectorTime {
|
||||
printAverageDetectorTime(e)
|
||||
}
|
||||
|
||||
if foundResults && *fail {
|
||||
if e.HasFoundResults() && *fail {
|
||||
logger.V(2).Info("exiting with code 183 because results were found")
|
||||
os.Exit(183)
|
||||
}
|
||||
}
|
||||
|
||||
// logFatalFunc returns a log.Fatal style function. Calling the returned
|
||||
// function will terminate the program without cleanup.
|
||||
func logFatalFunc(logger logr.Logger) func(error, string, ...any) {
|
||||
return func(err error, message string, keyAndVals ...any) {
|
||||
logger.Error(err, message, keyAndVals...)
|
||||
if err != nil {
|
||||
os.Exit(1)
|
||||
return
|
||||
}
|
||||
os.Exit(0)
|
||||
}
|
||||
}
|
||||
|
||||
func commaSeparatedToSlice(s []string) []string {
|
||||
var result []string
|
||||
for _, items := range s {
|
||||
@@ -537,26 +575,8 @@ func commaSeparatedToSlice(s []string) []string {
|
||||
|
||||
func printAverageDetectorTime(e *engine.Engine) {
|
||||
fmt.Fprintln(os.Stderr, "Average detector time is the measurement of average time spent on each detector when results are returned.")
|
||||
for detectorName, durations := range e.DetectorAvgTime() {
|
||||
var total time.Duration
|
||||
for _, d := range durations {
|
||||
total += d
|
||||
}
|
||||
avgDuration := total / time.Duration(len(durations))
|
||||
fmt.Fprintf(os.Stderr, "%s: %s\n", detectorName, avgDuration)
|
||||
}
|
||||
}
|
||||
|
||||
// logFatalFunc returns a log.Fatal style function. Calling the returned
|
||||
// function will terminate the program without cleanup.
|
||||
func logFatalFunc(logger logr.Logger) func(error, string, ...any) {
|
||||
return func(err error, message string, keyAndVals ...any) {
|
||||
logger.Error(err, message, keyAndVals...)
|
||||
if err != nil {
|
||||
os.Exit(1)
|
||||
return
|
||||
}
|
||||
os.Exit(0)
|
||||
for detectorName, duration := range e.GetDetectorsMetrics() {
|
||||
fmt.Fprintf(os.Stderr, "%s: %s\n", detectorName, duration)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -16,9 +16,14 @@ func IsDone(ctx context.Context) bool {
|
||||
// write would succeed, either operation will be performed randomly.
|
||||
func CancellableWrite[T any](ctx context.Context, ch chan<- T, item T) error {
|
||||
select {
|
||||
case ch <- item:
|
||||
return nil
|
||||
case <-ctx.Done():
|
||||
case <-ctx.Done(): // priority to context cancellation
|
||||
return ctx.Err()
|
||||
default:
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
case ch <- item:
|
||||
return nil
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
+26
-2
@@ -3,6 +3,7 @@ package common
|
||||
import (
|
||||
"crypto/tls"
|
||||
"crypto/x509"
|
||||
"io"
|
||||
"net"
|
||||
"net/http"
|
||||
"strings"
|
||||
@@ -75,6 +76,14 @@ func PinnedCertPool() *x509.CertPool {
|
||||
return trustedCerts
|
||||
}
|
||||
|
||||
type FakeTransport struct {
|
||||
CreateResponse func(req *http.Request) (*http.Response, error)
|
||||
}
|
||||
|
||||
func (t FakeTransport) RoundTrip(req *http.Request) (*http.Response, error) {
|
||||
return t.CreateResponse(req)
|
||||
}
|
||||
|
||||
type CustomTransport struct {
|
||||
T http.RoundTripper
|
||||
}
|
||||
@@ -91,6 +100,21 @@ func NewCustomTransport(T http.RoundTripper) *CustomTransport {
|
||||
return &CustomTransport{T}
|
||||
}
|
||||
|
||||
func ConstantResponseHttpClient(statusCode int, body string) *http.Client {
|
||||
return &http.Client{
|
||||
Timeout: DefaultResponseTimeout,
|
||||
Transport: FakeTransport{
|
||||
CreateResponse: func(req *http.Request) (*http.Response, error) {
|
||||
return &http.Response{
|
||||
Request: req,
|
||||
Body: io.NopCloser(strings.NewReader(body)),
|
||||
StatusCode: statusCode,
|
||||
}, nil
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func PinnedRetryableHttpClient() *http.Client {
|
||||
httpClient := retryablehttp.NewClient()
|
||||
httpClient.Logger = nil
|
||||
@@ -152,9 +176,9 @@ func SaneHttpClient() *http.Client {
|
||||
}
|
||||
|
||||
// SaneHttpClientTimeOut adds a custom timeout for some scanners
|
||||
func SaneHttpClientTimeOut(timeOutSeconds int64) *http.Client {
|
||||
func SaneHttpClientTimeOut(timeout time.Duration) *http.Client {
|
||||
httpClient := &http.Client{}
|
||||
httpClient.Timeout = time.Second * time.Duration(timeOutSeconds)
|
||||
httpClient.Timeout = timeout
|
||||
httpClient.Transport = NewCustomTransport(nil)
|
||||
return httpClient
|
||||
}
|
||||
|
||||
@@ -3,7 +3,9 @@ package common
|
||||
import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
"crypto/rand"
|
||||
"io"
|
||||
"math/big"
|
||||
"strings"
|
||||
)
|
||||
|
||||
@@ -49,3 +51,16 @@ func ResponseContainsSubstring(reader io.ReadCloser, target string) (bool, error
|
||||
}
|
||||
return false, nil
|
||||
}
|
||||
|
||||
var letters = []rune("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789")
|
||||
|
||||
// RandomID returns a random string of the given length.
|
||||
func RandomID(length int) string {
|
||||
b := make([]rune, length)
|
||||
for i := range b {
|
||||
randInt, _ := rand.Int(rand.Reader, big.NewInt(int64(len(letters))))
|
||||
b[i] = letters[randInt.Int64()]
|
||||
}
|
||||
|
||||
return string(b)
|
||||
}
|
||||
|
||||
+18
-52
@@ -2,45 +2,39 @@ package context
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"runtime/debug"
|
||||
"os"
|
||||
"time"
|
||||
|
||||
"github.com/go-logr/logr"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/log"
|
||||
)
|
||||
|
||||
var (
|
||||
// defaultLogger can be set via SetDefaultLogger.
|
||||
defaultLogger logr.Logger = logr.Discard()
|
||||
// It is initialized to write to stderr. To disable, you can call
|
||||
// SetDefaultLogger with logr.Discard().
|
||||
defaultLogger logr.Logger
|
||||
)
|
||||
|
||||
func init() {
|
||||
defaultLogger, _ = log.New("context", log.WithConsoleSink(os.Stderr))
|
||||
}
|
||||
|
||||
// Context wraps context.Context and includes an additional Logger() method.
|
||||
type Context interface {
|
||||
context.Context
|
||||
Logger() logr.Logger
|
||||
Parent() context.Context
|
||||
SetParent(ctx context.Context) Context
|
||||
}
|
||||
|
||||
// Parent returns the parent context.
|
||||
func (l logCtx) Parent() context.Context {
|
||||
return l.Context
|
||||
}
|
||||
|
||||
// SetParent sets the parent context on the context.
|
||||
func (l logCtx) SetParent(ctx context.Context) Context {
|
||||
l.Context = ctx
|
||||
return l
|
||||
}
|
||||
|
||||
type CancelFunc context.CancelFunc
|
||||
// CancelFunc is a type alias to context.CancelFunc to allow use as if they are
|
||||
// the same types.
|
||||
type CancelFunc = context.CancelFunc
|
||||
|
||||
// logCtx implements Context.
|
||||
type logCtx struct {
|
||||
// Embed context.Context to get all methods for free.
|
||||
context.Context
|
||||
log logr.Logger
|
||||
err *error
|
||||
}
|
||||
|
||||
// Logger returns a structured logger.
|
||||
@@ -48,13 +42,6 @@ func (l logCtx) Logger() logr.Logger {
|
||||
return l.log
|
||||
}
|
||||
|
||||
func (l logCtx) Err() error {
|
||||
if l.err != nil && *l.err != nil {
|
||||
return *l.err
|
||||
}
|
||||
return l.Context.Err()
|
||||
}
|
||||
|
||||
// Background returns context.Background with a default logger.
|
||||
func Background() Context {
|
||||
return logCtx{
|
||||
@@ -78,7 +65,7 @@ func WithCancel(parent Context) (Context, context.CancelFunc) {
|
||||
log: parent.Logger(),
|
||||
Context: ctx,
|
||||
}
|
||||
return captureCancelCallstack(lCtx, cancel)
|
||||
return lCtx, cancel
|
||||
}
|
||||
|
||||
// WithDeadline returns context.WithDeadline with the log object propagated and
|
||||
@@ -89,7 +76,7 @@ func WithDeadline(parent Context, d time.Time) (Context, context.CancelFunc) {
|
||||
log: parent.Logger().WithValues("deadline", d),
|
||||
Context: ctx,
|
||||
}
|
||||
return captureCancelCallstack(lCtx, cancel)
|
||||
return lCtx, cancel
|
||||
}
|
||||
|
||||
// WithTimeout returns context.WithTimeout with the log object propagated and
|
||||
@@ -100,7 +87,7 @@ func WithTimeout(parent Context, timeout time.Duration) (Context, context.Cancel
|
||||
log: parent.Logger().WithValues("timeout", timeout),
|
||||
Context: ctx,
|
||||
}
|
||||
return captureCancelCallstack(lCtx, cancel)
|
||||
return lCtx, cancel
|
||||
}
|
||||
|
||||
// WithValue returns context.WithValue with the log object propagated and
|
||||
@@ -145,30 +132,9 @@ func AddLogger(parent context.Context) Context {
|
||||
}
|
||||
|
||||
// SetupDefaultLogger sets the package-level global default logger that will be
|
||||
// used for Background and TODO contexts.
|
||||
// used for Background and TODO contexts. On startup, the default logger will
|
||||
// be configured to output logs to stderr. Use logr.Discard() to disable all
|
||||
// logs from Contexts.
|
||||
func SetDefaultLogger(l logr.Logger) {
|
||||
defaultLogger = l
|
||||
}
|
||||
|
||||
// captureCancelCallstack is a helper function to capture the callstack where
|
||||
// the cancel function was first called.
|
||||
func captureCancelCallstack(ctx logCtx, f context.CancelFunc) (Context, context.CancelFunc) {
|
||||
if ctx.err == nil {
|
||||
var err error
|
||||
ctx.err = &err
|
||||
}
|
||||
return ctx, func() {
|
||||
// We must check Err() before calling f() since f() sets the error.
|
||||
// If there's already an error, do nothing special.
|
||||
if ctx.Err() != nil {
|
||||
f()
|
||||
return
|
||||
}
|
||||
f()
|
||||
// Set the error with the stacktrace if the err pointer is non-nil.
|
||||
*ctx.err = fmt.Errorf(
|
||||
"%w (canceled at %v\n%s)",
|
||||
ctx.Err(), time.Now(), string(debug.Stack()),
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -158,7 +158,7 @@ func TestWithValues(t *testing.T) {
|
||||
assert.NotContains(t, logs[6], `what does this do?`)
|
||||
}
|
||||
|
||||
func TestDiscardLogger(t *testing.T) {
|
||||
func TestDefaultLogger(t *testing.T) {
|
||||
var panicked bool
|
||||
defer func() {
|
||||
if r := recover(); r != nil {
|
||||
@@ -170,45 +170,10 @@ func TestDiscardLogger(t *testing.T) {
|
||||
ctx.Logger().Info("this shouldn't panic")
|
||||
}
|
||||
|
||||
func TestErrCallstack(t *testing.T) {
|
||||
c, cancel := WithCancel(Background())
|
||||
ctx := c.(logCtx)
|
||||
func TestRace(t *testing.T) {
|
||||
ctx, cancel := WithCancel(Background())
|
||||
go cancel()
|
||||
go func() { _ = ctx.Err() }()
|
||||
cancel()
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
assert.Contains(t, ctx.Err().Error(), "TestErrCallstack")
|
||||
case <-time.After(1 * time.Second):
|
||||
assert.Fail(t, "context should be done")
|
||||
}
|
||||
}
|
||||
|
||||
func TestErrCallstackTimeout(t *testing.T) {
|
||||
ctx, cancel := WithTimeout(Background(), 10*time.Millisecond)
|
||||
defer cancel()
|
||||
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
// Deadline exceeded errors will not have a callstack from the cancel
|
||||
// function.
|
||||
assert.NotContains(t, ctx.Err().Error(), "TestErrCallstackTimeout")
|
||||
case <-time.After(1 * time.Second):
|
||||
assert.Fail(t, "context should be done")
|
||||
}
|
||||
}
|
||||
|
||||
func TestErrCallstackTimeoutCancel(t *testing.T) {
|
||||
ctx, cancel := WithTimeout(Background(), 10*time.Millisecond)
|
||||
|
||||
var err error
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
err = ctx.Err()
|
||||
case <-time.After(1 * time.Second):
|
||||
assert.Fail(t, "context should be done")
|
||||
}
|
||||
|
||||
// Calling cancel after deadline exceeded should not overwrite the original
|
||||
// error.
|
||||
cancel()
|
||||
assert.Equal(t, err, ctx.Err())
|
||||
_ = ctx.Err()
|
||||
}
|
||||
|
||||
@@ -2,6 +2,7 @@ package alchemy
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"strings"
|
||||
@@ -11,14 +12,15 @@ import (
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
|
||||
)
|
||||
|
||||
type Scanner struct{}
|
||||
type Scanner struct {
|
||||
client *http.Client
|
||||
}
|
||||
|
||||
// Ensure the Scanner satisfies the interface at compile time.
|
||||
var _ detectors.Detector = (*Scanner)(nil)
|
||||
|
||||
var (
|
||||
client = common.SaneHttpClient()
|
||||
|
||||
defaultClient = common.SaneHttpClient()
|
||||
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
|
||||
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"alchemy"}) + `\b([0-9a-zA-Z]{23}_[0-9a-zA-Z]{8})\b`)
|
||||
)
|
||||
@@ -47,6 +49,10 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
}
|
||||
|
||||
if verify {
|
||||
client := s.client
|
||||
if client == nil {
|
||||
client = defaultClient
|
||||
}
|
||||
req, err := http.NewRequestWithContext(ctx, "GET", "https://eth-mainnet.g.alchemy.com/v2/"+resMatch+"/getNFTs/?owner=vitalik.eth", nil)
|
||||
if err != nil {
|
||||
continue
|
||||
@@ -56,15 +62,21 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
defer res.Body.Close()
|
||||
if res.StatusCode >= 200 && res.StatusCode < 300 {
|
||||
s1.Verified = true
|
||||
} else if res.StatusCode == 401 {
|
||||
// The secret is determinately not verified (nothing to do)
|
||||
} else {
|
||||
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
|
||||
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
|
||||
continue
|
||||
}
|
||||
s1.VerificationError = fmt.Errorf("unexpected HTTP response status %d", res.StatusCode)
|
||||
}
|
||||
} else {
|
||||
s1.VerificationError = err
|
||||
}
|
||||
}
|
||||
|
||||
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
|
||||
if !s1.Verified && detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
|
||||
continue
|
||||
}
|
||||
|
||||
results = append(results, s1)
|
||||
}
|
||||
|
||||
|
||||
@@ -6,10 +6,11 @@ package alchemy
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"github.com/google/go-cmp/cmp"
|
||||
"github.com/google/go-cmp/cmp/cmpopts"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/kylelemons/godebug/pretty"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
@@ -32,11 +33,12 @@ func TestAlchemy_FromChunk(t *testing.T) {
|
||||
verify bool
|
||||
}
|
||||
tests := []struct {
|
||||
name string
|
||||
s Scanner
|
||||
args args
|
||||
want []detectors.Result
|
||||
wantErr bool
|
||||
name string
|
||||
s Scanner
|
||||
args args
|
||||
want []detectors.Result
|
||||
wantErr bool
|
||||
wantVerificationErr bool
|
||||
}{
|
||||
{
|
||||
name: "found, verified",
|
||||
@@ -52,7 +54,8 @@ func TestAlchemy_FromChunk(t *testing.T) {
|
||||
Verified: true,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
wantErr: false,
|
||||
wantVerificationErr: false,
|
||||
},
|
||||
{
|
||||
name: "found, unverified",
|
||||
@@ -68,7 +71,8 @@ func TestAlchemy_FromChunk(t *testing.T) {
|
||||
Verified: false,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
wantErr: false,
|
||||
wantVerificationErr: false,
|
||||
},
|
||||
{
|
||||
name: "not found",
|
||||
@@ -78,14 +82,48 @@ func TestAlchemy_FromChunk(t *testing.T) {
|
||||
data: []byte("You cannot find the secret within"),
|
||||
verify: true,
|
||||
},
|
||||
want: nil,
|
||||
wantErr: false,
|
||||
want: nil,
|
||||
wantErr: false,
|
||||
wantVerificationErr: false,
|
||||
},
|
||||
{
|
||||
name: "found, would be verified if not for timeout",
|
||||
s: Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a alchemy secret %s within", secret)),
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_Alchemy,
|
||||
Verified: false,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
wantVerificationErr: true,
|
||||
},
|
||||
{
|
||||
name: "found, verified but unexpected api surface",
|
||||
s: Scanner{client: common.ConstantResponseHttpClient(404, "")},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a alchemy secret %s within", secret)),
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_Alchemy,
|
||||
Verified: false,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
wantVerificationErr: true,
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
s := Scanner{}
|
||||
got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
|
||||
got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
|
||||
if (err != nil) != tt.wantErr {
|
||||
t.Errorf("Alchemy.FromData() error = %v, wantErr %v", err, tt.wantErr)
|
||||
return
|
||||
@@ -94,9 +132,12 @@ func TestAlchemy_FromChunk(t *testing.T) {
|
||||
if len(got[i].Raw) == 0 {
|
||||
t.Fatalf("no raw secret present: \n %+v", got[i])
|
||||
}
|
||||
got[i].Raw = nil
|
||||
if (got[i].VerificationError != nil) != tt.wantVerificationErr {
|
||||
t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError)
|
||||
}
|
||||
}
|
||||
if diff := pretty.Compare(got, tt.want); diff != "" {
|
||||
ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "VerificationError")
|
||||
if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {
|
||||
t.Errorf("Alchemy.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
|
||||
}
|
||||
})
|
||||
|
||||
@@ -18,7 +18,8 @@ import (
|
||||
)
|
||||
|
||||
type scanner struct {
|
||||
skipIDs map[string]struct{}
|
||||
verificationClient *http.Client
|
||||
skipIDs map[string]struct{}
|
||||
}
|
||||
|
||||
func New(opts ...func(*scanner)) *scanner {
|
||||
@@ -48,7 +49,7 @@ func WithSkipIDs(skipIDs []string) func(*scanner) {
|
||||
var _ detectors.Detector = (*scanner)(nil)
|
||||
|
||||
var (
|
||||
client = common.SaneHttpClient()
|
||||
defaultVerificationClient = common.SaneHttpClient()
|
||||
|
||||
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
|
||||
// Key types are from this list https://docs.aws.amazon.com/IAM/latest/UserGuide/reference_identifiers.html#identifiers-unique-ids
|
||||
@@ -123,8 +124,9 @@ func (s scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
host := "sts.amazonaws.com"
|
||||
region := "us-east-1"
|
||||
endpoint := "https://sts.amazonaws.com"
|
||||
datestamp := time.Now().UTC().Format("20060102")
|
||||
amzDate := time.Now().UTC().Format("20060102T150405Z0700")
|
||||
now := time.Now().UTC()
|
||||
datestamp := now.Format("20060102")
|
||||
amzDate := now.Format("20060102T150405Z0700")
|
||||
|
||||
req, err := http.NewRequestWithContext(ctx, method, endpoint, nil)
|
||||
if err != nil {
|
||||
@@ -171,6 +173,10 @@ func (s scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
req.Header.Add("Content-type", "application/x-www-form-urlencoded; charset=utf-8")
|
||||
req.URL.RawQuery = params.Encode()
|
||||
|
||||
client := s.verificationClient
|
||||
if client == nil {
|
||||
client = defaultVerificationClient
|
||||
}
|
||||
res, err := client.Do(req)
|
||||
if err == nil {
|
||||
|
||||
@@ -184,6 +190,8 @@ func (s scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
"user_id": identityInfo.GetCallerIdentityResponse.GetCallerIdentityResult.UserID,
|
||||
"arn": identityInfo.GetCallerIdentityResponse.GetCallerIdentityResult.Arn,
|
||||
}
|
||||
} else {
|
||||
s1.VerificationError = err
|
||||
}
|
||||
res.Body.Close()
|
||||
} else {
|
||||
@@ -192,7 +200,25 @@ func (s scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
continue
|
||||
}
|
||||
|
||||
if res.StatusCode != 403 {
|
||||
if res.StatusCode == 403 {
|
||||
var body awsErrorResponseBody
|
||||
err = json.NewDecoder(res.Body).Decode(&body)
|
||||
res.Body.Close()
|
||||
if err == nil {
|
||||
// All instances of the code I've seen in the wild are PascalCased but this check is
|
||||
// case-insensitive out of an abundance of caution
|
||||
if strings.EqualFold(body.Error.Code, "InvalidClientTokenId") {
|
||||
// determinate failure - nothing to do
|
||||
} else {
|
||||
// We see a surprising number of false-negative signature mismatch errors here
|
||||
// (The official SDK somehow elicits even more than just making the request
|
||||
// ourselves)
|
||||
s1.VerificationError = fmt.Errorf("request to %v returned status %d with an unexpected reason (%s: %s)", res.Request.URL, res.StatusCode, body.Error.Code, body.Error.Message)
|
||||
}
|
||||
} else {
|
||||
s1.VerificationError = fmt.Errorf("couldn't parse the sts response body (%v)", err)
|
||||
}
|
||||
} else {
|
||||
s1.VerificationError = fmt.Errorf("request to %v returned unexpected status %d", res.Request.URL, res.StatusCode)
|
||||
}
|
||||
}
|
||||
@@ -243,6 +269,15 @@ func awsCustomCleanResults(results []detectors.Result) []detectors.Result {
|
||||
return out
|
||||
}
|
||||
|
||||
type awsError struct {
|
||||
Code string `json:"Code"`
|
||||
Message string `json:"Message"`
|
||||
}
|
||||
|
||||
type awsErrorResponseBody struct {
|
||||
Error awsError `json:"Error"`
|
||||
}
|
||||
|
||||
type identityRes struct {
|
||||
GetCallerIdentityResponse struct {
|
||||
GetCallerIdentityResult struct {
|
||||
|
||||
@@ -17,6 +17,8 @@ import (
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
|
||||
)
|
||||
|
||||
var unverifiedSecretClient = common.ConstantResponseHttpClient(403, `{"Error": {"Code": "InvalidClientTokenId"} }`)
|
||||
|
||||
func TestAWS_FromChunk(t *testing.T) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
|
||||
defer cancel()
|
||||
@@ -69,7 +71,7 @@ func TestAWS_FromChunk(t *testing.T) {
|
||||
},
|
||||
{
|
||||
name: "found, unverified",
|
||||
s: scanner{},
|
||||
s: scanner{verificationClient: unverifiedSecretClient},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a aws secret %s within aws %s but not valid", inactiveSecret, id)), // the secret would satisfy the regex but not pass validation
|
||||
@@ -163,7 +165,7 @@ func TestAWS_FromChunk(t *testing.T) {
|
||||
},
|
||||
{
|
||||
name: "found, unverified, with leading +",
|
||||
s: scanner{},
|
||||
s: scanner{verificationClient: unverifiedSecretClient},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a aws secret %s within aws %s but not valid", "+HaNv9cTwheDKGJaws/+BMF2GgybQgBWdhcOOdfF", id)), // the secret would satisfy the regex but not pass validation
|
||||
@@ -194,9 +196,45 @@ func TestAWS_FromChunk(t *testing.T) {
|
||||
},
|
||||
{
|
||||
name: "found, would be verified if not for http timeout",
|
||||
s: scanner{},
|
||||
s: scanner{verificationClient: common.SaneHttpClientTimeOut(1 * time.Microsecond)},
|
||||
args: args{
|
||||
ctx: timeoutContext(1 * time.Microsecond),
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a aws secret %s within aws %s", secret, id)),
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_AWS,
|
||||
Verified: false,
|
||||
Redacted: "AKIASP2TPHJSQH3FJRUX",
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
wantVerificationError: true,
|
||||
},
|
||||
{
|
||||
name: "found, unverified due to unexpected http response status",
|
||||
s: scanner{verificationClient: common.ConstantResponseHttpClient(500, "internal server error")},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a aws secret %s within aws %s", secret, id)),
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_AWS,
|
||||
Verified: false,
|
||||
Redacted: "AKIASP2TPHJSQH3FJRUX",
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
wantVerificationError: true,
|
||||
},
|
||||
{
|
||||
name: "found, unverified due to unexpected 403 response reason",
|
||||
s: scanner{verificationClient: common.ConstantResponseHttpClient(403, `{"Error": {"Code": "SignatureDoesNotMatch"} }`)},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a aws secret %s within aws %s", secret, id)),
|
||||
verify: true,
|
||||
},
|
||||
@@ -224,7 +262,7 @@ func TestAWS_FromChunk(t *testing.T) {
|
||||
t.Fatalf("no raw secret present: \n %+v", got[i])
|
||||
}
|
||||
if (got[i].VerificationError != nil) != tt.wantVerificationError {
|
||||
t.Fatalf("verification error = %v, wantVerificationError %v", got[i].VerificationError, tt.wantVerificationError)
|
||||
t.Fatalf("wantVerificationError %v, verification error = %v", tt.wantVerificationError, got[i].VerificationError)
|
||||
}
|
||||
}
|
||||
ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "RawV2", "Raw", "VerificationError")
|
||||
@@ -249,8 +287,3 @@ func BenchmarkFromData(benchmark *testing.B) {
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func timeoutContext(timeout time.Duration) context.Context {
|
||||
c, _ := context.WithTimeout(context.Background(), timeout)
|
||||
return c
|
||||
}
|
||||
|
||||
@@ -20,7 +20,7 @@ var (
|
||||
client = common.SaneHttpClient()
|
||||
|
||||
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
|
||||
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"datadog", "dd"}) + `\b([a-zA-Z-0-9]{40})\b`)
|
||||
appPat = regexp.MustCompile(detectors.PrefixRegex([]string{"datadog", "dd"}) + `\b([a-zA-Z-0-9]{40})\b`)
|
||||
apiPat = regexp.MustCompile(detectors.PrefixRegex([]string{"datadog", "dd"}) + `\b([a-zA-Z-0-9]{32})\b`)
|
||||
)
|
||||
|
||||
@@ -34,24 +34,28 @@ func (s Scanner) Keywords() []string {
|
||||
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
|
||||
dataStr := string(data)
|
||||
|
||||
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
|
||||
appMatches := appPat.FindAllStringSubmatch(dataStr, -1)
|
||||
apiMatches := apiPat.FindAllStringSubmatch(dataStr, -1)
|
||||
|
||||
for _, match := range matches {
|
||||
if len(match) != 2 {
|
||||
for _, apiMatch := range apiMatches {
|
||||
if len(apiMatch) != 2 {
|
||||
continue
|
||||
}
|
||||
resMatch := strings.TrimSpace(match[1])
|
||||
for _, apimatch := range apiMatches {
|
||||
if len(apimatch) != 2 {
|
||||
resApiMatch := strings.TrimSpace(apiMatch[1])
|
||||
appIncluded := false
|
||||
for _, appMatch := range appMatches {
|
||||
if len(appMatch) != 2 {
|
||||
continue
|
||||
}
|
||||
resApiMatch := strings.TrimSpace(apimatch[1])
|
||||
resAppMatch := strings.TrimSpace(appMatch[1])
|
||||
|
||||
s1 := detectors.Result{
|
||||
DetectorType: detectorspb.DetectorType_DatadogToken,
|
||||
Raw: []byte(resMatch),
|
||||
RawV2: []byte(resMatch + resApiMatch),
|
||||
Raw: []byte(resAppMatch),
|
||||
RawV2: []byte(resAppMatch + resApiMatch),
|
||||
ExtraData: map[string]string{
|
||||
"Type": "Application+APIKey",
|
||||
},
|
||||
}
|
||||
|
||||
if verify {
|
||||
@@ -62,7 +66,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
}
|
||||
req.Header.Add("Content-Type", "application/json")
|
||||
req.Header.Add("DD-API-KEY", resApiMatch)
|
||||
req.Header.Add("DD-APPLICATION-KEY", resMatch)
|
||||
req.Header.Add("DD-APPLICATION-KEY", resAppMatch)
|
||||
res, err := client.Do(req)
|
||||
if err == nil {
|
||||
defer res.Body.Close()
|
||||
@@ -70,16 +74,49 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
s1.Verified = true
|
||||
} else {
|
||||
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
|
||||
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
|
||||
if detectors.IsKnownFalsePositive(resApiMatch, detectors.DefaultFalsePositives, true) {
|
||||
continue
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
appIncluded = true
|
||||
results = append(results, s1)
|
||||
}
|
||||
|
||||
if !appIncluded {
|
||||
s1 := detectors.Result{
|
||||
DetectorType: detectorspb.DetectorType_DatadogToken,
|
||||
Raw: []byte(resApiMatch),
|
||||
RawV2: []byte(resApiMatch),
|
||||
ExtraData: map[string]string{
|
||||
"Type": "APIKeyOnly",
|
||||
},
|
||||
}
|
||||
|
||||
if verify {
|
||||
|
||||
req, err := http.NewRequestWithContext(ctx, "GET", "https://api.datadoghq.com/api/v1/validate", nil)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
req.Header.Add("Content-Type", "application/json")
|
||||
req.Header.Add("DD-API-KEY", resApiMatch)
|
||||
res, err := client.Do(req)
|
||||
if err == nil {
|
||||
defer res.Body.Close()
|
||||
if res.StatusCode >= 200 && res.StatusCode < 300 {
|
||||
s1.Verified = true
|
||||
} else {
|
||||
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
|
||||
if detectors.IsKnownFalsePositive(resApiMatch, detectors.DefaultFalsePositives, true) {
|
||||
continue
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
results = append(results, s1)
|
||||
}
|
||||
}
|
||||
|
||||
return results, nil
|
||||
|
||||
@@ -19,11 +19,11 @@ import (
|
||||
func TestDatadogToken_FromChunk(t *testing.T) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
|
||||
defer cancel()
|
||||
testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
|
||||
testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")
|
||||
if err != nil {
|
||||
t.Fatalf("could not get test secrets from GCP: %s", err)
|
||||
}
|
||||
secret := testSecrets.MustGetField("DATADOGTOKEN_TOKEN")
|
||||
apiKey := testSecrets.MustGetField("DATADOGTOKEN_TOKEN")
|
||||
appKey := testSecrets.MustGetField("DATADOGTOKEN_APPKEY")
|
||||
inactiveAppKey := testSecrets.MustGetField("DATADOGTOKEN_INACTIVE")
|
||||
|
||||
@@ -44,13 +44,16 @@ func TestDatadogToken_FromChunk(t *testing.T) {
|
||||
s: Scanner{},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a datadogtoken secret %s within datadog %s", appKey, secret)),
|
||||
data: []byte(fmt.Sprintf("You can find a datadogtoken secret %s within datadog %s", appKey, apiKey)),
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_DatadogToken,
|
||||
Verified: true,
|
||||
ExtraData: map[string]string{
|
||||
"Type": "Application+APIKey",
|
||||
},
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
@@ -60,13 +63,35 @@ func TestDatadogToken_FromChunk(t *testing.T) {
|
||||
s: Scanner{},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a datadogtoken secret %s within but datadog %s not valid", inactiveAppKey, secret)), // the secret would satisfy the regex but not pass validation
|
||||
data: []byte(fmt.Sprintf("You can find a datadogtoken secret %s within but datadog %s not valid", inactiveAppKey, apiKey)), // the secret would satisfy the regex but not pass validation
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_DatadogToken,
|
||||
Verified: false,
|
||||
ExtraData: map[string]string{
|
||||
"Type": "Application+APIKey",
|
||||
},
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "api key found, verified",
|
||||
s: Scanner{},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a datadogtoken secret %s", apiKey)), // the secret would satisfy the regex but not pass validation
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_DatadogToken,
|
||||
Verified: true,
|
||||
ExtraData: map[string]string{
|
||||
"Type": "APIKeyOnly",
|
||||
},
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
@@ -96,6 +121,7 @@ func TestDatadogToken_FromChunk(t *testing.T) {
|
||||
t.Fatalf("no raw secret present: \n %+v", got[i])
|
||||
}
|
||||
got[i].Raw = nil
|
||||
got[i].RawV2 = nil
|
||||
}
|
||||
if diff := pretty.Compare(got, tt.want); diff != "" {
|
||||
t.Errorf("DatadogToken.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
|
||||
|
||||
@@ -0,0 +1,95 @@
|
||||
package dockerhub
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
|
||||
)
|
||||
|
||||
type Scanner struct{}
|
||||
|
||||
// Ensure the Scanner satisfies the interface at compile time.
|
||||
var _ detectors.Detector = (*Scanner)(nil)
|
||||
|
||||
var (
|
||||
client = common.SaneHttpClient()
|
||||
|
||||
// Can use email or username for login.
|
||||
usernamePat = regexp.MustCompile(`(?im)(?:user|usr|-u)\S{0,40}?[:=\s]{1,3}[ '"=]?([a-zA-Z0-9]{4,40})\b`)
|
||||
emailPat = regexp.MustCompile(common.EmailPattern)
|
||||
|
||||
// Can use password or personal access token (PAT) for login, but this scanner will only check for PATs.
|
||||
accessTokenPat = regexp.MustCompile(`\bdckr_pat_([a-zA-Z0-9_-]){27}\b`)
|
||||
)
|
||||
|
||||
// Keywords are used for efficiently pre-filtering chunks.
|
||||
// Use identifiers in the secret preferably, or the provider name.
|
||||
func (s Scanner) Keywords() []string {
|
||||
return []string{"dckr_pat_"}
|
||||
}
|
||||
|
||||
// FromData will find and optionally verify Dockerhub secrets in a given set of bytes.
|
||||
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
|
||||
dataStr := string(data)
|
||||
|
||||
emailMatches := emailPat.FindAllString(dataStr, -1)
|
||||
dataStr = emailPat.ReplaceAllString(dataStr, "")
|
||||
usernameMatches := usernamePat.FindAllStringSubmatch(dataStr, -1)
|
||||
|
||||
accessTokenMatches := accessTokenPat.FindAllString(dataStr, -1)
|
||||
|
||||
userMatches := emailMatches
|
||||
for _, usernameMatch := range usernameMatches {
|
||||
if len(usernameMatch) > 1 {
|
||||
userMatches = append(userMatches, usernameMatch[1])
|
||||
}
|
||||
}
|
||||
|
||||
for _, resUserMatch := range userMatches {
|
||||
for _, resAccessTokenMatch := range accessTokenMatches {
|
||||
|
||||
s1 := detectors.Result{
|
||||
DetectorType: detectorspb.DetectorType_Dockerhub,
|
||||
Raw: []byte(fmt.Sprintf("%s: %s", resUserMatch, resAccessTokenMatch)),
|
||||
}
|
||||
|
||||
if verify {
|
||||
payload := strings.NewReader(fmt.Sprintf(`{"username": "%s", "password": "%s"}`, resUserMatch, resAccessTokenMatch))
|
||||
|
||||
req, err := http.NewRequestWithContext(ctx, "GET", "https://hub.docker.com/v2/users/login", payload)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
req.Header.Add("Content-Type", "application/json")
|
||||
res, err := client.Do(req)
|
||||
if err == nil {
|
||||
defer res.Body.Close()
|
||||
body, err := io.ReadAll(res.Body)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
|
||||
// Valid credentials can still return a 401 status code if 2FA is enabled
|
||||
if (res.StatusCode >= 200 && res.StatusCode < 300) || (res.StatusCode == 401 && strings.Contains(string(body), "login_2fa_token")) {
|
||||
s1.Verified = true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
results = append(results, s1)
|
||||
}
|
||||
}
|
||||
|
||||
return results, nil
|
||||
}
|
||||
|
||||
func (s Scanner) Type() detectorspb.DetectorType {
|
||||
return detectorspb.DetectorType_Dockerhub
|
||||
}
|
||||
@@ -0,0 +1,138 @@
|
||||
//go:build detectors
|
||||
// +build detectors
|
||||
|
||||
package dockerhub
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/kylelemons/godebug/pretty"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
|
||||
)
|
||||
|
||||
func TestDockerhub_FromChunk(t *testing.T) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
|
||||
defer cancel()
|
||||
testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors4")
|
||||
if err != nil {
|
||||
t.Fatalf("could not get test secrets from GCP: %s", err)
|
||||
}
|
||||
username := testSecrets.MustGetField("DOCKERHUB_USERNAME")
|
||||
email := testSecrets.MustGetField("DOCKERHUB_EMAIL")
|
||||
pat := testSecrets.MustGetField("DOCKERHUB_PAT")
|
||||
inactivePat := testSecrets.MustGetField("DOCKERHUB_INACTIVE_PAT")
|
||||
|
||||
type args struct {
|
||||
ctx context.Context
|
||||
data []byte
|
||||
verify bool
|
||||
}
|
||||
tests := []struct {
|
||||
name string
|
||||
s Scanner
|
||||
args args
|
||||
want []detectors.Result
|
||||
wantErr bool
|
||||
}{
|
||||
{
|
||||
name: "found, verified",
|
||||
s: Scanner{},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("docker login -u %s -p %s", username, pat)),
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_Dockerhub,
|
||||
Verified: true,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "found, verified (email)",
|
||||
s: Scanner{},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("docker login -u %s -p %s", email, pat)),
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_Dockerhub,
|
||||
Verified: true,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "found, unverified",
|
||||
s: Scanner{},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("docker login -u %s -p %s", username, inactivePat)), // the secret would satisfy the regex but not pass validation
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_Dockerhub,
|
||||
Verified: false,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "not found",
|
||||
s: Scanner{},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte("You cannot find the secret within"),
|
||||
verify: true,
|
||||
},
|
||||
want: nil,
|
||||
wantErr: false,
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
s := Scanner{}
|
||||
got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
|
||||
if (err != nil) != tt.wantErr {
|
||||
t.Errorf("Dockerhub.FromData() error = %v, wantErr %v", err, tt.wantErr)
|
||||
return
|
||||
}
|
||||
for i := range got {
|
||||
if len(got[i].Raw) == 0 {
|
||||
t.Fatalf("no raw secret present: \n %+v", got[i])
|
||||
}
|
||||
got[i].Raw = nil
|
||||
}
|
||||
if diff := pretty.Compare(got, tt.want); diff != "" {
|
||||
t.Errorf("Dockerhub.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkFromData(benchmark *testing.B) {
|
||||
ctx := context.Background()
|
||||
s := Scanner{}
|
||||
for name, data := range detectors.MustGetBenchmarkData() {
|
||||
benchmark.Run(name, func(b *testing.B) {
|
||||
for n := 0; n < b.N; n++ {
|
||||
_, err := s.FromData(ctx, false, data)
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -1,8 +1,8 @@
|
||||
package ibmcloud
|
||||
package envoyapikey
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"strings"
|
||||
@@ -21,16 +21,16 @@ var (
|
||||
client = common.SaneHttpClient()
|
||||
|
||||
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
|
||||
keyPat = regexp.MustCompile(`\b([A-Za-z0-9_-]{44})\b`)
|
||||
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"envoy"}) + `\b([a-zA-Z0-9]{220})\b`)
|
||||
)
|
||||
|
||||
// Keywords are used for efficiently pre-filtering chunks.
|
||||
// Use identifiers in the secret preferably, or the provider name.
|
||||
func (s Scanner) Keywords() []string {
|
||||
return []string{"ibm"}
|
||||
return []string{"envoy"}
|
||||
}
|
||||
|
||||
// FromData will find and optionally verify IBM Cloud secrets in a given set of bytes.
|
||||
// FromData will find and optionally verify Envoy secrets in a given set of bytes.
|
||||
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
|
||||
dataStr := string(data)
|
||||
|
||||
@@ -43,22 +43,27 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
resMatch := strings.TrimSpace(match[1])
|
||||
|
||||
s1 := detectors.Result{
|
||||
DetectorType: detectorspb.DetectorType_Ibmcloud,
|
||||
DetectorType: detectorspb.DetectorType_EnvoyApiKey,
|
||||
Raw: []byte(resMatch),
|
||||
}
|
||||
|
||||
if verify {
|
||||
req, err := http.NewRequestWithContext(ctx, "GET", "https://api.ibmcloud.com/apps", nil)
|
||||
req, err := http.NewRequestWithContext(ctx, "GET", "https://api.envoy.com/v1/locations", nil)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
req.Header.Add("Accept", "application/vnd.ibmcloud+json; version=3")
|
||||
req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", resMatch))
|
||||
req.Header.Add("Accept", "application/vnd.envoy+json; version=3")
|
||||
req.Header.Add("X-Api-Key", resMatch)
|
||||
res, err := client.Do(req)
|
||||
if err == nil {
|
||||
defer res.Body.Close()
|
||||
if res.StatusCode >= 200 && res.StatusCode < 300 {
|
||||
s1.Verified = true
|
||||
body, _ := io.ReadAll(res.Body)
|
||||
|
||||
// Invalid API keys can also return status code 200, so check for presence of 'status 401' in response body.
|
||||
if res.StatusCode >= 200 && res.StatusCode < 300 || res.StatusCode == 403 {
|
||||
if !strings.Contains(string(body), `"status":401`) {
|
||||
s1.Verified = true
|
||||
}
|
||||
} else {
|
||||
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
|
||||
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
|
||||
@@ -75,5 +80,5 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
}
|
||||
|
||||
func (s Scanner) Type() detectorspb.DetectorType {
|
||||
return detectorspb.DetectorType_Ibmcloud
|
||||
return detectorspb.DetectorType_EnvoyApiKey
|
||||
}
|
||||
+12
-11
@@ -1,7 +1,7 @@
|
||||
//go:build detectors
|
||||
// +build detectors
|
||||
|
||||
package ibmcloud
|
||||
package envoyapikey
|
||||
|
||||
import (
|
||||
"context"
|
||||
@@ -10,21 +10,22 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/kylelemons/godebug/pretty"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
|
||||
)
|
||||
|
||||
func TestIbmcloud_FromChunk(t *testing.T) {
|
||||
func TestEnvoyapikey_FromChunk(t *testing.T) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
|
||||
defer cancel()
|
||||
testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")
|
||||
testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors4")
|
||||
if err != nil {
|
||||
t.Fatalf("could not get test secrets from GCP: %s", err)
|
||||
}
|
||||
secret := testSecrets.MustGetField("IBM_CLOUD_KEY")
|
||||
inactiveSecret := testSecrets.MustGetField("IBM_CLOUD_KEY_INACTIVE")
|
||||
secret := testSecrets.MustGetField("ENVOYAPIKEY")
|
||||
inactiveSecret := testSecrets.MustGetField("ENVOYAPIKEY_INACTIVE")
|
||||
|
||||
type args struct {
|
||||
ctx context.Context
|
||||
@@ -43,12 +44,12 @@ func TestIbmcloud_FromChunk(t *testing.T) {
|
||||
s: Scanner{},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a ibmcloud secret %s within", secret)),
|
||||
data: []byte(fmt.Sprintf("You can find a envoyapikey secret %s within", secret)),
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_IbmClou,
|
||||
DetectorType: detectorspb.DetectorType_EnvoyApiKey,
|
||||
Verified: true,
|
||||
},
|
||||
},
|
||||
@@ -59,12 +60,12 @@ func TestIbmcloud_FromChunk(t *testing.T) {
|
||||
s: Scanner{},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a ibmcloud secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
|
||||
data: []byte(fmt.Sprintf("You can find a envoyapikey secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_Ibmcloud,
|
||||
DetectorType: detectorspb.DetectorType_EnvoyApiKey,
|
||||
Verified: false,
|
||||
},
|
||||
},
|
||||
@@ -87,7 +88,7 @@ func TestIbmcloud_FromChunk(t *testing.T) {
|
||||
s := Scanner{}
|
||||
got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
|
||||
if (err != nil) != tt.wantErr {
|
||||
t.Errorf("Ibmcloud.FromData() error = %v, wantErr %v", err, tt.wantErr)
|
||||
t.Errorf("Envoyapikey.FromData() error = %v, wantErr %v", err, tt.wantErr)
|
||||
return
|
||||
}
|
||||
for i := range got {
|
||||
@@ -97,7 +98,7 @@ func TestIbmcloud_FromChunk(t *testing.T) {
|
||||
got[i].Raw = nil
|
||||
}
|
||||
if diff := pretty.Compare(got, tt.want); diff != "" {
|
||||
t.Errorf("Ibmcloud.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
|
||||
t.Errorf("Envoyapikey.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
|
||||
}
|
||||
})
|
||||
}
|
||||
+35
-12
@@ -2,6 +2,8 @@ package ftp
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"net/textproto"
|
||||
"net/url"
|
||||
"regexp"
|
||||
"strings"
|
||||
@@ -13,7 +15,17 @@ import (
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
|
||||
)
|
||||
|
||||
type Scanner struct{}
|
||||
const (
|
||||
// https://datatracker.ietf.org/doc/html/rfc959
|
||||
ftpNotLoggedIn = 530
|
||||
|
||||
defaultVerificationTimeout = 5 * time.Second
|
||||
)
|
||||
|
||||
type Scanner struct {
|
||||
// Verification timeout. Defaults to 5 seconds if unset.
|
||||
verificationTimeout time.Duration
|
||||
}
|
||||
|
||||
// Ensure the Scanner satisfies the interface at compile time.
|
||||
var _ detectors.Detector = (*Scanner)(nil)
|
||||
@@ -58,48 +70,59 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
rawURL.Path = ""
|
||||
redact := strings.TrimSpace(strings.Replace(rawURL.String(), password, "********", -1))
|
||||
|
||||
s := detectors.Result{
|
||||
r := detectors.Result{
|
||||
DetectorType: detectorspb.DetectorType_FTP,
|
||||
Raw: []byte(rawURL.String()),
|
||||
Redacted: redact,
|
||||
}
|
||||
|
||||
if verify {
|
||||
s.Verified = verifyFTP(ctx, parsedURL)
|
||||
timeout := s.verificationTimeout
|
||||
if timeout == 0 {
|
||||
timeout = defaultVerificationTimeout
|
||||
}
|
||||
verificationErr := verifyFTP(timeout, parsedURL)
|
||||
r.Verified = verificationErr == nil
|
||||
if !isErrDeterminate(verificationErr) {
|
||||
r.VerificationError = verificationErr
|
||||
}
|
||||
}
|
||||
|
||||
if !s.Verified {
|
||||
if !r.Verified {
|
||||
// Skip unverified findings where the password starts with a `$` - it's almost certainly a variable.
|
||||
if strings.HasPrefix(password, "$") {
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
if detectors.IsKnownFalsePositive(string(s.Raw), []detectors.FalsePositive{"@ftp.freebsd.org"}, false) {
|
||||
if detectors.IsKnownFalsePositive(string(r.Raw), []detectors.FalsePositive{"@ftp.freebsd.org"}, false) {
|
||||
continue
|
||||
}
|
||||
|
||||
results = append(results, s)
|
||||
results = append(results, r)
|
||||
}
|
||||
|
||||
return results, nil
|
||||
}
|
||||
|
||||
func verifyFTP(ctx context.Context, u *url.URL) bool {
|
||||
func isErrDeterminate(e error) bool {
|
||||
ftpErr := &textproto.Error{}
|
||||
return errors.As(e, &ftpErr) && ftpErr.Code == ftpNotLoggedIn
|
||||
}
|
||||
|
||||
func verifyFTP(timeout time.Duration, u *url.URL) error {
|
||||
host := u.Host
|
||||
if !strings.Contains(host, ":") {
|
||||
host = host + ":21"
|
||||
}
|
||||
|
||||
c, err := ftp.Dial(host, ftp.DialWithTimeout(5*time.Second))
|
||||
c, err := ftp.Dial(host, ftp.DialWithTimeout(timeout))
|
||||
if err != nil {
|
||||
return false
|
||||
return err
|
||||
}
|
||||
|
||||
password, _ := u.User.Password()
|
||||
err = c.Login(u.User.Username(), password)
|
||||
|
||||
return err == nil
|
||||
return c.Login(u.User.Username(), password)
|
||||
}
|
||||
|
||||
func (s Scanner) Type() detectorspb.DetectorType {
|
||||
|
||||
@@ -5,9 +5,11 @@ package ftp
|
||||
|
||||
import (
|
||||
"context"
|
||||
"github.com/google/go-cmp/cmp"
|
||||
"github.com/google/go-cmp/cmp/cmpopts"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/kylelemons/godebug/pretty"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
|
||||
)
|
||||
@@ -19,11 +21,12 @@ func TestFTP_FromChunk(t *testing.T) {
|
||||
verify bool
|
||||
}
|
||||
tests := []struct {
|
||||
name string
|
||||
s Scanner
|
||||
args args
|
||||
want []detectors.Result
|
||||
wantErr bool
|
||||
name string
|
||||
s Scanner
|
||||
args args
|
||||
want []detectors.Result
|
||||
wantErr bool
|
||||
wantVerificationErr bool
|
||||
}{
|
||||
{
|
||||
name: "bad scheme",
|
||||
@@ -48,7 +51,7 @@ func TestFTP_FromChunk(t *testing.T) {
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_FTP,
|
||||
Verified: true,
|
||||
Redacted: "ftp://dlpuser:*************************@ftp.dlptest.com",
|
||||
Redacted: "ftp://dlpuser:********@ftp.dlptest.com",
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
@@ -66,11 +69,49 @@ func TestFTP_FromChunk(t *testing.T) {
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_FTP,
|
||||
Verified: false,
|
||||
Redacted: "ftp://dlpuser:*******@ftp.dlptest.com",
|
||||
Redacted: "ftp://dlpuser:********@ftp.dlptest.com",
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "bad host",
|
||||
s: Scanner{},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
// https://dlptest.com/ftp-test/
|
||||
data: []byte("ftp://dlpuser:rNrKYTX9g7z3RgJRmxWuGHbeu@ftp.dlptest.com.badhost"),
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_FTP,
|
||||
Verified: false,
|
||||
Redacted: "ftp://dlpuser:********@ftp.dlptest.com.badhost",
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
wantVerificationErr: true,
|
||||
},
|
||||
{
|
||||
name: "timeout",
|
||||
s: Scanner{verificationTimeout: 1 * time.Microsecond},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
// https://dlptest.com/ftp-test/
|
||||
data: []byte("ftp://dlpuser:rNrKYTX9g7z3RgJRmxWuGHbeu@ftp.dlptest.com"),
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_FTP,
|
||||
Verified: false,
|
||||
Redacted: "ftp://dlpuser:********@ftp.dlptest.com",
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
wantVerificationErr: true,
|
||||
},
|
||||
{
|
||||
name: "blocked FP",
|
||||
s: Scanner{},
|
||||
@@ -84,8 +125,7 @@ func TestFTP_FromChunk(t *testing.T) {
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
s := Scanner{}
|
||||
got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
|
||||
got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
|
||||
if (err != nil) != tt.wantErr {
|
||||
t.Errorf("URI.FromData() error = %v, wantErr %v", err, tt.wantErr)
|
||||
return
|
||||
@@ -95,9 +135,14 @@ func TestFTP_FromChunk(t *testing.T) {
|
||||
// }
|
||||
for i := range got {
|
||||
got[i].Raw = nil
|
||||
|
||||
if (got[i].VerificationError != nil) != tt.wantVerificationErr {
|
||||
t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError)
|
||||
}
|
||||
}
|
||||
if diff := pretty.Compare(got, tt.want); diff != "" {
|
||||
t.Errorf("URI.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
|
||||
opts := cmpopts.IgnoreFields(detectors.Result{}, "VerificationError")
|
||||
if diff := cmp.Diff(got, tt.want, opts); diff != "" {
|
||||
t.Errorf("FTP.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
@@ -7,6 +7,7 @@ import (
|
||||
"net/http"
|
||||
"regexp"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
@@ -19,7 +20,7 @@ type Scanner struct{}
|
||||
var _ detectors.Detector = (*Scanner)(nil)
|
||||
|
||||
var (
|
||||
client = common.SaneHttpClientTimeOut(5)
|
||||
client = common.SaneHttpClientTimeOut(5 * time.Second)
|
||||
|
||||
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
|
||||
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"getemail"}) + `\b([a-zA-Z0-9-]{20})\b`)
|
||||
|
||||
@@ -0,0 +1,81 @@
|
||||
package github_oauth2
|
||||
|
||||
import (
|
||||
"context"
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
|
||||
"golang.org/x/oauth2/clientcredentials"
|
||||
"golang.org/x/oauth2/github"
|
||||
)
|
||||
|
||||
type Scanner struct{ detectors.EndpointSetter }
|
||||
|
||||
// Ensure the Scanner satisfies the interfaces at compile time.
|
||||
var _ detectors.Detector = (*Scanner)(nil)
|
||||
|
||||
var (
|
||||
// Oauth2 client ID and secret
|
||||
oauth2ClientIDPat = regexp.MustCompile(detectors.PrefixRegex([]string{"github"}) + `\b([a-f0-9]{20})\b`)
|
||||
oauth2ClientSecretPat = regexp.MustCompile(detectors.PrefixRegex([]string{"github"}) + `\b([a-f0-9]{40})\b`)
|
||||
)
|
||||
|
||||
const (
|
||||
githubBadVerificationCodeError = "bad_verification_code"
|
||||
)
|
||||
|
||||
// Keywords are used for efficiently pre-filtering chunks.
|
||||
// Use identifiers in the secret preferably, or the provider name.
|
||||
func (s Scanner) Keywords() []string {
|
||||
return []string{"github"}
|
||||
}
|
||||
|
||||
// FromData will find and optionally verify GitHub secrets in a given set of bytes.
|
||||
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
|
||||
dataStr := string(data)
|
||||
|
||||
// Oauth2 client ID and secret
|
||||
oauth2ClientIDMatches := oauth2ClientIDPat.FindAllStringSubmatch(dataStr, -1)
|
||||
oauth2ClientSecretMatches := oauth2ClientSecretPat.FindAllStringSubmatch(dataStr, -1)
|
||||
|
||||
for _, idMatch := range oauth2ClientIDMatches {
|
||||
if len(idMatch) != 2 {
|
||||
continue
|
||||
}
|
||||
for _, secretMatch := range oauth2ClientSecretMatches {
|
||||
if len(secretMatch) != 2 {
|
||||
continue
|
||||
}
|
||||
|
||||
s1 := detectors.Result{
|
||||
DetectorType: detectorspb.DetectorType_GitHubOauth2,
|
||||
Raw: []byte(idMatch[1]),
|
||||
RawV2: []byte(idMatch[1] + secretMatch[1]),
|
||||
}
|
||||
|
||||
config := &clientcredentials.Config{
|
||||
ClientID: idMatch[1],
|
||||
ClientSecret: secretMatch[1],
|
||||
TokenURL: github.Endpoint.TokenURL,
|
||||
}
|
||||
_, err := config.Token(ctx)
|
||||
if err != nil && strings.Contains(err.Error(), githubBadVerificationCodeError) {
|
||||
s1.Verified = true
|
||||
}
|
||||
|
||||
if !s1.Verified && detectors.IsKnownFalsePositive(string(s1.Raw), detectors.DefaultFalsePositives, true) {
|
||||
continue
|
||||
}
|
||||
|
||||
results = append(results, s1)
|
||||
}
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
func (s Scanner) Type() detectorspb.DetectorType {
|
||||
return detectorspb.DetectorType_GitHubOauth2
|
||||
}
|
||||
@@ -12,7 +12,11 @@ import (
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
|
||||
)
|
||||
|
||||
type Scanner struct{ detectors.EndpointSetter }
|
||||
type Scanner struct {
|
||||
client *http.Client
|
||||
|
||||
detectors.EndpointSetter
|
||||
}
|
||||
|
||||
// Ensure the Scanner satisfies the interfaces at compile time.
|
||||
var _ detectors.Detector = (*Scanner)(nil)
|
||||
@@ -23,7 +27,8 @@ func (Scanner) Version() int { return 1 }
|
||||
func (Scanner) DefaultEndpoint() string { return "https://gitlab.com" }
|
||||
|
||||
var (
|
||||
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"gitlab"}) + `\b((?:glpat|)[a-zA-Z0-9\-=_]{20,22})\b`)
|
||||
defaultClient = common.SaneHttpClient()
|
||||
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"gitlab"}) + `\b((?:glpat|)[a-zA-Z0-9\-=_]{20,22})\b`)
|
||||
)
|
||||
|
||||
// Keywords are used for efficiently pre-filtering chunks.
|
||||
@@ -59,7 +64,10 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
// one of these scopes has access to, so we just check an example endpoint for each scope. If any
|
||||
// of them contain data, we know we have a valid key, but if they all fail, we don't
|
||||
|
||||
client := common.SaneHttpClient()
|
||||
client := s.client
|
||||
if client == nil {
|
||||
client = defaultClient
|
||||
}
|
||||
for _, baseURL := range s.Endpoints(s.DefaultEndpoint()) {
|
||||
// test `read_user` scope
|
||||
req, err := http.NewRequestWithContext(ctx, "GET", baseURL+"/api/v4/user", nil)
|
||||
@@ -74,9 +82,19 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
// 200 means good key and has `read_user` scope
|
||||
// 403 means good key but not the right scope
|
||||
// 401 is bad key
|
||||
if res.StatusCode == http.StatusOK || res.StatusCode == http.StatusForbidden {
|
||||
switch res.StatusCode {
|
||||
case http.StatusOK:
|
||||
secret.Verified = true
|
||||
case http.StatusForbidden:
|
||||
// Good key but not the right scope
|
||||
secret.Verified = true
|
||||
case http.StatusUnauthorized:
|
||||
// Nothing to do; zero values are the ones we want
|
||||
default:
|
||||
secret.VerificationError = fmt.Errorf("unexpected HTTP response status %d", res.StatusCode)
|
||||
}
|
||||
} else {
|
||||
secret.VerificationError = err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -6,11 +6,11 @@ package gitlab
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"github.com/google/go-cmp/cmp"
|
||||
"github.com/google/go-cmp/cmp/cmpopts"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/kylelemons/godebug/pretty"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
|
||||
@@ -31,11 +31,12 @@ func TestGitlab_FromChunk(t *testing.T) {
|
||||
verify bool
|
||||
}
|
||||
tests := []struct {
|
||||
name string
|
||||
s Scanner
|
||||
args args
|
||||
want []detectors.Result
|
||||
wantErr bool
|
||||
name string
|
||||
s Scanner
|
||||
args args
|
||||
want []detectors.Result
|
||||
wantErr bool
|
||||
wantVerificationErr bool
|
||||
}{
|
||||
{
|
||||
name: "found",
|
||||
@@ -85,6 +86,56 @@ func TestGitlab_FromChunk(t *testing.T) {
|
||||
},
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "found, would be verified but for timeout",
|
||||
s: Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a gitlab super secret %s within", secret)),
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_Gitlab,
|
||||
Verified: false,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
wantVerificationErr: true,
|
||||
},
|
||||
{
|
||||
name: "found and valid but unexpected api response",
|
||||
s: Scanner{client: common.ConstantResponseHttpClient(500, "")},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a gitlab super secret %s within", secret)),
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_Gitlab,
|
||||
Verified: false,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
wantVerificationErr: true,
|
||||
},
|
||||
{
|
||||
name: "found, good key but wrong scope",
|
||||
s: Scanner{client: common.ConstantResponseHttpClient(403, "")},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a gitlab super secret %s within", secret)),
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_Gitlab,
|
||||
Verified: true,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "not found",
|
||||
s: Scanner{},
|
||||
@@ -99,8 +150,7 @@ func TestGitlab_FromChunk(t *testing.T) {
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
s := Scanner{}
|
||||
got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
|
||||
got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
|
||||
if (err != nil) != tt.wantErr {
|
||||
t.Errorf("Gitlab.FromData() error = %v, wantErr %v", err, tt.wantErr)
|
||||
return
|
||||
@@ -109,9 +159,12 @@ func TestGitlab_FromChunk(t *testing.T) {
|
||||
if len(got[i].Raw) == 0 {
|
||||
t.Fatal("no raw secret present")
|
||||
}
|
||||
got[i].Raw = nil
|
||||
if (got[i].VerificationError != nil) != tt.wantVerificationErr {
|
||||
t.Fatalf(" wantVerificationError = %v, verification error = %v,", tt.wantVerificationErr, got[i].VerificationError)
|
||||
}
|
||||
}
|
||||
if diff := pretty.Compare(got, tt.want); diff != "" {
|
||||
opts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "VerificationError")
|
||||
if diff := cmp.Diff(got, tt.want, opts); diff != "" {
|
||||
t.Errorf("Gitlab.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
|
||||
}
|
||||
})
|
||||
|
||||
@@ -0,0 +1,91 @@
|
||||
package huggingface
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
|
||||
)
|
||||
|
||||
type Scanner struct {
|
||||
client *http.Client
|
||||
}
|
||||
|
||||
// Ensure the Scanner satisfies the interface at compile time.
|
||||
var _ detectors.Detector = (*Scanner)(nil)
|
||||
|
||||
var (
|
||||
defaultClient = common.SaneHttpClient()
|
||||
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
|
||||
keyPat = regexp.MustCompile(`\bhf_[a-zA-Z]{34}\b`)
|
||||
)
|
||||
|
||||
// Keywords are used for efficiently pre-filtering chunks.
|
||||
// Use identifiers in the secret preferably, or the provider name.
|
||||
func (s Scanner) Keywords() []string {
|
||||
return []string{"huggingface", "hugging_face", "hf"} // Huggingface docs occasionally use "hf" instead of "huggingface"
|
||||
}
|
||||
|
||||
// FromData will find and optionally verify Huggingface secrets in a given set of bytes.
|
||||
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
|
||||
dataStr := string(data)
|
||||
|
||||
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
|
||||
|
||||
for _, match := range matches {
|
||||
if len(match) != 1 {
|
||||
continue
|
||||
}
|
||||
resMatch := strings.TrimSpace(match[0])
|
||||
|
||||
s1 := detectors.Result{
|
||||
DetectorType: detectorspb.DetectorType_HuggingFace,
|
||||
Raw: []byte(resMatch),
|
||||
}
|
||||
|
||||
if verify {
|
||||
client := s.client
|
||||
if client == nil {
|
||||
client = defaultClient
|
||||
}
|
||||
req, err := http.NewRequestWithContext(ctx, "GET", "https://huggingface.co/api/whoami-v2", nil)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
|
||||
req.Header.Set("Authorization", "Bearer "+resMatch)
|
||||
|
||||
res, err := client.Do(req)
|
||||
if err == nil {
|
||||
defer res.Body.Close()
|
||||
if res.StatusCode >= 200 && res.StatusCode < 300 {
|
||||
s1.Verified = true
|
||||
} else if res.StatusCode == 401 {
|
||||
// The secret is determinately not verified (nothing to do)
|
||||
} else {
|
||||
s1.VerificationError = fmt.Errorf("unexpected HTTP response status %d", res.StatusCode)
|
||||
}
|
||||
} else {
|
||||
s1.VerificationError = err
|
||||
}
|
||||
}
|
||||
|
||||
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
|
||||
if !s1.Verified && detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
|
||||
continue
|
||||
}
|
||||
|
||||
results = append(results, s1)
|
||||
}
|
||||
|
||||
return results, nil
|
||||
}
|
||||
|
||||
func (s Scanner) Type() detectorspb.DetectorType {
|
||||
return detectorspb.DetectorType_HuggingFace
|
||||
}
|
||||
@@ -0,0 +1,160 @@
|
||||
//go:build detectors
|
||||
// +build detectors
|
||||
|
||||
package huggingface
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"github.com/google/go-cmp/cmp"
|
||||
"github.com/google/go-cmp/cmp/cmpopts"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
|
||||
)
|
||||
|
||||
func TestHuggingface_FromChunk(t *testing.T) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
|
||||
defer cancel()
|
||||
testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")
|
||||
if err != nil {
|
||||
t.Fatalf("could not get test secrets from GCP: %s", err)
|
||||
}
|
||||
secret := testSecrets.MustGetField("HUGGINGFACE")
|
||||
inactiveSecret := testSecrets.MustGetField("HUGGINGFACE_INACTIVE")
|
||||
|
||||
type args struct {
|
||||
ctx context.Context
|
||||
data []byte
|
||||
verify bool
|
||||
}
|
||||
tests := []struct {
|
||||
name string
|
||||
s Scanner
|
||||
args args
|
||||
want []detectors.Result
|
||||
wantErr bool
|
||||
wantVerificationErr bool
|
||||
}{
|
||||
{
|
||||
name: "found, verified",
|
||||
s: Scanner{},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a huggingface secret %s within", secret)),
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_HuggingFace,
|
||||
Verified: true,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
wantVerificationErr: false,
|
||||
},
|
||||
{
|
||||
name: "found, unverified",
|
||||
s: Scanner{},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a huggingface secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_HuggingFace,
|
||||
Verified: false,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
wantVerificationErr: false,
|
||||
},
|
||||
{
|
||||
name: "not found",
|
||||
s: Scanner{},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte("You cannot find the secret within"),
|
||||
verify: true,
|
||||
},
|
||||
want: nil,
|
||||
wantErr: false,
|
||||
wantVerificationErr: false,
|
||||
},
|
||||
{
|
||||
name: "found, would be verified if not for timeout",
|
||||
s: Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a huggingface secret %s within", secret)),
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_HuggingFace,
|
||||
Verified: false,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
wantVerificationErr: true,
|
||||
},
|
||||
{
|
||||
name: "found, verified but unexpected api surface",
|
||||
s: Scanner{client: common.ConstantResponseHttpClient(404, "")},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a huggingface secret %s within", secret)),
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_HuggingFace,
|
||||
Verified: false,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
wantVerificationErr: true,
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
|
||||
if (err != nil) != tt.wantErr {
|
||||
t.Errorf("Huggingface.FromData() error = %v, wantErr %v", err, tt.wantErr)
|
||||
return
|
||||
}
|
||||
for i := range got {
|
||||
if len(got[i].Raw) == 0 {
|
||||
t.Fatalf("no raw secret present: \n %+v", got[i])
|
||||
}
|
||||
if (got[i].VerificationError != nil) != tt.wantVerificationErr {
|
||||
t.Fatalf(" wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError)
|
||||
}
|
||||
}
|
||||
ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "VerificationError")
|
||||
if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {
|
||||
t.Errorf("Huggingface.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkFromData(benchmark *testing.B) {
|
||||
ctx := context.Background()
|
||||
s := Scanner{}
|
||||
for name, data := range detectors.MustGetBenchmarkData() {
|
||||
benchmark.Run(name, func(b *testing.B) {
|
||||
for n := 0; n < b.N; n++ {
|
||||
_, err := s.FromData(ctx, false, data)
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
+28
-11
@@ -86,7 +86,14 @@ matchLoop:
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(ctx, 5*time.Second)
|
||||
defer cancel()
|
||||
s.Verified = j.ping(ctx)
|
||||
pingRes := j.ping(ctx)
|
||||
s.Verified = pingRes.err == nil
|
||||
// If there's a ping error that is marked as "determinate" we throw it away. We do this because this was the
|
||||
// behavior before tri-state verification was introduced and preserving it allows us to gradually migrate
|
||||
// detectors to use tri-state verification.
|
||||
if pingRes.err != nil && !pingRes.determinate {
|
||||
s.VerificationError = pingRes.err
|
||||
}
|
||||
// TODO: specialized redaction
|
||||
}
|
||||
|
||||
@@ -101,15 +108,15 @@ matchLoop:
|
||||
}
|
||||
|
||||
func tryRedactAnonymousJDBC(conn string) string {
|
||||
if s, ok := tryRedactBasicAuth(conn); ok {
|
||||
return s
|
||||
}
|
||||
if s, ok := tryRedactURLParams(conn); ok {
|
||||
return s
|
||||
}
|
||||
if s, ok := tryRedactODBC(conn); ok {
|
||||
return s
|
||||
}
|
||||
if s, ok := tryRedactBasicAuth(conn); ok {
|
||||
return s
|
||||
}
|
||||
if s, ok := tryRedactRegex(conn); ok {
|
||||
return s
|
||||
}
|
||||
@@ -158,8 +165,8 @@ func tryRedactODBC(conn string) (string, bool) {
|
||||
var found bool
|
||||
var newParams []string
|
||||
for _, param := range strings.Split(conn, ";") {
|
||||
key, val, _ := strings.Cut(param, "=")
|
||||
if strings.Contains(strings.ToLower(key), "pass") {
|
||||
key, val, isKvp := strings.Cut(param, "=")
|
||||
if isKvp && strings.Contains(strings.ToLower(key), "pass") {
|
||||
newParams = append(newParams, key+"="+strings.Repeat("*", len(val)))
|
||||
found = true
|
||||
continue
|
||||
@@ -198,8 +205,13 @@ var supportedSubprotocols = map[string]func(string) (jdbc, error){
|
||||
"sqlserver": parseSqlServer,
|
||||
}
|
||||
|
||||
type pingResult struct {
|
||||
err error
|
||||
determinate bool
|
||||
}
|
||||
|
||||
type jdbc interface {
|
||||
ping(context.Context) bool
|
||||
ping(context.Context) pingResult
|
||||
}
|
||||
|
||||
func newJDBC(conn string) (jdbc, error) {
|
||||
@@ -220,11 +232,16 @@ func newJDBC(conn string) (jdbc, error) {
|
||||
return parser(subname)
|
||||
}
|
||||
|
||||
func ping(ctx context.Context, driverName, conn string) bool {
|
||||
if err := pingErr(ctx, driverName, conn); err != nil {
|
||||
return false
|
||||
func ping(ctx context.Context, driverName string, isDeterminate func(error) bool, candidateConns ...string) pingResult {
|
||||
var indeterminateErrors []error
|
||||
for _, c := range candidateConns {
|
||||
err := pingErr(ctx, driverName, c)
|
||||
if err == nil || isDeterminate(err) {
|
||||
return pingResult{err, true}
|
||||
}
|
||||
indeterminateErrors = append(indeterminateErrors, err)
|
||||
}
|
||||
return true
|
||||
return pingResult{errors.Join(indeterminateErrors...), false}
|
||||
}
|
||||
|
||||
func pingErr(ctx context.Context, driverName, conn string) error {
|
||||
|
||||
@@ -105,6 +105,22 @@ func TestJdbcVerified(t *testing.T) {
|
||||
},
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "sql server verified",
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("jdbc:sqlserver://odbc:server=localhost;database=%s;password=%s", sqlServerDatabase, sqlServerPass)),
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_JDBC,
|
||||
Verified: true,
|
||||
Redacted: "jdbc:sqlserver://odbc:server=localhost;database=master;password=**************",
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
|
||||
@@ -5,6 +5,7 @@ package jdbc
|
||||
|
||||
import (
|
||||
"context"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"os"
|
||||
"testing"
|
||||
|
||||
@@ -207,6 +208,51 @@ func TestJdbc_FromDataWithIgnorePattern(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestJdbc_Redact(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
conn string
|
||||
want string
|
||||
}{
|
||||
{
|
||||
name: "basic auth'",
|
||||
conn: "//user:secret@tcp(127.0.0.1:3306)/",
|
||||
want: "//user:******@tcp(127.0.0.1:3306)/",
|
||||
},
|
||||
{
|
||||
name: "basic auth including raw string 'pass'",
|
||||
conn: "//wrongUser:wrongPass@tcp(127.0.0.1:3306)/",
|
||||
want: "//wrongUser:*********@tcp(127.0.0.1:3306)/",
|
||||
},
|
||||
{
|
||||
name: "basic auth including raw string 'pass' with unfortunate db name",
|
||||
conn: "//wrongUser:wrongPass@tcp(127.0.0.1:3306)/passwords",
|
||||
want: "//wrongUser:*********@tcp(127.0.0.1:3306)/passwords",
|
||||
},
|
||||
{
|
||||
name: "url param-style",
|
||||
conn: "jdbc:postgresql://localhost:5432/foo?sslmode=disable&password=p@ssw04d",
|
||||
want: "jdbc:postgresql://localhost:5432/foo?sslmode=disable&password=********",
|
||||
},
|
||||
{
|
||||
name: "odbc-style without server",
|
||||
conn: "//odbc:server=localhost;user id=sa;database=master;password=/p?s=sw&rd",
|
||||
want: "//odbc:server=localhost;user id=sa;database=master;password=**********",
|
||||
},
|
||||
{
|
||||
name: "odbc-style with server",
|
||||
conn: "jdbc:sqlserver://a.b.c.net;database=database-name;spring.datasource.password=super-secret-password",
|
||||
want: "jdbc:sqlserver://a.b.c.net;database=database-name;spring.datasource.password=*********************",
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := tryRedactAnonymousJDBC(tt.conn)
|
||||
assert.Equal(t, tt.want, got)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkFromData(benchmark *testing.B) {
|
||||
ctx := context.Background()
|
||||
s := Scanner{}
|
||||
|
||||
+28
-19
@@ -3,9 +3,8 @@ package jdbc
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"github.com/go-sql-driver/mysql"
|
||||
"strings"
|
||||
|
||||
_ "github.com/go-sql-driver/mysql"
|
||||
)
|
||||
|
||||
type mysqlJDBC struct {
|
||||
@@ -16,30 +15,40 @@ type mysqlJDBC struct {
|
||||
params string
|
||||
}
|
||||
|
||||
func (s *mysqlJDBC) ping(ctx context.Context) bool {
|
||||
if ping(ctx, "mysql", s.conn) {
|
||||
return true
|
||||
}
|
||||
// try building connection string (should be same as s.conn though)
|
||||
if ping(ctx, "mysql", s.build()) {
|
||||
return true
|
||||
}
|
||||
// try removing database
|
||||
s.database = ""
|
||||
return ping(ctx, "mysql", s.build())
|
||||
func (s *mysqlJDBC) ping(ctx context.Context) pingResult {
|
||||
return ping(ctx, "mysql", isMySQLErrorDeterminate,
|
||||
s.conn,
|
||||
buildMySQLConnectionString(s.host, s.database, s.userPass, s.params),
|
||||
buildMySQLConnectionString(s.host, "", s.userPass, s.params))
|
||||
}
|
||||
|
||||
func (s *mysqlJDBC) build() string {
|
||||
conn := s.host + "/" + s.database
|
||||
if s.userPass != "" {
|
||||
conn = s.userPass + "@" + conn
|
||||
func buildMySQLConnectionString(host, database, userPass, params string) string {
|
||||
conn := host + "/" + database
|
||||
if userPass != "" {
|
||||
conn = userPass + "@" + conn
|
||||
}
|
||||
if s.params != "" {
|
||||
conn = conn + "?" + s.params
|
||||
if params != "" {
|
||||
conn = conn + "?" + params
|
||||
}
|
||||
return conn
|
||||
}
|
||||
|
||||
func isMySQLErrorDeterminate(err error) bool {
|
||||
// MySQL error numbers from https://dev.mysql.com/doc/mysql-errors/8.0/en/server-error-reference.html
|
||||
if mySQLErr, isMySQLErr := err.(*mysql.MySQLError); isMySQLErr {
|
||||
switch mySQLErr.Number {
|
||||
case 1044:
|
||||
// User access denied to a particular database
|
||||
return false // "Indeterminate" so that other connection variations will be tried
|
||||
case 1045:
|
||||
// User access denied
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
func parseMySQL(subname string) (jdbc, error) {
|
||||
// expected form: [subprotocol:]//[user:password@]HOST[/DB][?key=val[&key=val]]
|
||||
hostAndDB, params, _ := strings.Cut(subname, "?")
|
||||
|
||||
@@ -21,45 +21,54 @@ const (
|
||||
)
|
||||
|
||||
func TestMySQL(t *testing.T) {
|
||||
type result struct {
|
||||
parseErr bool
|
||||
pingOk bool
|
||||
pingDeterminate bool
|
||||
}
|
||||
tests := []struct {
|
||||
input string
|
||||
wantErr bool
|
||||
wantPing bool
|
||||
input string
|
||||
want result
|
||||
}{
|
||||
{
|
||||
input: "",
|
||||
wantErr: true,
|
||||
input: "",
|
||||
want: result{parseErr: true},
|
||||
},
|
||||
{
|
||||
input: "//" + mysqlUser + ":" + mysqlPass + "@tcp(127.0.0.1:3306)/" + mysqlDatabase,
|
||||
wantPing: true,
|
||||
input: "//" + mysqlUser + ":" + mysqlPass + "@tcp(127.0.0.1:3306)/" + mysqlDatabase,
|
||||
want: result{pingOk: true, pingDeterminate: true},
|
||||
},
|
||||
{
|
||||
input: "//wrongUser:wrongPass@tcp(127.0.0.1:3306)/" + mysqlDatabase,
|
||||
wantPing: false,
|
||||
input: "//wrongUser:wrongPass@tcp(127.0.0.1:3306)/" + mysqlDatabase,
|
||||
want: result{pingOk: false, pingDeterminate: true},
|
||||
},
|
||||
{
|
||||
input: "//" + mysqlUser + ":wrongPass@tcp(127.0.0.1:3306)/" + mysqlDatabase,
|
||||
wantPing: false,
|
||||
input: "//" + mysqlUser + ":wrongPass@tcp(127.0.0.1:3306)/" + mysqlDatabase,
|
||||
want: result{pingOk: false, pingDeterminate: true},
|
||||
},
|
||||
{
|
||||
input: "//" + mysqlUser + ":" + mysqlPass + "@tcp(127.0.0.1:3306)/",
|
||||
wantPing: true,
|
||||
input: "//" + mysqlUser + ":" + mysqlPass + "@tcp(127.0.0.1:3306)/",
|
||||
want: result{pingOk: true, pingDeterminate: true},
|
||||
},
|
||||
{
|
||||
input: "//" + mysqlUser + ":" + mysqlPass + "@tcp(127.0.0.1:3306)/wrongDB",
|
||||
wantPing: true,
|
||||
input: "//" + mysqlUser + ":" + mysqlPass + "@tcp(127.0.0.1:3306)/wrongDB",
|
||||
want: result{pingOk: true, pingDeterminate: true},
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.input, func(t *testing.T) {
|
||||
j, err := parseMySQL(tt.input)
|
||||
if tt.wantErr {
|
||||
assert.Error(t, err)
|
||||
|
||||
if err != nil {
|
||||
got := result{parseErr: true}
|
||||
assert.Equal(t, tt.want, got)
|
||||
return
|
||||
}
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, tt.wantPing, j.ping(context.Background()))
|
||||
|
||||
pr := j.ping(context.Background())
|
||||
|
||||
got := result{pingOk: pr.err == nil, pingDeterminate: pr.determinate}
|
||||
assert.Equal(t, tt.want, got)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -4,9 +4,8 @@ import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"github.com/lib/pq"
|
||||
"strings"
|
||||
|
||||
_ "github.com/lib/pq"
|
||||
)
|
||||
|
||||
type postgresJDBC struct {
|
||||
@@ -14,37 +13,35 @@ type postgresJDBC struct {
|
||||
params map[string]string
|
||||
}
|
||||
|
||||
func (s *postgresJDBC) ping(ctx context.Context) bool {
|
||||
// try the provided connection string directly
|
||||
if ping(ctx, "postgres", s.conn) {
|
||||
return true
|
||||
}
|
||||
// try as a URL
|
||||
if ping(ctx, "postgres", "postgres://"+s.conn) {
|
||||
return true
|
||||
}
|
||||
// build a connection string
|
||||
data := map[string]string{
|
||||
// default user
|
||||
"user": "postgres",
|
||||
}
|
||||
for key, val := range s.params {
|
||||
if key == "host" {
|
||||
if h, p, found := strings.Cut(val, ":"); found {
|
||||
data["host"] = h
|
||||
data["port"] = p
|
||||
continue
|
||||
}
|
||||
func (s *postgresJDBC) ping(ctx context.Context) pingResult {
|
||||
// It is crucial that we try to build a connection string ourselves before using the one we found. This is because
|
||||
// if the found connection string doesn't include a username, the driver will attempt to connect using the current
|
||||
// user's name, which will fail in a way that looks like a determinate failure, thus terminating the waterfall. In
|
||||
// contrast, when we build a connection string ourselves, if there's no username, we try 'postgres' instead, which
|
||||
// actually has a chance of working.
|
||||
return ping(ctx, "postgres", isPostgresErrorDeterminate,
|
||||
buildPostgresConnectionString(s.params, true),
|
||||
buildPostgresConnectionString(s.params, false),
|
||||
s.conn,
|
||||
"postgres://"+s.conn)
|
||||
}
|
||||
|
||||
func isPostgresErrorDeterminate(err error) bool {
|
||||
// Postgres codes from https://www.postgresql.org/docs/current/errcodes-appendix.html
|
||||
if pqErr, isPostgresError := err.(*pq.Error); isPostgresError {
|
||||
switch pqErr.Code {
|
||||
case "28P01":
|
||||
// Invalid username/password
|
||||
return true
|
||||
case "3D000":
|
||||
// Unknown database
|
||||
return false // "Indeterminate" so that other connection variations will be tried
|
||||
case "3F000":
|
||||
// Unknown schema
|
||||
return false // "Indeterminate" so that other connection variations will be tried
|
||||
}
|
||||
data[key] = val
|
||||
}
|
||||
if ping(ctx, "postgres", joinKeyValues(data, " ")) {
|
||||
return true
|
||||
}
|
||||
if s.params["dbname"] != "" {
|
||||
delete(s.params, "dbname")
|
||||
return s.ping(ctx)
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
@@ -79,3 +76,25 @@ func parsePostgres(subname string) (jdbc, error) {
|
||||
|
||||
return &postgresJDBC{subname[2:], params}, nil
|
||||
}
|
||||
|
||||
func buildPostgresConnectionString(params map[string]string, includeDbName bool) string {
|
||||
data := map[string]string{
|
||||
// default user
|
||||
"user": "postgres",
|
||||
}
|
||||
for key, val := range params {
|
||||
if key == "host" {
|
||||
if h, p, found := strings.Cut(val, ":"); found {
|
||||
data["host"] = h
|
||||
data["port"] = p
|
||||
continue
|
||||
}
|
||||
}
|
||||
if key == "dbname" && !includeDbName {
|
||||
continue
|
||||
}
|
||||
data[key] = val
|
||||
}
|
||||
|
||||
return joinKeyValues(data, " ")
|
||||
}
|
||||
|
||||
@@ -20,49 +20,62 @@ const (
|
||||
)
|
||||
|
||||
func TestPostgres(t *testing.T) {
|
||||
type result struct {
|
||||
parseErr bool
|
||||
pingOk bool
|
||||
pingDeterminate bool
|
||||
}
|
||||
tests := []struct {
|
||||
input string
|
||||
wantErr bool
|
||||
wantPing bool
|
||||
input string
|
||||
want result
|
||||
}{
|
||||
{
|
||||
input: "//localhost:5432/foo?sslmode=disable&password=" + postgresPass,
|
||||
wantPing: true,
|
||||
input: "//localhost:5432/foo?sslmode=disable&password=" + postgresPass,
|
||||
want: result{pingOk: true, pingDeterminate: true},
|
||||
},
|
||||
{
|
||||
input: "//localhost:5432/foo?sslmode=disable&user=" + postgresUser + "&password=" + postgresPass,
|
||||
wantPing: true,
|
||||
input: "//localhost:5432/foo?sslmode=disable&user=" + postgresUser + "&password=" + postgresPass,
|
||||
want: result{pingOk: true, pingDeterminate: true},
|
||||
},
|
||||
{
|
||||
input: "//localhost/foo?sslmode=disable&port=5432&password=" + postgresPass,
|
||||
wantPing: true,
|
||||
input: "//localhost/foo?sslmode=disable&port=5432&password=" + postgresPass,
|
||||
want: result{pingOk: true, pingDeterminate: true},
|
||||
},
|
||||
{
|
||||
input: "//localhost:5432/foo?password=" + postgresPass,
|
||||
wantPing: false,
|
||||
input: "//localhost:5432/foo?password=" + postgresPass,
|
||||
want: result{pingOk: false, pingDeterminate: false},
|
||||
},
|
||||
{
|
||||
input: "//localhost:5432/foo?sslmode=disable&password=foo",
|
||||
wantPing: false,
|
||||
input: "//localhost:5432/foo?sslmode=disable&password=foo",
|
||||
want: result{pingOk: false, pingDeterminate: true},
|
||||
},
|
||||
{
|
||||
input: "//localhost:5432/foo?sslmode=disable&user=foo&password=" + postgresPass,
|
||||
wantPing: false,
|
||||
input: "//localhost:5432/foo?sslmode=disable&user=foo&password=" + postgresPass,
|
||||
want: result{pingOk: false, pingDeterminate: true},
|
||||
},
|
||||
{
|
||||
input: "invalid",
|
||||
wantErr: true,
|
||||
input: "//badhost:5432/foo?sslmode=disable&user=foo&password=" + postgresPass,
|
||||
want: result{pingOk: false, pingDeterminate: false},
|
||||
},
|
||||
{
|
||||
input: "invalid",
|
||||
want: result{parseErr: true},
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.input, func(t *testing.T) {
|
||||
j, err := parsePostgres(tt.input)
|
||||
if tt.wantErr {
|
||||
assert.Error(t, err)
|
||||
|
||||
if err != nil {
|
||||
got := result{parseErr: true}
|
||||
assert.Equal(t, tt.want, got)
|
||||
return
|
||||
}
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, tt.wantPing, j.ping(context.Background()))
|
||||
|
||||
pr := j.ping(context.Background())
|
||||
|
||||
got := result{pingOk: pr.err == nil, pingDeterminate: pr.determinate}
|
||||
assert.Equal(t, tt.want, got)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -14,12 +14,18 @@ type sqliteJDBC struct {
|
||||
testing bool
|
||||
}
|
||||
|
||||
func (s *sqliteJDBC) ping(ctx context.Context) bool {
|
||||
var cannotVerifySqliteError error = errors.New("sqlite credentials cannot be verified")
|
||||
|
||||
func (s *sqliteJDBC) ping(ctx context.Context) pingResult {
|
||||
if !s.testing {
|
||||
// sqlite is not a networked database, so we cannot verify
|
||||
return false
|
||||
return pingResult{cannotVerifySqliteError, true}
|
||||
}
|
||||
return ping(ctx, "sqlite3", s.filename)
|
||||
return ping(ctx, "sqlite3", isSqliteErrorDeterminate, s.filename)
|
||||
}
|
||||
|
||||
func isSqliteErrorDeterminate(err error) bool {
|
||||
return true
|
||||
}
|
||||
|
||||
func parseSqlite(subname string) (jdbc, error) {
|
||||
|
||||
@@ -39,7 +39,7 @@ func TestParseSqlite(t *testing.T) {
|
||||
assert.Error(t, err)
|
||||
} else {
|
||||
assert.NoError(t, err)
|
||||
assert.True(t, j.ping(context.Background()))
|
||||
assert.True(t, j.ping(context.Background()).err == nil)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
@@ -3,9 +3,8 @@ package jdbc
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
mssql "github.com/denisenkom/go-mssqldb"
|
||||
"strings"
|
||||
|
||||
_ "github.com/denisenkom/go-mssqldb"
|
||||
)
|
||||
|
||||
type sqlServerJDBC struct {
|
||||
@@ -13,15 +12,25 @@ type sqlServerJDBC struct {
|
||||
params map[string]string
|
||||
}
|
||||
|
||||
func (s *sqlServerJDBC) ping(ctx context.Context) bool {
|
||||
if ping(ctx, "mssql", s.conn) {
|
||||
return true
|
||||
func (s *sqlServerJDBC) ping(ctx context.Context) pingResult {
|
||||
return ping(ctx, "mssql", isSqlServerErrorDeterminate,
|
||||
joinKeyValues(s.params, ";"),
|
||||
s.conn,
|
||||
"sqlserver://"+s.conn)
|
||||
}
|
||||
|
||||
func isSqlServerErrorDeterminate(err error) bool {
|
||||
// Error numbers from https://learn.microsoft.com/en-us/sql/relational-databases/errors-events/database-engine-events-and-errors?view=sql-server-ver16
|
||||
if mssqlError, isMssqlError := err.(mssql.Error); isMssqlError {
|
||||
switch mssqlError.Number {
|
||||
case 18456:
|
||||
// Login failed
|
||||
// This is a determinate failure iff we tried to use a real user
|
||||
return mssqlError.Message != "login error: Login failed for user ''."
|
||||
}
|
||||
}
|
||||
if ping(ctx, "mssql", joinKeyValues(s.params, ";")) {
|
||||
return true
|
||||
}
|
||||
// try URL format
|
||||
return ping(ctx, "mssql", "sqlserver://"+s.conn)
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
func parseSqlServer(subname string) (jdbc, error) {
|
||||
|
||||
@@ -21,33 +21,50 @@ const (
|
||||
)
|
||||
|
||||
func TestSqlServer(t *testing.T) {
|
||||
type result struct {
|
||||
parseErr bool
|
||||
pingOk bool
|
||||
pingDeterminate bool
|
||||
}
|
||||
tests := []struct {
|
||||
input string
|
||||
wantErr bool
|
||||
wantPing bool
|
||||
input string
|
||||
want result
|
||||
}{
|
||||
{
|
||||
input: "",
|
||||
wantErr: true,
|
||||
input: "",
|
||||
want: result{parseErr: true},
|
||||
},
|
||||
{
|
||||
input: "//odbc:server=localhost;user id=sa;database=master;password=" + sqlServerPass,
|
||||
wantPing: true,
|
||||
input: "//server=localhost;user id=sa;database=master;password=" + sqlServerPass,
|
||||
want: result{pingOk: true, pingDeterminate: true},
|
||||
},
|
||||
{
|
||||
input: "//localhost;database= master;spring.datasource.password=" + sqlServerPass,
|
||||
wantPing: true,
|
||||
input: "//server=badhost;user id=sa;database=master;password=" + sqlServerPass,
|
||||
want: result{pingOk: false, pingDeterminate: false},
|
||||
},
|
||||
{
|
||||
input: "//localhost;database=master;spring.datasource.password=" + sqlServerPass,
|
||||
want: result{pingOk: true, pingDeterminate: true},
|
||||
},
|
||||
{
|
||||
input: "//localhost;database=master;spring.datasource.password=badpassword",
|
||||
want: result{pingOk: false, pingDeterminate: true},
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.input, func(t *testing.T) {
|
||||
j, err := parseSqlServer(tt.input)
|
||||
if tt.wantErr {
|
||||
assert.Error(t, err)
|
||||
|
||||
if err != nil {
|
||||
got := result{parseErr: true}
|
||||
assert.Equal(t, tt.want, got)
|
||||
return
|
||||
}
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, tt.wantPing, j.ping(context.Background()))
|
||||
|
||||
pr := j.ping(context.Background())
|
||||
|
||||
got := result{pingOk: pr.err == nil, pingDeterminate: pr.determinate}
|
||||
assert.Equal(t, tt.want, got)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -20,13 +20,13 @@ var (
|
||||
client = common.SaneHttpClient()
|
||||
|
||||
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
|
||||
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"launchdarkly"}) + `\b([a-z0-9-]{40})\b`)
|
||||
keyPat = regexp.MustCompile(`\b((?:api|sdk|mob)-[a-z0-9]{8}-[a-z0-9]{4}-4[a-z0-9]{3}-[a-z0-9]{4}-[a-z0-9]{12})\b`)
|
||||
)
|
||||
|
||||
// Keywords are used for efficiently pre-filtering chunks.
|
||||
// Use identifiers in the secret preferably, or the provider name.
|
||||
func (s Scanner) Keywords() []string {
|
||||
return []string{"launchdarkly"}
|
||||
return []string{"api", "sdk", "mob"}
|
||||
}
|
||||
|
||||
// FromData will find and optionally verify LaunchDarkly secrets in a given set of bytes.
|
||||
|
||||
+48
-28
@@ -3,6 +3,8 @@ package ldap
|
||||
import (
|
||||
"context"
|
||||
"crypto/tls"
|
||||
"fmt"
|
||||
"net"
|
||||
"net/url"
|
||||
"regexp"
|
||||
"strings"
|
||||
@@ -63,7 +65,11 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
}
|
||||
|
||||
if verify {
|
||||
s1.Verified = verifyLDAP(ctx, username[1], password[1], ldapURL)
|
||||
verificationErr := verifyLDAP(username[1], password[1], ldapURL)
|
||||
s1.Verified = verificationErr == nil
|
||||
if !isErrDeterminate(verificationErr) {
|
||||
s1.VerificationError = verificationErr
|
||||
}
|
||||
}
|
||||
|
||||
results = append(results, s1)
|
||||
@@ -89,7 +95,12 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
}
|
||||
|
||||
if verify {
|
||||
s1.Verified = verifyLDAP(ctx, username, password, ldapURL)
|
||||
verificationError := verifyLDAP(username, password, ldapURL)
|
||||
|
||||
s1.Verified = verificationError == nil
|
||||
if !isErrDeterminate(verificationError) {
|
||||
s1.VerificationError = verificationError
|
||||
}
|
||||
}
|
||||
|
||||
results = append(results, s1)
|
||||
@@ -98,7 +109,19 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
return results, nil
|
||||
}
|
||||
|
||||
func verifyLDAP(ctx context.Context, username, password string, ldapURL *url.URL) bool {
|
||||
func isErrDeterminate(err error) bool {
|
||||
switch e := err.(type) {
|
||||
case *ldap.Error:
|
||||
switch e.Err.(type) {
|
||||
case *net.OpError:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
return true
|
||||
}
|
||||
|
||||
func verifyLDAP(username, password string, ldapURL *url.URL) error {
|
||||
// Tests with non-TLS, TLS, and STARTTLS
|
||||
|
||||
ldap.DefaultTimeout = 5 * time.Second
|
||||
@@ -109,38 +132,35 @@ func verifyLDAP(ctx context.Context, username, password string, ldapURL *url.URL
|
||||
case "ldap":
|
||||
// Non-TLS dial
|
||||
l, err := ldap.DialURL(uri)
|
||||
if err == nil {
|
||||
defer l.Close()
|
||||
// Non-TLS verify
|
||||
err = l.Bind(username, password)
|
||||
if err == nil {
|
||||
return true
|
||||
}
|
||||
|
||||
// STARTTLS
|
||||
err = l.StartTLS(&tls.Config{InsecureSkipVerify: true})
|
||||
if err == nil {
|
||||
// STARTTLS verify
|
||||
err = l.Bind(username, password)
|
||||
if err == nil {
|
||||
return true
|
||||
}
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer l.Close()
|
||||
// Non-TLS verify
|
||||
err = l.Bind(username, password)
|
||||
if err == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
// STARTTLS
|
||||
err = l.StartTLS(&tls.Config{InsecureSkipVerify: true})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// STARTTLS verify
|
||||
return l.Bind(username, password)
|
||||
case "ldaps":
|
||||
// TLS dial
|
||||
l, err := ldap.DialTLS("tcp", uri, &tls.Config{InsecureSkipVerify: true})
|
||||
if err == nil {
|
||||
defer l.Close()
|
||||
// TLS verify
|
||||
err = l.Bind(username, password)
|
||||
if err == nil {
|
||||
return true
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer l.Close()
|
||||
// TLS verify
|
||||
return l.Bind(username, password)
|
||||
}
|
||||
|
||||
return false
|
||||
return fmt.Errorf("unknown ldap scheme %q", ldapURL.Scheme)
|
||||
}
|
||||
|
||||
func (s Scanner) Type() detectorspb.DetectorType {
|
||||
|
||||
@@ -7,13 +7,14 @@ import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"github.com/google/go-cmp/cmp"
|
||||
"github.com/google/go-cmp/cmp/cmpopts"
|
||||
"os"
|
||||
"os/exec"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/kylelemons/godebug/pretty"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
|
||||
@@ -60,11 +61,12 @@ func TestLdap_Integration_FromChunk(t *testing.T) {
|
||||
verify bool
|
||||
}
|
||||
tests := []struct {
|
||||
name string
|
||||
s Scanner
|
||||
args args
|
||||
want []detectors.Result
|
||||
wantErr bool
|
||||
name string
|
||||
s Scanner
|
||||
args args
|
||||
want []detectors.Result
|
||||
wantErr bool
|
||||
wantVerificationErr bool
|
||||
}{
|
||||
{
|
||||
name: "found with URI and separate user+password usage, verified",
|
||||
@@ -152,6 +154,26 @@ func TestLdap_Integration_FromChunk(t *testing.T) {
|
||||
},
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "inaccessible host",
|
||||
s: Scanner{},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(`
|
||||
ldap://badhost:1389
|
||||
binddn="cn=admin,dc=example,dc=org"
|
||||
pass="P@55w0rd"`),
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_LDAP,
|
||||
Verified: false,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
wantVerificationErr: true,
|
||||
},
|
||||
{
|
||||
name: "not found",
|
||||
s: Scanner{},
|
||||
@@ -176,9 +198,12 @@ func TestLdap_Integration_FromChunk(t *testing.T) {
|
||||
if len(got[i].Raw) == 0 {
|
||||
t.Fatalf("no raw secret present: \n %+v", got[i])
|
||||
}
|
||||
got[i].Raw = nil
|
||||
if (got[i].VerificationError != nil) != tt.wantVerificationErr {
|
||||
t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError)
|
||||
}
|
||||
}
|
||||
if diff := pretty.Compare(got, tt.want); diff != "" {
|
||||
ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "VerificationError")
|
||||
if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {
|
||||
t.Errorf("Ldap.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
|
||||
}
|
||||
})
|
||||
@@ -205,7 +230,7 @@ func startOpenLDAP() error {
|
||||
return nil
|
||||
case <-time.After(30 * time.Second):
|
||||
stopOpenLDAP()
|
||||
return errors.New("timeout waiting for postgres database to be ready")
|
||||
return errors.New("timeout waiting for ldap service to be ready")
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -6,6 +6,7 @@ import (
|
||||
"net/http"
|
||||
"regexp"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
@@ -18,7 +19,7 @@ type Scanner struct{}
|
||||
var _ detectors.Detector = (*Scanner)(nil)
|
||||
|
||||
var (
|
||||
client = common.SaneHttpClientTimeOut(5)
|
||||
client = common.SaneHttpClientTimeOut(5 * time.Second)
|
||||
|
||||
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
|
||||
keyPat = regexp.MustCompile(`(https:\/\/[a-zA-Z-0-9]+\.webhook\.office\.com\/webhookb2\/[a-zA-Z-0-9]{8}-[a-zA-Z-0-9]{4}-[a-zA-Z-0-9]{4}-[a-zA-Z-0-9]{4}-[a-zA-Z-0-9]{12}\@[a-zA-Z-0-9]{8}-[a-zA-Z-0-9]{4}-[a-zA-Z-0-9]{4}-[a-zA-Z-0-9]{4}-[a-zA-Z-0-9]{12}\/IncomingWebhook\/[a-zA-Z-0-9]{32}\/[a-zA-Z-0-9]{8}-[a-zA-Z-0-9]{4}-[a-zA-Z-0-9]{4}-[a-zA-Z-0-9]{4}-[a-zA-Z-0-9]{12})`)
|
||||
|
||||
@@ -2,6 +2,8 @@ package mongodb
|
||||
|
||||
import (
|
||||
"context"
|
||||
"go.mongodb.org/mongo-driver/x/mongo/driver/auth"
|
||||
"go.mongodb.org/mongo-driver/x/mongo/driver/topology"
|
||||
"regexp"
|
||||
"strings"
|
||||
"time"
|
||||
@@ -14,14 +16,17 @@ import (
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
|
||||
)
|
||||
|
||||
type Scanner struct{}
|
||||
type Scanner struct {
|
||||
timeout time.Duration // Zero value means "default timeout"
|
||||
}
|
||||
|
||||
// Ensure the Scanner satisfies the interface at compile time.
|
||||
var _ detectors.Detector = (*Scanner)(nil)
|
||||
|
||||
var (
|
||||
defaultTimeout = 2 * time.Second
|
||||
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
|
||||
keyPat = regexp.MustCompile(`\b(mongodb(\+srv)?://[\S]{3,50}:([\S]{3,50})@[-.%\w\/:]+)\b`)
|
||||
keyPat = regexp.MustCompile(`\b(mongodb(\+srv)?://[\S]{3,50}:([\S]{3,88})@[-.%\w\/:]+)\b`)
|
||||
// TODO: Add support for sharded cluster, replica set and Atlas Deployment.
|
||||
)
|
||||
|
||||
@@ -46,23 +51,15 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
}
|
||||
|
||||
if verify {
|
||||
func() {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
|
||||
defer cancel()
|
||||
client, err := mongo.Connect(ctx, options.Client().ApplyURI(resMatch))
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
defer func() {
|
||||
if err := client.Disconnect(ctx); err != nil {
|
||||
return
|
||||
}
|
||||
}()
|
||||
if err := client.Ping(ctx, readpref.Primary()); err != nil {
|
||||
return
|
||||
}
|
||||
s1.Verified = true
|
||||
}()
|
||||
timeout := s.timeout
|
||||
if timeout == 0 {
|
||||
timeout = defaultTimeout
|
||||
}
|
||||
err := verifyUri(resMatch, timeout)
|
||||
s1.Verified = err == nil
|
||||
if !isErrDeterminate(err) {
|
||||
s1.VerificationError = err
|
||||
}
|
||||
}
|
||||
results = append(results, s1)
|
||||
}
|
||||
@@ -70,6 +67,33 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
return results, nil
|
||||
}
|
||||
|
||||
func isErrDeterminate(err error) bool {
|
||||
switch e := err.(type) {
|
||||
case topology.ConnectionError:
|
||||
switch e.Unwrap().(type) {
|
||||
case *auth.Error:
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
func verifyUri(uri string, timeout time.Duration) error {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), timeout)
|
||||
defer cancel()
|
||||
client, err := mongo.Connect(ctx, options.Client().ApplyURI(uri))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer func() {
|
||||
_ = client.Disconnect(ctx)
|
||||
}()
|
||||
return client.Ping(ctx, readpref.Primary())
|
||||
}
|
||||
|
||||
func (s Scanner) Type() detectorspb.DetectorType {
|
||||
return detectorspb.DetectorType_MongoDB
|
||||
}
|
||||
|
||||
@@ -6,6 +6,7 @@ package mongodb
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
@@ -34,11 +35,12 @@ func TestMongoDB_FromChunk(t *testing.T) {
|
||||
verify bool
|
||||
}
|
||||
tests := []struct {
|
||||
name string
|
||||
s Scanner
|
||||
args args
|
||||
want []detectors.Result
|
||||
wantErr bool
|
||||
name string
|
||||
s Scanner
|
||||
args args
|
||||
want []detectors.Result
|
||||
wantErr bool
|
||||
wantVerificationErr bool
|
||||
}{
|
||||
{
|
||||
name: "found, verified",
|
||||
@@ -72,6 +74,40 @@ func TestMongoDB_FromChunk(t *testing.T) {
|
||||
},
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "found, would be verified but for connection timeout",
|
||||
s: Scanner{timeout: 1 * time.Microsecond},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a mongodb secret %s within", secret)),
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_MongoDB,
|
||||
Verified: false,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
wantVerificationErr: true,
|
||||
},
|
||||
{
|
||||
name: "found, bad host",
|
||||
s: Scanner{},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a mongodb secret %s within", strings.ReplaceAll(secret, ".mongodb.net", ".mongodb.net.bad"))),
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_MongoDB,
|
||||
Verified: false,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
wantVerificationErr: true,
|
||||
},
|
||||
{
|
||||
name: "not found",
|
||||
s: Scanner{},
|
||||
@@ -86,8 +122,7 @@ func TestMongoDB_FromChunk(t *testing.T) {
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
s := Scanner{}
|
||||
got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
|
||||
got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
|
||||
if (err != nil) != tt.wantErr {
|
||||
t.Errorf("MongoDB.FromData() error = %v, wantErr %v", err, tt.wantErr)
|
||||
return
|
||||
@@ -97,8 +132,11 @@ func TestMongoDB_FromChunk(t *testing.T) {
|
||||
t.Fatalf("no raw secret present: \n %+v", got[i])
|
||||
}
|
||||
got[i].Raw = nil
|
||||
if (got[i].VerificationError != nil) != tt.wantVerificationErr {
|
||||
t.Fatalf("wantVerificationErr = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError)
|
||||
}
|
||||
}
|
||||
ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "RawV2")
|
||||
ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "RawV2", "VerificationError")
|
||||
if diff := cmp.Diff(tt.want, got, ignoreOpts); diff != "" {
|
||||
t.Errorf("MongoDB.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
|
||||
}
|
||||
|
||||
@@ -36,7 +36,6 @@ func (s Scanner) Keywords() []string {
|
||||
// FromData will find and optionally verify NpmToken secrets in a given set of bytes.
|
||||
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
|
||||
dataStr := string(data)
|
||||
|
||||
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
|
||||
for _, match := range matches {
|
||||
if len(match) != 2 {
|
||||
|
||||
@@ -19,8 +19,8 @@ type Scanner struct{}
|
||||
var _ detectors.Detector = (*Scanner)(nil)
|
||||
|
||||
var (
|
||||
domainPat = regexp.MustCompile(`[a-z0-9-]{1,40}\.okta(?:preview|-emea){0,1}\.com`)
|
||||
tokenPat = regexp.MustCompile(`00[a-zA-Z0-9_-]{40}`)
|
||||
domainPat = regexp.MustCompile(`\b[a-z0-9-]{1,40}\.okta(?:preview|-emea){0,1}\.com\b`)
|
||||
tokenPat = regexp.MustCompile(`\b00[a-zA-Z0-9_-]{40}\b`)
|
||||
// TODO: Oauth client secrets
|
||||
)
|
||||
|
||||
|
||||
@@ -52,7 +52,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
}
|
||||
|
||||
if verify {
|
||||
req, err := http.NewRequestWithContext(ctx, "GET", "https://api.opsgenie.com/v2/users", nil)
|
||||
req, err := http.NewRequestWithContext(ctx, "GET", "https://api.opsgenie.com/v2/alerts", nil)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
|
||||
@@ -22,8 +22,8 @@ var (
|
||||
client = common.SaneHttpClient()
|
||||
|
||||
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
|
||||
idPat = regexp.MustCompile(`\b([A-Za-z0-9_\.]{7}-[A-Za-z0-9_\.]{72})\b`)
|
||||
keyPat = regexp.MustCompile(`\b([A-Za-z0-9_\.]{69}-[A-Za-z0-9_\.]{10})\b`)
|
||||
idPat = regexp.MustCompile(`\b([A-Za-z0-9_\.]{7}-[A-Za-z0-9_\.]{72}|[A-Za-z0-9_\.]{5}-[A-Za-z0-9_\.]{38})\b`)
|
||||
keyPat = regexp.MustCompile(`\b([A-Za-z0-9_\.\-]{44,80})\b`)
|
||||
)
|
||||
|
||||
// Keywords are used for efficiently pre-filtering chunks.
|
||||
@@ -39,24 +39,24 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
|
||||
idmatches := idPat.FindAllStringSubmatch(dataStr, -1)
|
||||
|
||||
for _, match := range matches {
|
||||
if len(match) != 2 {
|
||||
for _, idMatch := range idmatches {
|
||||
if len(idMatch) != 2 {
|
||||
continue
|
||||
}
|
||||
resMatch := strings.TrimSpace(match[1])
|
||||
for _, idMatch := range idmatches {
|
||||
if len(idMatch) != 2 {
|
||||
resIDMatch := strings.TrimSpace(idMatch[1])
|
||||
for _, secretMatch := range matches {
|
||||
if len(secretMatch) != 2 {
|
||||
continue
|
||||
}
|
||||
residMatch := strings.TrimSpace(idMatch[1])
|
||||
resSecretMatch := strings.TrimSpace(secretMatch[1])
|
||||
|
||||
s1 := detectors.Result{
|
||||
DetectorType: detectorspb.DetectorType_PaypalOauth,
|
||||
Raw: []byte(resMatch),
|
||||
Raw: []byte(resSecretMatch),
|
||||
}
|
||||
|
||||
if verify {
|
||||
data := fmt.Sprintf("%s:%s", residMatch, resMatch)
|
||||
data := fmt.Sprintf("%s:%s", resIDMatch, resSecretMatch)
|
||||
encoded := b64.StdEncoding.EncodeToString([]byte(data))
|
||||
payload := strings.NewReader("grant_type=client_credentials")
|
||||
req, err := http.NewRequestWithContext(ctx, "POST", "https://api-m.sandbox.paypal.com/v1/oauth2/token", payload)
|
||||
@@ -74,7 +74,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
s1.Verified = true
|
||||
} else {
|
||||
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
|
||||
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
|
||||
if detectors.IsKnownFalsePositive(resIDMatch, detectors.DefaultFalsePositives, true) {
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
@@ -19,7 +19,7 @@ import (
|
||||
func TestPaypalOauth_FromChunk(t *testing.T) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
|
||||
defer cancel()
|
||||
testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
|
||||
testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")
|
||||
if err != nil {
|
||||
t.Fatalf("could not get test secrets from GCP: %s", err)
|
||||
}
|
||||
@@ -27,6 +27,9 @@ func TestPaypalOauth_FromChunk(t *testing.T) {
|
||||
inactiveSecret := testSecrets.MustGetField("PAYPALOAUTH_SECRET_INACTIVE")
|
||||
id := testSecrets.MustGetField("PAYPALOAUTH_CLIENTID")
|
||||
|
||||
newId := testSecrets.MustGetField("PAYPALOAUTH_NEW_INACTIVE_CLIENTID")
|
||||
newSecret := testSecrets.MustGetField("PAYPALOAUTH_NEW_INACTIVE_SECRET")
|
||||
|
||||
type args struct {
|
||||
ctx context.Context
|
||||
data []byte
|
||||
@@ -52,6 +55,10 @@ func TestPaypalOauth_FromChunk(t *testing.T) {
|
||||
DetectorType: detectorspb.DetectorType_PaypalOauth,
|
||||
Verified: true,
|
||||
},
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_PaypalOauth,
|
||||
Verified: false,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
},
|
||||
@@ -68,6 +75,30 @@ func TestPaypalOauth_FromChunk(t *testing.T) {
|
||||
DetectorType: detectorspb.DetectorType_PaypalOauth,
|
||||
Verified: false,
|
||||
},
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_PaypalOauth,
|
||||
Verified: false,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "new format, unverified",
|
||||
s: Scanner{},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a paypaloauth secret %s within %s but not valid", newSecret, newId)), // the secret would satisfy the regex but not pass validation
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_PaypalOauth,
|
||||
Verified: false,
|
||||
},
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_PaypalOauth,
|
||||
Verified: false,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
},
|
||||
|
||||
@@ -2,6 +2,7 @@ package pubnubpublishkey
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"strings"
|
||||
@@ -11,17 +12,19 @@ import (
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
|
||||
)
|
||||
|
||||
type Scanner struct{}
|
||||
type Scanner struct {
|
||||
client *http.Client
|
||||
}
|
||||
|
||||
// Ensure the Scanner satisfies the interface at compile time.
|
||||
var _ detectors.Detector = (*Scanner)(nil)
|
||||
|
||||
var (
|
||||
client = common.SaneHttpClient()
|
||||
defaultClient = common.SaneHttpClient()
|
||||
|
||||
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
|
||||
pubPat = regexp.MustCompile(`\b(pub-c-[0-9a-z]{8}-[0-9a-z]{4}-[a-z0-9]{4}-[a-z0-9]{4}-[a-z0-9]{12})\b`)
|
||||
subPat = regexp.MustCompile(`\b(sub-c-[0-9a-z]{8}-[a-z]{4}-[a-z0-9]{4}-[a-z0-9]{4}-[a-z0-9]{12})\b`)
|
||||
subPat = regexp.MustCompile(`\b(sub-c-[0-9a-z]{8}-[a-z0-9]{4}-[a-z0-9]{4}-[a-z0-9]{4}-[a-z0-9]{12})\b`)
|
||||
)
|
||||
|
||||
// Keywords are used for efficiently pre-filtering chunks.
|
||||
@@ -60,19 +63,31 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
client := s.client
|
||||
if client == nil {
|
||||
client = defaultClient
|
||||
}
|
||||
res, err := client.Do(req)
|
||||
if err == nil {
|
||||
defer res.Body.Close()
|
||||
if res.StatusCode >= 200 && res.StatusCode < 300 {
|
||||
s1.Verified = true
|
||||
} else if res.StatusCode == 400 || res.StatusCode == 403 {
|
||||
// 403 is suggested by the API docs (https://www.pubnub.com/docs/sdks/rest-api/send-signal-to-channel)
|
||||
// 400 is what actually seems to be coming back for invalid credentials
|
||||
// There's nothing to do here, because zero values of Result are what we want
|
||||
} else {
|
||||
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
|
||||
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
|
||||
continue
|
||||
}
|
||||
s1.VerificationError = fmt.Errorf("unexpected HTTP response status %d", res.StatusCode)
|
||||
}
|
||||
} else {
|
||||
s1.VerificationError = err
|
||||
}
|
||||
}
|
||||
// This function will check false positives for common test words, but also it will make sure the key
|
||||
// appears 'random' enough to be a real key.
|
||||
if !s1.Verified && detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
|
||||
continue
|
||||
}
|
||||
|
||||
results = append(results, s1)
|
||||
}
|
||||
|
||||
@@ -6,10 +6,11 @@ package pubnubpublishkey
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"github.com/google/go-cmp/cmp"
|
||||
"github.com/google/go-cmp/cmp/cmpopts"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/kylelemons/godebug/pretty"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
@@ -33,11 +34,12 @@ func TestPubNubPublishKey_FromChunk(t *testing.T) {
|
||||
verify bool
|
||||
}
|
||||
tests := []struct {
|
||||
name string
|
||||
s Scanner
|
||||
args args
|
||||
want []detectors.Result
|
||||
wantErr bool
|
||||
name string
|
||||
s Scanner
|
||||
args args
|
||||
want []detectors.Result
|
||||
wantErr bool
|
||||
wantVerificationErr bool
|
||||
}{
|
||||
{
|
||||
name: "found, verified",
|
||||
@@ -71,6 +73,40 @@ func TestPubNubPublishKey_FromChunk(t *testing.T) {
|
||||
},
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "found, would be verified but for timeout",
|
||||
s: Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a pubnub secret %s within pubnub %s", pub, sub)),
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_PubNubPublishKey,
|
||||
Verified: false,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
wantVerificationErr: true,
|
||||
},
|
||||
{
|
||||
name: "found, valid but unexpected api response",
|
||||
s: Scanner{client: common.ConstantResponseHttpClient(500, "")},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a pubnub secret %s within pubnub %s", pub, sub)),
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_PubNubPublishKey,
|
||||
Verified: false,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
wantVerificationErr: true,
|
||||
},
|
||||
{
|
||||
name: "not found",
|
||||
s: Scanner{},
|
||||
@@ -85,8 +121,7 @@ func TestPubNubPublishKey_FromChunk(t *testing.T) {
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
s := Scanner{}
|
||||
got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
|
||||
got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
|
||||
if (err != nil) != tt.wantErr {
|
||||
t.Errorf("PubNubPublishKey.FromData() error = %v, wantErr %v", err, tt.wantErr)
|
||||
return
|
||||
@@ -95,9 +130,12 @@ func TestPubNubPublishKey_FromChunk(t *testing.T) {
|
||||
if len(got[i].Raw) == 0 {
|
||||
t.Fatalf("no raw secret present: \n %+v", got[i])
|
||||
}
|
||||
got[i].Raw = nil
|
||||
if (got[i].VerificationError != nil) != tt.wantVerificationErr {
|
||||
t.Fatalf(" wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError)
|
||||
}
|
||||
}
|
||||
if diff := pretty.Compare(got, tt.want); diff != "" {
|
||||
opts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "VerificationError")
|
||||
if diff := cmp.Diff(got, tt.want, opts); diff != "" {
|
||||
t.Errorf("PubNubPublishKey.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
|
||||
}
|
||||
})
|
||||
|
||||
@@ -20,7 +20,7 @@ var (
|
||||
client = common.SaneHttpClient()
|
||||
|
||||
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
|
||||
keyPat = regexp.MustCompile(`\b(sub-c-[0-9a-z]{8}-[a-z]{4}-[a-z0-9]{4}-[a-z0-9]{4}-[a-z0-9]{12})\b`)
|
||||
keyPat = regexp.MustCompile(`\b(sub-c-[0-9a-z]{8}-[a-z0-9]{4}-[a-z0-9]{4}-[a-z0-9]{4}-[a-z0-9]{12})\b`)
|
||||
)
|
||||
|
||||
// Keywords are used for efficiently pre-filtering chunks.
|
||||
|
||||
@@ -7,6 +7,7 @@ import (
|
||||
"net/http"
|
||||
"regexp"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
@@ -19,7 +20,7 @@ type Scanner struct{}
|
||||
var _ detectors.Detector = (*Scanner)(nil)
|
||||
|
||||
var (
|
||||
client = common.SaneHttpClientTimeOut(10)
|
||||
client = common.SaneHttpClientTimeOut(10 * time.Second)
|
||||
|
||||
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
|
||||
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"scrapersite"}) + `\b([a-zA-Z0-9]{45})\b`)
|
||||
|
||||
@@ -7,6 +7,7 @@ import (
|
||||
"net/http"
|
||||
"regexp"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
@@ -19,7 +20,7 @@ type Scanner struct{}
|
||||
var _ detectors.Detector = (*Scanner)(nil)
|
||||
|
||||
var (
|
||||
client = common.SaneHttpClientTimeOut(10)
|
||||
client = common.SaneHttpClientTimeOut(10 * time.Second)
|
||||
|
||||
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
|
||||
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"screenshotlayer"}) + `\b([a-zA-Z0-9_]{32})\b`)
|
||||
|
||||
@@ -2,6 +2,8 @@ package shodankey
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"io"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"strings"
|
||||
@@ -48,31 +50,58 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
}
|
||||
|
||||
if verify {
|
||||
req, err := http.NewRequestWithContext(ctx, "GET", "https://api.shodan.io/shodan/host/count?key="+resMatch+"&query=port:22&facets=org,os", nil)
|
||||
if err != nil {
|
||||
s1.Verified = verifyToken(ctx, client, resMatch)
|
||||
if !s1.Verified && detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
|
||||
continue
|
||||
}
|
||||
|
||||
res, err := client.Do(req)
|
||||
if err == nil {
|
||||
defer res.Body.Close()
|
||||
if res.StatusCode >= 200 && res.StatusCode < 300 {
|
||||
s1.Verified = true
|
||||
} else {
|
||||
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
|
||||
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
|
||||
continue
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
results = append(results, s1)
|
||||
}
|
||||
|
||||
return results, nil
|
||||
}
|
||||
|
||||
type shodanInfoRes struct {
|
||||
ScanCredits int `json:"scan_credits"`
|
||||
UsageLimits struct {
|
||||
ScanCredits int `json:"scan_credits"`
|
||||
QueryCredits int `json:"query_credits"`
|
||||
MonitoredIps int `json:"monitored_ips"`
|
||||
} `json:"usage_limits"`
|
||||
Plan string `json:"plan"`
|
||||
HTTPS bool `json:"https"`
|
||||
Unlocked bool `json:"unlocked"`
|
||||
QueryCredits int `json:"query_credits"`
|
||||
MonitoredIps int `json:"monitored_ips"`
|
||||
UnlockedLeft int `json:"unlocked_left"`
|
||||
Telnet bool `json:"telnet"`
|
||||
}
|
||||
|
||||
func verifyToken(ctx context.Context, client *http.Client, token string) bool {
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, "https://api.shodan.io/api-info?key="+token, nil)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
|
||||
res, err := client.Do(req)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
defer res.Body.Close()
|
||||
|
||||
if res.StatusCode < 200 || res.StatusCode >= 300 {
|
||||
return false
|
||||
}
|
||||
|
||||
bytes, err := io.ReadAll(res.Body)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
|
||||
var info shodanInfoRes
|
||||
return json.Unmarshal(bytes, &info) == nil
|
||||
}
|
||||
|
||||
func (s Scanner) Type() detectorspb.DetectorType {
|
||||
return detectorspb.DetectorType_ShodanKey
|
||||
}
|
||||
|
||||
@@ -10,9 +10,9 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/kylelemons/godebug/pretty"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
|
||||
)
|
||||
|
||||
|
||||
@@ -20,7 +20,7 @@ var (
|
||||
client = common.SaneHttpClient()
|
||||
|
||||
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
|
||||
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"timezoneapi"}) + `\b([a-zA-Z0-9]{20})\b`)
|
||||
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"timezoneapi"}) + `\b([a-zA-Z]{20})\b`)
|
||||
)
|
||||
|
||||
// Keywords are used for efficiently pre-filtering chunks.
|
||||
@@ -54,8 +54,12 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
|
||||
res, err := client.Do(req)
|
||||
if err == nil {
|
||||
verifiedBodyResponse, err := common.ResponseContainsSubstring(res.Body, "date")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer res.Body.Close()
|
||||
if res.StatusCode >= 200 && res.StatusCode < 300 {
|
||||
if res.StatusCode >= 200 && res.StatusCode < 300 && verifiedBodyResponse {
|
||||
s1.Verified = true
|
||||
} else {
|
||||
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
|
||||
|
||||
@@ -0,0 +1,108 @@
|
||||
package trufflehogenterprise
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
|
||||
)
|
||||
|
||||
type Scanner struct{}
|
||||
|
||||
// Ensure the Scanner satisfies the interface at compile time.
|
||||
var _ detectors.Detector = (*Scanner)(nil)
|
||||
|
||||
var (
|
||||
client = common.SaneHttpClient()
|
||||
|
||||
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
|
||||
keyPat = regexp.MustCompile(`\bthog-key-[0-9a-f]{16}\b`)
|
||||
secretPat = regexp.MustCompile(`\bthog-secret-[0-9a-f]{32}\b`)
|
||||
hostnamePat = regexp.MustCompile(`\b[a-z]+-[a-z]+-[a-z]+\.[a-z][0-9]\.[a-z]+\.trufflehog\.org\b`)
|
||||
)
|
||||
|
||||
// Keywords are used for efficiently pre-filtering chunks.
|
||||
// Use identifiers in the secret preferably, or the provider name.
|
||||
func (s Scanner) Keywords() []string {
|
||||
return []string{"thog"}
|
||||
}
|
||||
|
||||
// FromData will find and optionally verify Trufflehog Enterprise secrets in a given set of bytes.
|
||||
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
|
||||
dataStr := string(data)
|
||||
|
||||
keyMatches := keyPat.FindAllStringSubmatch(dataStr, -1)
|
||||
secretMatches := secretPat.FindAllStringSubmatch(dataStr, -1)
|
||||
hostnameMatches := hostnamePat.FindAllStringSubmatch(dataStr, -1)
|
||||
|
||||
for _, keyMatch := range keyMatches {
|
||||
if len(keyMatch) != 1 {
|
||||
continue
|
||||
}
|
||||
resKeyMatch := strings.TrimSpace(keyMatch[0])
|
||||
for _, secretMatch := range secretMatches {
|
||||
|
||||
if len(secretMatch) != 1 {
|
||||
continue
|
||||
}
|
||||
resSecretMatch := strings.TrimSpace(secretMatch[0])
|
||||
|
||||
for _, hostnameMatch := range hostnameMatches {
|
||||
if len(hostnameMatch) != 1 {
|
||||
continue
|
||||
}
|
||||
|
||||
resHostnameMatch := strings.TrimSpace(hostnameMatch[0])
|
||||
|
||||
s1 := detectors.Result{
|
||||
DetectorType: detectorspb.DetectorType_TrufflehogEnterprise,
|
||||
Raw: []byte(resKeyMatch),
|
||||
}
|
||||
|
||||
if verify {
|
||||
endpoint := fmt.Sprintf("https://%s/api/v1/sources", resHostnameMatch)
|
||||
req, err := http.NewRequestWithContext(ctx, "GET", endpoint, nil)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
|
||||
req.Header.Add("Accept", "application/vnd.trufflehogenterprise+json; version=3")
|
||||
req.Header.Add("X-Thog-Secret", resSecretMatch)
|
||||
req.Header.Add("X-Thog-Key", resKeyMatch)
|
||||
|
||||
res, err := client.Do(req)
|
||||
if err == nil {
|
||||
verifiedBodyResponse, err := common.ResponseContainsSubstring(res.Body, "data")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
defer res.Body.Close()
|
||||
|
||||
if res.StatusCode >= 200 && res.StatusCode < 300 && verifiedBodyResponse {
|
||||
s1.Verified = true
|
||||
} else {
|
||||
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
|
||||
if detectors.IsKnownFalsePositive(resSecretMatch, detectors.DefaultFalsePositives, true) {
|
||||
continue
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
results = append(results, s1)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return results, nil
|
||||
}
|
||||
|
||||
func (s Scanner) Type() detectorspb.DetectorType {
|
||||
return detectorspb.DetectorType_TrufflehogEnterprise
|
||||
}
|
||||
@@ -0,0 +1,121 @@
|
||||
//go:build detectors
|
||||
// +build detectors
|
||||
|
||||
package trufflehogenterprise
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/kylelemons/godebug/pretty"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
|
||||
)
|
||||
|
||||
func TestTrufflehogenterprise_FromChunk(t *testing.T) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
|
||||
defer cancel()
|
||||
testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")
|
||||
if err != nil {
|
||||
t.Fatalf("could not get test secrets from GCP: %s", err)
|
||||
}
|
||||
host := testSecrets.MustGetField("THOG_HOSTNAME")
|
||||
key := testSecrets.MustGetField("THOG_WEB_KEY")
|
||||
secret := testSecrets.MustGetField("THOG_WEB_SECRET")
|
||||
inactiveSecret := testSecrets.MustGetField("THOG_WEB_INACTIVE_SECRET")
|
||||
|
||||
type args struct {
|
||||
ctx context.Context
|
||||
data []byte
|
||||
verify bool
|
||||
}
|
||||
tests := []struct {
|
||||
name string
|
||||
s Scanner
|
||||
args args
|
||||
want []detectors.Result
|
||||
wantErr bool
|
||||
}{
|
||||
{
|
||||
name: "found, verified",
|
||||
s: Scanner{},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a thog secret %s for %s with key %s within", secret, host, key)),
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_TrufflehogEnterprise,
|
||||
Verified: true,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "found, unverified",
|
||||
s: Scanner{},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a trufflehogenterprise secret %s for %s with key %s within but not valid", inactiveSecret, host, key)), // the secret would satisfy the regex but not pass validation
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_TrufflehogEnterprise,
|
||||
Verified: false,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "not found",
|
||||
s: Scanner{},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte("You cannot find the secret within"),
|
||||
verify: true,
|
||||
},
|
||||
want: nil,
|
||||
wantErr: false,
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
s := Scanner{}
|
||||
got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
|
||||
if (err != nil) != tt.wantErr {
|
||||
t.Errorf("Trufflehogenterprise.FromData() error = %v, wantErr %v", err, tt.wantErr)
|
||||
return
|
||||
}
|
||||
for i := range got {
|
||||
if len(got[i].Raw) == 0 {
|
||||
t.Fatalf("no raw secret present: \n %+v", got[i])
|
||||
}
|
||||
got[i].Raw = nil
|
||||
}
|
||||
if diff := pretty.Compare(got, tt.want); diff != "" {
|
||||
t.Errorf("Trufflehogenterprise.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkFromData(benchmark *testing.B) {
|
||||
ctx := context.Background()
|
||||
s := Scanner{}
|
||||
for name, data := range detectors.MustGetBenchmarkData() {
|
||||
benchmark.Run(name, func(b *testing.B) {
|
||||
for n := 0; n < b.N; n++ {
|
||||
_, err := s.FromData(ctx, false, data)
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -3,9 +3,7 @@ package twilio
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
@@ -53,14 +51,8 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
}
|
||||
|
||||
if verify {
|
||||
|
||||
form := url.Values{}
|
||||
form.Add("FriendlyName", "MyServiceName")
|
||||
req, err := http.NewRequestWithContext(
|
||||
ctx, "POST", "https://verify.twilio.com/v2/Services",
|
||||
|
||||
strings.NewReader(form.Encode()),
|
||||
)
|
||||
ctx, "GET", "https://verify.twilio.com/v2/Services", nil)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
|
||||
@@ -53,7 +53,8 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
password := match[1]
|
||||
|
||||
// Skip findings where the password only has "*" characters, this is a redacted password
|
||||
if strings.Trim(password, "*") == "" {
|
||||
// Also include the url encoded "*" characters: "%2A"
|
||||
if strings.Trim(password, "*") == "" || strings.Trim(password, "%2A") == "" {
|
||||
continue
|
||||
}
|
||||
|
||||
|
||||
@@ -31,14 +31,14 @@ func TestURI_FromChunk(t *testing.T) {
|
||||
s: Scanner{},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a uri secret %s within", "https://user:pass@www.httpwatch.com/httpgallery/authentication/authenticatedimage/default.aspx")),
|
||||
data: []byte(fmt.Sprintf("You can find a uri secret %s within", "https://user:pass@httpwatch.com/httpgallery/authentication/authenticatedimage/default.aspx")),
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_URI,
|
||||
Verified: false,
|
||||
Redacted: "https://user:********@www.httpwatch.com",
|
||||
Redacted: "https://user:********@httpwatch.com",
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
@@ -87,6 +87,17 @@ func TestURI_FromChunk(t *testing.T) {
|
||||
},
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "nothing found, password was redacted two different ways",
|
||||
s: Scanner{},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("Both %s and %s have been redacted within", "https://httpwatch::********@www.httpwatch.com/httpgallery/authentication/authenticatedimage/default.aspx?foo=bar", "https://httpwatch::%2A%2A%2A%2A%2A%2A@www.httpwatch.com/httpgallery/authentication/authenticatedimage/default.aspx?foo=bar")),
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{},
|
||||
wantErr: false,
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
|
||||
@@ -1,10 +1,7 @@
|
||||
package virustotal
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"io"
|
||||
"mime/multipart"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"strings"
|
||||
@@ -50,43 +47,36 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
}
|
||||
|
||||
if verify {
|
||||
body := &bytes.Buffer{}
|
||||
writer := multipart.NewWriter(body)
|
||||
fw, err := writer.CreateFormField("url")
|
||||
if err != nil {
|
||||
s1.Verified = verifyToken(ctx, client, resMatch)
|
||||
if !s1.Verified && detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
|
||||
continue
|
||||
}
|
||||
_, err = io.Copy(fw, strings.NewReader("https://www.amazon.com"))
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
writer.Close()
|
||||
req, err := http.NewRequestWithContext(ctx, "POST", "https://www.virustotal.com/api/v3/urls", bytes.NewReader(body.Bytes()))
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
req.Header.Add("Content-Type", writer.FormDataContentType())
|
||||
req.Header.Add("x-apikey", resMatch)
|
||||
res, err := client.Do(req)
|
||||
if err == nil {
|
||||
defer res.Body.Close()
|
||||
if res.StatusCode >= 200 && res.StatusCode < 300 {
|
||||
s1.Verified = true
|
||||
} else {
|
||||
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
|
||||
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
|
||||
continue
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
results = append(results, s1)
|
||||
}
|
||||
|
||||
return results, nil
|
||||
}
|
||||
|
||||
func verifyToken(ctx context.Context, client *http.Client, token string) bool {
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, "https://www.virustotal.com/api/v3/metadata", nil)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
req.Header.Add("x-apikey", token)
|
||||
|
||||
res, err := client.Do(req)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
defer res.Body.Close()
|
||||
|
||||
if res.StatusCode < 200 || res.StatusCode >= 300 {
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func (s Scanner) Type() detectorspb.DetectorType {
|
||||
return detectorspb.DetectorType_VirusTotal
|
||||
}
|
||||
|
||||
@@ -10,9 +10,9 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/kylelemons/godebug/pretty"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
|
||||
)
|
||||
|
||||
|
||||
@@ -6,6 +6,7 @@ import (
|
||||
"net/http"
|
||||
"regexp"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
@@ -18,7 +19,7 @@ type Scanner struct{}
|
||||
var _ detectors.Detector = (*Scanner)(nil)
|
||||
|
||||
var (
|
||||
client = common.SaneHttpClientTimeOut(5)
|
||||
client = common.SaneHttpClientTimeOut(5 * time.Second)
|
||||
|
||||
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
|
||||
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"zenserp"}) + `\b([0-9a-z-]{36})\b`)
|
||||
|
||||
+12
-20
@@ -1,16 +1,14 @@
|
||||
package engine
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"runtime"
|
||||
|
||||
"github.com/go-errors/errors"
|
||||
"google.golang.org/protobuf/proto"
|
||||
"google.golang.org/protobuf/types/known/anypb"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/context"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/sources/circleci"
|
||||
)
|
||||
|
||||
@@ -29,23 +27,17 @@ func (e *Engine) ScanCircleCI(ctx context.Context, token string) error {
|
||||
return err
|
||||
}
|
||||
|
||||
circleSource := circleci.Source{}
|
||||
ctx = context.WithValues(ctx,
|
||||
"source_type", circleSource.Type().String(),
|
||||
"source_name", "Circle CI",
|
||||
)
|
||||
err = circleSource.Init(ctx, "trufflehog - Circle CI", 0, int64(sourcespb.SourceType_SOURCE_TYPE_CIRCLECI), true, &conn, runtime.NumCPU())
|
||||
handle, err := e.sourceManager.Enroll(ctx, "trufflehog - Circle CI", new(circleci.Source).Type(),
|
||||
func(ctx context.Context, jobID, sourceID int64) (sources.Source, error) {
|
||||
circleSource := circleci.Source{}
|
||||
if err := circleSource.Init(ctx, "trufflehog - Circle CI", jobID, sourceID, true, &conn, runtime.NumCPU()); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &circleSource, nil
|
||||
})
|
||||
if err != nil {
|
||||
return errors.WrapPrefix(err, "failed to init Circle CI source", 0)
|
||||
return err
|
||||
}
|
||||
|
||||
e.sourcesWg.Go(func() error {
|
||||
defer common.RecoverWithExit(ctx)
|
||||
err := circleSource.Chunks(ctx, e.ChunksChan())
|
||||
if err != nil {
|
||||
return fmt.Errorf("error scanning CircleCI: %w", err)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
return nil
|
||||
_, err = e.sourceManager.ScheduleRun(ctx, handle)
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -1,6 +1,8 @@
|
||||
package engine
|
||||
|
||||
import (
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/envoyapikey"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/abbysale"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/abuseipdb"
|
||||
@@ -159,6 +161,7 @@ import (
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/convertkit"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/convier"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/copper"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/couchbase"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/countrylayer"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/courier"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/coveralls"
|
||||
@@ -198,6 +201,7 @@ import (
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/disqus"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/ditto"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/dnscheck"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/dockerhub"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/docparser"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/documo"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/docusign"
|
||||
@@ -552,6 +556,7 @@ import (
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/satismeterwritekey"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/saucelabs"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/scalewaykey"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/github_oauth2"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/scalr"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/scrapeowl"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/scraperapi"
|
||||
@@ -1523,6 +1528,10 @@ func DefaultDetectors() []detectors.Detector {
|
||||
prefect.Scanner{},
|
||||
buildkitev2.Scanner{},
|
||||
opsgenie.Scanner{},
|
||||
dockerhub.Scanner{},
|
||||
couchbase.Scanner{},
|
||||
envoyapikey.Scanner{},
|
||||
github_oauth2.Scanner{},
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
+12
-22
@@ -1,38 +1,28 @@
|
||||
package engine
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"runtime"
|
||||
|
||||
"github.com/go-errors/errors"
|
||||
"google.golang.org/protobuf/types/known/anypb"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/context"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/sources/docker"
|
||||
)
|
||||
|
||||
// ScanDocker scans a given docker connection.
|
||||
func (e *Engine) ScanDocker(ctx context.Context, conn *anypb.Any) error {
|
||||
dockerSource := docker.Source{}
|
||||
|
||||
ctx = context.WithValues(ctx,
|
||||
"source_type", dockerSource.Type().String(),
|
||||
"source_name", "docker",
|
||||
)
|
||||
err := dockerSource.Init(ctx, "trufflehog - docker", 0, int64(sourcespb.SourceType_SOURCE_TYPE_DOCKER), true, conn, runtime.NumCPU())
|
||||
handle, err := e.sourceManager.Enroll(ctx, "trufflehog - docker", new(docker.Source).Type(),
|
||||
func(ctx context.Context, jobID, sourceID int64) (sources.Source, error) {
|
||||
dockerSource := docker.Source{}
|
||||
if err := dockerSource.Init(ctx, "trufflehog - docker", jobID, sourceID, true, conn, runtime.NumCPU()); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &dockerSource, nil
|
||||
})
|
||||
if err != nil {
|
||||
return errors.WrapPrefix(err, "could not init docker source", 0)
|
||||
return err
|
||||
}
|
||||
|
||||
e.sourcesWg.Go(func() error {
|
||||
defer common.RecoverWithExit(ctx)
|
||||
err := dockerSource.Chunks(ctx, e.ChunksChan())
|
||||
if err != nil {
|
||||
return fmt.Errorf("error scanning docker image: %w", err)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
return nil
|
||||
_, err = e.sourceManager.ScheduleRun(ctx, handle)
|
||||
return err
|
||||
}
|
||||
|
||||
+379
-186
@@ -10,8 +10,8 @@ import (
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
ahocorasick "github.com/petar-dambovaliev/aho-corasick"
|
||||
"golang.org/x/sync/errgroup"
|
||||
ahocorasick "github.com/BobuSumisu/aho-corasick"
|
||||
lru "github.com/hashicorp/golang-lru"
|
||||
"google.golang.org/protobuf/proto"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
@@ -19,36 +19,80 @@ import (
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/context"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/decoders"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/output"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/source_metadatapb"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
|
||||
)
|
||||
|
||||
type Engine struct {
|
||||
concurrency int
|
||||
chunks chan *sources.Chunk
|
||||
results chan detectors.ResultWithMetadata
|
||||
decoders []decoders.Decoder
|
||||
detectors map[bool][]detectors.Detector
|
||||
chunksScanned uint64
|
||||
bytesScanned uint64
|
||||
// Metrics for the scan engine for external consumption.
|
||||
type Metrics struct {
|
||||
BytesScanned uint64
|
||||
ChunksScanned uint64
|
||||
VerifiedSecretsFound uint64
|
||||
UnverifiedSecretsFound uint64
|
||||
AvgDetectorTime map[string]time.Duration
|
||||
|
||||
scanStartTime time.Time
|
||||
ScanDuration time.Duration
|
||||
}
|
||||
|
||||
// runtimeMetrics for the scan engine for internal use by the engine.
|
||||
type runtimeMetrics struct {
|
||||
mu sync.RWMutex
|
||||
Metrics
|
||||
detectorAvgTime sync.Map
|
||||
sourcesWg *errgroup.Group
|
||||
workersWg sync.WaitGroup
|
||||
}
|
||||
|
||||
// Printer is used to format found results and output them to the user. Ex JSON, plain text, etc.
|
||||
// Please note printer implementations SHOULD BE thread safe.
|
||||
type Printer interface {
|
||||
Print(ctx context.Context, r *detectors.ResultWithMetadata) error
|
||||
}
|
||||
|
||||
type Engine struct {
|
||||
// CLI flags.
|
||||
concurrency uint8
|
||||
decoders []decoders.Decoder
|
||||
detectors map[bool][]detectors.Detector
|
||||
// filterUnverified is used to reduce the number of unverified results.
|
||||
// If there are multiple unverified results for the same chunk for the same detector,
|
||||
// only the first one will be kept.
|
||||
filterUnverified bool
|
||||
filterUnverified bool
|
||||
onlyVerified bool
|
||||
printAvgDetectorTime bool
|
||||
|
||||
// prefilter is a ahocorasick struct used for doing efficient string
|
||||
// matching given a set of words (keywords from the rules in the config)
|
||||
prefilter ahocorasick.AhoCorasick
|
||||
prefilter ahocorasick.Trie
|
||||
|
||||
// Engine synchronization primitives.
|
||||
sourceManager *sources.SourceManager
|
||||
results chan detectors.ResultWithMetadata
|
||||
detectableChunksChan chan detectableChunk
|
||||
workersWg sync.WaitGroup
|
||||
wgDetectorWorkers sync.WaitGroup
|
||||
WgNotifier sync.WaitGroup
|
||||
|
||||
// Runtime information.
|
||||
metrics runtimeMetrics
|
||||
// numFoundResults is used to keep track of the number of results found.
|
||||
numFoundResults uint32
|
||||
|
||||
// printer provides a method for formatting and outputting search results.
|
||||
// The specific implementation (e.g., JSON, plain text)
|
||||
// should be set during initialization based on user preference or program requirements.
|
||||
printer Printer
|
||||
|
||||
// dedupeCache is used to deduplicate results by comparing the
|
||||
// detector type, raw result, and source metadata
|
||||
dedupeCache *lru.Cache
|
||||
}
|
||||
|
||||
type EngineOption func(*Engine)
|
||||
|
||||
func WithConcurrency(concurrency int) EngineOption {
|
||||
func WithConcurrency(concurrency uint8) EngineOption {
|
||||
return func(e *Engine) {
|
||||
e.concurrency = concurrency
|
||||
}
|
||||
@@ -83,6 +127,26 @@ func WithFilterUnverified(filter bool) EngineOption {
|
||||
}
|
||||
}
|
||||
|
||||
// WithOnlyVerified sets the onlyVerified flag on the engine. If set to true,
|
||||
// the engine will only print verified results.
|
||||
func WithOnlyVerified(onlyVerified bool) EngineOption {
|
||||
return func(e *Engine) {
|
||||
e.onlyVerified = onlyVerified
|
||||
}
|
||||
}
|
||||
|
||||
// WithPrintAvgDetectorTime sets the printAvgDetectorTime flag on the engine. If set to
|
||||
// true, the engine will print the average time taken by each detector.
|
||||
// This option allows us to measure the time taken for each detector ONLY if
|
||||
// the engine is configured to print the results.
|
||||
// Calculating the average time taken by each detector is an expensive operation
|
||||
// and should be avoided unless specified by the user.
|
||||
func WithPrintAvgDetectorTime(printAvgDetectorTime bool) EngineOption {
|
||||
return func(e *Engine) {
|
||||
e.printAvgDetectorTime = printAvgDetectorTime
|
||||
}
|
||||
}
|
||||
|
||||
// WithFilterDetectors applies a filter to the configured list of detectors. If
|
||||
// the filterFunc returns true, the detector will be included for scanning.
|
||||
// This option applies to the existing list of detectors configured, so the
|
||||
@@ -98,6 +162,13 @@ func WithFilterDetectors(filterFunc func(detectors.Detector) bool) EngineOption
|
||||
}
|
||||
}
|
||||
|
||||
// WithPrinter sets the Printer on the engine.
|
||||
func WithPrinter(printer Printer) EngineOption {
|
||||
return func(e *Engine) {
|
||||
e.printer = printer
|
||||
}
|
||||
}
|
||||
|
||||
func filterDetectors(filterFunc func(detectors.Detector) bool, input []detectors.Detector) []detectors.Detector {
|
||||
var output []detectors.Detector
|
||||
for _, detector := range input {
|
||||
@@ -108,11 +179,103 @@ func filterDetectors(filterFunc func(detectors.Detector) bool, input []detectors
|
||||
return output
|
||||
}
|
||||
|
||||
func Start(ctx context.Context, options ...EngineOption) *Engine {
|
||||
// HasFoundResults returns true if any results are found.
|
||||
func (e *Engine) HasFoundResults() bool {
|
||||
return atomic.LoadUint32(&e.numFoundResults) > 0
|
||||
}
|
||||
|
||||
// GetMetrics returns a copy of Metrics.
|
||||
// It's safe for concurrent use, and the caller can't modify the original data.
|
||||
func (e *Engine) GetMetrics() Metrics {
|
||||
e.metrics.mu.RLock()
|
||||
defer e.metrics.mu.RUnlock()
|
||||
|
||||
result := e.metrics.Metrics
|
||||
result.AvgDetectorTime = make(map[string]time.Duration, len(e.metrics.AvgDetectorTime))
|
||||
|
||||
for detectorName, durations := range e.DetectorAvgTime() {
|
||||
var total time.Duration
|
||||
for _, d := range durations {
|
||||
total += d
|
||||
}
|
||||
avgDuration := total / time.Duration(len(durations))
|
||||
result.AvgDetectorTime[detectorName] = avgDuration
|
||||
}
|
||||
|
||||
result.ScanDuration = e.metrics.getScanDuration()
|
||||
|
||||
return result
|
||||
}
|
||||
|
||||
// GetDetectorsMetrics returns a copy of the average time taken by each detector.
|
||||
func (e *Engine) GetDetectorsMetrics() map[string]time.Duration {
|
||||
e.metrics.mu.RLock()
|
||||
defer e.metrics.mu.RUnlock()
|
||||
|
||||
result := make(map[string]time.Duration, len(DefaultDetectors()))
|
||||
for detectorName, durations := range e.DetectorAvgTime() {
|
||||
var total time.Duration
|
||||
for _, d := range durations {
|
||||
total += d
|
||||
}
|
||||
avgDuration := total / time.Duration(len(durations))
|
||||
result[detectorName] = avgDuration
|
||||
}
|
||||
|
||||
return result
|
||||
}
|
||||
|
||||
// getScanDuration returns the duration of the scan.
|
||||
// If the scan is still running, it returns the time since the scan started.
|
||||
func (m *Metrics) getScanDuration() time.Duration {
|
||||
if m.ScanDuration == 0 {
|
||||
return time.Since(m.scanStartTime)
|
||||
}
|
||||
|
||||
return m.ScanDuration
|
||||
}
|
||||
|
||||
// DetectorAvgTime returns the average time taken by each detector.
|
||||
func (e *Engine) DetectorAvgTime() map[string][]time.Duration {
|
||||
logger := context.Background().Logger()
|
||||
avgTime := map[string][]time.Duration{}
|
||||
e.metrics.detectorAvgTime.Range(func(k, v interface{}) bool {
|
||||
key, ok := k.(string)
|
||||
if !ok {
|
||||
logger.Info("expected detectorAvgTime key to be a string")
|
||||
return true
|
||||
}
|
||||
|
||||
value, ok := v.([]time.Duration)
|
||||
if !ok {
|
||||
logger.Info("expected detectorAvgTime value to be []time.Duration")
|
||||
return true
|
||||
}
|
||||
avgTime[key] = value
|
||||
return true
|
||||
})
|
||||
return avgTime
|
||||
}
|
||||
|
||||
// Start the engine with options.
|
||||
func Start(ctx context.Context, options ...EngineOption) (*Engine, error) {
|
||||
const (
|
||||
defaultChannelBuffer = 1
|
||||
// TODO (ahrav): Determine the optimal cache size.
|
||||
cacheSize = 512 // number of entries in the LRU cache
|
||||
)
|
||||
|
||||
cache, err := lru.New(cacheSize)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to initialize LRU cache: %w", err)
|
||||
}
|
||||
|
||||
e := &Engine{
|
||||
chunks: make(chan *sources.Chunk),
|
||||
results: make(chan detectors.ResultWithMetadata),
|
||||
detectorAvgTime: sync.Map{},
|
||||
detectableChunksChan: make(chan detectableChunk, defaultChannelBuffer),
|
||||
results: make(chan detectors.ResultWithMetadata, defaultChannelBuffer),
|
||||
dedupeCache: cache,
|
||||
printer: new(output.PlainPrinter), // default printer
|
||||
metrics: runtimeMetrics{Metrics: Metrics{scanStartTime: time.Now()}},
|
||||
}
|
||||
|
||||
for _, option := range options {
|
||||
@@ -123,14 +286,15 @@ func Start(ctx context.Context, options ...EngineOption) *Engine {
|
||||
if e.concurrency == 0 {
|
||||
numCPU := runtime.NumCPU()
|
||||
ctx.Logger().Info("No concurrency specified, defaulting to max", "cpu", numCPU)
|
||||
e.concurrency = numCPU
|
||||
e.concurrency = uint8(numCPU)
|
||||
}
|
||||
ctx.Logger().V(2).Info("engine started", "workers", e.concurrency)
|
||||
ctx.Logger().V(3).Info("engine started", "workers", e.concurrency)
|
||||
|
||||
sourcesWg, egCtx := errgroup.WithContext(ctx)
|
||||
sourcesWg.SetLimit(e.concurrency)
|
||||
e.sourcesWg = sourcesWg
|
||||
ctx.SetParent(egCtx)
|
||||
// Create SourceManager.
|
||||
e.sourceManager = sources.NewManager(
|
||||
sources.WithConcurrentSources(int(e.concurrency)),
|
||||
sources.WithConcurrentUnits(int(e.concurrency)),
|
||||
)
|
||||
|
||||
if len(e.decoders) == 0 {
|
||||
e.decoders = decoders.DefaultDecoders()
|
||||
@@ -146,21 +310,19 @@ func Start(ctx context.Context, options ...EngineOption) *Engine {
|
||||
// on keywords
|
||||
keywords := []string{}
|
||||
for _, d := range e.detectors[false] {
|
||||
keywords = append(keywords, d.Keywords()...)
|
||||
for _, kw := range d.Keywords() {
|
||||
keywords = append(keywords, strings.ToLower(kw))
|
||||
}
|
||||
}
|
||||
for _, d := range e.detectors[true] {
|
||||
keywords = append(keywords, d.Keywords()...)
|
||||
for _, kw := range d.Keywords() {
|
||||
keywords = append(keywords, strings.ToLower(kw))
|
||||
}
|
||||
}
|
||||
builder := ahocorasick.NewAhoCorasickBuilder(ahocorasick.Opts{
|
||||
AsciiCaseInsensitive: true,
|
||||
MatchOnlyWholeWords: false,
|
||||
MatchKind: ahocorasick.LeftMostLongestMatch,
|
||||
DFA: true,
|
||||
})
|
||||
e.prefilter = builder.Build(keywords)
|
||||
e.prefilter = *ahocorasick.NewTrieBuilder().AddStrings(keywords).Build()
|
||||
|
||||
ctx.Logger().Info("loaded decoders", "count", len(e.decoders))
|
||||
ctx.Logger().Info("loaded detectors",
|
||||
ctx.Logger().V(3).Info("loaded decoders", "count", len(e.decoders))
|
||||
ctx.Logger().V(3).Info("loaded detectors",
|
||||
"total", len(e.detectors[true])+len(e.detectors[false]),
|
||||
"verification_enabled", len(e.detectors[true]),
|
||||
"verification_disabled", len(e.detectors[false]),
|
||||
@@ -181,102 +343,93 @@ func Start(ctx context.Context, options ...EngineOption) *Engine {
|
||||
}
|
||||
}
|
||||
|
||||
// Start the workers.
|
||||
for i := 0; i < e.concurrency; i++ {
|
||||
ctx.Logger().V(2).Info("starting scanner workers", "count", e.concurrency)
|
||||
// Run the Secret scanner workers and Notifier pipelines.
|
||||
for worker := uint64(0); worker < uint64(e.concurrency); worker++ {
|
||||
e.workersWg.Add(1)
|
||||
go func() {
|
||||
defer common.RecoverWithExit(ctx)
|
||||
ctx := context.WithValue(ctx, "secret_worker_id", common.RandomID(5))
|
||||
defer common.Recover(ctx)
|
||||
defer e.workersWg.Done()
|
||||
e.detectorWorker(ctx)
|
||||
}()
|
||||
}
|
||||
|
||||
return e
|
||||
const detectorWorkerMultiplier = 50
|
||||
ctx.Logger().V(2).Info("starting detector workers", "count", e.concurrency*detectorWorkerMultiplier)
|
||||
for worker := uint64(0); worker < uint64(e.concurrency*detectorWorkerMultiplier); worker++ {
|
||||
e.wgDetectorWorkers.Add(1)
|
||||
go func() {
|
||||
ctx := context.WithValue(ctx, "detector_worker_id", common.RandomID(5))
|
||||
defer common.Recover(ctx)
|
||||
defer e.wgDetectorWorkers.Done()
|
||||
e.detectChunks(ctx)
|
||||
}()
|
||||
}
|
||||
|
||||
// We want 1/4th of the notifier workers as the number of scanner workers.
|
||||
const notifierWorkerRatio = 4
|
||||
maxNotifierWorkers := 1
|
||||
if numWorkers := e.concurrency / notifierWorkerRatio; numWorkers > 0 {
|
||||
maxNotifierWorkers = int(numWorkers)
|
||||
}
|
||||
ctx.Logger().V(2).Info("starting notifier workers", "count", maxNotifierWorkers)
|
||||
for worker := 0; worker < maxNotifierWorkers; worker++ {
|
||||
e.WgNotifier.Add(1)
|
||||
go func() {
|
||||
ctx := context.WithValue(ctx, "notifier_worker_id", common.RandomID(5))
|
||||
defer common.Recover(ctx)
|
||||
defer e.WgNotifier.Done()
|
||||
e.notifyResults(ctx)
|
||||
}()
|
||||
}
|
||||
|
||||
return e, nil
|
||||
}
|
||||
|
||||
// Finish waits for running sources to complete and workers to finish scanning
|
||||
// chunks before closing their respective channels. Once Finish is called, no
|
||||
// more sources may be scanned by the engine.
|
||||
func (e *Engine) Finish(ctx context.Context, logFunc func(error, string, ...any)) {
|
||||
func (e *Engine) Finish(ctx context.Context) error {
|
||||
defer common.RecoverWithExit(ctx)
|
||||
// wait for the sources to finish putting chunks onto the chunks channel
|
||||
sourceErr := e.sourcesWg.Wait()
|
||||
if sourceErr != nil {
|
||||
logFunc(sourceErr, "error occurred while collecting chunks")
|
||||
}
|
||||
// Wait for the sources to finish putting chunks onto the chunks channel.
|
||||
err := e.sourceManager.Wait()
|
||||
|
||||
close(e.chunks)
|
||||
// wait for the workers to finish processing all of the chunks and putting
|
||||
// results onto the results channel
|
||||
e.workersWg.Wait()
|
||||
e.workersWg.Wait() // Wait for the workers to finish scanning chunks.
|
||||
close(e.detectableChunksChan)
|
||||
e.wgDetectorWorkers.Wait() // Wait for the detector workers to finish detecting chunks.
|
||||
|
||||
// TODO: re-evaluate whether this is needed and investigate why if so
|
||||
//
|
||||
// not entirely sure why results don't get processed without this pause
|
||||
// since we've put all results on the channel at this point.
|
||||
time.Sleep(time.Second)
|
||||
close(e.results)
|
||||
close(e.results) // Detector workers are done, close the results channel and call it a day.
|
||||
e.WgNotifier.Wait() // Wait for the notifier workers to finish notifying results.
|
||||
|
||||
e.metrics.ScanDuration = time.Since(e.metrics.scanStartTime)
|
||||
|
||||
return err
|
||||
}
|
||||
|
||||
func (e *Engine) ChunksChan() chan *sources.Chunk {
|
||||
return e.chunks
|
||||
func (e *Engine) ChunksChan() <-chan *sources.Chunk {
|
||||
return e.sourceManager.Chunks()
|
||||
}
|
||||
|
||||
func (e *Engine) ResultsChan() chan detectors.ResultWithMetadata {
|
||||
return e.results
|
||||
}
|
||||
|
||||
func (e *Engine) ChunksScanned() uint64 {
|
||||
return e.chunksScanned
|
||||
}
|
||||
|
||||
func (e *Engine) BytesScanned() uint64 {
|
||||
return e.bytesScanned
|
||||
}
|
||||
|
||||
func (e *Engine) dedupeAndSend(chunkResults []detectors.ResultWithMetadata) {
|
||||
dedupeMap := make(map[string]struct{})
|
||||
for _, result := range chunkResults {
|
||||
// dedupe by comparing the detector type, raw result, and source metadata
|
||||
// NOTE: in order for the PLAIN decoder to maintain precedence, make sure UTF8 is the first decoder in the
|
||||
// default decoders list
|
||||
key := fmt.Sprintf("%s%s%s%+v", result.DetectorType.String(), result.Raw, result.RawV2, result.SourceMetadata)
|
||||
if _, ok := dedupeMap[key]; ok {
|
||||
continue
|
||||
}
|
||||
dedupeMap[key] = struct{}{}
|
||||
e.results <- result
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
func (e *Engine) DetectorAvgTime() map[string][]time.Duration {
|
||||
logger := context.Background().Logger()
|
||||
avgTime := map[string][]time.Duration{}
|
||||
e.detectorAvgTime.Range(func(k, v interface{}) bool {
|
||||
key, ok := k.(string)
|
||||
if !ok {
|
||||
logger.Info("expected DetectorAvgTime key to be a string")
|
||||
return true
|
||||
}
|
||||
|
||||
value, ok := v.([]time.Duration)
|
||||
if !ok {
|
||||
logger.Info("expected DetectorAvgTime value to be []time.Duration")
|
||||
return true
|
||||
}
|
||||
avgTime[key] = value
|
||||
return true
|
||||
})
|
||||
return avgTime
|
||||
// detectableChunk is a decoded chunk that is ready to be scanned by its detector.
|
||||
type detectableChunk struct {
|
||||
detector detectors.Detector
|
||||
chunk sources.Chunk
|
||||
decoder detectorspb.DecoderType
|
||||
wgDoneFn func()
|
||||
}
|
||||
|
||||
func (e *Engine) detectorWorker(ctx context.Context) {
|
||||
for originalChunk := range e.chunks {
|
||||
var wgDetect sync.WaitGroup
|
||||
|
||||
for originalChunk := range e.ChunksChan() {
|
||||
for chunk := range sources.Chunker(originalChunk) {
|
||||
var chunkResults []detectors.ResultWithMetadata
|
||||
matchedKeywords := make(map[string]struct{})
|
||||
atomic.AddUint64(&e.bytesScanned, uint64(len(chunk.Data)))
|
||||
atomic.AddUint64(&e.metrics.BytesScanned, uint64(len(chunk.Data)))
|
||||
for _, decoder := range e.decoders {
|
||||
var decoderType detectorspb.DecoderType
|
||||
switch decoder.(type) {
|
||||
@@ -298,8 +451,8 @@ func (e *Engine) detectorWorker(ctx context.Context) {
|
||||
}
|
||||
|
||||
// build a map of all keywords that were matched in the chunk
|
||||
for _, m := range e.prefilter.FindAll(string(decoded.Data)) {
|
||||
matchedKeywords[strings.ToLower(string(decoded.Data[m.Start():m.End()]))] = struct{}{}
|
||||
for _, m := range e.prefilter.MatchString(strings.ToLower(string(decoded.Data))) {
|
||||
matchedKeywords[strings.ToLower(m.MatchString())] = struct{}{}
|
||||
}
|
||||
|
||||
for verify, detectorsSet := range e.detectors {
|
||||
@@ -316,106 +469,146 @@ func (e *Engine) detectorWorker(ctx context.Context) {
|
||||
continue
|
||||
}
|
||||
|
||||
start := time.Now()
|
||||
|
||||
results, err := func() ([]detectors.Result, error) {
|
||||
ctx, cancel := context.WithTimeout(ctx, time.Second*10)
|
||||
defer cancel()
|
||||
defer common.Recover(ctx)
|
||||
return detector.FromData(ctx, verify, decoded.Data)
|
||||
}()
|
||||
if err != nil {
|
||||
ctx.Logger().Error(err, "could not scan chunk",
|
||||
"source_type", decoded.SourceType.String(),
|
||||
"metadata", decoded.SourceMetadata,
|
||||
)
|
||||
continue
|
||||
}
|
||||
|
||||
if e.filterUnverified {
|
||||
results = detectors.CleanResults(results)
|
||||
}
|
||||
for _, result := range results {
|
||||
resultChunk := chunk
|
||||
ignoreLinePresent := false
|
||||
if SupportsLineNumbers(chunk.SourceType) {
|
||||
copyChunk := *chunk
|
||||
copyMetaDataClone := proto.Clone(chunk.SourceMetadata)
|
||||
if copyMetaData, ok := copyMetaDataClone.(*source_metadatapb.MetaData); ok {
|
||||
copyChunk.SourceMetadata = copyMetaData
|
||||
}
|
||||
fragStart, mdLine := FragmentFirstLine(©Chunk)
|
||||
ignoreLinePresent = SetResultLineNumber(©Chunk, &result, fragStart, mdLine)
|
||||
resultChunk = ©Chunk
|
||||
}
|
||||
if ignoreLinePresent {
|
||||
continue
|
||||
}
|
||||
result.DecoderType = decoderType
|
||||
chunkResults = append(chunkResults, detectors.CopyMetadata(resultChunk, result))
|
||||
|
||||
}
|
||||
if len(results) > 0 {
|
||||
elapsed := time.Since(start)
|
||||
detectorName := results[0].DetectorType.String()
|
||||
avgTimeI, ok := e.detectorAvgTime.Load(detectorName)
|
||||
var avgTime []time.Duration
|
||||
if ok {
|
||||
avgTime, ok = avgTimeI.([]time.Duration)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
}
|
||||
avgTime = append(avgTime, elapsed)
|
||||
e.detectorAvgTime.Store(detectorName, avgTime)
|
||||
decoded.Verify = verify
|
||||
wgDetect.Add(1)
|
||||
e.detectableChunksChan <- detectableChunk{
|
||||
chunk: *decoded,
|
||||
detector: detector,
|
||||
decoder: decoderType,
|
||||
wgDoneFn: wgDetect.Done,
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
e.dedupeAndSend(chunkResults)
|
||||
}
|
||||
atomic.AddUint64(&e.chunksScanned, 1)
|
||||
atomic.AddUint64(&e.metrics.ChunksScanned, 1)
|
||||
}
|
||||
wgDetect.Wait()
|
||||
}
|
||||
|
||||
func (e *Engine) detectChunks(ctx context.Context) {
|
||||
for data := range e.detectableChunksChan {
|
||||
e.detectChunk(ctx, data)
|
||||
}
|
||||
}
|
||||
|
||||
// lineNumberSupportedSources is a list of sources that support line numbers.
|
||||
// It is stored this way because slice consts are not supported.
|
||||
func lineNumberSupportedSources() []sourcespb.SourceType {
|
||||
return []sourcespb.SourceType{
|
||||
sourcespb.SourceType_SOURCE_TYPE_GIT,
|
||||
func (e *Engine) detectChunk(ctx context.Context, data detectableChunk) {
|
||||
var start time.Time
|
||||
if e.printAvgDetectorTime {
|
||||
start = time.Now()
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(ctx, time.Second*10)
|
||||
defer common.Recover(ctx)
|
||||
defer cancel()
|
||||
|
||||
results, err := data.detector.FromData(ctx, data.chunk.Verify, data.chunk.Data)
|
||||
if err != nil {
|
||||
ctx.Logger().Error(err, "error scanning chunk")
|
||||
}
|
||||
if e.printAvgDetectorTime && len(results) > 0 {
|
||||
elapsed := time.Since(start)
|
||||
detectorName := results[0].DetectorType.String()
|
||||
avgTimeI, ok := e.metrics.detectorAvgTime.Load(detectorName)
|
||||
var avgTime []time.Duration
|
||||
if ok {
|
||||
avgTime, ok = avgTimeI.([]time.Duration)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
}
|
||||
avgTime = append(avgTime, elapsed)
|
||||
e.metrics.detectorAvgTime.Store(detectorName, avgTime)
|
||||
}
|
||||
|
||||
if e.filterUnverified {
|
||||
results = detectors.CleanResults(results)
|
||||
}
|
||||
|
||||
for _, res := range results {
|
||||
e.processResult(data, res)
|
||||
}
|
||||
data.wgDoneFn()
|
||||
}
|
||||
|
||||
func (e *Engine) processResult(data detectableChunk, res detectors.Result) {
|
||||
ignoreLinePresent := false
|
||||
if SupportsLineNumbers(data.chunk.SourceType) {
|
||||
copyChunk := data.chunk
|
||||
copyMetaDataClone := proto.Clone(data.chunk.SourceMetadata)
|
||||
if copyMetaData, ok := copyMetaDataClone.(*source_metadatapb.MetaData); ok {
|
||||
copyChunk.SourceMetadata = copyMetaData
|
||||
}
|
||||
fragStart, mdLine := FragmentFirstLine(©Chunk)
|
||||
ignoreLinePresent = SetResultLineNumber(©Chunk, &res, fragStart, mdLine)
|
||||
data.chunk = copyChunk
|
||||
}
|
||||
if ignoreLinePresent {
|
||||
return
|
||||
}
|
||||
|
||||
secret := detectors.CopyMetadata(&data.chunk, res)
|
||||
secret.DecoderType = data.decoder
|
||||
e.results <- secret
|
||||
}
|
||||
|
||||
func (e *Engine) notifyResults(ctx context.Context) {
|
||||
for r := range e.ResultsChan() {
|
||||
if e.onlyVerified && !r.Verified {
|
||||
continue
|
||||
}
|
||||
atomic.AddUint32(&e.numFoundResults, 1)
|
||||
|
||||
key := fmt.Sprintf("%s%s%s%+v", r.DetectorType.String(), r.Raw, r.RawV2, r.SourceMetadata)
|
||||
if _, ok := e.dedupeCache.Get(key); ok {
|
||||
continue
|
||||
}
|
||||
e.dedupeCache.Add(key, struct{}{})
|
||||
|
||||
if r.Verified {
|
||||
atomic.AddUint64(&e.metrics.VerifiedSecretsFound, 1)
|
||||
} else {
|
||||
atomic.AddUint64(&e.metrics.UnverifiedSecretsFound, 1)
|
||||
}
|
||||
|
||||
if err := e.printer.Print(ctx, &r); err != nil {
|
||||
ctx.Logger().Error(err, "error printing result")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// SupportsLineNumbers determines if a line number can be found for a source type.
|
||||
func SupportsLineNumbers(sourceType sourcespb.SourceType) bool {
|
||||
switch sourceType {
|
||||
case sourcespb.SourceType_SOURCE_TYPE_GIT,
|
||||
sourcespb.SourceType_SOURCE_TYPE_GITHUB,
|
||||
sourcespb.SourceType_SOURCE_TYPE_GITLAB,
|
||||
sourcespb.SourceType_SOURCE_TYPE_BITBUCKET,
|
||||
sourcespb.SourceType_SOURCE_TYPE_GERRIT,
|
||||
sourcespb.SourceType_SOURCE_TYPE_GITHUB_UNAUTHENTICATED_ORG,
|
||||
sourcespb.SourceType_SOURCE_TYPE_PUBLIC_GIT,
|
||||
sourcespb.SourceType_SOURCE_TYPE_FILESYSTEM,
|
||||
sourcespb.SourceType_SOURCE_TYPE_FILESYSTEM:
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
// SupportsLineNumbers determines if a line number can be found for a source type.
|
||||
func SupportsLineNumbers(sourceType sourcespb.SourceType) bool {
|
||||
for _, i := range lineNumberSupportedSources() {
|
||||
if i == sourceType {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// FragmentLineOffset sets the line number for a provided source chunk with a given detector result.
|
||||
func FragmentLineOffset(chunk *sources.Chunk, result *detectors.Result) (int64, bool) {
|
||||
lines := bytes.Split(chunk.Data, []byte("\n"))
|
||||
for i, line := range lines {
|
||||
if bytes.Contains(line, result.Raw) {
|
||||
// if the line contains the ignore tag, we should ignore the result
|
||||
if bytes.Contains(line, []byte(ignoreTag)) {
|
||||
return int64(i), true
|
||||
}
|
||||
return int64(i), false
|
||||
}
|
||||
before, after, found := bytes.Cut(chunk.Data, result.Raw)
|
||||
if !found {
|
||||
return 0, false
|
||||
}
|
||||
return 0, false
|
||||
lineNumber := int64(bytes.Count(before, []byte("\n")))
|
||||
// If the line contains the ignore tag, we should ignore the result.
|
||||
endLine := bytes.Index(after, []byte("\n"))
|
||||
if endLine == -1 {
|
||||
endLine = len(after)
|
||||
}
|
||||
if bytes.Contains(after[:endLine], []byte(ignoreTag)) {
|
||||
return lineNumber, true
|
||||
}
|
||||
return lineNumber, false
|
||||
}
|
||||
|
||||
// FragmentFirstLine returns the first line number of a fragment along with a pointer to the value to update in the
|
||||
|
||||
@@ -1,10 +1,12 @@
|
||||
package engine
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"testing"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/decoders"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
|
||||
)
|
||||
|
||||
@@ -49,6 +51,39 @@ func TestFragmentLineOffset(t *testing.T) {
|
||||
expectedLine: 4,
|
||||
ignore: false,
|
||||
},
|
||||
{
|
||||
name: "match on consecutive lines",
|
||||
chunk: &sources.Chunk{
|
||||
Data: []byte("line1\nline2\ntrufflehog:ignore\nline4\nsecret\nhere\nline6"),
|
||||
},
|
||||
result: &detectors.Result{
|
||||
Raw: []byte("secret\nhere"),
|
||||
},
|
||||
expectedLine: 4,
|
||||
ignore: false,
|
||||
},
|
||||
{
|
||||
name: "ignore on last consecutive lines",
|
||||
chunk: &sources.Chunk{
|
||||
Data: []byte("line1\nline2\nline3\nsecret\nhere // trufflehog:ignore\nline5"),
|
||||
},
|
||||
result: &detectors.Result{
|
||||
Raw: []byte("secret\nhere"),
|
||||
},
|
||||
expectedLine: 3,
|
||||
ignore: true,
|
||||
},
|
||||
{
|
||||
name: "ignore on last line",
|
||||
chunk: &sources.Chunk{
|
||||
Data: []byte("line1\nline2\nline3\nsecret here // trufflehog:ignore"),
|
||||
},
|
||||
result: &detectors.Result{
|
||||
Raw: []byte("secret here"),
|
||||
},
|
||||
expectedLine: 3,
|
||||
ignore: true,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
@@ -64,6 +99,43 @@ func TestFragmentLineOffset(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func setupFragmentLineOffsetBench(totalLines, needleLine int) (*sources.Chunk, *detectors.Result) {
|
||||
data := make([]byte, 0, 4096)
|
||||
needle := []byte("needle")
|
||||
for i := 0; i < totalLines; i++ {
|
||||
if i != needleLine {
|
||||
data = append(data, []byte(fmt.Sprintf("line%d\n", i))...)
|
||||
continue
|
||||
}
|
||||
data = append(data, needle...)
|
||||
data = append(data, '\n')
|
||||
}
|
||||
chunk := &sources.Chunk{Data: data}
|
||||
result := &detectors.Result{Raw: needle}
|
||||
return chunk, result
|
||||
}
|
||||
|
||||
func BenchmarkFragmentLineOffsetStart(b *testing.B) {
|
||||
chunk, result := setupFragmentLineOffsetBench(512, 2)
|
||||
for i := 0; i < b.N; i++ {
|
||||
_, _ = FragmentLineOffset(chunk, result)
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkFragmentLineOffsetMiddle(b *testing.B) {
|
||||
chunk, result := setupFragmentLineOffsetBench(512, 256)
|
||||
for i := 0; i < b.N; i++ {
|
||||
_, _ = FragmentLineOffset(chunk, result)
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkFragmentLineOffsetEnd(b *testing.B) {
|
||||
chunk, result := setupFragmentLineOffsetBench(512, 510)
|
||||
for i := 0; i < b.N; i++ {
|
||||
_, _ = FragmentLineOffset(chunk, result)
|
||||
}
|
||||
}
|
||||
|
||||
// Test to make sure that DefaultDecoders always returns the UTF8 decoder first.
|
||||
// Technically a decoder test but we want this to run and fail in CI
|
||||
func TestDefaultDecoders(t *testing.T) {
|
||||
@@ -72,3 +144,30 @@ func TestDefaultDecoders(t *testing.T) {
|
||||
t.Errorf("DefaultDecoders() = %v, expected UTF8 decoder to be first", ds)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSupportsLineNumbers(t *testing.T) {
|
||||
testCases := []struct {
|
||||
name string
|
||||
input sourcespb.SourceType
|
||||
expected bool
|
||||
}{
|
||||
{"Git source type", sourcespb.SourceType_SOURCE_TYPE_GIT, true},
|
||||
{"Github source type", sourcespb.SourceType_SOURCE_TYPE_GITHUB, true},
|
||||
{"Gitlab source type", sourcespb.SourceType_SOURCE_TYPE_GITLAB, true},
|
||||
}
|
||||
|
||||
for _, tc := range testCases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
if result := SupportsLineNumbers(tc.input); result != tc.expected {
|
||||
t.Errorf("Expected %v for input %v, got %v", tc.expected, tc.input, result)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkSupportsLineNumbersLoop(b *testing.B) {
|
||||
sourceType := sourcespb.SourceType_SOURCE_TYPE_GITHUB
|
||||
for i := 0; i < b.N; i++ {
|
||||
_ = SupportsLineNumbers(sourceType)
|
||||
}
|
||||
}
|
||||
|
||||
+12
-21
@@ -1,14 +1,11 @@
|
||||
package engine
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"runtime"
|
||||
|
||||
"github.com/go-errors/errors"
|
||||
"google.golang.org/protobuf/proto"
|
||||
"google.golang.org/protobuf/types/known/anypb"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/context"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
|
||||
@@ -27,24 +24,18 @@ func (e *Engine) ScanFileSystem(ctx context.Context, c sources.FilesystemConfig)
|
||||
return err
|
||||
}
|
||||
|
||||
fileSystemSource := filesystem.Source{}
|
||||
|
||||
ctx = context.WithValues(ctx,
|
||||
"source_type", fileSystemSource.Type().String(),
|
||||
"source_name", "filesystem",
|
||||
)
|
||||
err = fileSystemSource.Init(ctx, "trufflehog - filesystem", 0, int64(sourcespb.SourceType_SOURCE_TYPE_FILESYSTEM), true, &conn, runtime.NumCPU())
|
||||
handle, err := e.sourceManager.Enroll(ctx, "trufflehog - filesystem", new(filesystem.Source).Type(),
|
||||
func(ctx context.Context, jobID, sourceID int64) (sources.Source, error) {
|
||||
fileSystemSource := filesystem.Source{}
|
||||
fileSystemSource.WithFilter(c.Filter)
|
||||
if err := fileSystemSource.Init(ctx, "trufflehog - filesystem", jobID, sourceID, true, &conn, runtime.NumCPU()); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &fileSystemSource, nil
|
||||
})
|
||||
if err != nil {
|
||||
return errors.WrapPrefix(err, "could not init filesystem source", 0)
|
||||
return err
|
||||
}
|
||||
fileSystemSource.WithFilter(c.Filter)
|
||||
e.sourcesWg.Go(func() error {
|
||||
defer common.RecoverWithExit(ctx)
|
||||
err := fileSystemSource.Chunks(ctx, e.ChunksChan())
|
||||
if err != nil {
|
||||
return fmt.Errorf("error scanning filesystem: %w", err)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
return nil
|
||||
_, err = e.sourceManager.ScheduleRun(ctx, handle)
|
||||
return err
|
||||
}
|
||||
|
||||
+12
-17
@@ -6,7 +6,6 @@ import (
|
||||
"google.golang.org/protobuf/proto"
|
||||
"google.golang.org/protobuf/types/known/anypb"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/context"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
|
||||
@@ -45,23 +44,19 @@ func (e *Engine) ScanGCS(ctx context.Context, c sources.GCSConfig) error {
|
||||
return fmt.Errorf("failed to marshal GCS connection: %w", err)
|
||||
}
|
||||
|
||||
source := gcs.Source{}
|
||||
ctx = context.WithValues(ctx,
|
||||
"source_type", source.Type().String(),
|
||||
"source_name", "gcs",
|
||||
)
|
||||
if err = source.Init(ctx, "trufflehog - GCS", 0, 0, true, &conn, int(c.Concurrency)); err != nil {
|
||||
return fmt.Errorf("failed to initialize GCS source: %w", err)
|
||||
handle, err := e.sourceManager.Enroll(ctx, "trufflehog - gcs", new(gcs.Source).Type(),
|
||||
func(ctx context.Context, jobID, sourceID int64) (sources.Source, error) {
|
||||
gcsSource := gcs.Source{}
|
||||
if err := gcsSource.Init(ctx, "trufflehog - gcs", jobID, sourceID, true, &conn, int(c.Concurrency)); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &gcsSource, nil
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
e.sourcesWg.Go(func() error {
|
||||
defer common.RecoverWithExit(ctx)
|
||||
if err := source.Chunks(ctx, e.ChunksChan()); err != nil {
|
||||
return fmt.Errorf("could not scan GCS: %w", err)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
return nil
|
||||
_, err = e.sourceManager.ScheduleRun(ctx, handle)
|
||||
return err
|
||||
}
|
||||
|
||||
func isAuthValid(ctx context.Context, c sources.GCSConfig, connection *sourcespb.GCS) bool {
|
||||
|
||||
@@ -4,6 +4,8 @@ import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/context"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/decoders"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
|
||||
@@ -59,11 +61,13 @@ func TestScanGCS(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(context.TODO())
|
||||
defer cancel()
|
||||
|
||||
e := Start(ctx,
|
||||
e, err := Start(ctx,
|
||||
WithConcurrency(1),
|
||||
WithDecoders(decoders.DefaultDecoders()...),
|
||||
WithDetectors(false, DefaultDetectors()...),
|
||||
)
|
||||
assert.Nil(t, err)
|
||||
|
||||
go func() {
|
||||
resultCount := 0
|
||||
for range e.ResultsChan() {
|
||||
@@ -71,15 +75,15 @@ func TestScanGCS(t *testing.T) {
|
||||
}
|
||||
}()
|
||||
|
||||
err := e.ScanGCS(ctx, test.gcsConfig)
|
||||
err = e.ScanGCS(ctx, test.gcsConfig)
|
||||
if err != nil && !test.wantErr && !strings.Contains(err.Error(), "googleapi: Error 400: Bad Request") {
|
||||
t.Errorf("ScanGCS() got: %v, want: %v", err, nil)
|
||||
return
|
||||
}
|
||||
logFatalFunc := func(_ error, _ string, _ ...any) {
|
||||
t.Fatalf("error logging function should not have been called")
|
||||
if err := e.Finish(ctx); err != nil && !test.wantErr && !strings.Contains(err.Error(), "googleapi: Error 400: Bad Request") {
|
||||
t.Errorf("Finish() got: %v, want: %v", err, nil)
|
||||
return
|
||||
}
|
||||
e.Finish(ctx, logFatalFunc)
|
||||
|
||||
if err == nil && test.wantErr {
|
||||
t.Errorf("ScanGCS() got: %v, want: %v", err, "error")
|
||||
|
||||
+34
-41
@@ -1,14 +1,13 @@
|
||||
package engine
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"runtime"
|
||||
|
||||
gogit "github.com/go-git/go-git/v5"
|
||||
"google.golang.org/protobuf/proto"
|
||||
"google.golang.org/protobuf/types/known/anypb"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/context"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/source_metadatapb"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/sources/git"
|
||||
@@ -22,16 +21,6 @@ func (e *Engine) ScanGit(ctx context.Context, c sources.GitConfig) error {
|
||||
git.ScanOptionLogOptions(logOptions),
|
||||
}
|
||||
|
||||
options := &gogit.PlainOpenOptions{
|
||||
DetectDotGit: true,
|
||||
EnableDotGitCommonDir: true,
|
||||
}
|
||||
|
||||
repo, err := gogit.PlainOpenWithOptions(c.RepoPath, options)
|
||||
if err != nil {
|
||||
return fmt.Errorf("could not open repo: %s: %w", c.RepoPath, err)
|
||||
}
|
||||
|
||||
if c.MaxDepth != 0 {
|
||||
opts = append(opts, git.ScanOptionMaxDepth(int64(c.MaxDepth)))
|
||||
}
|
||||
@@ -44,35 +33,39 @@ func (e *Engine) ScanGit(ctx context.Context, c sources.GitConfig) error {
|
||||
if c.ExcludeGlobs != nil {
|
||||
opts = append(opts, git.ScanOptionExcludeGlobs(c.ExcludeGlobs))
|
||||
}
|
||||
if c.Bare {
|
||||
opts = append(opts, git.ScanOptionBare(c.Bare))
|
||||
}
|
||||
scanOptions := git.NewScanOptions(opts...)
|
||||
|
||||
gitSource := git.NewGit(sourcespb.SourceType_SOURCE_TYPE_GIT, 0, 0, "trufflehog - git", true, runtime.NumCPU(),
|
||||
func(file, email, commit, timestamp, repository string, line int64) *source_metadatapb.MetaData {
|
||||
return &source_metadatapb.MetaData{
|
||||
Data: &source_metadatapb.MetaData_Git{
|
||||
Git: &source_metadatapb.Git{
|
||||
Commit: commit,
|
||||
File: file,
|
||||
Email: email,
|
||||
Repository: repository,
|
||||
Timestamp: timestamp,
|
||||
Line: line,
|
||||
},
|
||||
},
|
||||
}
|
||||
})
|
||||
connection := &sourcespb.Git{
|
||||
// Using Directories here allows us to not pass any
|
||||
// authentication. Also by this point, the c.RepoPath should
|
||||
// still have been prepared and downloaded to a temporary
|
||||
// directory if it was a URL.
|
||||
Directories: []string{c.RepoPath},
|
||||
}
|
||||
var conn anypb.Any
|
||||
if err := anypb.MarshalFrom(&conn, connection, proto.MarshalOptions{}); err != nil {
|
||||
ctx.Logger().Error(err, "failed to marshal git connection")
|
||||
return err
|
||||
}
|
||||
|
||||
ctx = context.WithValues(ctx,
|
||||
"source_type", sourcespb.SourceType_SOURCE_TYPE_GIT.String(),
|
||||
"source_name", "git",
|
||||
)
|
||||
e.sourcesWg.Go(func() error {
|
||||
defer common.RecoverWithExit(ctx)
|
||||
err := gitSource.ScanRepo(ctx, repo, c.RepoPath, scanOptions, e.ChunksChan())
|
||||
if err != nil {
|
||||
return fmt.Errorf("could not scan repo: %w", err)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
return nil
|
||||
handle, err := e.sourceManager.Enroll(ctx, "trufflehog - git", new(git.Source).Type(),
|
||||
func(ctx context.Context, jobID, sourceID int64) (sources.Source, error) {
|
||||
gitSource := git.Source{}
|
||||
if err := gitSource.Init(ctx, "trufflehog - git", jobID, sourceID, true, &conn, runtime.NumCPU()); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
gitSource.WithScanOptions(scanOptions)
|
||||
// Don't try to clean up the provided directory. That's handled by the
|
||||
// caller of ScanGit.
|
||||
gitSource.WithPreserveTempDirs(true)
|
||||
return &gitSource, nil
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = e.sourceManager.ScheduleRun(ctx, handle)
|
||||
return err
|
||||
}
|
||||
|
||||
+32
-21
@@ -4,9 +4,12 @@ import (
|
||||
"os"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/context"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/decoders"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/source_metadatapb"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/sources/git"
|
||||
@@ -15,6 +18,14 @@ import (
|
||||
type expResult struct {
|
||||
B string
|
||||
LineNumber int64
|
||||
Verified bool
|
||||
}
|
||||
|
||||
type discardPrinter struct{}
|
||||
|
||||
func (p *discardPrinter) Print(context.Context, *detectors.ResultWithMetadata) error {
|
||||
// This method intentionally does nothing.
|
||||
return nil
|
||||
}
|
||||
|
||||
func TestGitEngine(t *testing.T) {
|
||||
@@ -38,27 +49,30 @@ func TestGitEngine(t *testing.T) {
|
||||
for tName, tTest := range map[string]testProfile{
|
||||
"all_secrets": {
|
||||
expected: map[string]expResult{
|
||||
"70001020fab32b1fcf2f1f0e5c66424eae649826": {"AKIAXYZDQCEN4B6JSJQI", 2},
|
||||
"84e9c75e388ae3e866e121087ea2dd45a71068f2": {"AKIAILE3JG6KMS3HZGCA", 4},
|
||||
"8afb0ecd4998b1179e428db5ebbcdc8221214432": {"369963c1434c377428ca8531fbc46c0c43d037a0", 3},
|
||||
"27fbead3bf883cdb7de9d7825ed401f28f9398f1": {"ffc7e0f9400fb6300167009e42d2f842cd7956e2", 7},
|
||||
"70001020fab32b1fcf2f1f0e5c66424eae649826": {"AKIAXYZDQCEN4B6JSJQI", 2, true},
|
||||
"84e9c75e388ae3e866e121087ea2dd45a71068f2": {"AKIAILE3JG6KMS3HZGCA", 4, true},
|
||||
"8afb0ecd4998b1179e428db5ebbcdc8221214432": {"369963c1434c377428ca8531fbc46c0c43d037a0", 3, false},
|
||||
"27fbead3bf883cdb7de9d7825ed401f28f9398f1": {"ffc7e0f9400fb6300167009e42d2f842cd7956e2", 7, false},
|
||||
},
|
||||
filter: common.FilterEmpty(),
|
||||
},
|
||||
"base_commit": {
|
||||
expected: map[string]expResult{
|
||||
"70001020fab32b1fcf2f1f0e5c66424eae649826": {"AKIAXYZDQCEN4B6JSJQI", 2},
|
||||
"70001020fab32b1fcf2f1f0e5c66424eae649826": {"AKIAXYZDQCEN4B6JSJQI", 2, true},
|
||||
},
|
||||
filter: common.FilterEmpty(),
|
||||
base: "2f251b8c1e72135a375b659951097ec7749d4af9",
|
||||
},
|
||||
} {
|
||||
t.Run(tName, func(t *testing.T) {
|
||||
e := Start(ctx,
|
||||
e, err := Start(ctx,
|
||||
WithConcurrency(1),
|
||||
WithDecoders(decoders.DefaultDecoders()...),
|
||||
WithDetectors(false, DefaultDetectors()...),
|
||||
WithDetectors(true, DefaultDetectors()...),
|
||||
WithPrinter(new(discardPrinter)),
|
||||
)
|
||||
assert.Nil(t, err)
|
||||
|
||||
cfg := sources.GitConfig{
|
||||
RepoPath: path,
|
||||
HeadRef: tTest.branch,
|
||||
@@ -70,11 +84,8 @@ func TestGitEngine(t *testing.T) {
|
||||
return
|
||||
}
|
||||
|
||||
logFatalFunc := func(_ error, _ string, _ ...any) {
|
||||
t.Fatalf("error logging function should not have been called")
|
||||
}
|
||||
go e.Finish(ctx, logFatalFunc)
|
||||
resultCount := 0
|
||||
// Wait for all the chunks to be processed.
|
||||
assert.Nil(t, e.Finish(ctx))
|
||||
for result := range e.ResultsChan() {
|
||||
switch meta := result.SourceMetadata.GetData().(type) {
|
||||
case *source_metadatapb.MetaData_Git:
|
||||
@@ -84,13 +95,14 @@ func TestGitEngine(t *testing.T) {
|
||||
if tTest.expected[meta.Git.Commit].LineNumber != result.SourceMetadata.GetGit().Line {
|
||||
t.Errorf("%s: unexpected line number. Got: %d, Expected: %d", tName, result.SourceMetadata.GetGit().Line, tTest.expected[meta.Git.Commit].LineNumber)
|
||||
}
|
||||
if tTest.expected[meta.Git.Commit].Verified != result.Verified {
|
||||
t.Errorf("%s: unexpected verification. Got: %v, Expected: %v", tName, result.Verified, tTest.expected[meta.Git.Commit].Verified)
|
||||
}
|
||||
}
|
||||
resultCount++
|
||||
|
||||
}
|
||||
if resultCount != len(tTest.expected) {
|
||||
t.Errorf("%s: unexpected number of results. Got: %d, Expected: %d", tName, resultCount, len(tTest.expected))
|
||||
}
|
||||
metrics := e.GetMetrics()
|
||||
assert.Equal(t, len(tTest.expected), int(metrics.VerifiedSecretsFound)+int(metrics.UnverifiedSecretsFound))
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -107,11 +119,13 @@ func BenchmarkGitEngine(b *testing.B) {
|
||||
ctx, cancel := context.WithCancel(ctx)
|
||||
defer cancel()
|
||||
|
||||
e := Start(ctx,
|
||||
e, err := Start(ctx,
|
||||
WithConcurrency(1),
|
||||
WithDecoders(decoders.DefaultDecoders()...),
|
||||
WithDetectors(false, DefaultDetectors()...),
|
||||
)
|
||||
assert.Nil(b, err)
|
||||
|
||||
go func() {
|
||||
resultCount := 0
|
||||
for range e.ResultsChan() {
|
||||
@@ -130,8 +144,5 @@ func BenchmarkGitEngine(b *testing.B) {
|
||||
return
|
||||
}
|
||||
}
|
||||
logFatalFunc := func(_ error, _ string, _ ...any) {
|
||||
b.Fatalf("error logging function should not have been called")
|
||||
}
|
||||
e.Finish(ctx, logFatalFunc)
|
||||
assert.Nil(b, e.Finish(ctx))
|
||||
}
|
||||
|
||||
+14
-25
@@ -1,13 +1,10 @@
|
||||
package engine
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
gogit "github.com/go-git/go-git/v5"
|
||||
"google.golang.org/protobuf/proto"
|
||||
"google.golang.org/protobuf/types/known/anypb"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/context"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
|
||||
@@ -17,8 +14,6 @@ import (
|
||||
|
||||
// ScanGitHub scans Github with the provided options.
|
||||
func (e *Engine) ScanGitHub(ctx context.Context, c sources.GithubConfig) error {
|
||||
source := github.Source{}
|
||||
|
||||
connection := sourcespb.GitHub{
|
||||
Endpoint: c.Endpoint,
|
||||
Organizations: c.Orgs,
|
||||
@@ -42,31 +37,25 @@ func (e *Engine) ScanGitHub(ctx context.Context, c sources.GithubConfig) error {
|
||||
return err
|
||||
}
|
||||
|
||||
ctx = context.WithValues(ctx,
|
||||
"source_type", source.Type().String(),
|
||||
"source_name", "github",
|
||||
)
|
||||
err = source.Init(ctx, "trufflehog - github", 0, 0, false, &conn, c.Concurrency)
|
||||
if err != nil {
|
||||
ctx.Logger().Error(err, "failed to initialize github source")
|
||||
return err
|
||||
}
|
||||
|
||||
logOptions := &gogit.LogOptions{}
|
||||
opts := []git.ScanOption{
|
||||
git.ScanOptionFilter(c.Filter),
|
||||
git.ScanOptionLogOptions(logOptions),
|
||||
}
|
||||
scanOptions := git.NewScanOptions(opts...)
|
||||
source.WithScanOptions(scanOptions)
|
||||
|
||||
e.sourcesWg.Go(func() error {
|
||||
defer common.RecoverWithExit(ctx)
|
||||
err := source.Chunks(ctx, e.ChunksChan())
|
||||
if err != nil {
|
||||
return fmt.Errorf("could not scan github: %w", err)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
return nil
|
||||
handle, err := e.sourceManager.Enroll(ctx, "trufflehog - github", new(github.Source).Type(),
|
||||
func(ctx context.Context, jobID, sourceID int64) (sources.Source, error) {
|
||||
githubSource := github.Source{}
|
||||
if err := githubSource.Init(ctx, "trufflehog - github", jobID, sourceID, true, &conn, c.Concurrency); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
githubSource.WithScanOptions(scanOptions)
|
||||
return &githubSource, nil
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = e.sourceManager.ScheduleRun(ctx, handle)
|
||||
return err
|
||||
}
|
||||
|
||||
+12
-20
@@ -4,12 +4,10 @@ import (
|
||||
"fmt"
|
||||
"runtime"
|
||||
|
||||
"github.com/go-errors/errors"
|
||||
gogit "github.com/go-git/go-git/v5"
|
||||
"google.golang.org/protobuf/proto"
|
||||
"google.golang.org/protobuf/types/known/anypb"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/context"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
|
||||
@@ -52,24 +50,18 @@ func (e *Engine) ScanGitLab(ctx context.Context, c sources.GitlabConfig) error {
|
||||
return err
|
||||
}
|
||||
|
||||
gitlabSource := gitlab.Source{}
|
||||
ctx = context.WithValues(ctx,
|
||||
"source_type", gitlabSource.Type().String(),
|
||||
"source_name", "gitlab",
|
||||
)
|
||||
err = gitlabSource.Init(ctx, "trufflehog - gitlab", 0, int64(sourcespb.SourceType_SOURCE_TYPE_GITLAB), true, &conn, runtime.NumCPU())
|
||||
handle, err := e.sourceManager.Enroll(ctx, "trufflehog - gitlab", new(gitlab.Source).Type(),
|
||||
func(ctx context.Context, jobID, sourceID int64) (sources.Source, error) {
|
||||
gitlabSource := gitlab.Source{}
|
||||
if err := gitlabSource.Init(ctx, "trufflehog - gitlab", jobID, sourceID, true, &conn, runtime.NumCPU()); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
gitlabSource.WithScanOptions(scanOptions)
|
||||
return &gitlabSource, nil
|
||||
})
|
||||
if err != nil {
|
||||
return errors.WrapPrefix(err, "could not init GitLab source", 0)
|
||||
return err
|
||||
}
|
||||
gitlabSource.WithScanOptions(scanOptions)
|
||||
|
||||
e.sourcesWg.Go(func() error {
|
||||
defer common.RecoverWithExit(ctx)
|
||||
err := gitlabSource.Chunks(ctx, e.ChunksChan())
|
||||
if err != nil {
|
||||
return fmt.Errorf("error scanning GitLab: %w", err)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
return nil
|
||||
_, err = e.sourceManager.ScheduleRun(ctx, handle)
|
||||
return err
|
||||
}
|
||||
|
||||
+11
-19
@@ -4,11 +4,9 @@ import (
|
||||
"fmt"
|
||||
"runtime"
|
||||
|
||||
"github.com/go-errors/errors"
|
||||
"google.golang.org/protobuf/proto"
|
||||
"google.golang.org/protobuf/types/known/anypb"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/context"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/credentialspb"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
|
||||
@@ -55,23 +53,17 @@ func (e *Engine) ScanS3(ctx context.Context, c sources.S3Config) error {
|
||||
return err
|
||||
}
|
||||
|
||||
s3Source := s3.Source{}
|
||||
ctx = context.WithValues(ctx,
|
||||
"source_type", s3Source.Type().String(),
|
||||
"source_name", "s3",
|
||||
)
|
||||
err = s3Source.Init(ctx, "trufflehog - s3", 0, int64(sourcespb.SourceType_SOURCE_TYPE_S3), true, &conn, runtime.NumCPU())
|
||||
handle, err := e.sourceManager.Enroll(ctx, "trufflehog - s3", new(s3.Source).Type(),
|
||||
func(ctx context.Context, jobID, sourceID int64) (sources.Source, error) {
|
||||
s3Source := s3.Source{}
|
||||
if err := s3Source.Init(ctx, "trufflehog - s3", jobID, sourceID, true, &conn, runtime.NumCPU()); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &s3Source, nil
|
||||
})
|
||||
if err != nil {
|
||||
return errors.WrapPrefix(err, "failed to init S3 source", 0)
|
||||
return err
|
||||
}
|
||||
|
||||
e.sourcesWg.Go(func() error {
|
||||
defer common.RecoverWithExit(ctx)
|
||||
err := s3Source.Chunks(ctx, e.ChunksChan())
|
||||
if err != nil {
|
||||
return fmt.Errorf("error scanning S3: %w", err)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
return nil
|
||||
_, err = e.sourceManager.ScheduleRun(ctx, handle)
|
||||
return err
|
||||
}
|
||||
|
||||
+12
-20
@@ -1,14 +1,12 @@
|
||||
package engine
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
|
||||
"github.com/go-errors/errors"
|
||||
"google.golang.org/protobuf/proto"
|
||||
"google.golang.org/protobuf/types/known/anypb"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/context"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
|
||||
@@ -42,25 +40,19 @@ func (e *Engine) ScanSyslog(ctx context.Context, c sources.SyslogConfig) error {
|
||||
if err != nil {
|
||||
return errors.WrapPrefix(err, "error unmarshalling connection", 0)
|
||||
}
|
||||
source := syslog.Source{}
|
||||
ctx = context.WithValues(ctx,
|
||||
"source_type", source.Type().String(),
|
||||
"source_name", "syslog",
|
||||
)
|
||||
err = source.Init(ctx, "trufflehog - syslog", 0, 0, false, &conn, c.Concurrency)
|
||||
source.InjectConnection(connection)
|
||||
|
||||
handle, err := e.sourceManager.Enroll(ctx, "trufflehog - syslog", new(syslog.Source).Type(),
|
||||
func(ctx context.Context, jobID, sourceID int64) (sources.Source, error) {
|
||||
syslogSource := syslog.Source{}
|
||||
if err := syslogSource.Init(ctx, "trufflehog - syslog", jobID, sourceID, true, &conn, c.Concurrency); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
syslogSource.InjectConnection(connection)
|
||||
return &syslogSource, nil
|
||||
})
|
||||
if err != nil {
|
||||
ctx.Logger().Error(err, "failed to initialize syslog source")
|
||||
return err
|
||||
}
|
||||
|
||||
e.sourcesWg.Go(func() error {
|
||||
defer common.RecoverWithExit(ctx)
|
||||
err := source.Chunks(ctx, e.ChunksChan())
|
||||
if err != nil {
|
||||
return fmt.Errorf("could not scan syslog: %w", err)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
return nil
|
||||
_, err = e.sourceManager.ScheduleRun(ctx, handle)
|
||||
return err
|
||||
}
|
||||
|
||||
+391
-124
@@ -5,6 +5,7 @@ import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
@@ -51,6 +52,49 @@ type Parser struct {
|
||||
dateFormat string
|
||||
}
|
||||
|
||||
type ParseState int
|
||||
|
||||
const (
|
||||
Initial ParseState = iota
|
||||
CommitLine
|
||||
MergeLine
|
||||
AuthorLine
|
||||
DateLine
|
||||
MessageStartLine
|
||||
MessageLine
|
||||
MessageEndLine
|
||||
DiffLine
|
||||
ModeLine
|
||||
IndexLine
|
||||
FromFileLine
|
||||
ToFileLine
|
||||
BinaryFileLine
|
||||
HunkLineNumberLine
|
||||
HunkContentLine
|
||||
ParseFailure
|
||||
)
|
||||
|
||||
func (state ParseState) String() string {
|
||||
return [...]string{
|
||||
"Initial",
|
||||
"CommitLine",
|
||||
"MergeLine",
|
||||
"AuthorLine",
|
||||
"DateLine",
|
||||
"MessageStartLine",
|
||||
"MessageLine",
|
||||
"MessageEndLine",
|
||||
"DiffLine",
|
||||
"ModeLine",
|
||||
"IndexLine",
|
||||
"FromFileLine",
|
||||
"ToFileLine",
|
||||
"BinaryFileLine",
|
||||
"HunkLineNumberLine",
|
||||
"HunkContentLine",
|
||||
}[state]
|
||||
}
|
||||
|
||||
// WithMaxDiffSize sets maxDiffSize option. Diffs larger than maxDiffSize will
|
||||
// be truncated.
|
||||
func WithMaxDiffSize(maxDiffSize int) Option {
|
||||
@@ -117,7 +161,7 @@ func (c1 *Commit) Equal(c2 *Commit) bool {
|
||||
}
|
||||
|
||||
// RepoPath parses the output of the `git log` command for the `source` path.
|
||||
func (c *Parser) RepoPath(ctx context.Context, source string, head string, abbreviatedLog bool, excludedGlobs []string) (chan Commit, error) {
|
||||
func (c *Parser) RepoPath(ctx context.Context, source string, head string, abbreviatedLog bool, excludedGlobs []string, isBare bool) (chan Commit, error) {
|
||||
args := []string{"-C", source, "log", "-p", "--full-history", "--date=format:%a %b %d %H:%M:%S %Y %z"}
|
||||
if abbreviatedLog {
|
||||
args = append(args, "--diff-filter=AM")
|
||||
@@ -134,10 +178,24 @@ func (c *Parser) RepoPath(ctx context.Context, source string, head string, abbre
|
||||
cmd := exec.Command("git", args...)
|
||||
absPath, err := filepath.Abs(source)
|
||||
if err == nil {
|
||||
cmd.Env = append(cmd.Env, fmt.Sprintf("GIT_DIR=%s", filepath.Join(absPath, ".git")))
|
||||
if !isBare {
|
||||
cmd.Env = append(cmd.Env, "GIT_DIR="+filepath.Join(absPath, ".git"))
|
||||
} else {
|
||||
cmd.Env = append(cmd.Env,
|
||||
"GIT_DIR="+absPath,
|
||||
)
|
||||
// We need those variables to handle incoming commits
|
||||
// while using trufflehog in pre-receive hooks
|
||||
if dir := os.Getenv("GIT_OBJECT_DIRECTORY"); dir != "" {
|
||||
cmd.Env = append(cmd.Env, "GIT_OBJECT_DIRECTORY="+dir)
|
||||
}
|
||||
if dir := os.Getenv("GIT_ALTERNATE_OBJECT_DIRECTORIES"); dir != "" {
|
||||
cmd.Env = append(cmd.Env, "GIT_ALTERNATE_OBJECT_DIRECTORIES="+dir)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return c.executeCommand(ctx, cmd)
|
||||
return c.executeCommand(ctx, cmd, false)
|
||||
}
|
||||
|
||||
// Staged parses the output of the `git diff` command for the `source` path.
|
||||
@@ -152,11 +210,11 @@ func (c *Parser) Staged(ctx context.Context, source string) (chan Commit, error)
|
||||
cmd.Env = append(cmd.Env, fmt.Sprintf("GIT_DIR=%s", filepath.Join(absPath, ".git")))
|
||||
}
|
||||
|
||||
return c.executeCommand(ctx, cmd)
|
||||
return c.executeCommand(ctx, cmd, true)
|
||||
}
|
||||
|
||||
// executeCommand runs an exec.Cmd, reads stdout and stderr, and waits for the Cmd to complete.
|
||||
func (c *Parser) executeCommand(ctx context.Context, cmd *exec.Cmd) (chan Commit, error) {
|
||||
func (c *Parser) executeCommand(ctx context.Context, cmd *exec.Cmd, isStaged bool) (chan Commit, error) {
|
||||
commitChan := make(chan Commit, 64)
|
||||
|
||||
stdOut, err := cmd.StdoutPipe()
|
||||
@@ -181,7 +239,7 @@ func (c *Parser) executeCommand(ctx context.Context, cmd *exec.Cmd) (chan Commit
|
||||
}()
|
||||
|
||||
go func() {
|
||||
c.FromReader(ctx, stdOut, commitChan)
|
||||
c.FromReader(ctx, stdOut, commitChan, isStaged)
|
||||
if err := cmd.Wait(); err != nil {
|
||||
ctx.Logger().V(2).Info("Error waiting for git command to complete.", "error", err)
|
||||
}
|
||||
@@ -190,14 +248,15 @@ func (c *Parser) executeCommand(ctx context.Context, cmd *exec.Cmd) (chan Commit
|
||||
return commitChan, nil
|
||||
}
|
||||
|
||||
func (c *Parser) FromReader(ctx context.Context, stdOut io.Reader, commitChan chan Commit) {
|
||||
func (c *Parser) FromReader(ctx context.Context, stdOut io.Reader, commitChan chan Commit, isStaged bool) {
|
||||
outReader := bufio.NewReader(stdOut)
|
||||
var (
|
||||
currentCommit *Commit
|
||||
currentDiff *Diff
|
||||
recentlyPassedAuthor bool
|
||||
totalLogSize int
|
||||
currentCommit *Commit
|
||||
currentDiff Diff
|
||||
|
||||
totalLogSize int
|
||||
)
|
||||
var latestState = Initial
|
||||
|
||||
defer common.RecoverWithExit(ctx)
|
||||
defer close(commitChan)
|
||||
@@ -205,15 +264,19 @@ func (c *Parser) FromReader(ctx context.Context, stdOut io.Reader, commitChan ch
|
||||
if common.IsDone(ctx) {
|
||||
break
|
||||
}
|
||||
|
||||
line, err := outReader.ReadBytes([]byte("\n")[0])
|
||||
if err != nil && len(line) == 0 {
|
||||
break
|
||||
}
|
||||
|
||||
switch {
|
||||
case isCommitLine(line):
|
||||
case isCommitLine(isStaged, latestState, line):
|
||||
latestState = CommitLine
|
||||
|
||||
// If there is a currentDiff, add it to currentCommit.
|
||||
if currentDiff != nil && currentDiff.Content.Len() > 0 {
|
||||
currentCommit.Diffs = append(currentCommit.Diffs, *currentDiff)
|
||||
if currentDiff.Content.Len() > 0 {
|
||||
currentCommit.Diffs = append(currentCommit.Diffs, currentDiff)
|
||||
currentCommit.Size += currentDiff.Content.Len()
|
||||
}
|
||||
// If there is a currentCommit, send it to the channel.
|
||||
@@ -222,7 +285,7 @@ func (c *Parser) FromReader(ctx context.Context, stdOut io.Reader, commitChan ch
|
||||
totalLogSize += currentCommit.Size
|
||||
}
|
||||
// Create a new currentDiff and currentCommit
|
||||
currentDiff = &Diff{}
|
||||
currentDiff = Diff{}
|
||||
currentCommit = &Commit{
|
||||
Message: strings.Builder{},
|
||||
}
|
||||
@@ -230,22 +293,39 @@ func (c *Parser) FromReader(ctx context.Context, stdOut io.Reader, commitChan ch
|
||||
if len(line) >= 47 {
|
||||
currentCommit.Hash = string(line[7:47])
|
||||
}
|
||||
case isAuthorLine(line):
|
||||
case isMergeLine(isStaged, latestState, line):
|
||||
latestState = MergeLine
|
||||
case isAuthorLine(isStaged, latestState, line):
|
||||
latestState = AuthorLine
|
||||
|
||||
currentCommit.Author = strings.TrimRight(string(line[8:]), "\n")
|
||||
recentlyPassedAuthor = true
|
||||
case isDateLine(line):
|
||||
case isDateLine(isStaged, latestState, line):
|
||||
latestState = DateLine
|
||||
|
||||
date, err := time.Parse(c.dateFormat, strings.TrimSpace(string(line[6:])))
|
||||
if err != nil {
|
||||
ctx.Logger().V(2).Info("Could not parse date from git stream.", "error", err)
|
||||
}
|
||||
currentCommit.Date = date
|
||||
case isDiffLine(line):
|
||||
case isMessageStartLine(isStaged, latestState, line):
|
||||
latestState = MessageStartLine
|
||||
// NoOp
|
||||
case isMessageLine(isStaged, latestState, line):
|
||||
latestState = MessageLine
|
||||
|
||||
currentCommit.Message.Write(line[4:])
|
||||
case isMessageEndLine(isStaged, latestState, line):
|
||||
latestState = MessageEndLine
|
||||
// NoOp
|
||||
case isDiffLine(isStaged, latestState, line):
|
||||
latestState = DiffLine
|
||||
|
||||
// This should never be nil, but check in case the stdin stream is messed up.
|
||||
if currentCommit == nil {
|
||||
currentCommit = &Commit{}
|
||||
}
|
||||
if currentDiff != nil && currentDiff.Content.Len() > 0 {
|
||||
currentCommit.Diffs = append(currentCommit.Diffs, *currentDiff)
|
||||
if currentDiff.Content.Len() > 0 {
|
||||
currentCommit.Diffs = append(currentCommit.Diffs, currentDiff)
|
||||
// If the currentDiff is over 1GB, drop it into the channel so it isn't held in memory waiting for more commits.
|
||||
totalSize := 0
|
||||
for _, diff := range currentCommit.Diffs {
|
||||
@@ -266,37 +346,36 @@ func (c *Parser) FromReader(ctx context.Context, stdOut io.Reader, commitChan ch
|
||||
currentCommit.Message.WriteString(oldCommit.Message.String())
|
||||
}
|
||||
}
|
||||
currentDiff = &Diff{}
|
||||
case isModeLine(line):
|
||||
currentDiff = Diff{}
|
||||
case isModeLine(isStaged, latestState, line):
|
||||
latestState = ModeLine
|
||||
// NoOp
|
||||
case isIndexLine(line):
|
||||
recentlyPassedAuthor = false
|
||||
case isIndexLine(isStaged, latestState, line):
|
||||
latestState = IndexLine
|
||||
// NoOp
|
||||
case isPlusFileLine(line):
|
||||
currentDiff.PathB = strings.TrimRight(strings.TrimRight(string(line[6:]), "\n"), "\t") // Trim the newline and tab characters. https://github.com/trufflesecurity/trufflehog/issues/1060
|
||||
case isMinusFileLine(line):
|
||||
// NoOp
|
||||
case isPlusDiffLine(line):
|
||||
currentDiff.Content.Write(line[1:])
|
||||
case isMinusDiffLine(line):
|
||||
// NoOp. We only care about additions.
|
||||
case (isMessageLine(line) && recentlyPassedAuthor):
|
||||
currentCommit.Message.Write(line[4:])
|
||||
case isContextDiffLine(line):
|
||||
currentDiff.Content.Write([]byte("\n"))
|
||||
case isBinaryLine(line):
|
||||
case isBinaryLine(isStaged, latestState, line):
|
||||
latestState = BinaryFileLine
|
||||
|
||||
currentDiff.IsBinary = true
|
||||
currentDiff.PathB = pathFromBinaryLine(line)
|
||||
case isLineNumberDiffLine(line):
|
||||
if currentDiff != nil && currentDiff.Content.Len() > 0 {
|
||||
currentCommit.Diffs = append(currentCommit.Diffs, *currentDiff)
|
||||
case isFromFileLine(isStaged, latestState, line):
|
||||
latestState = FromFileLine
|
||||
// NoOp
|
||||
case isToFileLine(isStaged, latestState, line):
|
||||
latestState = ToFileLine
|
||||
|
||||
// TODO: Is this fix still required?
|
||||
currentDiff.PathB = strings.TrimRight(strings.TrimRight(string(line[6:]), "\n"), "\t") // Trim the newline and tab characters. https://github.com/trufflesecurity/trufflehog/issues/1060
|
||||
case isHunkLineNumberLine(isStaged, latestState, line):
|
||||
latestState = HunkLineNumberLine
|
||||
|
||||
if currentDiff.Content.Len() > 0 {
|
||||
currentCommit.Diffs = append(currentCommit.Diffs, currentDiff)
|
||||
}
|
||||
newDiff := &Diff{
|
||||
currentDiff = Diff{
|
||||
PathB: currentDiff.PathB,
|
||||
}
|
||||
|
||||
currentDiff = newDiff
|
||||
|
||||
words := bytes.Split(line, []byte(" "))
|
||||
if len(words) >= 3 {
|
||||
startSlice := bytes.Split(words[2], []byte(","))
|
||||
@@ -305,7 +384,54 @@ func (c *Parser) FromReader(ctx context.Context, stdOut io.Reader, commitChan ch
|
||||
currentDiff.LineStart = lineStart
|
||||
}
|
||||
}
|
||||
case isHunkContextLine(isStaged, latestState, line):
|
||||
if latestState != HunkContentLine {
|
||||
latestState = HunkContentLine
|
||||
}
|
||||
// TODO: Why do we care about this? It creates empty lines in the diff. If there are no plusLines, it's just newlines.
|
||||
currentDiff.Content.Write([]byte("\n"))
|
||||
case isHunkPlusLine(isStaged, latestState, line):
|
||||
if latestState != HunkContentLine {
|
||||
latestState = HunkContentLine
|
||||
}
|
||||
|
||||
currentDiff.Content.Write(line[1:])
|
||||
case isHunkMinusLine(isStaged, latestState, line):
|
||||
if latestState != HunkContentLine {
|
||||
latestState = HunkContentLine
|
||||
}
|
||||
// NoOp. We only care about additions.
|
||||
case isHunkNewlineWarningLine(isStaged, latestState, line):
|
||||
if latestState != HunkContentLine {
|
||||
latestState = HunkContentLine
|
||||
}
|
||||
// NoOp
|
||||
case isHunkEmptyLine(isStaged, latestState, line):
|
||||
if latestState != HunkContentLine {
|
||||
latestState = HunkContentLine
|
||||
}
|
||||
// NoOp
|
||||
case isCommitSeparatorLine(isStaged, latestState, line):
|
||||
// NoOp
|
||||
default:
|
||||
// Skip ahead until we find the next diff or commit.
|
||||
if latestState == ParseFailure {
|
||||
continue
|
||||
}
|
||||
|
||||
// Here be dragons...
|
||||
// Build an informative error message.
|
||||
var err error
|
||||
if currentCommit != nil && currentCommit.Hash != "" {
|
||||
err = fmt.Errorf(`failed to parse line "%s" after state "%s" (commit=%s)`, line, latestState, currentCommit.Hash)
|
||||
} else {
|
||||
err = fmt.Errorf(`failed to parse line "%s" after state "%s"`, line, latestState)
|
||||
}
|
||||
ctx.Logger().V(2).Error(err, "Recovering at the latest commit or diff...\n")
|
||||
|
||||
latestState = ParseFailure
|
||||
}
|
||||
|
||||
if currentDiff.Content.Len() > c.maxDiffSize {
|
||||
ctx.Logger().V(2).Info(fmt.Sprintf(
|
||||
"Diff for %s exceeded MaxDiffSize(%d)", currentDiff.PathB, c.maxDiffSize,
|
||||
@@ -313,130 +439,157 @@ func (c *Parser) FromReader(ctx context.Context, stdOut io.Reader, commitChan ch
|
||||
break
|
||||
}
|
||||
}
|
||||
cleanupParse(currentCommit, currentDiff, commitChan, &totalLogSize)
|
||||
cleanupParse(currentCommit, ¤tDiff, commitChan, &totalLogSize)
|
||||
|
||||
ctx.Logger().V(2).Info("finished parsing git log.", "total_log_size", totalLogSize)
|
||||
}
|
||||
|
||||
func cleanupParse(currentCommit *Commit, currentDiff *Diff, commitChan chan Commit, totalLogSize *int) {
|
||||
if currentDiff != nil && currentDiff.Content.Len() > 0 {
|
||||
currentCommit.Diffs = append(currentCommit.Diffs, *currentDiff)
|
||||
func isMergeLine(isStaged bool, latestState ParseState, line []byte) bool {
|
||||
if isStaged || latestState != CommitLine {
|
||||
return false
|
||||
}
|
||||
if currentCommit != nil {
|
||||
commitChan <- *currentCommit
|
||||
if totalLogSize != nil {
|
||||
*totalLogSize += currentCommit.Size
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Date: Tue Aug 10 15:20:40 2021 +0100
|
||||
func isDateLine(line []byte) bool {
|
||||
if len(line) > 7 && bytes.Equal(line[:5], []byte("Date:")) {
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// Author: Bill Rich <bill.rich@trufflesec.com>
|
||||
func isAuthorLine(line []byte) bool {
|
||||
if len(line) > 8 && bytes.Equal(line[:7], []byte("Author:")) {
|
||||
if len(line) > 6 && bytes.Equal(line[:6], []byte("Merge:")) {
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// commit 7a95bbf0199e280a0e42dbb1d1a3f56cdd0f6e05
|
||||
func isCommitLine(line []byte) bool {
|
||||
if len(line) > 7 && bytes.Equal(line[:6], []byte("commit")) {
|
||||
func isCommitLine(isStaged bool, latestState ParseState, line []byte) bool {
|
||||
if isStaged || !(latestState == Initial ||
|
||||
latestState == MessageStartLine ||
|
||||
latestState == MessageEndLine ||
|
||||
latestState == ModeLine ||
|
||||
latestState == IndexLine ||
|
||||
latestState == BinaryFileLine ||
|
||||
latestState == ToFileLine ||
|
||||
latestState == HunkContentLine ||
|
||||
latestState == ParseFailure) {
|
||||
return false
|
||||
}
|
||||
|
||||
if len(line) > 7 && bytes.Equal(line[:7], []byte("commit ")) {
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// diff --git a/internal/addrs/move_endpoint_module.go b/internal/addrs/move_endpoint_module.go
|
||||
func isDiffLine(line []byte) bool {
|
||||
if len(line) > 5 && bytes.Equal(line[:4], []byte("diff")) {
|
||||
// Author: Bill Rich <bill.rich@trufflesec.com>
|
||||
func isAuthorLine(isStaged bool, latestState ParseState, line []byte) bool {
|
||||
if isStaged || !(latestState == CommitLine || latestState == MergeLine) {
|
||||
return false
|
||||
}
|
||||
if len(line) > 8 && bytes.Equal(line[:7], []byte("Author:")) {
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// index 1ed6fbee1..aea1e643a 100644
|
||||
func isIndexLine(line []byte) bool {
|
||||
if len(line) > 6 && bytes.Equal(line[:5], []byte("index")) {
|
||||
// Date: Tue Aug 10 15:20:40 2021 +0100
|
||||
func isDateLine(isStaged bool, latestState ParseState, line []byte) bool {
|
||||
if isStaged || latestState != AuthorLine {
|
||||
return false
|
||||
}
|
||||
if len(line) > 7 && bytes.Equal(line[:5], []byte("Date:")) {
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// new file mode 100644
|
||||
func isModeLine(line []byte) bool {
|
||||
if len(line) > 13 && bytes.Equal(line[:13], []byte("new file mode")) {
|
||||
return true
|
||||
// Line directly after Date with only a newline.
|
||||
func isMessageStartLine(isStaged bool, latestState ParseState, line []byte) bool {
|
||||
if isStaged || latestState != DateLine {
|
||||
return false
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// --- a/internal/addrs/move_endpoint_module.go
|
||||
func isMinusFileLine(line []byte) bool {
|
||||
if len(line) >= 6 && bytes.Equal(line[:3], []byte("---")) {
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// +++ b/internal/addrs/move_endpoint_module.go
|
||||
func isPlusFileLine(line []byte) bool {
|
||||
if len(line) >= 6 && bytes.Equal(line[:4], []byte("+++ ")) {
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// +fmt.Println("ok")
|
||||
func isPlusDiffLine(line []byte) bool {
|
||||
if len(line) >= 1 && bytes.Equal(line[:1], []byte("+")) {
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// -fmt.Println("ok")
|
||||
func isMinusDiffLine(line []byte) bool {
|
||||
if len(line) >= 1 && bytes.Equal(line[:1], []byte("-")) {
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// fmt.Println("ok")
|
||||
func isContextDiffLine(line []byte) bool {
|
||||
if len(line) >= 1 && bytes.Equal(line[:1], []byte(" ")) {
|
||||
// TODO: Improve the implementation of this and isMessageEndLine
|
||||
if len(strings.TrimRight(string(line[:]), "\r\n")) == 0 {
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// Line that starts with 4 spaces
|
||||
func isMessageLine(line []byte) bool {
|
||||
func isMessageLine(isStaged bool, latestState ParseState, line []byte) bool {
|
||||
if isStaged || !(latestState == MessageStartLine || latestState == MessageLine) {
|
||||
return false
|
||||
}
|
||||
if len(line) > 4 && bytes.Equal(line[:4], []byte(" ")) {
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// Binary files /dev/null and b/plugin.sig differ
|
||||
func isBinaryLine(line []byte) bool {
|
||||
if len(line) > 7 && bytes.Equal(line[:6], []byte("Binary")) {
|
||||
// Line directly after MessageLine with only a newline.
|
||||
func isMessageEndLine(isStaged bool, latestState ParseState, line []byte) bool {
|
||||
if isStaged || latestState != MessageLine {
|
||||
return false
|
||||
}
|
||||
if len(strings.TrimRight(string(line[:]), "\r\n")) == 0 {
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// @@ -298 +298 @@ func maxRetryErrorHandler(resp *http.Response, err error, numTries int)
|
||||
func isLineNumberDiffLine(line []byte) bool {
|
||||
if len(line) >= 8 && bytes.Equal(line[:2], []byte("@@")) {
|
||||
// diff --git a/internal/addrs/move_endpoint_module.go b/internal/addrs/move_endpoint_module.go
|
||||
func isDiffLine(isStaged bool, latestState ParseState, line []byte) bool {
|
||||
if !(latestState == MessageStartLine || // Empty commit messages can go from MessageStart->Diff
|
||||
latestState == MessageEndLine ||
|
||||
latestState == BinaryFileLine ||
|
||||
latestState == IndexLine ||
|
||||
latestState == HunkContentLine ||
|
||||
latestState == ParseFailure) {
|
||||
if latestState == Initial && !isStaged {
|
||||
return false
|
||||
}
|
||||
}
|
||||
if len(line) > 11 && bytes.Equal(line[:11], []byte("diff --git ")) {
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// old mode 100644
|
||||
// new mode 100755
|
||||
// new file mode 100644
|
||||
// similarity index 100%
|
||||
// rename from old.txt
|
||||
// rename to new.txt
|
||||
// deleted file mode 100644
|
||||
func isModeLine(isStaged bool, latestState ParseState, line []byte) bool {
|
||||
if !(latestState == DiffLine || latestState == ModeLine) {
|
||||
return false
|
||||
}
|
||||
// This could probably be better written.
|
||||
if (len(line) > 17 && bytes.Equal(line[:17], []byte("deleted file mode"))) ||
|
||||
(len(line) > 16 && bytes.Equal(line[:16], []byte("similarity index"))) ||
|
||||
(len(line) > 13 && bytes.Equal(line[:13], []byte("new file mode"))) ||
|
||||
(len(line) > 11 && bytes.Equal(line[:11], []byte("rename from"))) ||
|
||||
(len(line) > 9 && bytes.Equal(line[:9], []byte("rename to"))) ||
|
||||
(len(line) > 8 && bytes.Equal(line[:8], []byte("old mode"))) ||
|
||||
(len(line) > 8 && bytes.Equal(line[:8], []byte("new mode"))) {
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// index 1ed6fbee1..aea1e643a 100644
|
||||
// index 00000000..e69de29b
|
||||
func isIndexLine(isStaged bool, latestState ParseState, line []byte) bool {
|
||||
if !(latestState == DiffLine || latestState == ModeLine) {
|
||||
return false
|
||||
}
|
||||
if len(line) > 6 && bytes.Equal(line[:6], []byte("index ")) {
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// Binary files /dev/null and b/plugin.sig differ
|
||||
func isBinaryLine(isStaged bool, latestState ParseState, line []byte) bool {
|
||||
if latestState != IndexLine {
|
||||
return false
|
||||
}
|
||||
if len(line) > 7 && bytes.Equal(line[:6], []byte("Binary")) {
|
||||
return true
|
||||
}
|
||||
return false
|
||||
@@ -453,3 +606,117 @@ func pathFromBinaryLine(line []byte) string {
|
||||
bRaw := sbytes[1]
|
||||
return string(bRaw[:len(bRaw)-7]) // drop the "b/" and " differ"
|
||||
}
|
||||
|
||||
// --- a/internal/addrs/move_endpoint_module.go
|
||||
func isFromFileLine(isStaged bool, latestState ParseState, line []byte) bool {
|
||||
if latestState != IndexLine {
|
||||
return false
|
||||
}
|
||||
if len(line) >= 6 && bytes.Equal(line[:4], []byte("--- ")) {
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// +++ b/internal/addrs/move_endpoint_module.go
|
||||
func isToFileLine(isStaged bool, latestState ParseState, line []byte) bool {
|
||||
if latestState != FromFileLine {
|
||||
return false
|
||||
}
|
||||
if len(line) >= 6 && bytes.Equal(line[:4], []byte("+++ ")) {
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// @@ -298 +298 @@ func maxRetryErrorHandler(resp *http.Response, err error, numTries int)
|
||||
func isHunkLineNumberLine(isStaged bool, latestState ParseState, line []byte) bool {
|
||||
if !(latestState == ToFileLine || latestState == HunkContentLine) {
|
||||
return false
|
||||
}
|
||||
if len(line) >= 8 && bytes.Equal(line[:2], []byte("@@")) {
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// fmt.Println("ok")
|
||||
// (There's a space before `fmt` that gets removed by the formatter.)
|
||||
func isHunkContextLine(isStaged bool, latestState ParseState, line []byte) bool {
|
||||
if !(latestState == HunkLineNumberLine || latestState == HunkContentLine) {
|
||||
return false
|
||||
}
|
||||
if len(line) >= 1 && bytes.Equal(line[:1], []byte(" ")) {
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// +fmt.Println("ok")
|
||||
func isHunkPlusLine(isStaged bool, latestState ParseState, line []byte) bool {
|
||||
if !(latestState == HunkLineNumberLine || latestState == HunkContentLine) {
|
||||
return false
|
||||
}
|
||||
if len(line) >= 1 && bytes.Equal(line[:1], []byte("+")) {
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// -fmt.Println("ok")
|
||||
func isHunkMinusLine(isStaged bool, latestState ParseState, line []byte) bool {
|
||||
if !(latestState == HunkLineNumberLine || latestState == HunkContentLine) {
|
||||
return false
|
||||
}
|
||||
if len(line) >= 1 && bytes.Equal(line[:1], []byte("-")) {
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// \ No newline at end of file
|
||||
func isHunkNewlineWarningLine(isStaged bool, latestState ParseState, line []byte) bool {
|
||||
if latestState != HunkContentLine {
|
||||
return false
|
||||
}
|
||||
if len(line) >= 27 && bytes.Equal(line[:27], []byte("\\ No newline at end of file")) {
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// Newline after hunk, or an empty line, e.g.
|
||||
// +}
|
||||
//
|
||||
// commit 00920984e3435057f09cee5468850f7546dfa637 (tag: v3.42.0)
|
||||
func isHunkEmptyLine(isStaged bool, latestState ParseState, line []byte) bool {
|
||||
if !(latestState == HunkLineNumberLine || latestState == HunkContentLine) {
|
||||
return false
|
||||
}
|
||||
// TODO: Can this also be `\n\r`?
|
||||
if len(line) == 1 && bytes.Equal(line[:1], []byte("\n")) {
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func isCommitSeparatorLine(isStaged bool, latestState ParseState, line []byte) bool {
|
||||
if (latestState == ModeLine || latestState == IndexLine || latestState == BinaryFileLine || latestState == ToFileLine) &&
|
||||
len(line) == 1 && bytes.Equal(line[:1], []byte("\n")) {
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func cleanupParse(currentCommit *Commit, currentDiff *Diff, commitChan chan Commit, totalLogSize *int) {
|
||||
// Ignore empty or binary diffs (this condition may be redundant).
|
||||
if currentDiff != nil && currentDiff.Content.Len() > 0 {
|
||||
currentCommit.Diffs = append(currentCommit.Diffs, *currentDiff)
|
||||
}
|
||||
if currentCommit != nil {
|
||||
commitChan <- *currentCommit
|
||||
if totalLogSize != nil {
|
||||
*totalLogSize += currentCommit.Size
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
+1786
-224
File diff suppressed because it is too large
Load Diff
+12
-4
@@ -7,12 +7,20 @@ import (
|
||||
"github.com/pkg/errors"
|
||||
)
|
||||
|
||||
type provider string
|
||||
|
||||
const (
|
||||
providerGithub provider = "Github"
|
||||
providerGitlab provider = "Gitlab"
|
||||
providerBitbucket provider = "Bitbucket"
|
||||
)
|
||||
|
||||
func NormalizeBitbucketRepo(repoURL string) (string, error) {
|
||||
if !strings.HasPrefix(repoURL, "https") {
|
||||
return "", errors.New("Bitbucket requires https repo urls: e.g. https://bitbucket.org/org/repo.git")
|
||||
}
|
||||
|
||||
return NormalizeOrgRepoURL("Bitbucket", repoURL)
|
||||
return NormalizeOrgRepoURL(providerBitbucket, repoURL)
|
||||
}
|
||||
|
||||
func NormalizeGerritProject(project string) (string, error) {
|
||||
@@ -20,7 +28,7 @@ func NormalizeGerritProject(project string) (string, error) {
|
||||
}
|
||||
|
||||
func NormalizeGithubRepo(repoURL string) (string, error) {
|
||||
return NormalizeOrgRepoURL("Github", repoURL)
|
||||
return NormalizeOrgRepoURL(providerGithub, repoURL)
|
||||
}
|
||||
|
||||
func NormalizeGitlabRepo(repoURL string) (string, error) {
|
||||
@@ -28,12 +36,12 @@ func NormalizeGitlabRepo(repoURL string) (string, error) {
|
||||
return "", errors.New("Gitlab requires http/https repo urls: e.g. https://gitlab.com/org/repo.git")
|
||||
}
|
||||
|
||||
return NormalizeOrgRepoURL("Gitlab", repoURL)
|
||||
return NormalizeOrgRepoURL(providerGitlab, repoURL)
|
||||
}
|
||||
|
||||
// NormalizeOrgRepoURL attempts to normalize repos for any provider using the example.com/org/repo style.
|
||||
// e.g. %s, Gitlab and Bitbucket
|
||||
func NormalizeOrgRepoURL(provider, repoURL string) (string, error) {
|
||||
func NormalizeOrgRepoURL(provider provider, repoURL string) (string, error) {
|
||||
if strings.HasSuffix(repoURL, ".git") {
|
||||
return repoURL, nil
|
||||
}
|
||||
|
||||
+10
-10
@@ -8,22 +8,22 @@ import (
|
||||
|
||||
func Test_NormalizeOrgRepoURL(t *testing.T) {
|
||||
tests := map[string]struct {
|
||||
Provider string
|
||||
Provider provider
|
||||
Repo string
|
||||
Out string
|
||||
Err error
|
||||
}{
|
||||
"github is good": {Provider: "Github", Repo: "https://github.com/org/repo", Out: "https://github.com/org/repo.git", Err: nil},
|
||||
"gitlab is good": {Provider: "Gitlab", Repo: "https://gitlab.com/org/repo", Out: "https://gitlab.com/org/repo.git", Err: nil},
|
||||
"bitbucket is good": {Provider: "Bitbucket", Repo: "https://bitbucket.com/org/repo", Out: "https://bitbucket.com/org/repo.git", Err: nil},
|
||||
"github is good": {Provider: providerGithub, Repo: "https://github.com/org/repo", Out: "https://github.com/org/repo.git", Err: nil},
|
||||
"gitlab is good": {Provider: providerGitlab, Repo: "https://gitlab.com/org/repo", Out: "https://gitlab.com/org/repo.git", Err: nil},
|
||||
"bitbucket is good": {Provider: providerBitbucket, Repo: "https://bitbucket.com/org/repo", Out: "https://bitbucket.com/org/repo.git", Err: nil},
|
||||
"example provider is good": {Provider: "example", Repo: "https://example.com/org/repo", Out: "https://example.com/org/repo.git", Err: nil},
|
||||
"example provider problem": {Provider: "example", Repo: "https://example.com/org", Out: "", Err: errors.Errorf("example repo appears to be missing the repo name. Org: %q Repo url: %q", "org", "https://example.com/org")},
|
||||
"no path": {Provider: "Github", Repo: "https://github.com", Out: "", Err: errors.Errorf("Github repo appears to be missing the path. Repo url: %q", "https://github.com")},
|
||||
"org but no repo": {Provider: "Github", Repo: "https://github.com/org", Out: "", Err: errors.Errorf("Github repo appears to be missing the repo name. Org: %q Repo url: %q", "org", "https://github.com/org")},
|
||||
"org but no repo with slash": {Provider: "Github", Repo: "https://github.com/org/", Out: "", Err: errors.Errorf("Github repo appears to be missing the repo name. Org: %q Repo url: %q", "org", "https://github.com/org/")},
|
||||
"two slashes": {Provider: "Github", Repo: "https://github.com//", Out: "", Err: errors.Errorf("Github repo appears to be missing the org name. Repo url: %q", "https://github.com//")},
|
||||
"repo with trailing slash": {Provider: "Github", Repo: "https://github.com/org/repo/", Out: "", Err: errors.Errorf("Github repo contains a trailing slash. Repo url: %q", "https://github.com/org/repo/")},
|
||||
"too many url path parts": {Provider: "Github", Repo: "https://github.com/org/repo/unknown/", Out: "", Err: errors.Errorf("Github repo contains a trailing slash. Repo url: %q", "https://github.com/org/repo/unknown/")},
|
||||
"no path": {Provider: providerGithub, Repo: "https://github.com", Out: "", Err: errors.Errorf("Github repo appears to be missing the path. Repo url: %q", "https://github.com")},
|
||||
"org but no repo": {Provider: providerGithub, Repo: "https://github.com/org", Out: "", Err: errors.Errorf("Github repo appears to be missing the repo name. Org: %q Repo url: %q", "org", "https://github.com/org")},
|
||||
"org but no repo with slash": {Provider: providerGithub, Repo: "https://github.com/org/", Out: "", Err: errors.Errorf("Github repo appears to be missing the repo name. Org: %q Repo url: %q", "org", "https://github.com/org/")},
|
||||
"two slashes": {Provider: providerGithub, Repo: "https://github.com//", Out: "", Err: errors.Errorf("Github repo appears to be missing the org name. Repo url: %q", "https://github.com//")},
|
||||
"repo with trailing slash": {Provider: providerGithub, Repo: "https://github.com/org/repo/", Out: "", Err: errors.Errorf("Github repo contains a trailing slash. Repo url: %q", "https://github.com/org/repo/")},
|
||||
"too many url path parts": {Provider: providerGithub, Repo: "https://github.com/org/repo/unknown/", Out: "", Err: errors.Errorf("Github repo contains a trailing slash. Repo url: %q", "https://github.com/org/repo/unknown/")},
|
||||
}
|
||||
|
||||
for name, tt := range tests {
|
||||
|
||||
@@ -4,14 +4,19 @@ import (
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"sync"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/context"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
|
||||
)
|
||||
|
||||
var dedupeCache = make(map[string]struct{})
|
||||
|
||||
func PrintGitHubActionsOutput(r *detectors.ResultWithMetadata) error {
|
||||
// GitHubActionsPrinter is a printer that prints results in GitHub Actions format.
|
||||
type GitHubActionsPrinter struct{ mu sync.Mutex }
|
||||
|
||||
func (p *GitHubActionsPrinter) Print(_ context.Context, r *detectors.ResultWithMetadata) error {
|
||||
out := gitHubActionsOutputFormat{
|
||||
DetectorType: r.Result.DetectorType.String(),
|
||||
DecoderType: r.Result.DecoderType.String(),
|
||||
@@ -52,6 +57,8 @@ func PrintGitHubActionsOutput(r *detectors.ResultWithMetadata) error {
|
||||
}
|
||||
dedupeCache[key] = struct{}{}
|
||||
|
||||
p.mu.Lock()
|
||||
defer p.mu.Unlock()
|
||||
message := fmt.Sprintf("Found %s %s result 🐷🔑\n", verifiedStatus, out.DetectorType)
|
||||
if r.Result.DecoderType != detectorspb.DecoderType_PLAIN {
|
||||
message = fmt.Sprintf("Found %s %s result with %s encoding 🐷🔑\n", verifiedStatus, out.DetectorType, out.DecoderType)
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user