Compare commits
1 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| def91bd138 |
@@ -0,0 +1,5 @@
|
||||
{
|
||||
"image": "trufflesecurity/protos:1.18-0",
|
||||
"features": {
|
||||
}
|
||||
}
|
||||
@@ -6,7 +6,11 @@ labels: bug, needs triage
|
||||
assignees: trufflesecurity/product-eng
|
||||
---
|
||||
|
||||
Please review the [Community Note](https://github.com/trufflesecurity/trufflehog/blob/main/.github/community_note.md) before submitting
|
||||
### Community Note
|
||||
|
||||
* Please vote on this issue by adding a 👍 [reaction](https://blog.github.com/2016-03-10-add-reactions-to-pull-requests-issues-and-comments/) to the original issue to help the community and maintainers prioritize this request
|
||||
* Please do not leave "+1" or other comments that do not add relevant new information or questions, they generate extra noise for issue followers and do not help prioritize the request
|
||||
* If you are interested in working on this issue or have submitted a pull request, please leave a comment
|
||||
|
||||
### TruffleHog Version
|
||||
<!--- Please run `trufflehog --version` to show the version. If you are not running the latest version, please upgrade because your issue may have already been fixed. --->
|
||||
|
||||
@@ -6,27 +6,33 @@ labels: enhancement, needs triage
|
||||
assignees: trufflesecurity/product-eng
|
||||
---
|
||||
|
||||
Please review the [Community Note](https://github.com/trufflesecurity/trufflehog/blob/main/.github/community_note.md) before submitting
|
||||
### Community Note
|
||||
|
||||
* Please vote on this issue by adding a 👍 [reaction](https://blog.github.com/2016-03-10-add-reactions-to-pull-requests-issues-and-comments/) to the original issue to help the community and maintainers prioritize this request
|
||||
* Please do not leave "+1" or other comments that do not add relevant new information or questions, they generate extra noise for issue followers and do not help prioritize the request
|
||||
* If you are interested in working on this issue or have submitted a pull request, please leave a comment
|
||||
|
||||
### Description
|
||||
|
||||
## Description
|
||||
<!--- Please leave a helpful description of the feature request here. --->
|
||||
|
||||
### Preferred Solution
|
||||
<!--- A clear and concise description of what you want to happen. What
|
||||
information may be required and what would be the preferred way to provide it?
|
||||
## Problem to be Addressed
|
||||
<!--- A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] --->
|
||||
|
||||
## Description of the Preferred Solution
|
||||
<!--- A clear and concise description of what you want to happen. What
|
||||
information may be required and what would be the preferred way to provide it?
|
||||
What should the output include? --->
|
||||
|
||||
### Additional Context
|
||||
## Additional Context
|
||||
<!--- Add any other context or screenshots about the feature request here. --->
|
||||
|
||||
#### References
|
||||
### References
|
||||
|
||||
<!---
|
||||
Information about referencing Github Issues: https://help.github.com/articles/basic-writing-and-formatting-syntax/#referencing-issues-and-pull-requests
|
||||
|
||||
Are there any other GitHub issues (open or closed) or pull requests that should be linked here? Vendor blog posts or documentation? For example:
|
||||
|
||||
* #0000
|
||||
--->
|
||||
|
||||
|
||||
* #0000
|
||||
|
||||
@@ -2,11 +2,3 @@
|
||||
Please create an issue to collect feedback prior to feature additions. Please also reference that issue in any PRs.
|
||||
If possible try to keep PRs scoped to one feature, and add tests for new features.
|
||||
-->
|
||||
|
||||
### Description:
|
||||
Explain the purpose of the PR.
|
||||
|
||||
### Checklist:
|
||||
* [ ] Tests passing (`make test-community`)?
|
||||
* [ ] Lint passing (`make lint` this requires [golangci-lint](https://golangci-lint.run/usage/install/#local-installation))?
|
||||
|
||||
|
||||
@@ -1,7 +0,0 @@
|
||||
## Community Note
|
||||
|
||||
Please vote on this issue by adding a 👍 reaction to the original issue to help the community and maintainers prioritize this request.
|
||||
|
||||
Please do not leave "+1" or other comments that do not add relevant new information or questions, they generate extra noise for issue followers and do not help prioritize the request.
|
||||
|
||||
If you are interested in working on this issue or have submitted a pull request, please leave a comment.
|
||||
@@ -17,7 +17,7 @@ jobs:
|
||||
steps:
|
||||
- uses: actions/setup-go@v4
|
||||
with:
|
||||
go-version: '1.21'
|
||||
go-version: '1.20'
|
||||
- uses: actions/checkout@v3
|
||||
- name: golangci-lint
|
||||
uses: golangci/golangci-lint-action@v3
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
name: Performance Test
|
||||
name: Performance Test
|
||||
|
||||
on: [pull_request]
|
||||
|
||||
@@ -7,13 +7,13 @@ jobs:
|
||||
# skip if PR is from a fork.
|
||||
# TODO: this could probabaly be refactored a bit so that it runs on forks
|
||||
if: ${{ ! github.event.pull_request.head.repo.fork }}
|
||||
|
||||
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Install Go
|
||||
uses: actions/setup-go@v4
|
||||
with:
|
||||
go-version: '1.21'
|
||||
go-version: '1.20'
|
||||
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v3
|
||||
|
||||
@@ -35,7 +35,7 @@ jobs:
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v4
|
||||
with:
|
||||
go-version: '1.21'
|
||||
go-version: '1.20'
|
||||
- name: Run GoReleaser
|
||||
uses: goreleaser/goreleaser-action@v4
|
||||
with:
|
||||
|
||||
@@ -16,7 +16,7 @@ jobs:
|
||||
- name: Install Go
|
||||
uses: actions/setup-go@v4
|
||||
with:
|
||||
go-version: '1.21'
|
||||
go-version: '1.20'
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v3
|
||||
with:
|
||||
|
||||
@@ -11,7 +11,7 @@ jobs:
|
||||
- name: Install Go
|
||||
uses: actions/setup-go@v4
|
||||
with:
|
||||
go-version: '1.21'
|
||||
go-version: '1.20'
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v3
|
||||
- name: Smoke
|
||||
|
||||
@@ -18,7 +18,7 @@ jobs:
|
||||
steps:
|
||||
- uses: actions/setup-go@v4
|
||||
with:
|
||||
go-version: '1.21'
|
||||
go-version: '1.20'
|
||||
- uses: actions/checkout@v3
|
||||
- name: Run Snifftest
|
||||
run: make snifftest
|
||||
run: make snifftest
|
||||
@@ -21,7 +21,7 @@ jobs:
|
||||
- name: Install Go
|
||||
uses: actions/setup-go@v4
|
||||
with:
|
||||
go-version: '1.21'
|
||||
go-version: '1.20'
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v3
|
||||
- id: 'auth'
|
||||
@@ -42,7 +42,7 @@ jobs:
|
||||
- name: Install Go
|
||||
uses: actions/setup-go@v4
|
||||
with:
|
||||
go-version: '1.21'
|
||||
go-version: '1.20'
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v3
|
||||
- id: 'auth'
|
||||
@@ -52,8 +52,8 @@ jobs:
|
||||
service_account: 'github-ci-external@trufflehog-testing.iam.gserviceaccount.com'
|
||||
- name: Test
|
||||
run: make test-detectors
|
||||
continue-on-error: true
|
||||
test-community:
|
||||
continue-on-error: true
|
||||
test-fork:
|
||||
if: ${{ github.event.pull_request.head.repo.fork }}
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
@@ -63,8 +63,8 @@ jobs:
|
||||
- name: Install Go
|
||||
uses: actions/setup-go@v4
|
||||
with:
|
||||
go-version: '1.21'
|
||||
go-version: '1.20'
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v3
|
||||
- name: Test
|
||||
run: make test-community
|
||||
run: make test-forks
|
||||
|
||||
@@ -39,7 +39,7 @@ test-race:
|
||||
test-detectors:
|
||||
CGO_ENABLED=0 go test -tags=detectors -timeout=5m $(shell go list ./... | grep pkg/detectors)
|
||||
|
||||
test-community:
|
||||
test-forks:
|
||||
CGO_ENABLED=0 go test -timeout=5m $(shell go list ./... | grep -v /vendor/ | grep -v pkg/detectors | grep -v pkg/sources)
|
||||
|
||||
bench:
|
||||
|
||||
@@ -8,6 +8,7 @@
|
||||
|
||||
<div align="center">
|
||||
|
||||
[](https://github.com/trufflesecurity/trufflehog/actions)
|
||||
[](https://goreportcard.com/report/github.com/trufflesecurity/trufflehog/v3)
|
||||
[](/LICENSE)
|
||||
[](/pkg/detectors)
|
||||
@@ -263,9 +264,9 @@ jobs:
|
||||
extra_args: --debug --only-verified
|
||||
```
|
||||
|
||||
# Pre-commit Hook
|
||||
# Precommit Hook
|
||||
|
||||
Trufflehog can be used in a pre-commit hook to prevent credentials from leaking before they ever leave your computer.
|
||||
Trufflehog can be used in a precommit hook to prevent credentials from leaking before they ever leave your computer.
|
||||
An example `.pre-commit-config.yaml` is provided (see [pre-commit.com](https://pre-commit.com/) for installation).
|
||||
|
||||
```yaml
|
||||
|
||||
@@ -14,15 +14,10 @@ require (
|
||||
github.com/BobuSumisu/aho-corasick v1.0.3
|
||||
github.com/TheZeroSlave/zapsentry v1.17.0
|
||||
github.com/aws/aws-sdk-go v1.44.83
|
||||
github.com/aymanbagabas/go-osc52 v1.2.1
|
||||
github.com/bill-rich/disk-buffer-reader v0.1.7
|
||||
github.com/bill-rich/go-syslog v0.0.0-20220413021637-49edb52a574c
|
||||
github.com/bitfinexcom/bitfinex-api-go v0.0.0-20210608095005-9e0b26f200fb
|
||||
github.com/bradleyfalzon/ghinstallation/v2 v2.6.0
|
||||
github.com/charmbracelet/bubbles v0.16.1
|
||||
github.com/charmbracelet/bubbletea v0.24.1
|
||||
github.com/charmbracelet/glamour v0.6.0
|
||||
github.com/charmbracelet/lipgloss v0.7.1
|
||||
github.com/couchbase/gocb/v2 v2.6.3
|
||||
github.com/crewjam/rfc5424 v0.1.0
|
||||
github.com/denisenkom/go-mssqldb v0.12.3
|
||||
@@ -31,7 +26,7 @@ require (
|
||||
github.com/felixge/fgprof v0.9.3
|
||||
github.com/getsentry/sentry-go v0.22.0
|
||||
github.com/go-errors/errors v1.4.2
|
||||
github.com/go-git/go-git/v5 v5.8.1
|
||||
github.com/go-git/go-git/v5 v5.8.0
|
||||
github.com/go-ldap/ldap/v3 v3.4.5
|
||||
github.com/go-logr/logr v1.2.4
|
||||
github.com/go-logr/zapr v1.2.4
|
||||
@@ -39,11 +34,11 @@ require (
|
||||
github.com/go-sql-driver/mysql v1.7.1
|
||||
github.com/gobwas/glob v0.2.3
|
||||
github.com/golang-jwt/jwt v3.2.2+incompatible
|
||||
github.com/golang/mock v1.4.3
|
||||
github.com/google/go-cmp v0.5.9
|
||||
github.com/google/go-containerregistry v0.15.2
|
||||
github.com/google/go-github/v42 v42.0.0
|
||||
github.com/googleapis/gax-go/v2 v2.12.0
|
||||
github.com/h2non/filetype v1.1.3
|
||||
github.com/hashicorp/go-retryablehttp v0.7.4
|
||||
github.com/hashicorp/golang-lru v0.5.1
|
||||
github.com/jlaffaye/ftp v0.2.0
|
||||
@@ -51,11 +46,8 @@ require (
|
||||
github.com/jpillora/overseer v1.1.6
|
||||
github.com/kylelemons/godebug v1.1.0
|
||||
github.com/lib/pq v1.10.9
|
||||
github.com/lrstanley/bubblezone v0.0.0-20221222153816-e95291e2243e
|
||||
github.com/mattn/go-isatty v0.0.18
|
||||
github.com/mattn/go-sqlite3 v1.14.17
|
||||
github.com/mholt/archiver/v4 v4.0.0-alpha.8
|
||||
github.com/muesli/reflow v0.3.0
|
||||
github.com/patrickmn/go-cache v2.1.0+incompatible
|
||||
github.com/paulbellamy/ratecounter v0.2.0
|
||||
github.com/pkg/errors v0.9.1
|
||||
@@ -85,7 +77,6 @@ require (
|
||||
cloud.google.com/go/compute v1.20.1 // indirect
|
||||
cloud.google.com/go/compute/metadata v0.2.3 // indirect
|
||||
cloud.google.com/go/iam v1.1.0 // indirect
|
||||
dario.cat/mergo v1.0.0 // indirect
|
||||
github.com/Azure/go-autorest v14.2.0+incompatible // indirect
|
||||
github.com/Azure/go-autorest/autorest v0.11.24 // indirect
|
||||
github.com/Azure/go-autorest/autorest/adal v0.9.18 // indirect
|
||||
@@ -97,13 +88,9 @@ require (
|
||||
github.com/Microsoft/go-winio v0.6.1 // indirect
|
||||
github.com/ProtonMail/go-crypto v0.0.0-20230717121422-5aa5874ade95 // indirect
|
||||
github.com/acomagu/bufpipe v1.0.4 // indirect
|
||||
github.com/alecthomas/chroma v0.10.0 // indirect
|
||||
github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 // indirect
|
||||
github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 // indirect
|
||||
github.com/andybalholm/brotli v1.0.5 // indirect
|
||||
github.com/atotto/clipboard v0.1.4 // indirect
|
||||
github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect
|
||||
github.com/aymerick/douceur v0.2.0 // indirect
|
||||
github.com/benbjohnson/clock v1.1.0 // indirect
|
||||
github.com/beorn7/perks v1.0.1 // indirect
|
||||
github.com/bodgit/plumbing v1.2.0 // indirect
|
||||
@@ -112,12 +99,10 @@ require (
|
||||
github.com/cespare/xxhash/v2 v2.2.0 // indirect
|
||||
github.com/cloudflare/circl v1.3.3 // indirect
|
||||
github.com/connesc/cipherio v0.2.1 // indirect
|
||||
github.com/containerd/console v1.0.4-0.20230313162750-1ae8d489ac81 // indirect
|
||||
github.com/containerd/stargz-snapshotter/estargz v0.14.3 // indirect
|
||||
github.com/couchbase/gocbcore/v10 v10.2.4 // indirect
|
||||
github.com/davecgh/go-spew v1.1.1 // indirect
|
||||
github.com/dimchansky/utfbom v1.1.1 // indirect
|
||||
github.com/dlclark/regexp2 v1.4.0 // indirect
|
||||
github.com/docker/cli v23.0.5+incompatible // indirect
|
||||
github.com/docker/distribution v2.8.2+incompatible // indirect
|
||||
github.com/docker/docker v23.0.5+incompatible // indirect
|
||||
@@ -140,30 +125,24 @@ require (
|
||||
github.com/google/s2a-go v0.1.4 // indirect
|
||||
github.com/google/uuid v1.3.0 // indirect
|
||||
github.com/googleapis/enterprise-certificate-proxy v0.2.5 // indirect
|
||||
github.com/gorilla/css v1.0.0 // indirect
|
||||
github.com/h2non/parth v0.0.0-20190131123155-b4df798d6542 // indirect
|
||||
github.com/hashicorp/errwrap v1.0.0 // indirect
|
||||
github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
|
||||
github.com/hashicorp/go-multierror v1.1.1 // indirect
|
||||
github.com/hashicorp/golang-lru/v2 v2.0.4 // indirect
|
||||
github.com/imdario/mergo v0.3.15 // indirect
|
||||
github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect
|
||||
github.com/jmespath/go-jmespath v0.4.0 // indirect
|
||||
github.com/jpillora/s3 v1.1.4 // indirect
|
||||
github.com/kevinburke/ssh_config v1.2.0 // indirect
|
||||
github.com/klauspost/compress v1.16.5 // indirect
|
||||
github.com/klauspost/pgzip v1.2.5 // indirect
|
||||
github.com/lucasb-eyer/go-colorful v1.2.0 // indirect
|
||||
github.com/mattn/go-colorable v0.1.13 // indirect
|
||||
github.com/mattn/go-localereader v0.0.1 // indirect
|
||||
github.com/mattn/go-runewidth v0.0.14 // indirect
|
||||
github.com/mattn/go-isatty v0.0.17 // indirect
|
||||
github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
|
||||
github.com/microcosm-cc/bluemonday v1.0.23 // indirect
|
||||
github.com/mitchellh/go-homedir v1.1.0 // indirect
|
||||
github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe // indirect
|
||||
github.com/muesli/ansi v0.0.0-20211031195517-c9f0611b6c70 // indirect
|
||||
github.com/muesli/cancelreader v0.2.2 // indirect
|
||||
github.com/muesli/termenv v0.15.1 // indirect
|
||||
github.com/nwaples/rardecode/v2 v2.0.0-beta.2 // indirect
|
||||
github.com/olekukonko/tablewriter v0.0.5 // indirect
|
||||
github.com/onsi/ginkgo v1.16.5 // indirect
|
||||
github.com/onsi/gomega v1.23.0 // indirect
|
||||
github.com/opencontainers/go-digest v1.0.0 // indirect
|
||||
@@ -175,9 +154,7 @@ require (
|
||||
github.com/prometheus/client_model v0.3.0 // indirect
|
||||
github.com/prometheus/common v0.42.0 // indirect
|
||||
github.com/prometheus/procfs v0.10.1 // indirect
|
||||
github.com/rivo/uniseg v0.4.2 // indirect
|
||||
github.com/rogpeppe/go-internal v1.10.0 // indirect
|
||||
github.com/sahilm/fuzzy v0.1.0 // indirect
|
||||
github.com/sirupsen/logrus v1.9.0 // indirect
|
||||
github.com/skeema/knownhosts v1.2.0 // indirect
|
||||
github.com/therootcompany/xz v1.0.1 // indirect
|
||||
@@ -188,8 +165,6 @@ require (
|
||||
github.com/xdg-go/scram v1.1.2 // indirect
|
||||
github.com/xdg-go/stringprep v1.0.4 // indirect
|
||||
github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d // indirect
|
||||
github.com/yuin/goldmark v1.5.2 // indirect
|
||||
github.com/yuin/goldmark-emoji v1.0.1 // indirect
|
||||
github.com/yusufpapurcu/wmi v1.2.2 // indirect
|
||||
go.opencensus.io v0.24.0 // indirect
|
||||
go.uber.org/atomic v1.7.0 // indirect
|
||||
@@ -198,7 +173,6 @@ require (
|
||||
golang.org/x/mod v0.11.0 // indirect
|
||||
golang.org/x/net v0.12.0 // indirect
|
||||
golang.org/x/sys v0.10.0 // indirect
|
||||
golang.org/x/term v0.10.0 // indirect
|
||||
golang.org/x/time v0.3.0 // indirect
|
||||
golang.org/x/tools v0.10.0 // indirect
|
||||
golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect
|
||||
|
||||
@@ -27,8 +27,6 @@ cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiy
|
||||
cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos=
|
||||
cloud.google.com/go/storage v1.31.0 h1:+S3LjjEN2zZ+L5hOwj4+1OkGCsLVe0NzpXKQ1pSdTCI=
|
||||
cloud.google.com/go/storage v1.31.0/go.mod h1:81ams1PrhW16L4kF7qg+4mTq7SRs5HsbDTM0bWvrwJ0=
|
||||
dario.cat/mergo v1.0.0 h1:AGCNq9Evsj31mOgNPcLyXc+4PNABt905YmuqPYYpBWk=
|
||||
dario.cat/mergo v1.0.0/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk=
|
||||
dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/azcore v0.19.0/go.mod h1:h6H6c8enJmmocHUbLiiGY6sx7f9i+X3m1CHdd5c6Rdw=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/azidentity v0.11.0/go.mod h1:HcM1YX14R7CJcghJGOYCgdezslRSVzqwLf/q+4Y2r/0=
|
||||
@@ -68,8 +66,6 @@ github.com/TheZeroSlave/zapsentry v1.17.0 h1:RIQCG89U7vWWZVmmCxeUz/g32WEcAYXUrXH
|
||||
github.com/TheZeroSlave/zapsentry v1.17.0/go.mod h1:D1YMfSuu6xnkhwFXxrronesmsiyDhIqo+86I3Ok+r64=
|
||||
github.com/acomagu/bufpipe v1.0.4 h1:e3H4WUzM3npvo5uv95QuJM3cQspFNtFBzvJ2oNjKIDQ=
|
||||
github.com/acomagu/bufpipe v1.0.4/go.mod h1:mxdxdup/WdsKVreO5GpW4+M/1CE2sMG4jeGJ2sYmHc4=
|
||||
github.com/alecthomas/chroma v0.10.0 h1:7XDcGkCQopCNKjZHfYrNLraA+M7e0fMiJ/Mfikbfjek=
|
||||
github.com/alecthomas/chroma v0.10.0/go.mod h1:jtJATyUxlIORhUOFNA9NZDWGAQ8wpxQQqNSB4rjA/1s=
|
||||
github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 h1:JYp7IbQjafoB+tBA3gMyHYHrpOtNuDiK/uB5uXxq5wM=
|
||||
github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
|
||||
github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 h1:s6gZFSlWYmbqAuRjVTiNNhvNRfY2Wxp9nhfyel4rklc=
|
||||
@@ -81,17 +77,8 @@ github.com/andybalholm/brotli v1.0.5/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHG
|
||||
github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be h1:9AeTilPcZAjCFIImctFaOjnTIavg87rW78vTPkQqLI8=
|
||||
github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY=
|
||||
github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio=
|
||||
github.com/atotto/clipboard v0.1.4 h1:EH0zSVneZPSuFR11BlR9YppQTVDbh5+16AmcJi4g1z4=
|
||||
github.com/atotto/clipboard v0.1.4/go.mod h1:ZY9tmq7sm5xIbd9bOK4onWV4S6X0u6GY7Vn0Yu86PYI=
|
||||
github.com/aws/aws-sdk-go v1.44.83 h1:7+Rtc2Eio6EKUNoZeMV/IVxzVrY5oBQcNPtCcgIHYJA=
|
||||
github.com/aws/aws-sdk-go v1.44.83/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo=
|
||||
github.com/aymanbagabas/go-osc52 v1.0.3/go.mod h1:zT8H+Rk4VSabYN90pWyugflM3ZhpTZNC7cASDfUCdT4=
|
||||
github.com/aymanbagabas/go-osc52 v1.2.1 h1:q2sWUyDcozPLcLabEMd+a+7Ea2DitxZVN9hTxab9L4E=
|
||||
github.com/aymanbagabas/go-osc52 v1.2.1/go.mod h1:zT8H+Rk4VSabYN90pWyugflM3ZhpTZNC7cASDfUCdT4=
|
||||
github.com/aymanbagabas/go-osc52/v2 v2.0.1 h1:HwpRHbFMcZLEVr42D4p7XBqjyuxQH5SMiErDT4WkJ2k=
|
||||
github.com/aymanbagabas/go-osc52/v2 v2.0.1/go.mod h1:uYgXzlJ7ZpABp8OJ+exZzJJhRNQ2ASbcXHWsFqH8hp8=
|
||||
github.com/aymerick/douceur v0.2.0 h1:Mv+mAeH1Q+n9Fr+oyamOlAkUNPWPlA8PPGR0QAaYuPk=
|
||||
github.com/aymerick/douceur v0.2.0/go.mod h1:wlT5vV2O3h55X9m7iVYN0TBM0NH/MmbLnd30/FjWUq4=
|
||||
github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8=
|
||||
github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA=
|
||||
github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
|
||||
@@ -116,14 +103,6 @@ github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA
|
||||
github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
|
||||
github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44=
|
||||
github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
|
||||
github.com/charmbracelet/bubbles v0.16.1 h1:6uzpAAaT9ZqKssntbvZMlksWHruQLNxg49H5WdeuYSY=
|
||||
github.com/charmbracelet/bubbles v0.16.1/go.mod h1:2QCp9LFlEsBQMvIYERr7Ww2H2bA7xen1idUDIzm/+Xc=
|
||||
github.com/charmbracelet/bubbletea v0.24.1 h1:LpdYfnu+Qc6XtvMz6d/6rRY71yttHTP5HtrjMgWvixc=
|
||||
github.com/charmbracelet/bubbletea v0.24.1/go.mod h1:rK3g/2+T8vOSEkNHvtq40umJpeVYDn6bLaqbgzhL/hg=
|
||||
github.com/charmbracelet/glamour v0.6.0 h1:wi8fse3Y7nfcabbbDuwolqTqMQPMnVPeZhDM273bISc=
|
||||
github.com/charmbracelet/glamour v0.6.0/go.mod h1:taqWV4swIMMbWALc0m7AfE9JkPSU8om2538k9ITBxOc=
|
||||
github.com/charmbracelet/lipgloss v0.7.1 h1:17WMwi7N1b1rVWOjMT+rCh7sQkvDU75B2hbZpc5Kc1E=
|
||||
github.com/charmbracelet/lipgloss v0.7.1/go.mod h1:yG0k3giv8Qj8edTCbbg6AlQ5e8KNWpFujkNawKNhE2c=
|
||||
github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
|
||||
github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
|
||||
github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
|
||||
@@ -139,8 +118,6 @@ github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWH
|
||||
github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
|
||||
github.com/connesc/cipherio v0.2.1 h1:FGtpTPMbKNNWByNrr9aEBtaJtXjqOzkIXNYJp6OEycw=
|
||||
github.com/connesc/cipherio v0.2.1/go.mod h1:ukY0MWJDFnJEbXMQtOcn2VmTpRfzcTz4OoVrWGGJZcA=
|
||||
github.com/containerd/console v1.0.4-0.20230313162750-1ae8d489ac81 h1:q2hJAaP1k2wIvVRd/hEHD7lacgqrCPS+k8g1MndzfWY=
|
||||
github.com/containerd/console v1.0.4-0.20230313162750-1ae8d489ac81/go.mod h1:YynlIjWYF8myEu6sdkwKIvGQq+cOckRm6So2avqoYAk=
|
||||
github.com/containerd/stargz-snapshotter/estargz v0.14.3 h1:OqlDCK3ZVUO6C3B/5FSkDwbkEETK84kQgEeFwDC+62k=
|
||||
github.com/containerd/stargz-snapshotter/estargz v0.14.3/go.mod h1:KY//uOCIkSuNAHhJogcZtrNHdKrA99/FCCRjE3HD36o=
|
||||
github.com/couchbase/gocb/v2 v2.6.3 h1:5RsMo+RRfK0mVxHLAfpBz3/tHlgXZb1WBNItLk9Ab+c=
|
||||
@@ -161,8 +138,6 @@ github.com/denisenkom/go-mssqldb v0.12.3 h1:pBSGx9Tq67pBOTLmxNuirNTeB8Vjmf886Kx+
|
||||
github.com/denisenkom/go-mssqldb v0.12.3/go.mod h1:k0mtMFOnU+AihqFxPMiF05rtiDrorD1Vrm1KEz5hxDo=
|
||||
github.com/dimchansky/utfbom v1.1.1 h1:vV6w1AhK4VMnhBno/TPVCoK9U/LP0PkLCS9tbxHdi/U=
|
||||
github.com/dimchansky/utfbom v1.1.1/go.mod h1:SxdoEBH5qIqFocHMyGOXVAybYJdr71b1Q/j0mACtrfE=
|
||||
github.com/dlclark/regexp2 v1.4.0 h1:F1rxgk7p4uKjwIQxBs9oAXe5CqrXlCduYEJvrF4u93E=
|
||||
github.com/dlclark/regexp2 v1.4.0/go.mod h1:2pZnwuY/m+8K6iRw6wQdMtk+rH5tNGR1i55kozfMjCc=
|
||||
github.com/dnaeon/go-vcr v1.2.0/go.mod h1:R4UdLID7HZT3taECzJs4YgbbH6PIGXB6W/sc5OLb6RQ=
|
||||
github.com/docker/cli v23.0.5+incompatible h1:ufWmAOuD3Vmr7JP2G5K3cyuNC4YZWiAsuDEvFVVDafE=
|
||||
github.com/docker/cli v23.0.5+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8=
|
||||
@@ -206,8 +181,8 @@ github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376/go.mod h1:an3vInlBmS
|
||||
github.com/go-git/go-billy/v5 v5.4.1 h1:Uwp5tDRkPr+l/TnbHOQzp+tmJfLceOlbVucgpTz8ix4=
|
||||
github.com/go-git/go-billy/v5 v5.4.1/go.mod h1:vjbugF6Fz7JIflbVpl1hJsGjSHNltrSw45YK/ukIvQg=
|
||||
github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20230305113008-0c11038e723f h1:Pz0DHeFij3XFhoBRGUDPzSJ+w2UcK5/0JvF8DRI58r8=
|
||||
github.com/go-git/go-git/v5 v5.8.1 h1:Zo79E4p7TRk0xoRgMq0RShiTHGKcKI4+DI6BfJc/Q+A=
|
||||
github.com/go-git/go-git/v5 v5.8.1/go.mod h1:FHFuoD6yGz5OSKEBK+aWN9Oah0q54Jxl0abmj6GnqAo=
|
||||
github.com/go-git/go-git/v5 v5.8.0 h1:Rc543s6Tyq+YcyPwZRvU4jzZGM8rB/wWu94TnTIYALQ=
|
||||
github.com/go-git/go-git/v5 v5.8.0/go.mod h1:coJHKEOk5kUClpsNlXrUvPrDxY3w3gjHvhcZd8Fodw8=
|
||||
github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
|
||||
github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
|
||||
github.com/go-ldap/ldap/v3 v3.4.5 h1:ekEKmaDrpvR2yf5Nc/DClsGG9lAmdDixe44mLzlW5r8=
|
||||
@@ -311,12 +286,8 @@ github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+
|
||||
github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
|
||||
github.com/googleapis/gax-go/v2 v2.12.0 h1:A+gCJKdRfqXkr+BIRGtZLibNXf0m1f9E4HG56etFpas=
|
||||
github.com/googleapis/gax-go/v2 v2.12.0/go.mod h1:y+aIqrI5eb1YGMVJfuV3185Ts/D7qKpsEkdD5+I6QGU=
|
||||
github.com/gorilla/css v1.0.0 h1:BQqNyPTi50JCFMTw/b67hByjMVXZRwGha6wxVGkeihY=
|
||||
github.com/gorilla/css v1.0.0/go.mod h1:Dn721qIggHpt4+EFCcTLTU/vk5ySda2ReITrtgBl60c=
|
||||
github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
|
||||
github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw=
|
||||
github.com/h2non/filetype v1.1.3 h1:FKkx9QbD7HR/zjK1Ia5XiBsq9zdLi5Kf3zGyFTAFkGg=
|
||||
github.com/h2non/filetype v1.1.3/go.mod h1:319b3zT68BvV+WRj7cwy856M2ehB3HqNOt6sy1HndBY=
|
||||
github.com/h2non/parth v0.0.0-20190131123155-b4df798d6542 h1:2VTzZjLZBgl62/EtslCrtky5vbi9dd7HrQPQIx6wqiw=
|
||||
github.com/h2non/parth v0.0.0-20190131123155-b4df798d6542/go.mod h1:Ow0tF8D4Kplbc8s8sSb3V2oUCygFHVp8gC3Dn6U4MNI=
|
||||
github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA=
|
||||
@@ -332,9 +303,13 @@ github.com/hashicorp/go-retryablehttp v0.7.4/go.mod h1:Jy/gPYAdjqffZ/yFGCFV2doI5
|
||||
github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
|
||||
github.com/hashicorp/golang-lru v0.5.1 h1:0hERBMJE1eitiLkihrMvRVBYAkpHzc/J3QdDN+dAcgU=
|
||||
github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
|
||||
github.com/hashicorp/golang-lru/v2 v2.0.4 h1:7GHuZcgid37q8o5i3QI9KMT4nCWQQ3Kx3Ov6bb9MfK0=
|
||||
github.com/hashicorp/golang-lru/v2 v2.0.4/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM=
|
||||
github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
|
||||
github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
|
||||
github.com/ianlancetaylor/demangle v0.0.0-20210905161508-09a460cdf81d/go.mod h1:aYm2/VgdVmcIU8iMfdMvDMsRAQjcfZSKFby6HOFvi/w=
|
||||
github.com/imdario/mergo v0.3.15 h1:M8XP7IuFNsqUx6VPK2P9OSmsYsI/YFaGil0uD21V3dM=
|
||||
github.com/imdario/mergo v0.3.15/go.mod h1:WBLT9ZmE3lPoWsEzCh9LPo3TiwVN+ZKEjmz+hD27ysY=
|
||||
github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOlocH6Fxy8MmwDt+yVQYULKfN0RoTN8A=
|
||||
github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i6rXxKeerYnT8Nvf0QmHCRC1n8sfWVwXF2Frvo=
|
||||
github.com/jlaffaye/ftp v0.2.0 h1:lXNvW7cBu7R/68bknOX3MrRIIqZ61zELs1P2RAiA3lg=
|
||||
@@ -370,46 +345,24 @@ github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0
|
||||
github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
|
||||
github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw=
|
||||
github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
|
||||
github.com/lrstanley/bubblezone v0.0.0-20221222153816-e95291e2243e h1:XoxHx8K6ZKoMtjzWOMDuM69LCdjDDsTOtTfWGrT/fns=
|
||||
github.com/lrstanley/bubblezone v0.0.0-20221222153816-e95291e2243e/go.mod h1:v5lEwWaguF1o2MW/ucO0ZIA/IZymdBYJJ+2cMRLE7LU=
|
||||
github.com/lucasb-eyer/go-colorful v1.2.0 h1:1nnpGOrhyZZuNyfu1QjKiUICQ74+3FNCN69Aj6K7nkY=
|
||||
github.com/lucasb-eyer/go-colorful v1.2.0/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i7ruzyGqttikkLy0=
|
||||
github.com/matryer/is v1.2.0 h1:92UTHpy8CDwaJ08GqLDzhhuixiBUUD1p3AU6PHddz4A=
|
||||
github.com/matryer/is v1.2.0/go.mod h1:2fLPjFQM9rhQ15aVEtbuwhJinnOqrmgXPNdZsdwlWXA=
|
||||
github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=
|
||||
github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg=
|
||||
github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
|
||||
github.com/mattn/go-isatty v0.0.18 h1:DOKFKCQ7FNG2L1rbrmstDN4QVRdS89Nkh85u68Uwp98=
|
||||
github.com/mattn/go-isatty v0.0.18/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
|
||||
github.com/mattn/go-localereader v0.0.1 h1:ygSAOl7ZXTx4RdPYinUpg6W99U8jWvWi9Ye2JC/oIi4=
|
||||
github.com/mattn/go-localereader v0.0.1/go.mod h1:8fBrzywKY7BI3czFoHkuzRoWE9C+EiG4R1k4Cjx5p88=
|
||||
github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI=
|
||||
github.com/mattn/go-runewidth v0.0.12/go.mod h1:RAqKPSqVFrSLVXbA8x7dzmKdmGzieGRCM46jaSJTDAk=
|
||||
github.com/mattn/go-runewidth v0.0.14 h1:+xnbZSEeDbOIg5/mE6JF0w6n9duR1l3/WmbinWVwUuU=
|
||||
github.com/mattn/go-runewidth v0.0.14/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
|
||||
github.com/mattn/go-isatty v0.0.17 h1:BTarxUcIeDqL27Mc+vyvdWYSL28zpIhv3RoTdsLMPng=
|
||||
github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
|
||||
github.com/mattn/go-sqlite3 v1.14.17 h1:mCRHCLDUBXgpKAqIKsaAaAsrAlbkeomtRFKXh2L6YIM=
|
||||
github.com/mattn/go-sqlite3 v1.14.17/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg=
|
||||
github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo=
|
||||
github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4=
|
||||
github.com/mholt/archiver/v4 v4.0.0-alpha.8 h1:tRGQuDVPh66WCOelqe6LIGh0gwmfwxUrSSDunscGsRM=
|
||||
github.com/mholt/archiver/v4 v4.0.0-alpha.8/go.mod h1:5f7FUYGXdJWUjESffJaYR4R60VhnHxb2X3T1teMyv5A=
|
||||
github.com/microcosm-cc/bluemonday v1.0.21/go.mod h1:ytNkv4RrDrLJ2pqlsSI46O6IVXmZOBBD4SaJyDwwTkM=
|
||||
github.com/microcosm-cc/bluemonday v1.0.23 h1:SMZe2IGa0NuHvnVNAZ+6B38gsTbi5e4sViiWJyDDqFY=
|
||||
github.com/microcosm-cc/bluemonday v1.0.23/go.mod h1:mN70sk7UkkF8TUr2IGBpNN0jAgStuPzlK76QuruE/z4=
|
||||
github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y=
|
||||
github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
|
||||
github.com/modocache/gover v0.0.0-20171022184752-b58185e213c5/go.mod h1:caMODM3PzxT8aQXRPkAt8xlV/e7d7w8GM5g0fa5F0D8=
|
||||
github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe h1:iruDEfMl2E6fbMZ9s0scYfZQ84/6SPL6zC8ACM2oIL0=
|
||||
github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc=
|
||||
github.com/muesli/ansi v0.0.0-20211031195517-c9f0611b6c70 h1:kMlmsLSbjkikxQJ1IPwaM+7LJ9ltFu/fi8CRzvSnQmA=
|
||||
github.com/muesli/ansi v0.0.0-20211031195517-c9f0611b6c70/go.mod h1:fQuZ0gauxyBcmsdE3ZT4NasjaRdxmbCS0jRHsrWu3Ho=
|
||||
github.com/muesli/cancelreader v0.2.2 h1:3I4Kt4BQjOR54NavqnDogx/MIoWBFa0StPA8ELUXHmA=
|
||||
github.com/muesli/cancelreader v0.2.2/go.mod h1:3XuTXfFS2VjM+HTLZY9Ak0l6eUKfijIfMUZ4EgX0QYo=
|
||||
github.com/muesli/reflow v0.3.0 h1:IFsN6K9NfGtjeggFP+68I4chLZV2yIKsXJFNZ+eWh6s=
|
||||
github.com/muesli/reflow v0.3.0/go.mod h1:pbwTDkVPibjO2kyvBQRBxTWEEGDGq0FlB1BIKtnHY/8=
|
||||
github.com/muesli/termenv v0.13.0/go.mod h1:sP1+uffeLaEYpyOTb8pLCUctGcGLnoFjSn4YJK5e2bc=
|
||||
github.com/muesli/termenv v0.15.1 h1:UzuTb/+hhlBugQz28rpzey4ZuKcZ03MeKsoG7IJZIxs=
|
||||
github.com/muesli/termenv v0.15.1/go.mod h1:HeAQPTzpfs016yGtA4g00CsdYnVLJvxsS4ANqrZs2sQ=
|
||||
github.com/nbio/st v0.0.0-20140626010706-e9e8d9816f32 h1:W6apQkHrMkS0Muv8G/TipAy/FJl/rCYT0+EuS8+Z0z4=
|
||||
github.com/nbio/st v0.0.0-20140626010706-e9e8d9816f32/go.mod h1:9wM+0iRr9ahx58uYLpLIr5fm8diHn0JbqRycJi6w0Ms=
|
||||
github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
|
||||
@@ -418,8 +371,6 @@ github.com/nwaples/rardecode/v2 v2.0.0-beta.2/go.mod h1:yntwv/HfMc/Hbvtq9I19D1n5
|
||||
github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A=
|
||||
github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE=
|
||||
github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU=
|
||||
github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec=
|
||||
github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY=
|
||||
github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
|
||||
github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk=
|
||||
github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE=
|
||||
@@ -461,18 +412,12 @@ github.com/prometheus/procfs v0.10.1 h1:kYK1Va/YMlutzCGazswoHKo//tZVlFpKYh+Pymzi
|
||||
github.com/prometheus/procfs v0.10.1/go.mod h1:nwNm2aOCAYw8uTR/9bWRREkZFxAUcWzPHWJq+XBB/FM=
|
||||
github.com/rabbitmq/amqp091-go v1.8.1 h1:RejT1SBUim5doqcL6s7iN6SBmsQqyTgXb1xMlH0h1hA=
|
||||
github.com/rabbitmq/amqp091-go v1.8.1/go.mod h1:+jPrT9iY2eLjRaMSRHUhc3z14E/l85kv/f+6luSD3pc=
|
||||
github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
|
||||
github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
|
||||
github.com/rivo/uniseg v0.4.2 h1:YwD0ulJSJytLpiaWua0sBDusfsCZohxjxzVTYjwxfV8=
|
||||
github.com/rivo/uniseg v0.4.2/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
|
||||
github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ=
|
||||
github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
|
||||
github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ=
|
||||
github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog=
|
||||
github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
|
||||
github.com/rwcarlsen/goexif v0.0.0-20190401172101-9e8deecbddbd/go.mod h1:hPqNNc0+uJM6H+SuU8sEs5K5IQeKccPqeSjfgcKGgPk=
|
||||
github.com/sahilm/fuzzy v0.1.0 h1:FzWGaw2Opqyu+794ZQ9SYifWv2EIXpwP4q8dY1kDAwI=
|
||||
github.com/sahilm/fuzzy v0.1.0/go.mod h1:VFvziUEIMCrT6A6tw2RFIXPXXmzXbOsSHF0DOI8ZK9Y=
|
||||
github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
|
||||
github.com/sergi/go-diff v1.3.1 h1:xkr+Oxo4BOQKmkn/B9eMK0g5Kg/983T9DqqPHwYqD+8=
|
||||
github.com/sergi/go-diff v1.3.1/go.mod h1:aMJSSKb2lpPvRNec0+w3fl7LP9IOFzdc9Pa4NFbPK1I=
|
||||
@@ -528,10 +473,6 @@ github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7Jul
|
||||
github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
|
||||
github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
|
||||
github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
|
||||
github.com/yuin/goldmark v1.5.2 h1:ALmeCk/px5FSm1MAcFBAsVKZjDuMVj8Tm7FFIlMJnqU=
|
||||
github.com/yuin/goldmark v1.5.2/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
|
||||
github.com/yuin/goldmark-emoji v1.0.1 h1:ctuWEyzGBwiucEqxzwe0SOYDXPAucOrE9NQC18Wa1os=
|
||||
github.com/yuin/goldmark-emoji v1.0.1/go.mod h1:2w1E6FEWLcDQkoTE+7HU6QF1F6SLlNGjRIBbIZQFqkQ=
|
||||
github.com/yusufpapurcu/wmi v1.2.2 h1:KBNDSne4vP5mbSWnJbO+51IMOXJB67QiYCSBrubbPRg=
|
||||
github.com/yusufpapurcu/wmi v1.2.2/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0=
|
||||
go.mongodb.org/mongo-driver v1.12.0 h1:aPx33jmn/rQuJXPQLZQ8NtfPQG8CaqgLThFtqRb0PiE=
|
||||
@@ -629,7 +570,6 @@ golang.org/x/net v0.0.0-20210610132358-84b48f89b13b/go.mod h1:9nx3DQGgdP8bBQD5qx
|
||||
golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
|
||||
golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
|
||||
golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
|
||||
golang.org/x/net v0.0.0-20221002022538-bcab6841153b/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk=
|
||||
golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY=
|
||||
golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
|
||||
golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc=
|
||||
@@ -690,10 +630,8 @@ golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBc
|
||||
golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220906165534-d0df966e6959/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
@@ -708,7 +646,6 @@ golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
|
||||
golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U=
|
||||
golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo=
|
||||
golang.org/x/term v0.10.0 h1:3R7pNqamzBraeqj/Tj8qt1aQ2HpmlC+Cx/qL/7hn4/c=
|
||||
golang.org/x/term v0.10.0/go.mod h1:lpqdcUyK/oCiQxvxVrppt5ggO2KCZ5QblwqPnfZ6d5o=
|
||||
golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
|
||||
@@ -13,7 +13,6 @@ import (
|
||||
"github.com/felixge/fgprof"
|
||||
"github.com/go-logr/logr"
|
||||
"github.com/jpillora/overseer"
|
||||
"github.com/mattn/go-isatty"
|
||||
"google.golang.org/protobuf/types/known/anypb"
|
||||
"gopkg.in/alecthomas/kingpin.v2"
|
||||
|
||||
@@ -29,7 +28,6 @@ import (
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/sources/git"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/tui"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/updater"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/version"
|
||||
)
|
||||
@@ -68,7 +66,6 @@ var (
|
||||
gitScanSinceCommit = gitScan.Flag("since-commit", "Commit to start scan from.").String()
|
||||
gitScanBranch = gitScan.Flag("branch", "Branch to scan.").String()
|
||||
gitScanMaxDepth = gitScan.Flag("max-depth", "Maximum depth of commits to scan.").Int()
|
||||
gitScanBare = gitScan.Flag("bare", "Scan bare repository (e.g. useful while using in pre-receive hooks)").Bool()
|
||||
_ = gitScan.Flag("allow", "No-op flag for backwards compat.").Bool()
|
||||
_ = gitScan.Flag("entropy", "No-op flag for backwards compat.").Bool()
|
||||
_ = gitScan.Flag("regex", "No-op flag for backwards compat.").Bool()
|
||||
@@ -146,18 +143,6 @@ func init() {
|
||||
}
|
||||
|
||||
cli.Version("trufflehog " + version.BuildVersion)
|
||||
|
||||
if len(os.Args) <= 1 && isatty.IsTerminal(os.Stdout.Fd()) {
|
||||
args := tui.Run()
|
||||
if len(args) == 0 {
|
||||
os.Exit(0)
|
||||
}
|
||||
|
||||
// Overwrite the Args slice so overseer works properly.
|
||||
os.Args = os.Args[:1]
|
||||
os.Args = append(os.Args, args...)
|
||||
}
|
||||
|
||||
cmd = kingpin.MustParse(cli.Parse(os.Args[1:]))
|
||||
|
||||
switch {
|
||||
@@ -392,7 +377,6 @@ func run(state overseer.State) {
|
||||
HeadRef: *gitScanBranch,
|
||||
BaseRef: *gitScanSinceCommit,
|
||||
MaxDepth: *gitScanMaxDepth,
|
||||
Bare: *gitScanBare,
|
||||
Filter: filter,
|
||||
ExcludeGlobs: excludedGlobs,
|
||||
}
|
||||
@@ -533,7 +517,6 @@ func run(state overseer.State) {
|
||||
"bytes", metrics.BytesScanned,
|
||||
"verified_secrets", metrics.VerifiedSecretsFound,
|
||||
"unverified_secrets", metrics.UnverifiedSecretsFound,
|
||||
"scan_duration", metrics.ScanDuration.String(),
|
||||
)
|
||||
|
||||
if *printAvgDetectorTime {
|
||||
|
||||
@@ -16,14 +16,9 @@ func IsDone(ctx context.Context) bool {
|
||||
// write would succeed, either operation will be performed randomly.
|
||||
func CancellableWrite[T any](ctx context.Context, ch chan<- T, item T) error {
|
||||
select {
|
||||
case <-ctx.Done(): // priority to context cancellation
|
||||
case ch <- item:
|
||||
return nil
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
default:
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
case ch <- item:
|
||||
return nil
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
+42
-4
@@ -2,7 +2,10 @@ package context
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"runtime/debug"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/go-logr/logr"
|
||||
@@ -34,7 +37,9 @@ type CancelFunc = context.CancelFunc
|
||||
type logCtx struct {
|
||||
// Embed context.Context to get all methods for free.
|
||||
context.Context
|
||||
log logr.Logger
|
||||
log logr.Logger
|
||||
err *error
|
||||
errLock *sync.Mutex
|
||||
}
|
||||
|
||||
// Logger returns a structured logger.
|
||||
@@ -42,6 +47,13 @@ func (l logCtx) Logger() logr.Logger {
|
||||
return l.log
|
||||
}
|
||||
|
||||
func (l logCtx) Err() error {
|
||||
if l.err != nil && *l.err != nil {
|
||||
return *l.err
|
||||
}
|
||||
return l.Context.Err()
|
||||
}
|
||||
|
||||
// Background returns context.Background with a default logger.
|
||||
func Background() Context {
|
||||
return logCtx{
|
||||
@@ -65,7 +77,7 @@ func WithCancel(parent Context) (Context, context.CancelFunc) {
|
||||
log: parent.Logger(),
|
||||
Context: ctx,
|
||||
}
|
||||
return lCtx, cancel
|
||||
return captureCancelCallstack(lCtx, cancel)
|
||||
}
|
||||
|
||||
// WithDeadline returns context.WithDeadline with the log object propagated and
|
||||
@@ -76,7 +88,7 @@ func WithDeadline(parent Context, d time.Time) (Context, context.CancelFunc) {
|
||||
log: parent.Logger().WithValues("deadline", d),
|
||||
Context: ctx,
|
||||
}
|
||||
return lCtx, cancel
|
||||
return captureCancelCallstack(lCtx, cancel)
|
||||
}
|
||||
|
||||
// WithTimeout returns context.WithTimeout with the log object propagated and
|
||||
@@ -87,7 +99,7 @@ func WithTimeout(parent Context, timeout time.Duration) (Context, context.Cancel
|
||||
log: parent.Logger().WithValues("timeout", timeout),
|
||||
Context: ctx,
|
||||
}
|
||||
return lCtx, cancel
|
||||
return captureCancelCallstack(lCtx, cancel)
|
||||
}
|
||||
|
||||
// WithValue returns context.WithValue with the log object propagated and
|
||||
@@ -138,3 +150,29 @@ func AddLogger(parent context.Context) Context {
|
||||
func SetDefaultLogger(l logr.Logger) {
|
||||
defaultLogger = l
|
||||
}
|
||||
|
||||
// captureCancelCallstack is a helper function to capture the callstack where
|
||||
// the cancel function was first called.
|
||||
func captureCancelCallstack(ctx logCtx, f context.CancelFunc) (Context, context.CancelFunc) {
|
||||
if ctx.err == nil {
|
||||
var err error
|
||||
ctx.err = &err
|
||||
ctx.errLock = &sync.Mutex{}
|
||||
}
|
||||
return ctx, func() {
|
||||
ctx.errLock.Lock()
|
||||
defer ctx.errLock.Unlock()
|
||||
// We must check Err() before calling f() since f() sets the error.
|
||||
// If there's already an error, do nothing special.
|
||||
if ctx.Err() != nil {
|
||||
f()
|
||||
return
|
||||
}
|
||||
f()
|
||||
// Set the error with the stacktrace if the err pointer is non-nil.
|
||||
*ctx.err = fmt.Errorf(
|
||||
"%w (canceled at %v\n%s)",
|
||||
ctx.Err(), time.Now(), string(debug.Stack()),
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -170,10 +170,51 @@ func TestDefaultLogger(t *testing.T) {
|
||||
ctx.Logger().Info("this shouldn't panic")
|
||||
}
|
||||
|
||||
func TestRace(t *testing.T) {
|
||||
ctx, cancel := WithCancel(Background())
|
||||
go cancel()
|
||||
go func() { _ = ctx.Err() }()
|
||||
func TestErrCallstack(t *testing.T) {
|
||||
c, cancel := WithCancel(Background())
|
||||
ctx := c.(logCtx)
|
||||
cancel()
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
assert.Contains(t, ctx.Err().Error(), "TestErrCallstack")
|
||||
case <-time.After(1 * time.Second):
|
||||
assert.Fail(t, "context should be done")
|
||||
}
|
||||
}
|
||||
|
||||
func TestErrCallstackTimeout(t *testing.T) {
|
||||
ctx, cancel := WithTimeout(Background(), 10*time.Millisecond)
|
||||
defer cancel()
|
||||
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
// Deadline exceeded errors will not have a callstack from the cancel
|
||||
// function.
|
||||
assert.NotContains(t, ctx.Err().Error(), "TestErrCallstackTimeout")
|
||||
case <-time.After(1 * time.Second):
|
||||
assert.Fail(t, "context should be done")
|
||||
}
|
||||
}
|
||||
|
||||
func TestErrCallstackTimeoutCancel(t *testing.T) {
|
||||
ctx, cancel := WithTimeout(Background(), 10*time.Millisecond)
|
||||
|
||||
var err error
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
err = ctx.Err()
|
||||
case <-time.After(1 * time.Second):
|
||||
assert.Fail(t, "context should be done")
|
||||
}
|
||||
|
||||
// Calling cancel after deadline exceeded should not overwrite the original
|
||||
// error.
|
||||
cancel()
|
||||
assert.Equal(t, err, ctx.Err())
|
||||
}
|
||||
|
||||
func TestRace(t *testing.T) {
|
||||
_, cancel := WithCancel(Background())
|
||||
go cancel()
|
||||
cancel()
|
||||
_ = ctx.Err()
|
||||
}
|
||||
|
||||
@@ -65,7 +65,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
} else if res.StatusCode == 401 {
|
||||
// The secret is determinately not verified (nothing to do)
|
||||
} else {
|
||||
s1.VerificationError = fmt.Errorf("unexpected HTTP response status %d", res.StatusCode)
|
||||
s1.VerificationError = fmt.Errorf("request to %v returned unexpected status %d", res.Request.URL, res.StatusCode)
|
||||
}
|
||||
} else {
|
||||
s1.VerificationError = err
|
||||
|
||||
@@ -6,11 +6,10 @@ package alchemy
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"github.com/google/go-cmp/cmp"
|
||||
"github.com/google/go-cmp/cmp/cmpopts"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/kylelemons/godebug/pretty"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
@@ -132,12 +131,13 @@ func TestAlchemy_FromChunk(t *testing.T) {
|
||||
if len(got[i].Raw) == 0 {
|
||||
t.Fatalf("no raw secret present: \n %+v", got[i])
|
||||
}
|
||||
got[i].Raw = nil
|
||||
if (got[i].VerificationError != nil) != tt.wantVerificationErr {
|
||||
t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError)
|
||||
t.Fatalf("verification error = %v, wantVerificationError %v", got[i].VerificationError, tt.wantVerificationErr)
|
||||
}
|
||||
got[i].VerificationError = nil
|
||||
}
|
||||
ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "VerificationError")
|
||||
if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {
|
||||
if diff := pretty.Compare(got, tt.want); diff != "" {
|
||||
t.Errorf("Alchemy.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
|
||||
}
|
||||
})
|
||||
|
||||
@@ -20,7 +20,7 @@ var (
|
||||
client = common.SaneHttpClient()
|
||||
|
||||
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
|
||||
appPat = regexp.MustCompile(detectors.PrefixRegex([]string{"datadog", "dd"}) + `\b([a-zA-Z-0-9]{40})\b`)
|
||||
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"datadog", "dd"}) + `\b([a-zA-Z-0-9]{40})\b`)
|
||||
apiPat = regexp.MustCompile(detectors.PrefixRegex([]string{"datadog", "dd"}) + `\b([a-zA-Z-0-9]{32})\b`)
|
||||
)
|
||||
|
||||
@@ -34,28 +34,24 @@ func (s Scanner) Keywords() []string {
|
||||
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
|
||||
dataStr := string(data)
|
||||
|
||||
appMatches := appPat.FindAllStringSubmatch(dataStr, -1)
|
||||
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
|
||||
apiMatches := apiPat.FindAllStringSubmatch(dataStr, -1)
|
||||
|
||||
for _, apiMatch := range apiMatches {
|
||||
if len(apiMatch) != 2 {
|
||||
for _, match := range matches {
|
||||
if len(match) != 2 {
|
||||
continue
|
||||
}
|
||||
resApiMatch := strings.TrimSpace(apiMatch[1])
|
||||
appIncluded := false
|
||||
for _, appMatch := range appMatches {
|
||||
if len(appMatch) != 2 {
|
||||
resMatch := strings.TrimSpace(match[1])
|
||||
for _, apimatch := range apiMatches {
|
||||
if len(apimatch) != 2 {
|
||||
continue
|
||||
}
|
||||
resAppMatch := strings.TrimSpace(appMatch[1])
|
||||
resApiMatch := strings.TrimSpace(apimatch[1])
|
||||
|
||||
s1 := detectors.Result{
|
||||
DetectorType: detectorspb.DetectorType_DatadogToken,
|
||||
Raw: []byte(resAppMatch),
|
||||
RawV2: []byte(resAppMatch + resApiMatch),
|
||||
ExtraData: map[string]string{
|
||||
"Type": "Application+APIKey",
|
||||
},
|
||||
Raw: []byte(resMatch),
|
||||
RawV2: []byte(resMatch + resApiMatch),
|
||||
}
|
||||
|
||||
if verify {
|
||||
@@ -66,7 +62,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
}
|
||||
req.Header.Add("Content-Type", "application/json")
|
||||
req.Header.Add("DD-API-KEY", resApiMatch)
|
||||
req.Header.Add("DD-APPLICATION-KEY", resAppMatch)
|
||||
req.Header.Add("DD-APPLICATION-KEY", resMatch)
|
||||
res, err := client.Do(req)
|
||||
if err == nil {
|
||||
defer res.Body.Close()
|
||||
@@ -74,49 +70,16 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
s1.Verified = true
|
||||
} else {
|
||||
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
|
||||
if detectors.IsKnownFalsePositive(resApiMatch, detectors.DefaultFalsePositives, true) {
|
||||
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
|
||||
continue
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
appIncluded = true
|
||||
|
||||
results = append(results, s1)
|
||||
}
|
||||
|
||||
if !appIncluded {
|
||||
s1 := detectors.Result{
|
||||
DetectorType: detectorspb.DetectorType_DatadogToken,
|
||||
Raw: []byte(resApiMatch),
|
||||
RawV2: []byte(resApiMatch),
|
||||
ExtraData: map[string]string{
|
||||
"Type": "APIKeyOnly",
|
||||
},
|
||||
}
|
||||
|
||||
if verify {
|
||||
|
||||
req, err := http.NewRequestWithContext(ctx, "GET", "https://api.datadoghq.com/api/v1/validate", nil)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
req.Header.Add("Content-Type", "application/json")
|
||||
req.Header.Add("DD-API-KEY", resApiMatch)
|
||||
res, err := client.Do(req)
|
||||
if err == nil {
|
||||
defer res.Body.Close()
|
||||
if res.StatusCode >= 200 && res.StatusCode < 300 {
|
||||
s1.Verified = true
|
||||
} else {
|
||||
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
|
||||
if detectors.IsKnownFalsePositive(resApiMatch, detectors.DefaultFalsePositives, true) {
|
||||
continue
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
results = append(results, s1)
|
||||
}
|
||||
}
|
||||
|
||||
return results, nil
|
||||
|
||||
@@ -19,11 +19,11 @@ import (
|
||||
func TestDatadogToken_FromChunk(t *testing.T) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
|
||||
defer cancel()
|
||||
testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")
|
||||
testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
|
||||
if err != nil {
|
||||
t.Fatalf("could not get test secrets from GCP: %s", err)
|
||||
}
|
||||
apiKey := testSecrets.MustGetField("DATADOGTOKEN_TOKEN")
|
||||
secret := testSecrets.MustGetField("DATADOGTOKEN_TOKEN")
|
||||
appKey := testSecrets.MustGetField("DATADOGTOKEN_APPKEY")
|
||||
inactiveAppKey := testSecrets.MustGetField("DATADOGTOKEN_INACTIVE")
|
||||
|
||||
@@ -44,16 +44,13 @@ func TestDatadogToken_FromChunk(t *testing.T) {
|
||||
s: Scanner{},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a datadogtoken secret %s within datadog %s", appKey, apiKey)),
|
||||
data: []byte(fmt.Sprintf("You can find a datadogtoken secret %s within datadog %s", appKey, secret)),
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_DatadogToken,
|
||||
Verified: true,
|
||||
ExtraData: map[string]string{
|
||||
"Type": "Application+APIKey",
|
||||
},
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
@@ -63,35 +60,13 @@ func TestDatadogToken_FromChunk(t *testing.T) {
|
||||
s: Scanner{},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a datadogtoken secret %s within but datadog %s not valid", inactiveAppKey, apiKey)), // the secret would satisfy the regex but not pass validation
|
||||
data: []byte(fmt.Sprintf("You can find a datadogtoken secret %s within but datadog %s not valid", inactiveAppKey, secret)), // the secret would satisfy the regex but not pass validation
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_DatadogToken,
|
||||
Verified: false,
|
||||
ExtraData: map[string]string{
|
||||
"Type": "Application+APIKey",
|
||||
},
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "api key found, verified",
|
||||
s: Scanner{},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a datadogtoken secret %s", apiKey)), // the secret would satisfy the regex but not pass validation
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_DatadogToken,
|
||||
Verified: true,
|
||||
ExtraData: map[string]string{
|
||||
"Type": "APIKeyOnly",
|
||||
},
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
@@ -121,7 +96,6 @@ func TestDatadogToken_FromChunk(t *testing.T) {
|
||||
t.Fatalf("no raw secret present: \n %+v", got[i])
|
||||
}
|
||||
got[i].Raw = nil
|
||||
got[i].RawV2 = nil
|
||||
}
|
||||
if diff := pretty.Compare(got, tt.want); diff != "" {
|
||||
t.Errorf("DatadogToken.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
|
||||
|
||||
+12
-35
@@ -2,8 +2,6 @@ package ftp
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"net/textproto"
|
||||
"net/url"
|
||||
"regexp"
|
||||
"strings"
|
||||
@@ -15,17 +13,7 @@ import (
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
|
||||
)
|
||||
|
||||
const (
|
||||
// https://datatracker.ietf.org/doc/html/rfc959
|
||||
ftpNotLoggedIn = 530
|
||||
|
||||
defaultVerificationTimeout = 5 * time.Second
|
||||
)
|
||||
|
||||
type Scanner struct {
|
||||
// Verification timeout. Defaults to 5 seconds if unset.
|
||||
verificationTimeout time.Duration
|
||||
}
|
||||
type Scanner struct{}
|
||||
|
||||
// Ensure the Scanner satisfies the interface at compile time.
|
||||
var _ detectors.Detector = (*Scanner)(nil)
|
||||
@@ -70,59 +58,48 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
rawURL.Path = ""
|
||||
redact := strings.TrimSpace(strings.Replace(rawURL.String(), password, "********", -1))
|
||||
|
||||
r := detectors.Result{
|
||||
s := detectors.Result{
|
||||
DetectorType: detectorspb.DetectorType_FTP,
|
||||
Raw: []byte(rawURL.String()),
|
||||
Redacted: redact,
|
||||
}
|
||||
|
||||
if verify {
|
||||
timeout := s.verificationTimeout
|
||||
if timeout == 0 {
|
||||
timeout = defaultVerificationTimeout
|
||||
}
|
||||
verificationErr := verifyFTP(timeout, parsedURL)
|
||||
r.Verified = verificationErr == nil
|
||||
if !isErrDeterminate(verificationErr) {
|
||||
r.VerificationError = verificationErr
|
||||
}
|
||||
s.Verified = verifyFTP(ctx, parsedURL)
|
||||
}
|
||||
|
||||
if !r.Verified {
|
||||
if !s.Verified {
|
||||
// Skip unverified findings where the password starts with a `$` - it's almost certainly a variable.
|
||||
if strings.HasPrefix(password, "$") {
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
if detectors.IsKnownFalsePositive(string(r.Raw), []detectors.FalsePositive{"@ftp.freebsd.org"}, false) {
|
||||
if detectors.IsKnownFalsePositive(string(s.Raw), []detectors.FalsePositive{"@ftp.freebsd.org"}, false) {
|
||||
continue
|
||||
}
|
||||
|
||||
results = append(results, r)
|
||||
results = append(results, s)
|
||||
}
|
||||
|
||||
return results, nil
|
||||
}
|
||||
|
||||
func isErrDeterminate(e error) bool {
|
||||
ftpErr := &textproto.Error{}
|
||||
return errors.As(e, &ftpErr) && ftpErr.Code == ftpNotLoggedIn
|
||||
}
|
||||
|
||||
func verifyFTP(timeout time.Duration, u *url.URL) error {
|
||||
func verifyFTP(ctx context.Context, u *url.URL) bool {
|
||||
host := u.Host
|
||||
if !strings.Contains(host, ":") {
|
||||
host = host + ":21"
|
||||
}
|
||||
|
||||
c, err := ftp.Dial(host, ftp.DialWithTimeout(timeout))
|
||||
c, err := ftp.Dial(host, ftp.DialWithTimeout(5*time.Second))
|
||||
if err != nil {
|
||||
return err
|
||||
return false
|
||||
}
|
||||
|
||||
password, _ := u.User.Password()
|
||||
return c.Login(u.User.Username(), password)
|
||||
err = c.Login(u.User.Username(), password)
|
||||
|
||||
return err == nil
|
||||
}
|
||||
|
||||
func (s Scanner) Type() detectorspb.DetectorType {
|
||||
|
||||
@@ -5,11 +5,9 @@ package ftp
|
||||
|
||||
import (
|
||||
"context"
|
||||
"github.com/google/go-cmp/cmp"
|
||||
"github.com/google/go-cmp/cmp/cmpopts"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/kylelemons/godebug/pretty"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
|
||||
)
|
||||
@@ -21,12 +19,11 @@ func TestFTP_FromChunk(t *testing.T) {
|
||||
verify bool
|
||||
}
|
||||
tests := []struct {
|
||||
name string
|
||||
s Scanner
|
||||
args args
|
||||
want []detectors.Result
|
||||
wantErr bool
|
||||
wantVerificationErr bool
|
||||
name string
|
||||
s Scanner
|
||||
args args
|
||||
want []detectors.Result
|
||||
wantErr bool
|
||||
}{
|
||||
{
|
||||
name: "bad scheme",
|
||||
@@ -51,7 +48,7 @@ func TestFTP_FromChunk(t *testing.T) {
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_FTP,
|
||||
Verified: true,
|
||||
Redacted: "ftp://dlpuser:********@ftp.dlptest.com",
|
||||
Redacted: "ftp://dlpuser:*************************@ftp.dlptest.com",
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
@@ -69,49 +66,11 @@ func TestFTP_FromChunk(t *testing.T) {
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_FTP,
|
||||
Verified: false,
|
||||
Redacted: "ftp://dlpuser:********@ftp.dlptest.com",
|
||||
Redacted: "ftp://dlpuser:*******@ftp.dlptest.com",
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "bad host",
|
||||
s: Scanner{},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
// https://dlptest.com/ftp-test/
|
||||
data: []byte("ftp://dlpuser:rNrKYTX9g7z3RgJRmxWuGHbeu@ftp.dlptest.com.badhost"),
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_FTP,
|
||||
Verified: false,
|
||||
Redacted: "ftp://dlpuser:********@ftp.dlptest.com.badhost",
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
wantVerificationErr: true,
|
||||
},
|
||||
{
|
||||
name: "timeout",
|
||||
s: Scanner{verificationTimeout: 1 * time.Microsecond},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
// https://dlptest.com/ftp-test/
|
||||
data: []byte("ftp://dlpuser:rNrKYTX9g7z3RgJRmxWuGHbeu@ftp.dlptest.com"),
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_FTP,
|
||||
Verified: false,
|
||||
Redacted: "ftp://dlpuser:********@ftp.dlptest.com",
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
wantVerificationErr: true,
|
||||
},
|
||||
{
|
||||
name: "blocked FP",
|
||||
s: Scanner{},
|
||||
@@ -125,7 +84,8 @@ func TestFTP_FromChunk(t *testing.T) {
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
|
||||
s := Scanner{}
|
||||
got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
|
||||
if (err != nil) != tt.wantErr {
|
||||
t.Errorf("URI.FromData() error = %v, wantErr %v", err, tt.wantErr)
|
||||
return
|
||||
@@ -135,14 +95,9 @@ func TestFTP_FromChunk(t *testing.T) {
|
||||
// }
|
||||
for i := range got {
|
||||
got[i].Raw = nil
|
||||
|
||||
if (got[i].VerificationError != nil) != tt.wantVerificationErr {
|
||||
t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError)
|
||||
}
|
||||
}
|
||||
opts := cmpopts.IgnoreFields(detectors.Result{}, "VerificationError")
|
||||
if diff := cmp.Diff(got, tt.want, opts); diff != "" {
|
||||
t.Errorf("FTP.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
|
||||
if diff := pretty.Compare(got, tt.want); diff != "" {
|
||||
t.Errorf("URI.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
@@ -1,81 +0,0 @@
|
||||
package github_oauth2
|
||||
|
||||
import (
|
||||
"context"
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
|
||||
"golang.org/x/oauth2/clientcredentials"
|
||||
"golang.org/x/oauth2/github"
|
||||
)
|
||||
|
||||
type Scanner struct{ detectors.EndpointSetter }
|
||||
|
||||
// Ensure the Scanner satisfies the interfaces at compile time.
|
||||
var _ detectors.Detector = (*Scanner)(nil)
|
||||
|
||||
var (
|
||||
// Oauth2 client ID and secret
|
||||
oauth2ClientIDPat = regexp.MustCompile(detectors.PrefixRegex([]string{"github"}) + `\b([a-f0-9]{20})\b`)
|
||||
oauth2ClientSecretPat = regexp.MustCompile(detectors.PrefixRegex([]string{"github"}) + `\b([a-f0-9]{40})\b`)
|
||||
)
|
||||
|
||||
const (
|
||||
githubBadVerificationCodeError = "bad_verification_code"
|
||||
)
|
||||
|
||||
// Keywords are used for efficiently pre-filtering chunks.
|
||||
// Use identifiers in the secret preferably, or the provider name.
|
||||
func (s Scanner) Keywords() []string {
|
||||
return []string{"github"}
|
||||
}
|
||||
|
||||
// FromData will find and optionally verify GitHub secrets in a given set of bytes.
|
||||
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
|
||||
dataStr := string(data)
|
||||
|
||||
// Oauth2 client ID and secret
|
||||
oauth2ClientIDMatches := oauth2ClientIDPat.FindAllStringSubmatch(dataStr, -1)
|
||||
oauth2ClientSecretMatches := oauth2ClientSecretPat.FindAllStringSubmatch(dataStr, -1)
|
||||
|
||||
for _, idMatch := range oauth2ClientIDMatches {
|
||||
if len(idMatch) != 2 {
|
||||
continue
|
||||
}
|
||||
for _, secretMatch := range oauth2ClientSecretMatches {
|
||||
if len(secretMatch) != 2 {
|
||||
continue
|
||||
}
|
||||
|
||||
s1 := detectors.Result{
|
||||
DetectorType: detectorspb.DetectorType_GitHubOauth2,
|
||||
Raw: []byte(idMatch[1]),
|
||||
RawV2: []byte(idMatch[1] + secretMatch[1]),
|
||||
}
|
||||
|
||||
config := &clientcredentials.Config{
|
||||
ClientID: idMatch[1],
|
||||
ClientSecret: secretMatch[1],
|
||||
TokenURL: github.Endpoint.TokenURL,
|
||||
}
|
||||
_, err := config.Token(ctx)
|
||||
if err != nil && strings.Contains(err.Error(), githubBadVerificationCodeError) {
|
||||
s1.Verified = true
|
||||
}
|
||||
|
||||
if !s1.Verified && detectors.IsKnownFalsePositive(string(s1.Raw), detectors.DefaultFalsePositives, true) {
|
||||
continue
|
||||
}
|
||||
|
||||
results = append(results, s1)
|
||||
}
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
func (s Scanner) Type() detectorspb.DetectorType {
|
||||
return detectorspb.DetectorType_GitHubOauth2
|
||||
}
|
||||
@@ -12,11 +12,7 @@ import (
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
|
||||
)
|
||||
|
||||
type Scanner struct {
|
||||
client *http.Client
|
||||
|
||||
detectors.EndpointSetter
|
||||
}
|
||||
type Scanner struct{ detectors.EndpointSetter }
|
||||
|
||||
// Ensure the Scanner satisfies the interfaces at compile time.
|
||||
var _ detectors.Detector = (*Scanner)(nil)
|
||||
@@ -27,8 +23,7 @@ func (Scanner) Version() int { return 1 }
|
||||
func (Scanner) DefaultEndpoint() string { return "https://gitlab.com" }
|
||||
|
||||
var (
|
||||
defaultClient = common.SaneHttpClient()
|
||||
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"gitlab"}) + `\b((?:glpat|)[a-zA-Z0-9\-=_]{20,22})\b`)
|
||||
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"gitlab"}) + `\b((?:glpat|)[a-zA-Z0-9\-=_]{20,22})\b`)
|
||||
)
|
||||
|
||||
// Keywords are used for efficiently pre-filtering chunks.
|
||||
@@ -64,10 +59,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
// one of these scopes has access to, so we just check an example endpoint for each scope. If any
|
||||
// of them contain data, we know we have a valid key, but if they all fail, we don't
|
||||
|
||||
client := s.client
|
||||
if client == nil {
|
||||
client = defaultClient
|
||||
}
|
||||
client := common.SaneHttpClient()
|
||||
for _, baseURL := range s.Endpoints(s.DefaultEndpoint()) {
|
||||
// test `read_user` scope
|
||||
req, err := http.NewRequestWithContext(ctx, "GET", baseURL+"/api/v4/user", nil)
|
||||
@@ -82,19 +74,9 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
// 200 means good key and has `read_user` scope
|
||||
// 403 means good key but not the right scope
|
||||
// 401 is bad key
|
||||
switch res.StatusCode {
|
||||
case http.StatusOK:
|
||||
if res.StatusCode == http.StatusOK || res.StatusCode == http.StatusForbidden {
|
||||
secret.Verified = true
|
||||
case http.StatusForbidden:
|
||||
// Good key but not the right scope
|
||||
secret.Verified = true
|
||||
case http.StatusUnauthorized:
|
||||
// Nothing to do; zero values are the ones we want
|
||||
default:
|
||||
secret.VerificationError = fmt.Errorf("unexpected HTTP response status %d", res.StatusCode)
|
||||
}
|
||||
} else {
|
||||
secret.VerificationError = err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -6,11 +6,11 @@ package gitlab
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"github.com/google/go-cmp/cmp"
|
||||
"github.com/google/go-cmp/cmp/cmpopts"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/kylelemons/godebug/pretty"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
|
||||
@@ -31,12 +31,11 @@ func TestGitlab_FromChunk(t *testing.T) {
|
||||
verify bool
|
||||
}
|
||||
tests := []struct {
|
||||
name string
|
||||
s Scanner
|
||||
args args
|
||||
want []detectors.Result
|
||||
wantErr bool
|
||||
wantVerificationErr bool
|
||||
name string
|
||||
s Scanner
|
||||
args args
|
||||
want []detectors.Result
|
||||
wantErr bool
|
||||
}{
|
||||
{
|
||||
name: "found",
|
||||
@@ -86,56 +85,6 @@ func TestGitlab_FromChunk(t *testing.T) {
|
||||
},
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "found, would be verified but for timeout",
|
||||
s: Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a gitlab super secret %s within", secret)),
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_Gitlab,
|
||||
Verified: false,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
wantVerificationErr: true,
|
||||
},
|
||||
{
|
||||
name: "found and valid but unexpected api response",
|
||||
s: Scanner{client: common.ConstantResponseHttpClient(500, "")},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a gitlab super secret %s within", secret)),
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_Gitlab,
|
||||
Verified: false,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
wantVerificationErr: true,
|
||||
},
|
||||
{
|
||||
name: "found, good key but wrong scope",
|
||||
s: Scanner{client: common.ConstantResponseHttpClient(403, "")},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a gitlab super secret %s within", secret)),
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_Gitlab,
|
||||
Verified: true,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "not found",
|
||||
s: Scanner{},
|
||||
@@ -150,7 +99,8 @@ func TestGitlab_FromChunk(t *testing.T) {
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
|
||||
s := Scanner{}
|
||||
got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
|
||||
if (err != nil) != tt.wantErr {
|
||||
t.Errorf("Gitlab.FromData() error = %v, wantErr %v", err, tt.wantErr)
|
||||
return
|
||||
@@ -159,12 +109,9 @@ func TestGitlab_FromChunk(t *testing.T) {
|
||||
if len(got[i].Raw) == 0 {
|
||||
t.Fatal("no raw secret present")
|
||||
}
|
||||
if (got[i].VerificationError != nil) != tt.wantVerificationErr {
|
||||
t.Fatalf(" wantVerificationError = %v, verification error = %v,", tt.wantVerificationErr, got[i].VerificationError)
|
||||
}
|
||||
got[i].Raw = nil
|
||||
}
|
||||
opts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "VerificationError")
|
||||
if diff := cmp.Diff(got, tt.want, opts); diff != "" {
|
||||
if diff := pretty.Compare(got, tt.want); diff != "" {
|
||||
t.Errorf("Gitlab.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
|
||||
}
|
||||
})
|
||||
|
||||
@@ -1,91 +0,0 @@
|
||||
package huggingface
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
|
||||
)
|
||||
|
||||
type Scanner struct {
|
||||
client *http.Client
|
||||
}
|
||||
|
||||
// Ensure the Scanner satisfies the interface at compile time.
|
||||
var _ detectors.Detector = (*Scanner)(nil)
|
||||
|
||||
var (
|
||||
defaultClient = common.SaneHttpClient()
|
||||
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
|
||||
keyPat = regexp.MustCompile(`\bhf_[a-zA-Z]{34}\b`)
|
||||
)
|
||||
|
||||
// Keywords are used for efficiently pre-filtering chunks.
|
||||
// Use identifiers in the secret preferably, or the provider name.
|
||||
func (s Scanner) Keywords() []string {
|
||||
return []string{"huggingface", "hugging_face", "hf"} // Huggingface docs occasionally use "hf" instead of "huggingface"
|
||||
}
|
||||
|
||||
// FromData will find and optionally verify Huggingface secrets in a given set of bytes.
|
||||
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
|
||||
dataStr := string(data)
|
||||
|
||||
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
|
||||
|
||||
for _, match := range matches {
|
||||
if len(match) != 1 {
|
||||
continue
|
||||
}
|
||||
resMatch := strings.TrimSpace(match[0])
|
||||
|
||||
s1 := detectors.Result{
|
||||
DetectorType: detectorspb.DetectorType_HuggingFace,
|
||||
Raw: []byte(resMatch),
|
||||
}
|
||||
|
||||
if verify {
|
||||
client := s.client
|
||||
if client == nil {
|
||||
client = defaultClient
|
||||
}
|
||||
req, err := http.NewRequestWithContext(ctx, "GET", "https://huggingface.co/api/whoami-v2", nil)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
|
||||
req.Header.Set("Authorization", "Bearer "+resMatch)
|
||||
|
||||
res, err := client.Do(req)
|
||||
if err == nil {
|
||||
defer res.Body.Close()
|
||||
if res.StatusCode >= 200 && res.StatusCode < 300 {
|
||||
s1.Verified = true
|
||||
} else if res.StatusCode == 401 {
|
||||
// The secret is determinately not verified (nothing to do)
|
||||
} else {
|
||||
s1.VerificationError = fmt.Errorf("unexpected HTTP response status %d", res.StatusCode)
|
||||
}
|
||||
} else {
|
||||
s1.VerificationError = err
|
||||
}
|
||||
}
|
||||
|
||||
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
|
||||
if !s1.Verified && detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
|
||||
continue
|
||||
}
|
||||
|
||||
results = append(results, s1)
|
||||
}
|
||||
|
||||
return results, nil
|
||||
}
|
||||
|
||||
func (s Scanner) Type() detectorspb.DetectorType {
|
||||
return detectorspb.DetectorType_HuggingFace
|
||||
}
|
||||
@@ -1,160 +0,0 @@
|
||||
//go:build detectors
|
||||
// +build detectors
|
||||
|
||||
package huggingface
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"github.com/google/go-cmp/cmp"
|
||||
"github.com/google/go-cmp/cmp/cmpopts"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
|
||||
)
|
||||
|
||||
func TestHuggingface_FromChunk(t *testing.T) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
|
||||
defer cancel()
|
||||
testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")
|
||||
if err != nil {
|
||||
t.Fatalf("could not get test secrets from GCP: %s", err)
|
||||
}
|
||||
secret := testSecrets.MustGetField("HUGGINGFACE")
|
||||
inactiveSecret := testSecrets.MustGetField("HUGGINGFACE_INACTIVE")
|
||||
|
||||
type args struct {
|
||||
ctx context.Context
|
||||
data []byte
|
||||
verify bool
|
||||
}
|
||||
tests := []struct {
|
||||
name string
|
||||
s Scanner
|
||||
args args
|
||||
want []detectors.Result
|
||||
wantErr bool
|
||||
wantVerificationErr bool
|
||||
}{
|
||||
{
|
||||
name: "found, verified",
|
||||
s: Scanner{},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a huggingface secret %s within", secret)),
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_HuggingFace,
|
||||
Verified: true,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
wantVerificationErr: false,
|
||||
},
|
||||
{
|
||||
name: "found, unverified",
|
||||
s: Scanner{},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a huggingface secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_HuggingFace,
|
||||
Verified: false,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
wantVerificationErr: false,
|
||||
},
|
||||
{
|
||||
name: "not found",
|
||||
s: Scanner{},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte("You cannot find the secret within"),
|
||||
verify: true,
|
||||
},
|
||||
want: nil,
|
||||
wantErr: false,
|
||||
wantVerificationErr: false,
|
||||
},
|
||||
{
|
||||
name: "found, would be verified if not for timeout",
|
||||
s: Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a huggingface secret %s within", secret)),
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_HuggingFace,
|
||||
Verified: false,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
wantVerificationErr: true,
|
||||
},
|
||||
{
|
||||
name: "found, verified but unexpected api surface",
|
||||
s: Scanner{client: common.ConstantResponseHttpClient(404, "")},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a huggingface secret %s within", secret)),
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_HuggingFace,
|
||||
Verified: false,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
wantVerificationErr: true,
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
|
||||
if (err != nil) != tt.wantErr {
|
||||
t.Errorf("Huggingface.FromData() error = %v, wantErr %v", err, tt.wantErr)
|
||||
return
|
||||
}
|
||||
for i := range got {
|
||||
if len(got[i].Raw) == 0 {
|
||||
t.Fatalf("no raw secret present: \n %+v", got[i])
|
||||
}
|
||||
if (got[i].VerificationError != nil) != tt.wantVerificationErr {
|
||||
t.Fatalf(" wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError)
|
||||
}
|
||||
}
|
||||
ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "VerificationError")
|
||||
if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {
|
||||
t.Errorf("Huggingface.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkFromData(benchmark *testing.B) {
|
||||
ctx := context.Background()
|
||||
s := Scanner{}
|
||||
for name, data := range detectors.MustGetBenchmarkData() {
|
||||
benchmark.Run(name, func(b *testing.B) {
|
||||
for n := 0; n < b.N; n++ {
|
||||
_, err := s.FromData(ctx, false, data)
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
+26
-46
@@ -3,8 +3,6 @@ package ldap
|
||||
import (
|
||||
"context"
|
||||
"crypto/tls"
|
||||
"fmt"
|
||||
"net"
|
||||
"net/url"
|
||||
"regexp"
|
||||
"strings"
|
||||
@@ -65,11 +63,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
}
|
||||
|
||||
if verify {
|
||||
verificationErr := verifyLDAP(username[1], password[1], ldapURL)
|
||||
s1.Verified = verificationErr == nil
|
||||
if !isErrDeterminate(verificationErr) {
|
||||
s1.VerificationError = verificationErr
|
||||
}
|
||||
s1.Verified = verifyLDAP(ctx, username[1], password[1], ldapURL)
|
||||
}
|
||||
|
||||
results = append(results, s1)
|
||||
@@ -95,12 +89,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
}
|
||||
|
||||
if verify {
|
||||
verificationError := verifyLDAP(username, password, ldapURL)
|
||||
|
||||
s1.Verified = verificationError == nil
|
||||
if !isErrDeterminate(verificationError) {
|
||||
s1.VerificationError = verificationError
|
||||
}
|
||||
s1.Verified = verifyLDAP(ctx, username, password, ldapURL)
|
||||
}
|
||||
|
||||
results = append(results, s1)
|
||||
@@ -109,19 +98,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
return results, nil
|
||||
}
|
||||
|
||||
func isErrDeterminate(err error) bool {
|
||||
switch e := err.(type) {
|
||||
case *ldap.Error:
|
||||
switch e.Err.(type) {
|
||||
case *net.OpError:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
return true
|
||||
}
|
||||
|
||||
func verifyLDAP(username, password string, ldapURL *url.URL) error {
|
||||
func verifyLDAP(ctx context.Context, username, password string, ldapURL *url.URL) bool {
|
||||
// Tests with non-TLS, TLS, and STARTTLS
|
||||
|
||||
ldap.DefaultTimeout = 5 * time.Second
|
||||
@@ -132,35 +109,38 @@ func verifyLDAP(username, password string, ldapURL *url.URL) error {
|
||||
case "ldap":
|
||||
// Non-TLS dial
|
||||
l, err := ldap.DialURL(uri)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer l.Close()
|
||||
// Non-TLS verify
|
||||
err = l.Bind(username, password)
|
||||
if err == nil {
|
||||
return nil
|
||||
}
|
||||
defer l.Close()
|
||||
// Non-TLS verify
|
||||
err = l.Bind(username, password)
|
||||
if err == nil {
|
||||
return true
|
||||
}
|
||||
|
||||
// STARTTLS
|
||||
err = l.StartTLS(&tls.Config{InsecureSkipVerify: true})
|
||||
if err != nil {
|
||||
return err
|
||||
// STARTTLS
|
||||
err = l.StartTLS(&tls.Config{InsecureSkipVerify: true})
|
||||
if err == nil {
|
||||
// STARTTLS verify
|
||||
err = l.Bind(username, password)
|
||||
if err == nil {
|
||||
return true
|
||||
}
|
||||
}
|
||||
}
|
||||
// STARTTLS verify
|
||||
return l.Bind(username, password)
|
||||
case "ldaps":
|
||||
// TLS dial
|
||||
l, err := ldap.DialTLS("tcp", uri, &tls.Config{InsecureSkipVerify: true})
|
||||
if err != nil {
|
||||
return err
|
||||
if err == nil {
|
||||
defer l.Close()
|
||||
// TLS verify
|
||||
err = l.Bind(username, password)
|
||||
if err == nil {
|
||||
return true
|
||||
}
|
||||
}
|
||||
defer l.Close()
|
||||
// TLS verify
|
||||
return l.Bind(username, password)
|
||||
}
|
||||
|
||||
return fmt.Errorf("unknown ldap scheme %q", ldapURL.Scheme)
|
||||
return false
|
||||
}
|
||||
|
||||
func (s Scanner) Type() detectorspb.DetectorType {
|
||||
|
||||
@@ -7,14 +7,13 @@ import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"github.com/google/go-cmp/cmp"
|
||||
"github.com/google/go-cmp/cmp/cmpopts"
|
||||
"os"
|
||||
"os/exec"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/kylelemons/godebug/pretty"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
|
||||
@@ -61,12 +60,11 @@ func TestLdap_Integration_FromChunk(t *testing.T) {
|
||||
verify bool
|
||||
}
|
||||
tests := []struct {
|
||||
name string
|
||||
s Scanner
|
||||
args args
|
||||
want []detectors.Result
|
||||
wantErr bool
|
||||
wantVerificationErr bool
|
||||
name string
|
||||
s Scanner
|
||||
args args
|
||||
want []detectors.Result
|
||||
wantErr bool
|
||||
}{
|
||||
{
|
||||
name: "found with URI and separate user+password usage, verified",
|
||||
@@ -154,26 +152,6 @@ func TestLdap_Integration_FromChunk(t *testing.T) {
|
||||
},
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "inaccessible host",
|
||||
s: Scanner{},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(`
|
||||
ldap://badhost:1389
|
||||
binddn="cn=admin,dc=example,dc=org"
|
||||
pass="P@55w0rd"`),
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_LDAP,
|
||||
Verified: false,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
wantVerificationErr: true,
|
||||
},
|
||||
{
|
||||
name: "not found",
|
||||
s: Scanner{},
|
||||
@@ -198,12 +176,9 @@ func TestLdap_Integration_FromChunk(t *testing.T) {
|
||||
if len(got[i].Raw) == 0 {
|
||||
t.Fatalf("no raw secret present: \n %+v", got[i])
|
||||
}
|
||||
if (got[i].VerificationError != nil) != tt.wantVerificationErr {
|
||||
t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError)
|
||||
}
|
||||
got[i].Raw = nil
|
||||
}
|
||||
ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "VerificationError")
|
||||
if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {
|
||||
if diff := pretty.Compare(got, tt.want); diff != "" {
|
||||
t.Errorf("Ldap.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
|
||||
}
|
||||
})
|
||||
@@ -230,7 +205,7 @@ func startOpenLDAP() error {
|
||||
return nil
|
||||
case <-time.After(30 * time.Second):
|
||||
stopOpenLDAP()
|
||||
return errors.New("timeout waiting for ldap service to be ready")
|
||||
return errors.New("timeout waiting for postgres database to be ready")
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -52,7 +52,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
}
|
||||
|
||||
if verify {
|
||||
req, err := http.NewRequestWithContext(ctx, "GET", "https://api.opsgenie.com/v2/alerts", nil)
|
||||
req, err := http.NewRequestWithContext(ctx, "GET", "https://api.opsgenie.com/v2/users", nil)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
|
||||
@@ -22,8 +22,8 @@ var (
|
||||
client = common.SaneHttpClient()
|
||||
|
||||
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
|
||||
idPat = regexp.MustCompile(`\b([A-Za-z0-9_\.]{7}-[A-Za-z0-9_\.]{72}|[A-Za-z0-9_\.]{5}-[A-Za-z0-9_\.]{38})\b`)
|
||||
keyPat = regexp.MustCompile(`\b([A-Za-z0-9_\.\-]{44,80})\b`)
|
||||
idPat = regexp.MustCompile(`\b([A-Za-z0-9_\.]{7}-[A-Za-z0-9_\.]{72})\b`)
|
||||
keyPat = regexp.MustCompile(`\b([A-Za-z0-9_\.]{69}-[A-Za-z0-9_\.]{10})\b`)
|
||||
)
|
||||
|
||||
// Keywords are used for efficiently pre-filtering chunks.
|
||||
@@ -39,24 +39,24 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
|
||||
idmatches := idPat.FindAllStringSubmatch(dataStr, -1)
|
||||
|
||||
for _, idMatch := range idmatches {
|
||||
if len(idMatch) != 2 {
|
||||
for _, match := range matches {
|
||||
if len(match) != 2 {
|
||||
continue
|
||||
}
|
||||
resIDMatch := strings.TrimSpace(idMatch[1])
|
||||
for _, secretMatch := range matches {
|
||||
if len(secretMatch) != 2 {
|
||||
resMatch := strings.TrimSpace(match[1])
|
||||
for _, idMatch := range idmatches {
|
||||
if len(idMatch) != 2 {
|
||||
continue
|
||||
}
|
||||
resSecretMatch := strings.TrimSpace(secretMatch[1])
|
||||
residMatch := strings.TrimSpace(idMatch[1])
|
||||
|
||||
s1 := detectors.Result{
|
||||
DetectorType: detectorspb.DetectorType_PaypalOauth,
|
||||
Raw: []byte(resSecretMatch),
|
||||
Raw: []byte(resMatch),
|
||||
}
|
||||
|
||||
if verify {
|
||||
data := fmt.Sprintf("%s:%s", resIDMatch, resSecretMatch)
|
||||
data := fmt.Sprintf("%s:%s", residMatch, resMatch)
|
||||
encoded := b64.StdEncoding.EncodeToString([]byte(data))
|
||||
payload := strings.NewReader("grant_type=client_credentials")
|
||||
req, err := http.NewRequestWithContext(ctx, "POST", "https://api-m.sandbox.paypal.com/v1/oauth2/token", payload)
|
||||
@@ -74,7 +74,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
s1.Verified = true
|
||||
} else {
|
||||
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
|
||||
if detectors.IsKnownFalsePositive(resIDMatch, detectors.DefaultFalsePositives, true) {
|
||||
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
@@ -19,7 +19,7 @@ import (
|
||||
func TestPaypalOauth_FromChunk(t *testing.T) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
|
||||
defer cancel()
|
||||
testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")
|
||||
testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
|
||||
if err != nil {
|
||||
t.Fatalf("could not get test secrets from GCP: %s", err)
|
||||
}
|
||||
@@ -27,9 +27,6 @@ func TestPaypalOauth_FromChunk(t *testing.T) {
|
||||
inactiveSecret := testSecrets.MustGetField("PAYPALOAUTH_SECRET_INACTIVE")
|
||||
id := testSecrets.MustGetField("PAYPALOAUTH_CLIENTID")
|
||||
|
||||
newId := testSecrets.MustGetField("PAYPALOAUTH_NEW_INACTIVE_CLIENTID")
|
||||
newSecret := testSecrets.MustGetField("PAYPALOAUTH_NEW_INACTIVE_SECRET")
|
||||
|
||||
type args struct {
|
||||
ctx context.Context
|
||||
data []byte
|
||||
@@ -55,10 +52,6 @@ func TestPaypalOauth_FromChunk(t *testing.T) {
|
||||
DetectorType: detectorspb.DetectorType_PaypalOauth,
|
||||
Verified: true,
|
||||
},
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_PaypalOauth,
|
||||
Verified: false,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
},
|
||||
@@ -75,30 +68,6 @@ func TestPaypalOauth_FromChunk(t *testing.T) {
|
||||
DetectorType: detectorspb.DetectorType_PaypalOauth,
|
||||
Verified: false,
|
||||
},
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_PaypalOauth,
|
||||
Verified: false,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "new format, unverified",
|
||||
s: Scanner{},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a paypaloauth secret %s within %s but not valid", newSecret, newId)), // the secret would satisfy the regex but not pass validation
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_PaypalOauth,
|
||||
Verified: false,
|
||||
},
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_PaypalOauth,
|
||||
Verified: false,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
},
|
||||
|
||||
@@ -2,7 +2,6 @@ package pubnubpublishkey
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"strings"
|
||||
@@ -12,15 +11,13 @@ import (
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
|
||||
)
|
||||
|
||||
type Scanner struct {
|
||||
client *http.Client
|
||||
}
|
||||
type Scanner struct{}
|
||||
|
||||
// Ensure the Scanner satisfies the interface at compile time.
|
||||
var _ detectors.Detector = (*Scanner)(nil)
|
||||
|
||||
var (
|
||||
defaultClient = common.SaneHttpClient()
|
||||
client = common.SaneHttpClient()
|
||||
|
||||
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
|
||||
pubPat = regexp.MustCompile(`\b(pub-c-[0-9a-z]{8}-[0-9a-z]{4}-[a-z0-9]{4}-[a-z0-9]{4}-[a-z0-9]{12})\b`)
|
||||
@@ -63,31 +60,19 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
client := s.client
|
||||
if client == nil {
|
||||
client = defaultClient
|
||||
}
|
||||
res, err := client.Do(req)
|
||||
if err == nil {
|
||||
defer res.Body.Close()
|
||||
if res.StatusCode >= 200 && res.StatusCode < 300 {
|
||||
s1.Verified = true
|
||||
} else if res.StatusCode == 400 || res.StatusCode == 403 {
|
||||
// 403 is suggested by the API docs (https://www.pubnub.com/docs/sdks/rest-api/send-signal-to-channel)
|
||||
// 400 is what actually seems to be coming back for invalid credentials
|
||||
// There's nothing to do here, because zero values of Result are what we want
|
||||
} else {
|
||||
s1.VerificationError = fmt.Errorf("unexpected HTTP response status %d", res.StatusCode)
|
||||
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
|
||||
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
|
||||
continue
|
||||
}
|
||||
}
|
||||
} else {
|
||||
s1.VerificationError = err
|
||||
}
|
||||
}
|
||||
// This function will check false positives for common test words, but also it will make sure the key
|
||||
// appears 'random' enough to be a real key.
|
||||
if !s1.Verified && detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
|
||||
continue
|
||||
}
|
||||
|
||||
results = append(results, s1)
|
||||
}
|
||||
|
||||
@@ -6,11 +6,10 @@ package pubnubpublishkey
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"github.com/google/go-cmp/cmp"
|
||||
"github.com/google/go-cmp/cmp/cmpopts"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/kylelemons/godebug/pretty"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
@@ -34,12 +33,11 @@ func TestPubNubPublishKey_FromChunk(t *testing.T) {
|
||||
verify bool
|
||||
}
|
||||
tests := []struct {
|
||||
name string
|
||||
s Scanner
|
||||
args args
|
||||
want []detectors.Result
|
||||
wantErr bool
|
||||
wantVerificationErr bool
|
||||
name string
|
||||
s Scanner
|
||||
args args
|
||||
want []detectors.Result
|
||||
wantErr bool
|
||||
}{
|
||||
{
|
||||
name: "found, verified",
|
||||
@@ -73,40 +71,6 @@ func TestPubNubPublishKey_FromChunk(t *testing.T) {
|
||||
},
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "found, would be verified but for timeout",
|
||||
s: Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a pubnub secret %s within pubnub %s", pub, sub)),
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_PubNubPublishKey,
|
||||
Verified: false,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
wantVerificationErr: true,
|
||||
},
|
||||
{
|
||||
name: "found, valid but unexpected api response",
|
||||
s: Scanner{client: common.ConstantResponseHttpClient(500, "")},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a pubnub secret %s within pubnub %s", pub, sub)),
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_PubNubPublishKey,
|
||||
Verified: false,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
wantVerificationErr: true,
|
||||
},
|
||||
{
|
||||
name: "not found",
|
||||
s: Scanner{},
|
||||
@@ -121,7 +85,8 @@ func TestPubNubPublishKey_FromChunk(t *testing.T) {
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
|
||||
s := Scanner{}
|
||||
got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
|
||||
if (err != nil) != tt.wantErr {
|
||||
t.Errorf("PubNubPublishKey.FromData() error = %v, wantErr %v", err, tt.wantErr)
|
||||
return
|
||||
@@ -130,12 +95,9 @@ func TestPubNubPublishKey_FromChunk(t *testing.T) {
|
||||
if len(got[i].Raw) == 0 {
|
||||
t.Fatalf("no raw secret present: \n %+v", got[i])
|
||||
}
|
||||
if (got[i].VerificationError != nil) != tt.wantVerificationErr {
|
||||
t.Fatalf(" wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError)
|
||||
}
|
||||
got[i].Raw = nil
|
||||
}
|
||||
opts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "VerificationError")
|
||||
if diff := cmp.Diff(got, tt.want, opts); diff != "" {
|
||||
if diff := pretty.Compare(got, tt.want); diff != "" {
|
||||
t.Errorf("PubNubPublishKey.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
|
||||
}
|
||||
})
|
||||
|
||||
@@ -1,113 +0,0 @@
|
||||
package salesforce
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
|
||||
)
|
||||
|
||||
const (
|
||||
currentVersion = "58.0" // current Salesforce version
|
||||
)
|
||||
|
||||
type Scanner struct {
|
||||
client *http.Client
|
||||
}
|
||||
|
||||
// Ensure the Scanner satisfies the interface at compile time.
|
||||
var _ detectors.Detector = (*Scanner)(nil)
|
||||
|
||||
var (
|
||||
defaultClient = common.SaneHttpClient()
|
||||
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
|
||||
accessTokenPat = regexp.MustCompile(`\b00[a-zA-Z0-9]{13}![a-zA-Z0-9_.]{96}\b`)
|
||||
instancePat = regexp.MustCompile(`\bhttps://[0-9a-zA-Z-\.]{1,100}\.my\.salesforce\.com\b`)
|
||||
)
|
||||
|
||||
// Keywords are used for efficiently pre-filtering chunks.
|
||||
// Use identifiers in the secret preferably, or the provider name.
|
||||
func (s Scanner) Keywords() []string {
|
||||
return []string{"salesforce"}
|
||||
}
|
||||
|
||||
// FromData will find and optionally verify Salesforce secrets in a given set of bytes.
|
||||
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
|
||||
dataStr := string(data)
|
||||
|
||||
instanceMatches := instancePat.FindAllStringSubmatch(dataStr, -1)
|
||||
tokenMatches := accessTokenPat.FindAllStringSubmatch(dataStr, -1)
|
||||
|
||||
fmt.Printf("instanceMatches: %v\n", instanceMatches)
|
||||
|
||||
for _, instance := range instanceMatches {
|
||||
if len(instance) != 1 {
|
||||
continue
|
||||
}
|
||||
|
||||
instanceMatch := strings.TrimSpace(instance[0])
|
||||
|
||||
for _, token := range tokenMatches {
|
||||
if len(token) != 1 {
|
||||
continue
|
||||
}
|
||||
|
||||
tokenMatch := strings.TrimSpace(token[0])
|
||||
|
||||
s1 := detectors.Result{
|
||||
DetectorType: detectorspb.DetectorType_Salesforce,
|
||||
Raw: []byte(tokenMatch),
|
||||
}
|
||||
|
||||
if verify {
|
||||
client := s.client
|
||||
if client == nil {
|
||||
client = defaultClient
|
||||
}
|
||||
|
||||
req, err := http.NewRequestWithContext(ctx, "GET", instanceMatch+"/services/data/v"+currentVersion+"/query?q=SELECT+name+from+Account", nil)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
req.Header.Set("Authorization", "Bearer "+tokenMatch)
|
||||
|
||||
res, err := client.Do(req)
|
||||
|
||||
if err != nil {
|
||||
// End execution, append Detector Result if request fails to prevent panic on response body checks
|
||||
s1.VerificationError = err
|
||||
results = append(results, s1)
|
||||
continue
|
||||
}
|
||||
|
||||
verifiedBodyResponse, err := common.ResponseContainsSubstring(res.Body, "records")
|
||||
|
||||
defer res.Body.Close()
|
||||
if res.StatusCode >= 200 && res.StatusCode < 300 && verifiedBodyResponse {
|
||||
s1.Verified = true
|
||||
} else if res.StatusCode >= 200 && res.StatusCode < 300 && !verifiedBodyResponse {
|
||||
s1.Verified = false
|
||||
} else if res.StatusCode == 401 {
|
||||
s1.Verified = false
|
||||
} else {
|
||||
s1.VerificationError = fmt.Errorf("request to %v returned status %d with error %+v", res.Request.URL, res.StatusCode, err)
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
results = append(results, s1)
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
return results, nil
|
||||
}
|
||||
|
||||
func (s Scanner) Type() detectorspb.DetectorType {
|
||||
return detectorspb.DetectorType_Salesforce
|
||||
}
|
||||
@@ -1,167 +0,0 @@
|
||||
//go:build detectors
|
||||
// +build detectors
|
||||
|
||||
package salesforce
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"github.com/google/go-cmp/cmp"
|
||||
"github.com/google/go-cmp/cmp/cmpopts"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
|
||||
)
|
||||
|
||||
func TestSalesforce_FromChunk(t *testing.T) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
|
||||
defer cancel()
|
||||
testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")
|
||||
if err != nil {
|
||||
t.Fatalf("could not get test secrets from GCP: %s", err)
|
||||
}
|
||||
instance := testSecrets.MustGetField("SALESFORCE_INSTANCE")
|
||||
// Salesforce secrets are not valid for long, so we may need to regenerate them.
|
||||
// Steps to regenerate:
|
||||
// 1. Install Salesforce CLI
|
||||
// 2. Authenticate with `sfdx org web login`. This will open a browser window. Use the credentials in the detection test accounts 1Pass vault.
|
||||
// 3. Run `sfdx org display --targetusername zubairkhan@trufflesec.com`to obtain new token.
|
||||
// Source: https://developer.salesforce.com/docs/atlas.en-us.api_rest.meta/api_rest/quickstart_oauth.htm
|
||||
token := testSecrets.MustGetField("SALESFORCE_TOKEN")
|
||||
inactiveToken := testSecrets.MustGetField("SALESFORCE_INACTIVE_TOKEN")
|
||||
|
||||
type args struct {
|
||||
ctx context.Context
|
||||
data []byte
|
||||
verify bool
|
||||
}
|
||||
tests := []struct {
|
||||
name string
|
||||
s Scanner
|
||||
args args
|
||||
want []detectors.Result
|
||||
wantErr bool
|
||||
wantVerificationErr bool
|
||||
}{
|
||||
{
|
||||
name: "found, verified",
|
||||
s: Scanner{},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a salesforce secret within %s for %s", token, instance)),
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_Salesforce,
|
||||
Verified: true,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
wantVerificationErr: false,
|
||||
},
|
||||
{
|
||||
name: "found, unverified",
|
||||
s: Scanner{},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a salesforce secret %s within but not valid for %s", inactiveToken, instance)), // the secret would satisfy the regex but not pass validation
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_Salesforce,
|
||||
Verified: false,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
wantVerificationErr: false,
|
||||
},
|
||||
{
|
||||
name: "not found",
|
||||
s: Scanner{},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte("You cannot find the secret within"),
|
||||
verify: true,
|
||||
},
|
||||
want: nil,
|
||||
wantErr: false,
|
||||
wantVerificationErr: false,
|
||||
},
|
||||
{
|
||||
name: "found, would be verified if not for timeout",
|
||||
s: Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a salesforce secret within %s for %s", token, instance)),
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_Salesforce,
|
||||
Verified: false,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
wantVerificationErr: true,
|
||||
},
|
||||
{
|
||||
name: "found, verified but unexpected api surface",
|
||||
s: Scanner{client: common.ConstantResponseHttpClient(404, "")},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a salesforce secret within %s for %s", token, instance)),
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_Salesforce,
|
||||
Verified: false,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
wantVerificationErr: true,
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
|
||||
if (err != nil) != tt.wantErr {
|
||||
t.Errorf("Salesforce.FromData() error = %v, wantErr %v", err, tt.wantErr)
|
||||
}
|
||||
for i := range got {
|
||||
if len(got[i].Raw) == 0 {
|
||||
t.Fatalf("no raw secret present: \n %+v", got[i])
|
||||
}
|
||||
|
||||
if (got[i].VerificationError != nil) != tt.wantVerificationErr {
|
||||
t.Fatalf(" wantVerificationError = %v, verification error = %v,", tt.wantVerificationErr, got[i].VerificationError)
|
||||
}
|
||||
}
|
||||
ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "VerificationError")
|
||||
if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {
|
||||
t.Errorf("Salesforce.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkFromData(benchmark *testing.B) {
|
||||
ctx := context.Background()
|
||||
s := Scanner{}
|
||||
for name, data := range detectors.MustGetBenchmarkData() {
|
||||
benchmark.Run(name, func(b *testing.B) {
|
||||
for n := 0; n < b.N; n++ {
|
||||
_, err := s.FromData(ctx, false, data)
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -20,7 +20,7 @@ var (
|
||||
client = common.SaneHttpClient()
|
||||
|
||||
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
|
||||
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"timezoneapi"}) + `\b([a-zA-Z]{20})\b`)
|
||||
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"timezoneapi"}) + `\b([a-zA-Z0-9]{20})\b`)
|
||||
)
|
||||
|
||||
// Keywords are used for efficiently pre-filtering chunks.
|
||||
@@ -54,12 +54,8 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
|
||||
res, err := client.Do(req)
|
||||
if err == nil {
|
||||
verifiedBodyResponse, err := common.ResponseContainsSubstring(res.Body, "date")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer res.Body.Close()
|
||||
if res.StatusCode >= 200 && res.StatusCode < 300 && verifiedBodyResponse {
|
||||
if res.StatusCode >= 200 && res.StatusCode < 300 {
|
||||
s1.Verified = true
|
||||
} else {
|
||||
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
|
||||
|
||||
+20
-12
@@ -1,14 +1,16 @@
|
||||
package engine
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"runtime"
|
||||
|
||||
"github.com/go-errors/errors"
|
||||
"google.golang.org/protobuf/proto"
|
||||
"google.golang.org/protobuf/types/known/anypb"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/context"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/sources/circleci"
|
||||
)
|
||||
|
||||
@@ -27,17 +29,23 @@ func (e *Engine) ScanCircleCI(ctx context.Context, token string) error {
|
||||
return err
|
||||
}
|
||||
|
||||
handle, err := e.sourceManager.Enroll(ctx, "trufflehog - Circle CI", new(circleci.Source).Type(),
|
||||
func(ctx context.Context, jobID, sourceID int64) (sources.Source, error) {
|
||||
circleSource := circleci.Source{}
|
||||
if err := circleSource.Init(ctx, "trufflehog - Circle CI", jobID, sourceID, true, &conn, runtime.NumCPU()); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &circleSource, nil
|
||||
})
|
||||
circleSource := circleci.Source{}
|
||||
ctx = context.WithValues(ctx,
|
||||
"source_type", circleSource.Type().String(),
|
||||
"source_name", "Circle CI",
|
||||
)
|
||||
err = circleSource.Init(ctx, "trufflehog - Circle CI", 0, int64(sourcespb.SourceType_SOURCE_TYPE_CIRCLECI), true, &conn, runtime.NumCPU())
|
||||
if err != nil {
|
||||
return err
|
||||
return errors.WrapPrefix(err, "failed to init Circle CI source", 0)
|
||||
}
|
||||
_, err = e.sourceManager.ScheduleRun(ctx, handle)
|
||||
return err
|
||||
|
||||
e.sourcesWg.Go(func() error {
|
||||
defer common.RecoverWithExit(ctx)
|
||||
err := circleSource.Chunks(ctx, e.ChunksChan())
|
||||
if err != nil {
|
||||
return fmt.Errorf("error scanning CircleCI: %w", err)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -556,7 +556,6 @@ import (
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/satismeterwritekey"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/saucelabs"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/scalewaykey"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/github_oauth2"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/scalr"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/scrapeowl"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/scraperapi"
|
||||
@@ -1531,7 +1530,6 @@ func DefaultDetectors() []detectors.Detector {
|
||||
dockerhub.Scanner{},
|
||||
couchbase.Scanner{},
|
||||
envoyapikey.Scanner{},
|
||||
github_oauth2.Scanner{},
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
+22
-12
@@ -1,28 +1,38 @@
|
||||
package engine
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"runtime"
|
||||
|
||||
"github.com/go-errors/errors"
|
||||
"google.golang.org/protobuf/types/known/anypb"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/context"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/sources/docker"
|
||||
)
|
||||
|
||||
// ScanDocker scans a given docker connection.
|
||||
func (e *Engine) ScanDocker(ctx context.Context, conn *anypb.Any) error {
|
||||
handle, err := e.sourceManager.Enroll(ctx, "trufflehog - docker", new(docker.Source).Type(),
|
||||
func(ctx context.Context, jobID, sourceID int64) (sources.Source, error) {
|
||||
dockerSource := docker.Source{}
|
||||
if err := dockerSource.Init(ctx, "trufflehog - docker", jobID, sourceID, true, conn, runtime.NumCPU()); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &dockerSource, nil
|
||||
})
|
||||
dockerSource := docker.Source{}
|
||||
|
||||
ctx = context.WithValues(ctx,
|
||||
"source_type", dockerSource.Type().String(),
|
||||
"source_name", "docker",
|
||||
)
|
||||
err := dockerSource.Init(ctx, "trufflehog - docker", 0, int64(sourcespb.SourceType_SOURCE_TYPE_DOCKER), true, conn, runtime.NumCPU())
|
||||
if err != nil {
|
||||
return err
|
||||
return errors.WrapPrefix(err, "could not init docker source", 0)
|
||||
}
|
||||
_, err = e.sourceManager.ScheduleRun(ctx, handle)
|
||||
return err
|
||||
|
||||
e.sourcesWg.Go(func() error {
|
||||
defer common.RecoverWithExit(ctx)
|
||||
err := dockerSource.Chunks(ctx, e.ChunksChan())
|
||||
if err != nil {
|
||||
return fmt.Errorf("error scanning docker image: %w", err)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
return nil
|
||||
}
|
||||
|
||||
+27
-47
@@ -12,6 +12,7 @@ import (
|
||||
|
||||
ahocorasick "github.com/BobuSumisu/aho-corasick"
|
||||
lru "github.com/hashicorp/golang-lru"
|
||||
"golang.org/x/sync/errgroup"
|
||||
"google.golang.org/protobuf/proto"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
@@ -33,9 +34,6 @@ type Metrics struct {
|
||||
VerifiedSecretsFound uint64
|
||||
UnverifiedSecretsFound uint64
|
||||
AvgDetectorTime map[string]time.Duration
|
||||
|
||||
scanStartTime time.Time
|
||||
ScanDuration time.Duration
|
||||
}
|
||||
|
||||
// runtimeMetrics for the scan engine for internal use by the engine.
|
||||
@@ -46,16 +44,18 @@ type runtimeMetrics struct {
|
||||
}
|
||||
|
||||
// Printer is used to format found results and output them to the user. Ex JSON, plain text, etc.
|
||||
// Please note printer implementations SHOULD BE thread safe.
|
||||
type Printer interface {
|
||||
Print(ctx context.Context, r *detectors.ResultWithMetadata) error
|
||||
}
|
||||
|
||||
type Engine struct {
|
||||
// CLI flags.
|
||||
concurrency uint8
|
||||
chunks chan *sources.Chunk
|
||||
results chan detectors.ResultWithMetadata
|
||||
decoders []decoders.Decoder
|
||||
detectors map[bool][]detectors.Detector
|
||||
sourcesWg *errgroup.Group
|
||||
workersWg sync.WaitGroup
|
||||
// filterUnverified is used to reduce the number of unverified results.
|
||||
// If there are multiple unverified results for the same chunk for the same detector,
|
||||
// only the first one will be kept.
|
||||
@@ -67,16 +67,13 @@ type Engine struct {
|
||||
// matching given a set of words (keywords from the rules in the config)
|
||||
prefilter ahocorasick.Trie
|
||||
|
||||
// Engine synchronization primitives.
|
||||
sourceManager *sources.SourceManager
|
||||
results chan detectors.ResultWithMetadata
|
||||
detectableChunksChan chan detectableChunk
|
||||
workersWg sync.WaitGroup
|
||||
wgDetectorWorkers sync.WaitGroup
|
||||
WgNotifier sync.WaitGroup
|
||||
|
||||
// Runtime information.
|
||||
// Runtime metrics.
|
||||
metrics runtimeMetrics
|
||||
|
||||
// numFoundResults is used to keep track of the number of results found.
|
||||
numFoundResults uint32
|
||||
|
||||
@@ -202,8 +199,6 @@ func (e *Engine) GetMetrics() Metrics {
|
||||
result.AvgDetectorTime[detectorName] = avgDuration
|
||||
}
|
||||
|
||||
result.ScanDuration = e.metrics.getScanDuration()
|
||||
|
||||
return result
|
||||
}
|
||||
|
||||
@@ -225,16 +220,6 @@ func (e *Engine) GetDetectorsMetrics() map[string]time.Duration {
|
||||
return result
|
||||
}
|
||||
|
||||
// getScanDuration returns the duration of the scan.
|
||||
// If the scan is still running, it returns the time since the scan started.
|
||||
func (m *Metrics) getScanDuration() time.Duration {
|
||||
if m.ScanDuration == 0 {
|
||||
return time.Since(m.scanStartTime)
|
||||
}
|
||||
|
||||
return m.ScanDuration
|
||||
}
|
||||
|
||||
// DetectorAvgTime returns the average time taken by each detector.
|
||||
func (e *Engine) DetectorAvgTime() map[string][]time.Duration {
|
||||
logger := context.Background().Logger()
|
||||
@@ -271,11 +256,12 @@ func Start(ctx context.Context, options ...EngineOption) (*Engine, error) {
|
||||
}
|
||||
|
||||
e := &Engine{
|
||||
chunks: make(chan *sources.Chunk, defaultChannelBuffer),
|
||||
detectableChunksChan: make(chan detectableChunk, defaultChannelBuffer),
|
||||
results: make(chan detectors.ResultWithMetadata, defaultChannelBuffer),
|
||||
sourcesWg: &errgroup.Group{},
|
||||
dedupeCache: cache,
|
||||
printer: new(output.PlainPrinter), // default printer
|
||||
metrics: runtimeMetrics{Metrics: Metrics{scanStartTime: time.Now()}},
|
||||
}
|
||||
|
||||
for _, option := range options {
|
||||
@@ -290,11 +276,8 @@ func Start(ctx context.Context, options ...EngineOption) (*Engine, error) {
|
||||
}
|
||||
ctx.Logger().V(3).Info("engine started", "workers", e.concurrency)
|
||||
|
||||
// Create SourceManager.
|
||||
e.sourceManager = sources.NewManager(
|
||||
sources.WithConcurrentSources(int(e.concurrency)),
|
||||
sources.WithConcurrentUnits(int(e.concurrency)),
|
||||
)
|
||||
// Limit number of concurrent goroutines dedicated to chunking a source.
|
||||
e.sourcesWg.SetLimit(int(e.concurrency))
|
||||
|
||||
if len(e.decoders) == 0 {
|
||||
e.decoders = decoders.DefaultDecoders()
|
||||
@@ -393,7 +376,9 @@ func Start(ctx context.Context, options ...EngineOption) (*Engine, error) {
|
||||
func (e *Engine) Finish(ctx context.Context) error {
|
||||
defer common.RecoverWithExit(ctx)
|
||||
// Wait for the sources to finish putting chunks onto the chunks channel.
|
||||
err := e.sourceManager.Wait()
|
||||
err := e.sourcesWg.Wait()
|
||||
|
||||
close(e.chunks) // Source workers are done.
|
||||
|
||||
e.workersWg.Wait() // Wait for the workers to finish scanning chunks.
|
||||
close(e.detectableChunksChan)
|
||||
@@ -402,13 +387,11 @@ func (e *Engine) Finish(ctx context.Context) error {
|
||||
close(e.results) // Detector workers are done, close the results channel and call it a day.
|
||||
e.WgNotifier.Wait() // Wait for the notifier workers to finish notifying results.
|
||||
|
||||
e.metrics.ScanDuration = time.Since(e.metrics.scanStartTime)
|
||||
|
||||
return err
|
||||
}
|
||||
|
||||
func (e *Engine) ChunksChan() <-chan *sources.Chunk {
|
||||
return e.sourceManager.Chunks()
|
||||
func (e *Engine) ChunksChan() chan *sources.Chunk {
|
||||
return e.chunks
|
||||
}
|
||||
|
||||
func (e *Engine) ResultsChan() chan detectors.ResultWithMetadata {
|
||||
@@ -426,7 +409,7 @@ type detectableChunk struct {
|
||||
func (e *Engine) detectorWorker(ctx context.Context) {
|
||||
var wgDetect sync.WaitGroup
|
||||
|
||||
for originalChunk := range e.ChunksChan() {
|
||||
for originalChunk := range e.chunks {
|
||||
for chunk := range sources.Chunker(originalChunk) {
|
||||
matchedKeywords := make(map[string]struct{})
|
||||
atomic.AddUint64(&e.metrics.BytesScanned, uint64(len(chunk.Data)))
|
||||
@@ -595,20 +578,17 @@ func SupportsLineNumbers(sourceType sourcespb.SourceType) bool {
|
||||
|
||||
// FragmentLineOffset sets the line number for a provided source chunk with a given detector result.
|
||||
func FragmentLineOffset(chunk *sources.Chunk, result *detectors.Result) (int64, bool) {
|
||||
before, after, found := bytes.Cut(chunk.Data, result.Raw)
|
||||
if !found {
|
||||
return 0, false
|
||||
lines := bytes.Split(chunk.Data, []byte("\n"))
|
||||
for i, line := range lines {
|
||||
if bytes.Contains(line, result.Raw) {
|
||||
// if the line contains the ignore tag, we should ignore the result
|
||||
if bytes.Contains(line, []byte(ignoreTag)) {
|
||||
return int64(i), true
|
||||
}
|
||||
return int64(i), false
|
||||
}
|
||||
}
|
||||
lineNumber := int64(bytes.Count(before, []byte("\n")))
|
||||
// If the line contains the ignore tag, we should ignore the result.
|
||||
endLine := bytes.Index(after, []byte("\n"))
|
||||
if endLine == -1 {
|
||||
endLine = len(after)
|
||||
}
|
||||
if bytes.Contains(after[:endLine], []byte(ignoreTag)) {
|
||||
return lineNumber, true
|
||||
}
|
||||
return lineNumber, false
|
||||
return 0, false
|
||||
}
|
||||
|
||||
// FragmentFirstLine returns the first line number of a fragment along with a pointer to the value to update in the
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
package engine
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"testing"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/decoders"
|
||||
@@ -51,39 +50,6 @@ func TestFragmentLineOffset(t *testing.T) {
|
||||
expectedLine: 4,
|
||||
ignore: false,
|
||||
},
|
||||
{
|
||||
name: "match on consecutive lines",
|
||||
chunk: &sources.Chunk{
|
||||
Data: []byte("line1\nline2\ntrufflehog:ignore\nline4\nsecret\nhere\nline6"),
|
||||
},
|
||||
result: &detectors.Result{
|
||||
Raw: []byte("secret\nhere"),
|
||||
},
|
||||
expectedLine: 4,
|
||||
ignore: false,
|
||||
},
|
||||
{
|
||||
name: "ignore on last consecutive lines",
|
||||
chunk: &sources.Chunk{
|
||||
Data: []byte("line1\nline2\nline3\nsecret\nhere // trufflehog:ignore\nline5"),
|
||||
},
|
||||
result: &detectors.Result{
|
||||
Raw: []byte("secret\nhere"),
|
||||
},
|
||||
expectedLine: 3,
|
||||
ignore: true,
|
||||
},
|
||||
{
|
||||
name: "ignore on last line",
|
||||
chunk: &sources.Chunk{
|
||||
Data: []byte("line1\nline2\nline3\nsecret here // trufflehog:ignore"),
|
||||
},
|
||||
result: &detectors.Result{
|
||||
Raw: []byte("secret here"),
|
||||
},
|
||||
expectedLine: 3,
|
||||
ignore: true,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
@@ -99,43 +65,6 @@ func TestFragmentLineOffset(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func setupFragmentLineOffsetBench(totalLines, needleLine int) (*sources.Chunk, *detectors.Result) {
|
||||
data := make([]byte, 0, 4096)
|
||||
needle := []byte("needle")
|
||||
for i := 0; i < totalLines; i++ {
|
||||
if i != needleLine {
|
||||
data = append(data, []byte(fmt.Sprintf("line%d\n", i))...)
|
||||
continue
|
||||
}
|
||||
data = append(data, needle...)
|
||||
data = append(data, '\n')
|
||||
}
|
||||
chunk := &sources.Chunk{Data: data}
|
||||
result := &detectors.Result{Raw: needle}
|
||||
return chunk, result
|
||||
}
|
||||
|
||||
func BenchmarkFragmentLineOffsetStart(b *testing.B) {
|
||||
chunk, result := setupFragmentLineOffsetBench(512, 2)
|
||||
for i := 0; i < b.N; i++ {
|
||||
_, _ = FragmentLineOffset(chunk, result)
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkFragmentLineOffsetMiddle(b *testing.B) {
|
||||
chunk, result := setupFragmentLineOffsetBench(512, 256)
|
||||
for i := 0; i < b.N; i++ {
|
||||
_, _ = FragmentLineOffset(chunk, result)
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkFragmentLineOffsetEnd(b *testing.B) {
|
||||
chunk, result := setupFragmentLineOffsetBench(512, 510)
|
||||
for i := 0; i < b.N; i++ {
|
||||
_, _ = FragmentLineOffset(chunk, result)
|
||||
}
|
||||
}
|
||||
|
||||
// Test to make sure that DefaultDecoders always returns the UTF8 decoder first.
|
||||
// Technically a decoder test but we want this to run and fail in CI
|
||||
func TestDefaultDecoders(t *testing.T) {
|
||||
|
||||
+36
-12
@@ -1,11 +1,14 @@
|
||||
package engine
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"runtime"
|
||||
|
||||
"github.com/go-errors/errors"
|
||||
"google.golang.org/protobuf/proto"
|
||||
"google.golang.org/protobuf/types/known/anypb"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/context"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
|
||||
@@ -24,18 +27,39 @@ func (e *Engine) ScanFileSystem(ctx context.Context, c sources.FilesystemConfig)
|
||||
return err
|
||||
}
|
||||
|
||||
handle, err := e.sourceManager.Enroll(ctx, "trufflehog - filesystem", new(filesystem.Source).Type(),
|
||||
func(ctx context.Context, jobID, sourceID int64) (sources.Source, error) {
|
||||
fileSystemSource := filesystem.Source{}
|
||||
fileSystemSource.WithFilter(c.Filter)
|
||||
if err := fileSystemSource.Init(ctx, "trufflehog - filesystem", jobID, sourceID, true, &conn, runtime.NumCPU()); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &fileSystemSource, nil
|
||||
})
|
||||
fileSystemSource := filesystem.Source{}
|
||||
fileSystemSource.WithFilter(c.Filter)
|
||||
|
||||
// TODO: This is how it would be used; everything after these comments would be removed.
|
||||
// handle, err := e.sourceManager.Enroll(ctx, "trufflehog - filesystem", fileSystemSource.Type(),
|
||||
// func(ctx context.Context, jobID, sourceID int64) (sources.Source, error) {
|
||||
// fileSystemSource := filesystem.Source{}
|
||||
// fileSystemSource.WithFilter(c.Filter)
|
||||
// if err := fileSystemSource.Init(ctx, "trufflehog - filesystem", jobID, sourceID, true, &conn, runtime.NumCPU()); err != nil {
|
||||
// return nil, err
|
||||
// }
|
||||
// return &fileSystemSource, nil
|
||||
// })
|
||||
// if err != nil {
|
||||
// return err
|
||||
// }
|
||||
// return e.sourceManager.ScheduleRun(ctx, handle)
|
||||
|
||||
ctx = context.WithValues(ctx,
|
||||
"source_type", fileSystemSource.Type().String(),
|
||||
"source_name", "filesystem",
|
||||
)
|
||||
err = fileSystemSource.Init(ctx, "trufflehog - filesystem", 0, int64(sourcespb.SourceType_SOURCE_TYPE_FILESYSTEM), true, &conn, runtime.NumCPU())
|
||||
if err != nil {
|
||||
return err
|
||||
return errors.WrapPrefix(err, "could not init filesystem source", 0)
|
||||
}
|
||||
_, err = e.sourceManager.ScheduleRun(ctx, handle)
|
||||
return err
|
||||
e.sourcesWg.Go(func() error {
|
||||
defer common.RecoverWithExit(ctx)
|
||||
err := fileSystemSource.Chunks(ctx, e.ChunksChan())
|
||||
if err != nil {
|
||||
return fmt.Errorf("error scanning filesystem: %w", err)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
return nil
|
||||
}
|
||||
|
||||
+17
-12
@@ -6,6 +6,7 @@ import (
|
||||
"google.golang.org/protobuf/proto"
|
||||
"google.golang.org/protobuf/types/known/anypb"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/context"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
|
||||
@@ -44,19 +45,23 @@ func (e *Engine) ScanGCS(ctx context.Context, c sources.GCSConfig) error {
|
||||
return fmt.Errorf("failed to marshal GCS connection: %w", err)
|
||||
}
|
||||
|
||||
handle, err := e.sourceManager.Enroll(ctx, "trufflehog - gcs", new(gcs.Source).Type(),
|
||||
func(ctx context.Context, jobID, sourceID int64) (sources.Source, error) {
|
||||
gcsSource := gcs.Source{}
|
||||
if err := gcsSource.Init(ctx, "trufflehog - gcs", jobID, sourceID, true, &conn, int(c.Concurrency)); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &gcsSource, nil
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
source := gcs.Source{}
|
||||
ctx = context.WithValues(ctx,
|
||||
"source_type", source.Type().String(),
|
||||
"source_name", "gcs",
|
||||
)
|
||||
if err = source.Init(ctx, "trufflehog - GCS", 0, 0, true, &conn, int(c.Concurrency)); err != nil {
|
||||
return fmt.Errorf("failed to initialize GCS source: %w", err)
|
||||
}
|
||||
_, err = e.sourceManager.ScheduleRun(ctx, handle)
|
||||
return err
|
||||
|
||||
e.sourcesWg.Go(func() error {
|
||||
defer common.RecoverWithExit(ctx)
|
||||
if err := source.Chunks(ctx, e.ChunksChan()); err != nil {
|
||||
return fmt.Errorf("could not scan GCS: %w", err)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
return nil
|
||||
}
|
||||
|
||||
func isAuthValid(ctx context.Context, c sources.GCSConfig, connection *sourcespb.GCS) bool {
|
||||
|
||||
@@ -80,10 +80,7 @@ func TestScanGCS(t *testing.T) {
|
||||
t.Errorf("ScanGCS() got: %v, want: %v", err, nil)
|
||||
return
|
||||
}
|
||||
if err := e.Finish(ctx); err != nil && !test.wantErr && !strings.Contains(err.Error(), "googleapi: Error 400: Bad Request") {
|
||||
t.Errorf("Finish() got: %v, want: %v", err, nil)
|
||||
return
|
||||
}
|
||||
assert.Nil(t, e.Finish(ctx))
|
||||
|
||||
if err == nil && test.wantErr {
|
||||
t.Errorf("ScanGCS() got: %v, want: %v", err, "error")
|
||||
|
||||
+26
-20
@@ -1,12 +1,15 @@
|
||||
package engine
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"runtime"
|
||||
|
||||
"github.com/go-errors/errors"
|
||||
gogit "github.com/go-git/go-git/v5"
|
||||
"google.golang.org/protobuf/proto"
|
||||
"google.golang.org/protobuf/types/known/anypb"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/context"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
|
||||
@@ -33,9 +36,6 @@ func (e *Engine) ScanGit(ctx context.Context, c sources.GitConfig) error {
|
||||
if c.ExcludeGlobs != nil {
|
||||
opts = append(opts, git.ScanOptionExcludeGlobs(c.ExcludeGlobs))
|
||||
}
|
||||
if c.Bare {
|
||||
opts = append(opts, git.ScanOptionBare(c.Bare))
|
||||
}
|
||||
scanOptions := git.NewScanOptions(opts...)
|
||||
|
||||
connection := &sourcespb.Git{
|
||||
@@ -46,26 +46,32 @@ func (e *Engine) ScanGit(ctx context.Context, c sources.GitConfig) error {
|
||||
Directories: []string{c.RepoPath},
|
||||
}
|
||||
var conn anypb.Any
|
||||
if err := anypb.MarshalFrom(&conn, connection, proto.MarshalOptions{}); err != nil {
|
||||
err := anypb.MarshalFrom(&conn, connection, proto.MarshalOptions{})
|
||||
if err != nil {
|
||||
ctx.Logger().Error(err, "failed to marshal git connection")
|
||||
return err
|
||||
}
|
||||
|
||||
handle, err := e.sourceManager.Enroll(ctx, "trufflehog - git", new(git.Source).Type(),
|
||||
func(ctx context.Context, jobID, sourceID int64) (sources.Source, error) {
|
||||
gitSource := git.Source{}
|
||||
if err := gitSource.Init(ctx, "trufflehog - git", jobID, sourceID, true, &conn, runtime.NumCPU()); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
gitSource.WithScanOptions(scanOptions)
|
||||
// Don't try to clean up the provided directory. That's handled by the
|
||||
// caller of ScanGit.
|
||||
gitSource.WithPreserveTempDirs(true)
|
||||
return &gitSource, nil
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
gitSource := git.Source{}
|
||||
if err := gitSource.Init(ctx, "trufflehog - git", 0, 0, true, &conn, runtime.NumCPU()); err != nil {
|
||||
return errors.WrapPrefix(err, "could not init git source", 0)
|
||||
}
|
||||
_, err = e.sourceManager.ScheduleRun(ctx, handle)
|
||||
return err
|
||||
gitSource.WithScanOptions(scanOptions)
|
||||
// Don't try to clean up the provided directory. That's handled by the
|
||||
// caller of ScanGit.
|
||||
gitSource.WithPreserveTempDirs(true)
|
||||
|
||||
ctx = context.WithValues(ctx,
|
||||
"source_type", sourcespb.SourceType_SOURCE_TYPE_GIT.String(),
|
||||
"source_name", "git",
|
||||
)
|
||||
e.sourcesWg.Go(func() error {
|
||||
defer common.RecoverWithExit(ctx)
|
||||
err := gitSource.Chunks(ctx, e.ChunksChan())
|
||||
if err != nil {
|
||||
return fmt.Errorf("could not scan repo: %w", err)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
return nil
|
||||
}
|
||||
|
||||
+25
-14
@@ -1,10 +1,13 @@
|
||||
package engine
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
gogit "github.com/go-git/go-git/v5"
|
||||
"google.golang.org/protobuf/proto"
|
||||
"google.golang.org/protobuf/types/known/anypb"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/context"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
|
||||
@@ -14,6 +17,8 @@ import (
|
||||
|
||||
// ScanGitHub scans Github with the provided options.
|
||||
func (e *Engine) ScanGitHub(ctx context.Context, c sources.GithubConfig) error {
|
||||
source := github.Source{}
|
||||
|
||||
connection := sourcespb.GitHub{
|
||||
Endpoint: c.Endpoint,
|
||||
Organizations: c.Orgs,
|
||||
@@ -37,25 +42,31 @@ func (e *Engine) ScanGitHub(ctx context.Context, c sources.GithubConfig) error {
|
||||
return err
|
||||
}
|
||||
|
||||
ctx = context.WithValues(ctx,
|
||||
"source_type", source.Type().String(),
|
||||
"source_name", "github",
|
||||
)
|
||||
err = source.Init(ctx, "trufflehog - github", 0, 0, false, &conn, c.Concurrency)
|
||||
if err != nil {
|
||||
ctx.Logger().Error(err, "failed to initialize github source")
|
||||
return err
|
||||
}
|
||||
|
||||
logOptions := &gogit.LogOptions{}
|
||||
opts := []git.ScanOption{
|
||||
git.ScanOptionFilter(c.Filter),
|
||||
git.ScanOptionLogOptions(logOptions),
|
||||
}
|
||||
scanOptions := git.NewScanOptions(opts...)
|
||||
source.WithScanOptions(scanOptions)
|
||||
|
||||
handle, err := e.sourceManager.Enroll(ctx, "trufflehog - github", new(github.Source).Type(),
|
||||
func(ctx context.Context, jobID, sourceID int64) (sources.Source, error) {
|
||||
githubSource := github.Source{}
|
||||
if err := githubSource.Init(ctx, "trufflehog - github", jobID, sourceID, true, &conn, c.Concurrency); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
githubSource.WithScanOptions(scanOptions)
|
||||
return &githubSource, nil
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = e.sourceManager.ScheduleRun(ctx, handle)
|
||||
return err
|
||||
e.sourcesWg.Go(func() error {
|
||||
defer common.RecoverWithExit(ctx)
|
||||
err := source.Chunks(ctx, e.ChunksChan())
|
||||
if err != nil {
|
||||
return fmt.Errorf("could not scan github: %w", err)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
return nil
|
||||
}
|
||||
|
||||
+20
-12
@@ -4,10 +4,12 @@ import (
|
||||
"fmt"
|
||||
"runtime"
|
||||
|
||||
"github.com/go-errors/errors"
|
||||
gogit "github.com/go-git/go-git/v5"
|
||||
"google.golang.org/protobuf/proto"
|
||||
"google.golang.org/protobuf/types/known/anypb"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/context"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
|
||||
@@ -50,18 +52,24 @@ func (e *Engine) ScanGitLab(ctx context.Context, c sources.GitlabConfig) error {
|
||||
return err
|
||||
}
|
||||
|
||||
handle, err := e.sourceManager.Enroll(ctx, "trufflehog - gitlab", new(gitlab.Source).Type(),
|
||||
func(ctx context.Context, jobID, sourceID int64) (sources.Source, error) {
|
||||
gitlabSource := gitlab.Source{}
|
||||
if err := gitlabSource.Init(ctx, "trufflehog - gitlab", jobID, sourceID, true, &conn, runtime.NumCPU()); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
gitlabSource.WithScanOptions(scanOptions)
|
||||
return &gitlabSource, nil
|
||||
})
|
||||
gitlabSource := gitlab.Source{}
|
||||
ctx = context.WithValues(ctx,
|
||||
"source_type", gitlabSource.Type().String(),
|
||||
"source_name", "gitlab",
|
||||
)
|
||||
err = gitlabSource.Init(ctx, "trufflehog - gitlab", 0, int64(sourcespb.SourceType_SOURCE_TYPE_GITLAB), true, &conn, runtime.NumCPU())
|
||||
if err != nil {
|
||||
return err
|
||||
return errors.WrapPrefix(err, "could not init GitLab source", 0)
|
||||
}
|
||||
_, err = e.sourceManager.ScheduleRun(ctx, handle)
|
||||
return err
|
||||
gitlabSource.WithScanOptions(scanOptions)
|
||||
|
||||
e.sourcesWg.Go(func() error {
|
||||
defer common.RecoverWithExit(ctx)
|
||||
err := gitlabSource.Chunks(ctx, e.ChunksChan())
|
||||
if err != nil {
|
||||
return fmt.Errorf("error scanning GitLab: %w", err)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
return nil
|
||||
}
|
||||
|
||||
+19
-11
@@ -4,9 +4,11 @@ import (
|
||||
"fmt"
|
||||
"runtime"
|
||||
|
||||
"github.com/go-errors/errors"
|
||||
"google.golang.org/protobuf/proto"
|
||||
"google.golang.org/protobuf/types/known/anypb"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/context"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/credentialspb"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
|
||||
@@ -53,17 +55,23 @@ func (e *Engine) ScanS3(ctx context.Context, c sources.S3Config) error {
|
||||
return err
|
||||
}
|
||||
|
||||
handle, err := e.sourceManager.Enroll(ctx, "trufflehog - s3", new(s3.Source).Type(),
|
||||
func(ctx context.Context, jobID, sourceID int64) (sources.Source, error) {
|
||||
s3Source := s3.Source{}
|
||||
if err := s3Source.Init(ctx, "trufflehog - s3", jobID, sourceID, true, &conn, runtime.NumCPU()); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &s3Source, nil
|
||||
})
|
||||
s3Source := s3.Source{}
|
||||
ctx = context.WithValues(ctx,
|
||||
"source_type", s3Source.Type().String(),
|
||||
"source_name", "s3",
|
||||
)
|
||||
err = s3Source.Init(ctx, "trufflehog - s3", 0, int64(sourcespb.SourceType_SOURCE_TYPE_S3), true, &conn, runtime.NumCPU())
|
||||
if err != nil {
|
||||
return err
|
||||
return errors.WrapPrefix(err, "failed to init S3 source", 0)
|
||||
}
|
||||
_, err = e.sourceManager.ScheduleRun(ctx, handle)
|
||||
return err
|
||||
|
||||
e.sourcesWg.Go(func() error {
|
||||
defer common.RecoverWithExit(ctx)
|
||||
err := s3Source.Chunks(ctx, e.ChunksChan())
|
||||
if err != nil {
|
||||
return fmt.Errorf("error scanning S3: %w", err)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
return nil
|
||||
}
|
||||
|
||||
+20
-12
@@ -1,12 +1,14 @@
|
||||
package engine
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
|
||||
"github.com/go-errors/errors"
|
||||
"google.golang.org/protobuf/proto"
|
||||
"google.golang.org/protobuf/types/known/anypb"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/context"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
|
||||
@@ -40,19 +42,25 @@ func (e *Engine) ScanSyslog(ctx context.Context, c sources.SyslogConfig) error {
|
||||
if err != nil {
|
||||
return errors.WrapPrefix(err, "error unmarshalling connection", 0)
|
||||
}
|
||||
|
||||
handle, err := e.sourceManager.Enroll(ctx, "trufflehog - syslog", new(syslog.Source).Type(),
|
||||
func(ctx context.Context, jobID, sourceID int64) (sources.Source, error) {
|
||||
syslogSource := syslog.Source{}
|
||||
if err := syslogSource.Init(ctx, "trufflehog - syslog", jobID, sourceID, true, &conn, c.Concurrency); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
syslogSource.InjectConnection(connection)
|
||||
return &syslogSource, nil
|
||||
})
|
||||
source := syslog.Source{}
|
||||
ctx = context.WithValues(ctx,
|
||||
"source_type", source.Type().String(),
|
||||
"source_name", "syslog",
|
||||
)
|
||||
err = source.Init(ctx, "trufflehog - syslog", 0, 0, false, &conn, c.Concurrency)
|
||||
source.InjectConnection(connection)
|
||||
if err != nil {
|
||||
ctx.Logger().Error(err, "failed to initialize syslog source")
|
||||
return err
|
||||
}
|
||||
_, err = e.sourceManager.ScheduleRun(ctx, handle)
|
||||
return err
|
||||
|
||||
e.sourcesWg.Go(func() error {
|
||||
defer common.RecoverWithExit(ctx)
|
||||
err := source.Chunks(ctx, e.ChunksChan())
|
||||
if err != nil {
|
||||
return fmt.Errorf("could not scan syslog: %w", err)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -5,7 +5,6 @@ import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
@@ -161,7 +160,7 @@ func (c1 *Commit) Equal(c2 *Commit) bool {
|
||||
}
|
||||
|
||||
// RepoPath parses the output of the `git log` command for the `source` path.
|
||||
func (c *Parser) RepoPath(ctx context.Context, source string, head string, abbreviatedLog bool, excludedGlobs []string, isBare bool) (chan Commit, error) {
|
||||
func (c *Parser) RepoPath(ctx context.Context, source string, head string, abbreviatedLog bool, excludedGlobs []string) (chan Commit, error) {
|
||||
args := []string{"-C", source, "log", "-p", "--full-history", "--date=format:%a %b %d %H:%M:%S %Y %z"}
|
||||
if abbreviatedLog {
|
||||
args = append(args, "--diff-filter=AM")
|
||||
@@ -178,21 +177,7 @@ func (c *Parser) RepoPath(ctx context.Context, source string, head string, abbre
|
||||
cmd := exec.Command("git", args...)
|
||||
absPath, err := filepath.Abs(source)
|
||||
if err == nil {
|
||||
if !isBare {
|
||||
cmd.Env = append(cmd.Env, "GIT_DIR="+filepath.Join(absPath, ".git"))
|
||||
} else {
|
||||
cmd.Env = append(cmd.Env,
|
||||
"GIT_DIR="+absPath,
|
||||
)
|
||||
// We need those variables to handle incoming commits
|
||||
// while using trufflehog in pre-receive hooks
|
||||
if dir := os.Getenv("GIT_OBJECT_DIRECTORY"); dir != "" {
|
||||
cmd.Env = append(cmd.Env, "GIT_OBJECT_DIRECTORY="+dir)
|
||||
}
|
||||
if dir := os.Getenv("GIT_ALTERNATE_OBJECT_DIRECTORIES"); dir != "" {
|
||||
cmd.Env = append(cmd.Env, "GIT_ALTERNATE_OBJECT_DIRECTORIES="+dir)
|
||||
}
|
||||
}
|
||||
cmd.Env = append(cmd.Env, fmt.Sprintf("GIT_DIR=%s", filepath.Join(absPath, ".git")))
|
||||
}
|
||||
|
||||
return c.executeCommand(ctx, cmd, false)
|
||||
|
||||
+18
-295
@@ -6,13 +6,8 @@ import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/h2non/filetype"
|
||||
"github.com/mholt/archiver/v4"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
@@ -31,18 +26,14 @@ var (
|
||||
maxTimeout = time.Duration(30) * time.Second
|
||||
)
|
||||
|
||||
// Ensure the Archive satisfies the interfaces at compile time.
|
||||
var _ SpecializedHandler = (*Archive)(nil)
|
||||
|
||||
// Archive is a handler for extracting and decompressing archives.
|
||||
type Archive struct {
|
||||
size int
|
||||
currentDepth int
|
||||
size int
|
||||
}
|
||||
|
||||
// New sets a default maximum size and current size counter.
|
||||
func (a *Archive) New() {
|
||||
a.size = 0
|
||||
func (d *Archive) New() {
|
||||
d.size = 0
|
||||
}
|
||||
|
||||
// SetArchiveMaxSize sets the maximum size of the archive.
|
||||
@@ -61,14 +52,14 @@ func SetArchiveMaxTimeout(timeout time.Duration) {
|
||||
}
|
||||
|
||||
// FromFile extracts the files from an archive.
|
||||
func (a *Archive) FromFile(originalCtx context.Context, data io.Reader) chan []byte {
|
||||
archiveChan := make(chan []byte, 512)
|
||||
func (d *Archive) FromFile(originalCtx context.Context, data io.Reader) chan ([]byte) {
|
||||
archiveChan := make(chan ([]byte), 512)
|
||||
go func() {
|
||||
ctx, cancel := context.WithTimeout(originalCtx, maxTimeout)
|
||||
logger := logContext.AddLogger(ctx).Logger()
|
||||
defer cancel()
|
||||
defer close(archiveChan)
|
||||
err := a.openArchive(ctx, 0, data, archiveChan)
|
||||
err := d.openArchive(ctx, 0, data, archiveChan)
|
||||
if err != nil {
|
||||
if errors.Is(err, archiver.ErrNoMatch) {
|
||||
return
|
||||
@@ -80,7 +71,7 @@ func (a *Archive) FromFile(originalCtx context.Context, data io.Reader) chan []b
|
||||
}
|
||||
|
||||
// openArchive takes a reader and extracts the contents up to the maximum depth.
|
||||
func (a *Archive) openArchive(ctx context.Context, depth int, reader io.Reader, archiveChan chan []byte) error {
|
||||
func (d *Archive) openArchive(ctx context.Context, depth int, reader io.Reader, archiveChan chan ([]byte)) error {
|
||||
if depth >= maxDepth {
|
||||
return fmt.Errorf("max archive depth reached")
|
||||
}
|
||||
@@ -106,14 +97,14 @@ func (a *Archive) openArchive(ctx context.Context, depth int, reader io.Reader,
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fileBytes, err := a.ReadToMax(ctx, compReader)
|
||||
fileBytes, err := d.ReadToMax(ctx, compReader)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
newReader := bytes.NewReader(fileBytes)
|
||||
return a.openArchive(ctx, depth+1, newReader, archiveChan)
|
||||
return d.openArchive(ctx, depth+1, newReader, archiveChan)
|
||||
case archiver.Extractor:
|
||||
err := archive.Extract(context.WithValue(ctx, depthKey, depth+1), reader, nil, a.extractorHandler(archiveChan))
|
||||
err := archive.Extract(context.WithValue(ctx, depthKey, depth+1), reader, nil, d.extractorHandler(archiveChan))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -123,7 +114,7 @@ func (a *Archive) openArchive(ctx context.Context, depth int, reader io.Reader,
|
||||
}
|
||||
|
||||
// IsFiletype returns true if the provided reader is an archive.
|
||||
func (a *Archive) IsFiletype(ctx context.Context, reader io.Reader) (io.Reader, bool) {
|
||||
func (d *Archive) IsFiletype(ctx context.Context, reader io.Reader) (io.Reader, bool) {
|
||||
format, readerB, err := archiver.Identify("", reader)
|
||||
if err != nil {
|
||||
return readerB, false
|
||||
@@ -138,7 +129,7 @@ func (a *Archive) IsFiletype(ctx context.Context, reader io.Reader) (io.Reader,
|
||||
}
|
||||
|
||||
// extractorHandler is applied to each file in an archiver.Extractor file.
|
||||
func (a *Archive) extractorHandler(archiveChan chan []byte) func(context.Context, archiver.File) error {
|
||||
func (d *Archive) extractorHandler(archiveChan chan ([]byte)) func(context.Context, archiver.File) error {
|
||||
return func(ctx context.Context, f archiver.File) error {
|
||||
logger := logContext.AddLogger(ctx).Logger()
|
||||
logger.V(5).Info("Handling extracted file.", "filename", f.Name())
|
||||
@@ -151,13 +142,13 @@ func (a *Archive) extractorHandler(archiveChan chan []byte) func(context.Context
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fileBytes, err := a.ReadToMax(ctx, fReader)
|
||||
fileBytes, err := d.ReadToMax(ctx, fReader)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fileContent := bytes.NewReader(fileBytes)
|
||||
|
||||
err = a.openArchive(ctx, depth, fileContent, archiveChan)
|
||||
err = d.openArchive(ctx, depth, fileContent, archiveChan)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -166,7 +157,7 @@ func (a *Archive) extractorHandler(archiveChan chan []byte) func(context.Context
|
||||
}
|
||||
|
||||
// ReadToMax reads up to the max size.
|
||||
func (a *Archive) ReadToMax(ctx context.Context, reader io.Reader) (data []byte, err error) {
|
||||
func (d *Archive) ReadToMax(ctx context.Context, reader io.Reader) (data []byte, err error) {
|
||||
// Archiver v4 is in alpha and using an experimental version of
|
||||
// rardecode. There is a bug somewhere with rar decoder format 29
|
||||
// that can lead to a panic. An issue is open in rardecode repo
|
||||
@@ -184,7 +175,7 @@ func (a *Archive) ReadToMax(ctx context.Context, reader io.Reader) (data []byte,
|
||||
}
|
||||
}()
|
||||
fileContent := bytes.Buffer{}
|
||||
logger.V(5).Info("Remaining buffer capacity", "bytes", maxSize-a.size)
|
||||
logger.V(5).Info("Remaining buffer capacity", "bytes", maxSize-d.size)
|
||||
for i := 0; i <= maxSize/512; i++ {
|
||||
if common.IsDone(ctx) {
|
||||
return nil, ctx.Err()
|
||||
@@ -194,285 +185,17 @@ func (a *Archive) ReadToMax(ctx context.Context, reader io.Reader) (data []byte,
|
||||
if err != nil && !errors.Is(err, io.EOF) {
|
||||
return []byte{}, err
|
||||
}
|
||||
a.size += bRead
|
||||
d.size += bRead
|
||||
if len(fileChunk) > 0 {
|
||||
fileContent.Write(fileChunk[0:bRead])
|
||||
}
|
||||
if bRead < 512 {
|
||||
return fileContent.Bytes(), nil
|
||||
}
|
||||
if a.size >= maxSize && bRead == 512 {
|
||||
if d.size >= maxSize && bRead == 512 {
|
||||
logger.V(2).Info("Max archive size reached.")
|
||||
return fileContent.Bytes(), nil
|
||||
}
|
||||
}
|
||||
return fileContent.Bytes(), nil
|
||||
}
|
||||
|
||||
const (
|
||||
arMimeType = "application/x-unix-archive"
|
||||
rpmMimeType = "application/x-rpm"
|
||||
)
|
||||
|
||||
// Define a map of mime types to corresponding command-line tools
|
||||
var mimeTools = map[string][]string{
|
||||
arMimeType: {"ar"},
|
||||
rpmMimeType: {"rpm2cpio", "cpio"},
|
||||
}
|
||||
|
||||
// Check if the command-line tool is installed.
|
||||
func isToolInstalled(tool string) bool {
|
||||
_, err := exec.LookPath(tool)
|
||||
return err == nil
|
||||
}
|
||||
|
||||
// Ensure all tools are available for given mime type.
|
||||
func ensureToolsForMimeType(mimeType string) error {
|
||||
tools, exists := mimeTools[mimeType]
|
||||
if !exists {
|
||||
return fmt.Errorf("unsupported mime type")
|
||||
}
|
||||
|
||||
for _, tool := range tools {
|
||||
if !isToolInstalled(tool) {
|
||||
return fmt.Errorf("Required tool " + tool + " is not installed")
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// HandleSpecialized takes a file path and an io.Reader representing the input file,
|
||||
// and processes it based on its extension, such as handling Debian (.deb) and RPM (.rpm) packages.
|
||||
// It returns an io.Reader that can be used to read the processed content of the file,
|
||||
// and an error if any issues occurred during processing.
|
||||
// If the file is specialized, the returned boolean is true with no error.
|
||||
// The caller is responsible for closing the returned reader.
|
||||
func (a *Archive) HandleSpecialized(ctx logContext.Context, reader io.Reader) (io.Reader, bool, error) {
|
||||
mimeType, reader, err := determineMimeType(reader)
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
|
||||
switch mimeType {
|
||||
case arMimeType: // includes .deb files
|
||||
if err := ensureToolsForMimeType(mimeType); err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
reader, err = a.extractDebContent(ctx, reader)
|
||||
case rpmMimeType:
|
||||
if err := ensureToolsForMimeType(mimeType); err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
reader, err = a.extractRpmContent(ctx, reader)
|
||||
default:
|
||||
return reader, false, nil
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
return nil, false, fmt.Errorf("unable to extract file with MIME type %s: %w", mimeType, err)
|
||||
}
|
||||
return reader, true, nil
|
||||
}
|
||||
|
||||
// extractDebContent takes a .deb file as an io.Reader, extracts its contents
|
||||
// into a temporary directory, and returns a Reader for the extracted data archive.
|
||||
// It handles the extraction process by using the 'ar' command and manages temporary
|
||||
// files and directories for the operation.
|
||||
// The caller is responsible for closing the returned reader.
|
||||
func (a *Archive) extractDebContent(ctx logContext.Context, file io.Reader) (io.ReadCloser, error) {
|
||||
if a.currentDepth >= maxDepth {
|
||||
return nil, fmt.Errorf("max archive depth reached")
|
||||
}
|
||||
|
||||
tmpEnv, err := a.createTempEnv(ctx, file)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer os.Remove(tmpEnv.tempFileName)
|
||||
defer os.RemoveAll(tmpEnv.extractPath)
|
||||
|
||||
cmd := exec.Command("ar", "x", tmpEnv.tempFile.Name())
|
||||
cmd.Dir = tmpEnv.extractPath
|
||||
if err := executeCommand(cmd); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
handler := func(ctx logContext.Context, env tempEnv, file string) (string, error) {
|
||||
if strings.HasPrefix(file, "data.tar.") {
|
||||
return file, nil
|
||||
}
|
||||
return a.handleNestedFileMIME(ctx, env, file)
|
||||
}
|
||||
|
||||
dataArchiveName, err := a.handleExtractedFiles(ctx, tmpEnv, handler)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return openDataArchive(tmpEnv.extractPath, dataArchiveName)
|
||||
}
|
||||
|
||||
// extractRpmContent takes an .rpm file as an io.Reader, extracts its contents
|
||||
// into a temporary directory, and returns a Reader for the extracted data archive.
|
||||
// It handles the extraction process by using the 'rpm2cpio' and 'cpio' commands and manages temporary
|
||||
// files and directories for the operation.
|
||||
// The caller is responsible for closing the returned reader.
|
||||
func (a *Archive) extractRpmContent(ctx logContext.Context, file io.Reader) (io.ReadCloser, error) {
|
||||
if a.currentDepth >= maxDepth {
|
||||
return nil, fmt.Errorf("max archive depth reached")
|
||||
}
|
||||
|
||||
tmpEnv, err := a.createTempEnv(ctx, file)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer os.Remove(tmpEnv.tempFileName)
|
||||
defer os.RemoveAll(tmpEnv.extractPath)
|
||||
|
||||
// Use rpm2cpio to convert the RPM file to a cpio archive and then extract it using cpio command.
|
||||
cmd := exec.Command("sh", "-c", "rpm2cpio "+tmpEnv.tempFile.Name()+" | cpio -id")
|
||||
cmd.Dir = tmpEnv.extractPath
|
||||
if err := executeCommand(cmd); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
handler := func(ctx logContext.Context, env tempEnv, file string) (string, error) {
|
||||
if strings.HasSuffix(file, ".tar.gz") {
|
||||
return file, nil
|
||||
}
|
||||
return a.handleNestedFileMIME(ctx, env, file)
|
||||
}
|
||||
|
||||
dataArchiveName, err := a.handleExtractedFiles(ctx, tmpEnv, handler)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return openDataArchive(tmpEnv.extractPath, dataArchiveName)
|
||||
}
|
||||
|
||||
func (a *Archive) handleNestedFileMIME(ctx logContext.Context, tempEnv tempEnv, fileName string) (string, error) {
|
||||
nestedFile, err := os.Open(filepath.Join(tempEnv.extractPath, fileName))
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer nestedFile.Close()
|
||||
|
||||
mimeType, reader, err := determineMimeType(nestedFile)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("unable to determine MIME type of nested filename: %s, %w", nestedFile.Name(), err)
|
||||
}
|
||||
|
||||
switch mimeType {
|
||||
case arMimeType:
|
||||
_, _, err = a.HandleSpecialized(ctx, reader)
|
||||
case rpmMimeType:
|
||||
_, _, err = a.HandleSpecialized(ctx, reader)
|
||||
default:
|
||||
return "", nil
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("unable to extract file with MIME type %s: %w", mimeType, err)
|
||||
}
|
||||
|
||||
return fileName, nil
|
||||
}
|
||||
|
||||
// determineMimeType reads from the provided reader to detect the MIME type.
|
||||
// It returns the detected MIME type and a new reader that includes the read portion.
|
||||
func determineMimeType(reader io.Reader) (string, io.Reader, error) {
|
||||
buffer := make([]byte, 512)
|
||||
n, err := reader.Read(buffer)
|
||||
if err != nil {
|
||||
return "", nil, fmt.Errorf("unable to read file for MIME type detection: %w", err)
|
||||
}
|
||||
|
||||
// Create a new reader that starts with the buffer we just read
|
||||
// and continues with the rest of the original reader.
|
||||
reader = io.MultiReader(bytes.NewReader(buffer[:n]), reader)
|
||||
|
||||
kind, err := filetype.Match(buffer)
|
||||
if err != nil {
|
||||
return "", nil, fmt.Errorf("unable to determine file type: %w", err)
|
||||
}
|
||||
|
||||
return kind.MIME.Value, reader, nil
|
||||
}
|
||||
|
||||
// handleExtractedFiles processes each file in the extracted directory using a provided handler function.
|
||||
// The function iterates through the files, applying the handleFile function to each, and returns the name
|
||||
// of the data archive it finds. This centralizes the logic for handling specialized files such as .deb and .rpm
|
||||
// by using the appropriate handling function passed as an argument. This design allows for flexibility and reuse
|
||||
// of this function across various extraction processes in the package.
|
||||
func (a *Archive) handleExtractedFiles(ctx logContext.Context, env tempEnv, handleFile func(logContext.Context, tempEnv, string) (string, error)) (string, error) {
|
||||
extractedFiles, err := os.ReadDir(env.extractPath)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("unable to read extracted directory: %w", err)
|
||||
}
|
||||
|
||||
var dataArchiveName string
|
||||
for _, file := range extractedFiles {
|
||||
name, err := handleFile(ctx, env, file.Name())
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if name != "" {
|
||||
dataArchiveName = name
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
return dataArchiveName, nil
|
||||
}
|
||||
|
||||
type tempEnv struct {
|
||||
tempFile *os.File
|
||||
tempFileName string
|
||||
extractPath string
|
||||
}
|
||||
|
||||
// createTempEnv creates a temporary file and a temporary directory for extracting archives.
|
||||
// The caller is responsible for removing these temporary resources
|
||||
// (both the file and directory) when they are no longer needed.
|
||||
func (a *Archive) createTempEnv(ctx context.Context, file io.Reader) (tempEnv, error) {
|
||||
tempFile, err := os.CreateTemp("", "tmp")
|
||||
if err != nil {
|
||||
return tempEnv{}, fmt.Errorf("unable to create temporary file: %w", err)
|
||||
}
|
||||
|
||||
extractPath, err := os.MkdirTemp("", "tmp_archive")
|
||||
if err != nil {
|
||||
return tempEnv{}, fmt.Errorf("unable to create temporary directory: %w", err)
|
||||
}
|
||||
|
||||
b, err := a.ReadToMax(ctx, file)
|
||||
if err != nil {
|
||||
return tempEnv{}, err
|
||||
}
|
||||
|
||||
if _, err = tempFile.Write(b); err != nil {
|
||||
return tempEnv{}, fmt.Errorf("unable to write to temporary file: %w", err)
|
||||
}
|
||||
|
||||
return tempEnv{tempFile: tempFile, tempFileName: tempFile.Name(), extractPath: extractPath}, nil
|
||||
}
|
||||
|
||||
func executeCommand(cmd *exec.Cmd) error {
|
||||
var stderr bytes.Buffer
|
||||
cmd.Stderr = &stderr
|
||||
if err := cmd.Run(); err != nil {
|
||||
return fmt.Errorf("unable to execute command %v: %w; error: %s", cmd.String(), err, stderr.String())
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func openDataArchive(extractPath string, dataArchiveName string) (io.ReadCloser, error) {
|
||||
dataArchivePath := filepath.Join(extractPath, dataArchiveName)
|
||||
dataFile, err := os.Open(dataArchivePath)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("unable to open file: %w", err)
|
||||
}
|
||||
return dataFile, nil
|
||||
}
|
||||
|
||||
@@ -2,9 +2,7 @@ package handlers
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io"
|
||||
"net/http"
|
||||
"os"
|
||||
"regexp"
|
||||
"strings"
|
||||
"testing"
|
||||
@@ -12,7 +10,6 @@ import (
|
||||
diskbufferreader "github.com/bill-rich/disk-buffer-reader"
|
||||
"github.com/stretchr/testify/assert"
|
||||
|
||||
logContext "github.com/trufflesecurity/trufflehog/v3/pkg/context"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
|
||||
)
|
||||
|
||||
@@ -125,39 +122,3 @@ func TestHandleFile(t *testing.T) {
|
||||
assert.True(t, HandleFile(context.Background(), reader, &sources.Chunk{}, ch))
|
||||
assert.Equal(t, 1, len(ch))
|
||||
}
|
||||
|
||||
func TestExtractDebContent(t *testing.T) {
|
||||
// Open the sample .deb file from the testdata folder.
|
||||
file, err := os.Open("testdata/test.deb")
|
||||
assert.Nil(t, err)
|
||||
defer file.Close()
|
||||
|
||||
ctx := logContext.AddLogger(context.Background())
|
||||
a := &Archive{}
|
||||
|
||||
reader, err := a.extractDebContent(ctx, file)
|
||||
assert.Nil(t, err)
|
||||
|
||||
content, err := io.ReadAll(reader)
|
||||
assert.Nil(t, err)
|
||||
expectedLength := 1015582
|
||||
assert.Equal(t, expectedLength, len(string(content)))
|
||||
}
|
||||
|
||||
func TestExtractRPMContent(t *testing.T) {
|
||||
// Open the sample .rpm file from the testdata folder.
|
||||
file, err := os.Open("testdata/test.rpm")
|
||||
assert.Nil(t, err)
|
||||
defer file.Close()
|
||||
|
||||
ctx := logContext.AddLogger(context.Background())
|
||||
a := &Archive{}
|
||||
|
||||
reader, err := a.extractRpmContent(ctx, file)
|
||||
assert.Nil(t, err)
|
||||
|
||||
content, err := io.ReadAll(reader)
|
||||
assert.Nil(t, err)
|
||||
expectedLength := 1822720
|
||||
assert.Equal(t, expectedLength, len(string(content)))
|
||||
}
|
||||
|
||||
+13
-53
@@ -4,9 +4,6 @@ import (
|
||||
"context"
|
||||
"io"
|
||||
|
||||
diskbufferreader "github.com/bill-rich/disk-buffer-reader"
|
||||
|
||||
logContext "github.com/trufflesecurity/trufflehog/v3/pkg/context"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
|
||||
)
|
||||
|
||||
@@ -16,76 +13,39 @@ func DefaultHandlers() []Handler {
|
||||
}
|
||||
}
|
||||
|
||||
// SpecializedHandler defines the interface for handlers that can process specialized archives.
|
||||
// It includes a method to handle specialized archives and determine if the file is of a special type.
|
||||
type SpecializedHandler interface {
|
||||
// HandleSpecialized examines the provided file reader within the context and determines if it is a specialized archive.
|
||||
// It returns a reader with any necessary modifications, a boolean indicating if the file was specialized,
|
||||
// and an error if something went wrong during processing.
|
||||
HandleSpecialized(logContext.Context, io.Reader) (io.Reader, bool, error)
|
||||
}
|
||||
|
||||
type Handler interface {
|
||||
FromFile(context.Context, io.Reader) chan ([]byte)
|
||||
IsFiletype(context.Context, io.Reader) (io.Reader, bool)
|
||||
New()
|
||||
}
|
||||
|
||||
// HandleFile processes a given file by selecting an appropriate handler from DefaultHandlers.
|
||||
// It first checks if the handler implements SpecializedHandler for any special processing,
|
||||
// then falls back to regular file type handling. If successful, it reads the file in chunks,
|
||||
// packages them in the provided chunk skeleton, and sends them to chunksChan.
|
||||
// The function returns true if processing was successful and false otherwise.
|
||||
// Context is used for cancellation, and the caller is responsible for canceling it if needed.
|
||||
func HandleFile(ctx context.Context, file io.Reader, chunkSkel *sources.Chunk, chunksChan chan *sources.Chunk) bool {
|
||||
aCtx := logContext.AddLogger(ctx)
|
||||
func HandleFile(ctx context.Context, file io.Reader, chunkSkel *sources.Chunk, chunksChan chan (*sources.Chunk)) bool {
|
||||
// Find a handler for this file.
|
||||
var handler Handler
|
||||
for _, h := range DefaultHandlers() {
|
||||
h.New()
|
||||
|
||||
// The re-reader is used to reset the file reader after checking if the handler implements SpecializedHandler.
|
||||
// This is necessary because the archive pkg doesn't correctly determine the file type when using
|
||||
// an io.MultiReader, which is used by the SpecializedHandler.
|
||||
reReader, err := diskbufferreader.New(file)
|
||||
if err != nil {
|
||||
aCtx.Logger().Error(err, "error creating re-reader reader")
|
||||
return false
|
||||
}
|
||||
defer reReader.Close()
|
||||
|
||||
// Check if the handler implements SpecializedHandler and process accordingly.
|
||||
if specialHandler, ok := h.(SpecializedHandler); ok {
|
||||
file, isSpecial, err := specialHandler.HandleSpecialized(aCtx, reReader)
|
||||
if isSpecial {
|
||||
return handleChunks(aCtx, h.FromFile(ctx, file), chunkSkel, chunksChan)
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
aCtx.Logger().Error(err, "error handling file")
|
||||
}
|
||||
}
|
||||
|
||||
if err := reReader.Reset(); err != nil {
|
||||
aCtx.Logger().Error(err, "error resetting re-reader")
|
||||
return false
|
||||
}
|
||||
reReader.Stop()
|
||||
var isType bool
|
||||
if file, isType = h.IsFiletype(aCtx, reReader); isType {
|
||||
return handleChunks(aCtx, h.FromFile(ctx, file), chunkSkel, chunksChan)
|
||||
if file, isType = h.IsFiletype(ctx, file); isType {
|
||||
handler = h
|
||||
break
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
if handler == nil {
|
||||
return false
|
||||
}
|
||||
|
||||
func handleChunks(ctx context.Context, handlerChan chan []byte, chunkSkel *sources.Chunk, chunksChan chan *sources.Chunk) bool {
|
||||
// Process the file and read all []byte chunks from handlerChan.
|
||||
handlerChan := handler.FromFile(ctx, file)
|
||||
for {
|
||||
select {
|
||||
case data, open := <-handlerChan:
|
||||
if !open {
|
||||
// We finished reading everything from handlerChan.
|
||||
return true
|
||||
}
|
||||
chunk := *chunkSkel
|
||||
chunk.Data = data
|
||||
// Send data on chunksChan.
|
||||
select {
|
||||
case chunksChan <- &chunk:
|
||||
case <-ctx.Done():
|
||||
|
||||
Vendored
BIN
Binary file not shown.
Vendored
BIN
Binary file not shown.
@@ -4,7 +4,6 @@ import (
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"sync"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/context"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
@@ -14,7 +13,7 @@ import (
|
||||
var dedupeCache = make(map[string]struct{})
|
||||
|
||||
// GitHubActionsPrinter is a printer that prints results in GitHub Actions format.
|
||||
type GitHubActionsPrinter struct{ mu sync.Mutex }
|
||||
type GitHubActionsPrinter struct{}
|
||||
|
||||
func (p *GitHubActionsPrinter) Print(_ context.Context, r *detectors.ResultWithMetadata) error {
|
||||
out := gitHubActionsOutputFormat{
|
||||
@@ -57,8 +56,6 @@ func (p *GitHubActionsPrinter) Print(_ context.Context, r *detectors.ResultWithM
|
||||
}
|
||||
dedupeCache[key] = struct{}{}
|
||||
|
||||
p.mu.Lock()
|
||||
defer p.mu.Unlock()
|
||||
message := fmt.Sprintf("Found %s %s result 🐷🔑\n", verifiedStatus, out.DetectorType)
|
||||
if r.Result.DecoderType != detectorspb.DecoderType_PLAIN {
|
||||
message = fmt.Sprintf("Found %s %s result with %s encoding 🐷🔑\n", verifiedStatus, out.DetectorType, out.DecoderType)
|
||||
|
||||
+1
-5
@@ -3,7 +3,6 @@ package output
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"sync"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/context"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
@@ -13,7 +12,7 @@ import (
|
||||
)
|
||||
|
||||
// JSONPrinter is a printer that prints results in JSON format.
|
||||
type JSONPrinter struct{ mu sync.Mutex }
|
||||
type JSONPrinter struct{}
|
||||
|
||||
func (p *JSONPrinter) Print(_ context.Context, r *detectors.ResultWithMetadata) error {
|
||||
v := &struct {
|
||||
@@ -61,9 +60,6 @@ func (p *JSONPrinter) Print(_ context.Context, r *detectors.ResultWithMetadata)
|
||||
if err != nil {
|
||||
return fmt.Errorf("could not marshal result: %w", err)
|
||||
}
|
||||
|
||||
p.mu.Lock()
|
||||
fmt.Println(string(out))
|
||||
p.mu.Unlock()
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -7,7 +7,6 @@ import (
|
||||
"net/url"
|
||||
"os"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
gogit "github.com/go-git/go-git/v5"
|
||||
"github.com/go-git/go-git/v5/plumbing"
|
||||
@@ -21,7 +20,7 @@ import (
|
||||
)
|
||||
|
||||
// LegacyJSONPrinter is a printer that prints results in legacy JSON format for backwards compatibility.
|
||||
type LegacyJSONPrinter struct{ mu sync.Mutex }
|
||||
type LegacyJSONPrinter struct{}
|
||||
|
||||
func (p *LegacyJSONPrinter) Print(ctx context.Context, r *detectors.ResultWithMetadata) error {
|
||||
var repo string
|
||||
@@ -53,10 +52,7 @@ func (p *LegacyJSONPrinter) Print(ctx context.Context, r *detectors.ResultWithMe
|
||||
if err != nil {
|
||||
return fmt.Errorf("could not marshal result: %w", err)
|
||||
}
|
||||
|
||||
p.mu.Lock()
|
||||
fmt.Println(string(out))
|
||||
p.mu.Unlock()
|
||||
return nil
|
||||
}
|
||||
|
||||
|
||||
+1
-4
@@ -5,7 +5,6 @@ import (
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
"github.com/fatih/color"
|
||||
"golang.org/x/text/cases"
|
||||
@@ -23,7 +22,7 @@ var (
|
||||
)
|
||||
|
||||
// PlainPrinter is a printer that prints results in plain text format.
|
||||
type PlainPrinter struct{ mu sync.Mutex }
|
||||
type PlainPrinter struct{}
|
||||
|
||||
func (p *PlainPrinter) Print(_ context.Context, r *detectors.ResultWithMetadata) error {
|
||||
out := outputFormat{
|
||||
@@ -40,8 +39,6 @@ func (p *PlainPrinter) Print(_ context.Context, r *detectors.ResultWithMetadata)
|
||||
}
|
||||
|
||||
printer := greenPrinter
|
||||
p.mu.Lock()
|
||||
defer p.mu.Unlock()
|
||||
|
||||
if out.Verified {
|
||||
yellowPrinter.Print("Found verified result 🐷🔑\n")
|
||||
|
||||
@@ -995,9 +995,6 @@ const (
|
||||
DetectorType_Dockerhub DetectorType = 921
|
||||
DetectorType_TrufflehogEnterprise DetectorType = 922
|
||||
DetectorType_EnvoyApiKey DetectorType = 923
|
||||
DetectorType_GitHubOauth2 DetectorType = 924
|
||||
DetectorType_Salesforce DetectorType = 925
|
||||
DetectorType_HuggingFace DetectorType = 926
|
||||
)
|
||||
|
||||
// Enum value maps for DetectorType.
|
||||
@@ -1923,9 +1920,6 @@ var (
|
||||
921: "Dockerhub",
|
||||
922: "TrufflehogEnterprise",
|
||||
923: "EnvoyApiKey",
|
||||
924: "GitHubOauth2",
|
||||
925: "Salesforce",
|
||||
926: "HuggingFace",
|
||||
}
|
||||
DetectorType_value = map[string]int32{
|
||||
"Alibaba": 0,
|
||||
@@ -2848,9 +2842,6 @@ var (
|
||||
"Dockerhub": 921,
|
||||
"TrufflehogEnterprise": 922,
|
||||
"EnvoyApiKey": 923,
|
||||
"GitHubOauth2": 924,
|
||||
"Salesforce": 925,
|
||||
"HuggingFace": 926,
|
||||
}
|
||||
)
|
||||
|
||||
@@ -3229,7 +3220,7 @@ var file_detectors_proto_rawDesc = []byte{
|
||||
0x44, 0x65, 0x63, 0x6f, 0x64, 0x65, 0x72, 0x54, 0x79, 0x70, 0x65, 0x12, 0x0b, 0x0a, 0x07, 0x55,
|
||||
0x4e, 0x4b, 0x4e, 0x4f, 0x57, 0x4e, 0x10, 0x00, 0x12, 0x09, 0x0a, 0x05, 0x50, 0x4c, 0x41, 0x49,
|
||||
0x4e, 0x10, 0x01, 0x12, 0x0a, 0x0a, 0x06, 0x42, 0x41, 0x53, 0x45, 0x36, 0x34, 0x10, 0x02, 0x12,
|
||||
0x09, 0x0a, 0x05, 0x55, 0x54, 0x46, 0x31, 0x36, 0x10, 0x03, 0x2a, 0xfc, 0x73, 0x0a, 0x0c, 0x44,
|
||||
0x09, 0x0a, 0x05, 0x55, 0x54, 0x46, 0x31, 0x36, 0x10, 0x03, 0x2a, 0xc6, 0x73, 0x0a, 0x0c, 0x44,
|
||||
0x65, 0x74, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x54, 0x79, 0x70, 0x65, 0x12, 0x0b, 0x0a, 0x07, 0x41,
|
||||
0x6c, 0x69, 0x62, 0x61, 0x62, 0x61, 0x10, 0x00, 0x12, 0x08, 0x0a, 0x04, 0x41, 0x4d, 0x51, 0x50,
|
||||
0x10, 0x01, 0x12, 0x07, 0x0a, 0x03, 0x41, 0x57, 0x53, 0x10, 0x02, 0x12, 0x09, 0x0a, 0x05, 0x41,
|
||||
@@ -4154,14 +4145,11 @@ var file_detectors_proto_rawDesc = []byte{
|
||||
0x68, 0x75, 0x62, 0x10, 0x99, 0x07, 0x12, 0x19, 0x0a, 0x14, 0x54, 0x72, 0x75, 0x66, 0x66, 0x6c,
|
||||
0x65, 0x68, 0x6f, 0x67, 0x45, 0x6e, 0x74, 0x65, 0x72, 0x70, 0x72, 0x69, 0x73, 0x65, 0x10, 0x9a,
|
||||
0x07, 0x12, 0x10, 0x0a, 0x0b, 0x45, 0x6e, 0x76, 0x6f, 0x79, 0x41, 0x70, 0x69, 0x4b, 0x65, 0x79,
|
||||
0x10, 0x9b, 0x07, 0x12, 0x11, 0x0a, 0x0c, 0x47, 0x69, 0x74, 0x48, 0x75, 0x62, 0x4f, 0x61, 0x75,
|
||||
0x74, 0x68, 0x32, 0x10, 0x9c, 0x07, 0x12, 0x0f, 0x0a, 0x0a, 0x53, 0x61, 0x6c, 0x65, 0x73, 0x66,
|
||||
0x6f, 0x72, 0x63, 0x65, 0x10, 0x9d, 0x07, 0x12, 0x10, 0x0a, 0x0b, 0x48, 0x75, 0x67, 0x67, 0x69,
|
||||
0x6e, 0x67, 0x46, 0x61, 0x63, 0x65, 0x10, 0x9e, 0x07, 0x42, 0x3d, 0x5a, 0x3b, 0x67, 0x69, 0x74,
|
||||
0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x74, 0x72, 0x75, 0x66, 0x66, 0x6c, 0x65, 0x73,
|
||||
0x65, 0x63, 0x75, 0x72, 0x69, 0x74, 0x79, 0x2f, 0x74, 0x72, 0x75, 0x66, 0x66, 0x6c, 0x65, 0x68,
|
||||
0x6f, 0x67, 0x2f, 0x76, 0x33, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x70, 0x62, 0x2f, 0x64, 0x65, 0x74,
|
||||
0x65, 0x63, 0x74, 0x6f, 0x72, 0x73, 0x70, 0x62, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
|
||||
0x10, 0x9b, 0x07, 0x42, 0x3d, 0x5a, 0x3b, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f,
|
||||
0x6d, 0x2f, 0x74, 0x72, 0x75, 0x66, 0x66, 0x6c, 0x65, 0x73, 0x65, 0x63, 0x75, 0x72, 0x69, 0x74,
|
||||
0x79, 0x2f, 0x74, 0x72, 0x75, 0x66, 0x66, 0x6c, 0x65, 0x68, 0x6f, 0x67, 0x2f, 0x76, 0x33, 0x2f,
|
||||
0x70, 0x6b, 0x67, 0x2f, 0x70, 0x62, 0x2f, 0x64, 0x65, 0x74, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x73,
|
||||
0x70, 0x62, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
|
||||
}
|
||||
|
||||
var (
|
||||
|
||||
+12
-120
@@ -5,8 +5,6 @@ import (
|
||||
"bytes"
|
||||
"errors"
|
||||
"io"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/context"
|
||||
)
|
||||
|
||||
const (
|
||||
@@ -27,7 +25,6 @@ func Chunker(originalChunk *Chunk) chan *Chunk {
|
||||
chunkChan <- originalChunk
|
||||
return
|
||||
}
|
||||
|
||||
r := bytes.NewReader(originalChunk.Data)
|
||||
reader := bufio.NewReaderSize(bufio.NewReader(r), ChunkSize)
|
||||
for {
|
||||
@@ -35,126 +32,21 @@ func Chunker(originalChunk *Chunk) chan *Chunk {
|
||||
chunk := *originalChunk
|
||||
chunkBytes = chunkBytes[:ChunkSize]
|
||||
n, err := reader.Read(chunkBytes)
|
||||
if n > 0 {
|
||||
peekData, _ := reader.Peek(TotalChunkSize - n)
|
||||
chunkBytes = append(chunkBytes[:n], peekData...)
|
||||
chunk.Data = chunkBytes
|
||||
chunkChan <- &chunk
|
||||
}
|
||||
if err != nil {
|
||||
if err != nil && !errors.Is(err, io.EOF) {
|
||||
break
|
||||
}
|
||||
if n == 0 {
|
||||
if errors.Is(err, io.EOF) {
|
||||
break
|
||||
}
|
||||
continue
|
||||
}
|
||||
peekData, _ := reader.Peek(PeekSize)
|
||||
copy(chunkBytes[n:], peekData)
|
||||
chunk.Data = chunkBytes[:n+len(peekData)]
|
||||
|
||||
chunkChan <- &chunk
|
||||
}
|
||||
}()
|
||||
return chunkChan
|
||||
}
|
||||
|
||||
type chunkReaderConfig struct {
|
||||
chunkSize int
|
||||
totalSize int
|
||||
peekSize int
|
||||
}
|
||||
|
||||
// ConfigOption is a function that configures a chunker.
|
||||
type ConfigOption func(*chunkReaderConfig)
|
||||
|
||||
// WithChunkSize sets the chunk size.
|
||||
func WithChunkSize(size int) ConfigOption {
|
||||
return func(c *chunkReaderConfig) {
|
||||
c.chunkSize = size
|
||||
}
|
||||
}
|
||||
|
||||
// WithPeekSize sets the peek size.
|
||||
func WithPeekSize(size int) ConfigOption {
|
||||
return func(c *chunkReaderConfig) {
|
||||
c.peekSize = size
|
||||
}
|
||||
}
|
||||
|
||||
// ChunkResult is the output unit of a ChunkReader,
|
||||
// it contains the data and error of a chunk.
|
||||
type ChunkResult struct {
|
||||
data []byte
|
||||
err error
|
||||
}
|
||||
|
||||
// Bytes for a ChunkResult.
|
||||
func (cr ChunkResult) Bytes() []byte {
|
||||
return cr.data
|
||||
}
|
||||
|
||||
// Error for a ChunkResult.
|
||||
func (cr ChunkResult) Error() error {
|
||||
return cr.err
|
||||
}
|
||||
|
||||
// ChunkReader reads chunks from a reader and returns a channel of chunks and a channel of errors.
|
||||
// The channel of chunks is closed when the reader is closed.
|
||||
// This should be used whenever a large amount of data is read from a reader.
|
||||
// Ex: reading attachments, archives, etc.
|
||||
type ChunkReader func(ctx context.Context, reader io.Reader) <-chan ChunkResult
|
||||
|
||||
// NewChunkReader returns a ChunkReader with the given options.
|
||||
func NewChunkReader(opts ...ConfigOption) ChunkReader {
|
||||
config := applyOptions(opts)
|
||||
return createReaderFn(config)
|
||||
}
|
||||
|
||||
func applyOptions(opts []ConfigOption) *chunkReaderConfig {
|
||||
// Set defaults.
|
||||
config := &chunkReaderConfig{
|
||||
chunkSize: ChunkSize, // default
|
||||
peekSize: PeekSize, // default
|
||||
}
|
||||
|
||||
for _, opt := range opts {
|
||||
opt(config)
|
||||
}
|
||||
|
||||
config.totalSize = config.chunkSize + config.peekSize
|
||||
|
||||
return config
|
||||
}
|
||||
|
||||
func createReaderFn(config *chunkReaderConfig) ChunkReader {
|
||||
return func(ctx context.Context, reader io.Reader) <-chan ChunkResult {
|
||||
return readInChunks(ctx, reader, config)
|
||||
}
|
||||
}
|
||||
|
||||
func readInChunks(ctx context.Context, reader io.Reader, config *chunkReaderConfig) <-chan ChunkResult {
|
||||
const channelSize = 1
|
||||
chunkReader := bufio.NewReaderSize(reader, config.chunkSize)
|
||||
chunkResultChan := make(chan ChunkResult, channelSize)
|
||||
|
||||
go func() {
|
||||
defer close(chunkResultChan)
|
||||
|
||||
for {
|
||||
chunkRes := ChunkResult{}
|
||||
chunkBytes := make([]byte, config.totalSize)
|
||||
chunkBytes = chunkBytes[:config.chunkSize]
|
||||
n, err := chunkReader.Read(chunkBytes)
|
||||
if n > 0 {
|
||||
peekData, _ := chunkReader.Peek(config.totalSize - n)
|
||||
chunkBytes = append(chunkBytes[:n], peekData...)
|
||||
chunkRes.data = chunkBytes
|
||||
}
|
||||
|
||||
// If there is an error other than EOF, or if we have read some bytes, send the chunk.
|
||||
if err != nil && !errors.Is(err, io.EOF) || n > 0 {
|
||||
if err != nil && !errors.Is(err, io.EOF) {
|
||||
ctx.Logger().Error(err, "error reading chunk")
|
||||
chunkRes.err = err
|
||||
}
|
||||
chunkResultChan <- chunkRes
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
}
|
||||
}()
|
||||
return chunkResultChan
|
||||
}
|
||||
|
||||
+47
-159
@@ -1,15 +1,13 @@
|
||||
package sources
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
"errors"
|
||||
"io"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
diskbufferreader "github.com/bill-rich/disk-buffer-reader"
|
||||
"github.com/stretchr/testify/assert"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/context"
|
||||
)
|
||||
|
||||
func TestChunker(t *testing.T) {
|
||||
@@ -20,39 +18,59 @@ func TestChunker(t *testing.T) {
|
||||
}
|
||||
defer reReader.Close()
|
||||
|
||||
baseChunks := make([]*Chunk, 0, 9)
|
||||
baseChunkCount := 0
|
||||
|
||||
// Count chunks from looping using chunk size.
|
||||
for {
|
||||
tmpChunk := make([]byte, ChunkSize)
|
||||
_, err := reReader.Read(tmpChunk)
|
||||
if err != nil {
|
||||
if errors.Is(err, io.EOF) {
|
||||
break
|
||||
}
|
||||
t.Fatal(err)
|
||||
}
|
||||
baseChunkCount++
|
||||
}
|
||||
_ = reReader.Reset()
|
||||
|
||||
// Get the first two chunks for comparing later.
|
||||
baseChunkOne := make([]byte, ChunkSize)
|
||||
baseChunkTwo := make([]byte, ChunkSize)
|
||||
|
||||
baseReader := bufio.NewReaderSize(reReader, ChunkSize)
|
||||
_, _ = baseReader.Read(baseChunkOne)
|
||||
peek, _ := baseReader.Peek(PeekSize)
|
||||
baseChunkOne = append(baseChunkOne, peek...)
|
||||
_, _ = baseReader.Read(baseChunkTwo)
|
||||
peek, _ = baseReader.Peek(PeekSize)
|
||||
baseChunkTwo = append(baseChunkTwo, peek...)
|
||||
|
||||
// Reset the reader to the beginning and use ChunkReader.
|
||||
_ = reReader.Reset()
|
||||
|
||||
testChunkCount := 0
|
||||
chunkData, _ := io.ReadAll(reReader)
|
||||
originalChunk := &Chunk{
|
||||
Data: chunkData,
|
||||
}
|
||||
|
||||
for chunk := range Chunker(originalChunk) {
|
||||
baseChunks = append(baseChunks, chunk)
|
||||
}
|
||||
|
||||
_ = reReader.Reset()
|
||||
|
||||
testChunks := make([]*Chunk, 0, 9)
|
||||
|
||||
testData, _ := io.ReadAll(reReader)
|
||||
testOriginalChunk := &Chunk{
|
||||
Data: testData,
|
||||
}
|
||||
|
||||
for chunk := range Chunker(testOriginalChunk) {
|
||||
testChunks = append(testChunks, chunk)
|
||||
}
|
||||
|
||||
if len(testChunks) != len(baseChunks) {
|
||||
t.Errorf("Wrong number of chunks received. Got %d, expected: %d.", len(testChunks), len(baseChunks))
|
||||
}
|
||||
|
||||
for i, baseChunk := range baseChunks {
|
||||
if !bytes.Equal(baseChunk.Data, testChunks[i].Data) {
|
||||
t.Errorf("Chunk %d did not match expected. Got: %d bytes, expected: %d bytes", i+1, len(testChunks[i].Data), len(baseChunk.Data))
|
||||
testChunkCount++
|
||||
switch testChunkCount {
|
||||
case 1:
|
||||
if !bytes.Equal(baseChunkOne, chunk.Data) {
|
||||
t.Errorf("First chunk did not match expected. Got: %d bytes, expected: %d bytes", len(chunk.Data), len(baseChunkOne))
|
||||
}
|
||||
case 2:
|
||||
if !bytes.Equal(baseChunkTwo, chunk.Data) {
|
||||
t.Errorf("Second chunk did not match expected. Got: %d bytes, expected: %d bytes", len(chunk.Data), len(baseChunkTwo))
|
||||
}
|
||||
}
|
||||
}
|
||||
if testChunkCount != baseChunkCount {
|
||||
t.Errorf("Wrong number of chunks received. Got %d, expected: %d.", testChunkCount, baseChunkCount)
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
func BenchmarkChunker(b *testing.B) {
|
||||
@@ -66,133 +84,3 @@ func BenchmarkChunker(b *testing.B) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewChunkedReader(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
input string
|
||||
chunkSize int
|
||||
peekSize int
|
||||
wantChunks []string
|
||||
wantErr bool
|
||||
}{
|
||||
{
|
||||
name: "Smaller data than default chunkSize and peekSize",
|
||||
input: "example input",
|
||||
chunkSize: ChunkSize,
|
||||
peekSize: PeekSize,
|
||||
wantChunks: []string{"example input"},
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "Reader with no data",
|
||||
input: "",
|
||||
chunkSize: 1024,
|
||||
peekSize: 512,
|
||||
wantChunks: []string{},
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "Smaller data than chunkSize and peekSize",
|
||||
input: "small data",
|
||||
chunkSize: 1024,
|
||||
peekSize: 512,
|
||||
wantChunks: []string{"small data"},
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "Equal to chunkSize",
|
||||
input: strings.Repeat("a", 1024),
|
||||
chunkSize: 1024,
|
||||
peekSize: 512,
|
||||
wantChunks: []string{strings.Repeat("a", 1024)},
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "Equal to chunkSize + peekSize",
|
||||
input: strings.Repeat("a", 1536),
|
||||
chunkSize: 1024,
|
||||
peekSize: 512,
|
||||
wantChunks: []string{strings.Repeat("a", 1536), strings.Repeat("a", 512)},
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "EOF during peeking",
|
||||
input: strings.Repeat("a", 1300),
|
||||
chunkSize: 1024,
|
||||
peekSize: 512,
|
||||
wantChunks: []string{strings.Repeat("a", 1300), strings.Repeat("a", 276)},
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "EOF during reading",
|
||||
input: strings.Repeat("a", 512),
|
||||
chunkSize: 1024,
|
||||
peekSize: 512,
|
||||
wantChunks: []string{strings.Repeat("a", 512)},
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "Equal to totalSize",
|
||||
input: strings.Repeat("a", 2048),
|
||||
chunkSize: 1024,
|
||||
peekSize: 1024,
|
||||
wantChunks: []string{strings.Repeat("a", 2048), strings.Repeat("a", 1024)},
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "Larger than totalSize",
|
||||
input: strings.Repeat("a", 4096),
|
||||
chunkSize: 1024,
|
||||
peekSize: 1024,
|
||||
wantChunks: []string{strings.Repeat("a", 2048), strings.Repeat("a", 2048), strings.Repeat("a", 2048), strings.Repeat("a", 1024)},
|
||||
wantErr: false,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
readerFunc := NewChunkReader(WithChunkSize(tt.chunkSize), WithPeekSize(tt.peekSize))
|
||||
reader := strings.NewReader(tt.input)
|
||||
ctx := context.Background()
|
||||
chunkResChan := readerFunc(ctx, reader)
|
||||
|
||||
var err error
|
||||
chunks := make([]string, 0)
|
||||
for data := range chunkResChan {
|
||||
chunks = append(chunks, string(data.Bytes()))
|
||||
err = data.Error()
|
||||
}
|
||||
|
||||
assert.Equal(t, tt.wantChunks, chunks, "Chunks do not match")
|
||||
if tt.wantErr {
|
||||
assert.Error(t, err, "Expected an error")
|
||||
} else {
|
||||
assert.NoError(t, err, "Unexpected error")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkChunkReader(b *testing.B) {
|
||||
var bigChunk = make([]byte, 1<<24) // 16MB
|
||||
|
||||
reader := bytes.NewReader(bigChunk)
|
||||
chunkReader := NewChunkReader(WithChunkSize(ChunkSize), WithPeekSize(PeekSize))
|
||||
|
||||
b.ReportAllocs()
|
||||
b.ResetTimer()
|
||||
|
||||
for i := 0; i < b.N; i++ {
|
||||
b.StartTimer()
|
||||
chunkResChan := chunkReader(context.Background(), reader)
|
||||
|
||||
// Drain the channel.
|
||||
for range chunkResChan {
|
||||
}
|
||||
|
||||
b.StopTimer()
|
||||
_, err := reader.Seek(0, 0)
|
||||
assert.Nil(b, err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -4,6 +4,7 @@ import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"sync/atomic"
|
||||
|
||||
@@ -218,7 +219,7 @@ func (s *Source) stepsForBuild(_ context.Context, proj project, bld build) ([]bu
|
||||
return bldRes.Steps, nil
|
||||
}
|
||||
|
||||
func (s *Source) chunkAction(ctx context.Context, proj project, bld build, act action, stepName string, chunksChan chan *sources.Chunk) error {
|
||||
func (s *Source) chunkAction(_ context.Context, proj project, bld build, act action, stepName string, chunksChan chan *sources.Chunk) error {
|
||||
req, err := http.NewRequest("GET", act.OutputURL, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -228,40 +229,35 @@ func (s *Source) chunkAction(ctx context.Context, proj project, bld build, act a
|
||||
return err
|
||||
}
|
||||
defer res.Body.Close()
|
||||
logOutput, err := io.ReadAll(res.Body)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
linkURL := fmt.Sprintf("https://app.circleci.com/pipelines/%s/%s/%s/%d", proj.VCS, proj.Username, proj.RepoName, bld.BuildNum)
|
||||
|
||||
chunkReader := sources.NewChunkReader()
|
||||
chunkResChan := chunkReader(ctx, res.Body)
|
||||
for data := range chunkResChan {
|
||||
chunk := &sources.Chunk{
|
||||
SourceType: s.Type(),
|
||||
SourceName: s.name,
|
||||
SourceID: s.SourceID(),
|
||||
Data: removeCircleSha1Line(data.Bytes()),
|
||||
SourceMetadata: &source_metadatapb.MetaData{
|
||||
Data: &source_metadatapb.MetaData_Circleci{
|
||||
Circleci: &source_metadatapb.CircleCI{
|
||||
VcsType: proj.VCS,
|
||||
Username: proj.Username,
|
||||
Repository: proj.RepoName,
|
||||
BuildNumber: int64(bld.BuildNum),
|
||||
BuildStep: stepName,
|
||||
Link: linkURL,
|
||||
},
|
||||
chunk := &sources.Chunk{
|
||||
SourceType: s.Type(),
|
||||
SourceName: s.name,
|
||||
SourceID: s.SourceID(),
|
||||
Data: removeCircleSha1Line(logOutput),
|
||||
SourceMetadata: &source_metadatapb.MetaData{
|
||||
Data: &source_metadatapb.MetaData_Circleci{
|
||||
Circleci: &source_metadatapb.CircleCI{
|
||||
VcsType: proj.VCS,
|
||||
Username: proj.Username,
|
||||
Repository: proj.RepoName,
|
||||
BuildNumber: int64(bld.BuildNum),
|
||||
BuildStep: stepName,
|
||||
Link: linkURL,
|
||||
},
|
||||
},
|
||||
Verify: s.verify,
|
||||
}
|
||||
chunk.Data = data.Bytes()
|
||||
if err := data.Error(); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := common.CancellableWrite(ctx, chunksChan, chunk); err != nil {
|
||||
return err
|
||||
}
|
||||
},
|
||||
Verify: s.verify,
|
||||
}
|
||||
|
||||
chunksChan <- chunk
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
|
||||
@@ -84,8 +84,6 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) err
|
||||
var img v1.Image
|
||||
var err error
|
||||
var base, tag string
|
||||
var hasDigest bool
|
||||
var imageName name.Reference
|
||||
|
||||
if strings.HasPrefix(image, "file://") {
|
||||
image = strings.TrimPrefix(image, "file://")
|
||||
@@ -95,18 +93,10 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) err
|
||||
return err
|
||||
}
|
||||
} else {
|
||||
base, tag, hasDigest = baseAndTagFromImage(image)
|
||||
|
||||
if hasDigest {
|
||||
imageName, err = name.NewDigest(image)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
} else {
|
||||
imageName, err = name.NewTag(image)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
base, tag = baseAndTagFromImage(image)
|
||||
imageName, err := name.NewTag(image)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
img, err = remote.Image(imageName, remoteOpts...)
|
||||
@@ -204,20 +194,10 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) err
|
||||
return nil
|
||||
}
|
||||
|
||||
func baseAndTagFromImage(image string) (base, tag string, hasDigest bool) {
|
||||
func baseAndTagFromImage(image string) (base, tag string) {
|
||||
regRepoDelimiter := "/"
|
||||
tagDelim := ":"
|
||||
digestDelim := "@"
|
||||
|
||||
// Split on digest first, if present.
|
||||
parts := strings.SplitN(image, digestDelim, 2)
|
||||
if len(parts) > 1 {
|
||||
base = parts[0]
|
||||
tag = parts[1]
|
||||
hasDigest = true
|
||||
return
|
||||
}
|
||||
parts = strings.Split(image, tagDelim)
|
||||
parts := strings.Split(image, tagDelim)
|
||||
// Verify that we aren't confusing a tag for a hostname w/ port for the purposes of weak validation.
|
||||
if len(parts) > 1 && !strings.Contains(parts[len(parts)-1], regRepoDelimiter) {
|
||||
base = strings.Join(parts[:len(parts)-1], tagDelim)
|
||||
|
||||
@@ -1,3 +1,5 @@
|
||||
//go:build integration
|
||||
|
||||
package docker
|
||||
|
||||
import (
|
||||
@@ -49,64 +51,3 @@ func TestDockerImageScan(t *testing.T) {
|
||||
|
||||
assert.Equal(t, 1, chunkCounter)
|
||||
}
|
||||
|
||||
func TestDockerImageScanWithDigest(t *testing.T) {
|
||||
dockerConn := &sourcespb.Docker{
|
||||
Credential: &sourcespb.Docker_Unauthenticated{
|
||||
Unauthenticated: &credentialspb.Unauthenticated{},
|
||||
},
|
||||
Images: []string{"trufflesecurity/secrets@sha256:864f6d41209462d8e37fc302ba1532656e265f7c361f11e29fed6ca1f4208e11"},
|
||||
}
|
||||
|
||||
conn := &anypb.Any{}
|
||||
err := conn.MarshalFrom(dockerConn)
|
||||
assert.NoError(t, err)
|
||||
|
||||
s := &Source{}
|
||||
err = s.Init(context.TODO(), "test source", 0, 0, false, conn, 1)
|
||||
assert.NoError(t, err)
|
||||
|
||||
var wg sync.WaitGroup
|
||||
chunksChan := make(chan *sources.Chunk, 1)
|
||||
chunkCounter := 0
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
for chunk := range chunksChan {
|
||||
assert.NotEmpty(t, chunk)
|
||||
chunkCounter++
|
||||
}
|
||||
}()
|
||||
|
||||
err = s.Chunks(context.TODO(), chunksChan)
|
||||
assert.NoError(t, err)
|
||||
|
||||
close(chunksChan)
|
||||
wg.Wait()
|
||||
|
||||
assert.Equal(t, 1, chunkCounter)
|
||||
}
|
||||
|
||||
func TestBaseAndTagFromImage(t *testing.T) {
|
||||
tests := []struct {
|
||||
image string
|
||||
wantBase string
|
||||
wantTag string
|
||||
wantDigest bool
|
||||
}{
|
||||
{"golang:1.16", "golang", "1.16", false},
|
||||
{"golang@sha256:abcdef", "golang", "sha256:abcdef", true},
|
||||
{"ghcr.io/golang:1.16", "ghcr.io/golang", "1.16", false},
|
||||
{"ghcr.io/golang:nightly", "ghcr.io/golang", "nightly", false},
|
||||
{"ghcr.io/golang", "ghcr.io/golang", "latest", false},
|
||||
{"ghcr.io/trufflesecurity/secrets", "ghcr.io/trufflesecurity/secrets", "latest", false},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
gotBase, gotTag, gotDigest := baseAndTagFromImage(tt.image)
|
||||
if gotBase != tt.wantBase || gotTag != tt.wantTag || gotDigest != tt.wantDigest {
|
||||
t.Errorf("baseAndTagFromImage(%q) = (%q, %q, %v), want (%q, %q, %v)",
|
||||
tt.image, gotBase, gotTag, gotDigest, tt.wantBase, tt.wantTag, tt.wantDigest)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
package filesystem
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/fs"
|
||||
@@ -22,6 +23,13 @@ import (
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
|
||||
)
|
||||
|
||||
const (
|
||||
// These buffer sizes are mainly driven by our largest credential size, which is GCP @ ~2.25KB.
|
||||
// Having a peek size larger than that ensures that we have complete credential coverage in our chunks.
|
||||
BufferSize = 10 * 1024 // 10KB
|
||||
PeekSize = 3 * 1024 // 3KB
|
||||
)
|
||||
|
||||
type Source struct {
|
||||
name string
|
||||
sourceId int64
|
||||
@@ -179,33 +187,34 @@ func (s *Source) scanFile(ctx context.Context, path string, chunksChan chan *sou
|
||||
}
|
||||
reReader.Stop()
|
||||
|
||||
chunkReader := sources.NewChunkReader()
|
||||
chunkResChan := chunkReader(ctx, reReader)
|
||||
for data := range chunkResChan {
|
||||
if err := data.Error(); err != nil {
|
||||
s.log.Error(err, "error reading chunk.")
|
||||
continue
|
||||
for {
|
||||
chunkBytes := make([]byte, BufferSize)
|
||||
reader := bufio.NewReaderSize(reReader, BufferSize)
|
||||
n, err := reader.Read(chunkBytes)
|
||||
if err != nil && !errors.Is(err, io.EOF) {
|
||||
break
|
||||
}
|
||||
|
||||
chunk := &sources.Chunk{
|
||||
SourceType: s.Type(),
|
||||
SourceName: s.name,
|
||||
SourceID: s.SourceID(),
|
||||
Data: data.Bytes(),
|
||||
SourceMetadata: &source_metadatapb.MetaData{
|
||||
Data: &source_metadatapb.MetaData_Filesystem{
|
||||
Filesystem: &source_metadatapb.Filesystem{
|
||||
File: sanitizer.UTF8(path),
|
||||
peekData, _ := reader.Peek(PeekSize)
|
||||
if n > 0 {
|
||||
chunksChan <- &sources.Chunk{
|
||||
SourceType: s.Type(),
|
||||
SourceName: s.name,
|
||||
SourceID: s.SourceID(),
|
||||
Data: append(chunkBytes[:n], peekData...),
|
||||
SourceMetadata: &source_metadatapb.MetaData{
|
||||
Data: &source_metadatapb.MetaData_Filesystem{
|
||||
Filesystem: &source_metadatapb.Filesystem{
|
||||
File: sanitizer.UTF8(path),
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
Verify: s.verify,
|
||||
Verify: s.verify,
|
||||
}
|
||||
}
|
||||
if err := common.CancellableWrite(ctx, chunksChan, chunk); err != nil {
|
||||
return err
|
||||
if errors.Is(err, io.EOF) {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
|
||||
@@ -1,14 +1,10 @@
|
||||
package filesystem
|
||||
|
||||
import (
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/go-logr/logr"
|
||||
"github.com/kylelemons/godebug/pretty"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"google.golang.org/protobuf/types/known/anypb"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/context"
|
||||
@@ -82,41 +78,3 @@ func TestSource_Scan(t *testing.T) {
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestScanFile(t *testing.T) {
|
||||
tmpfile, err := os.CreateTemp("", "example.txt")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer os.Remove(tmpfile.Name())
|
||||
|
||||
chunkSize := sources.ChunkSize
|
||||
secretPart1 := "SECRET"
|
||||
secretPart2 := "SPLIT"
|
||||
// Split the secret into two parts and pad the rest of the chunk with A's.
|
||||
data := strings.Repeat("A", chunkSize-len(secretPart1)) + secretPart1 + secretPart2 + strings.Repeat("A", chunkSize-len(secretPart2))
|
||||
|
||||
_, err = tmpfile.Write([]byte(data))
|
||||
assert.Nil(t, err)
|
||||
|
||||
err = tmpfile.Close()
|
||||
assert.Nil(t, err)
|
||||
|
||||
source := &Source{}
|
||||
chunksChan := make(chan *sources.Chunk, 2)
|
||||
|
||||
ctx := context.WithLogger(context.Background(), logr.Discard())
|
||||
go func() {
|
||||
defer close(chunksChan)
|
||||
err = source.scanFile(ctx, tmpfile.Name(), chunksChan)
|
||||
assert.Nil(t, err)
|
||||
}()
|
||||
|
||||
// Read from the channel and validate the secrets.
|
||||
foundSecret := ""
|
||||
for chunkCh := range chunksChan {
|
||||
foundSecret += string(chunkCh.Data)
|
||||
}
|
||||
|
||||
assert.Contains(t, foundSecret, secretPart1+secretPart2)
|
||||
}
|
||||
|
||||
+18
-28
@@ -4,6 +4,7 @@ import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/url"
|
||||
"os"
|
||||
"os/exec"
|
||||
@@ -25,7 +26,6 @@ import (
|
||||
"google.golang.org/protobuf/proto"
|
||||
"google.golang.org/protobuf/types/known/anypb"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/context"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/gitparse"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/handlers"
|
||||
@@ -115,9 +115,6 @@ func (s *Source) Init(aCtx context.Context, name string, jobId, sourceId int64,
|
||||
s.sourceId = sourceId
|
||||
s.jobId = jobId
|
||||
s.verify = verify
|
||||
if s.scanOptions == nil {
|
||||
s.scanOptions = &ScanOptions{}
|
||||
}
|
||||
|
||||
var conn sourcespb.Git
|
||||
if err := anypb.UnmarshalTo(connection, &conn, proto.UnmarshalOptions{}); err != nil {
|
||||
@@ -254,12 +251,12 @@ func (s *Source) scanDirs(ctx context.Context, chunksChan chan *sources.Chunk) e
|
||||
if len(gitDir) == 0 {
|
||||
continue
|
||||
}
|
||||
if !s.scanOptions.Bare && strings.HasSuffix(gitDir, "git") {
|
||||
if strings.HasSuffix(gitDir, "git") {
|
||||
// TODO: Figure out why we skip directories ending in "git".
|
||||
continue
|
||||
}
|
||||
// try paths instead of url
|
||||
repo, err := RepoFromPath(gitDir, s.scanOptions.Bare)
|
||||
repo, err := RepoFromPath(gitDir)
|
||||
if err != nil {
|
||||
ctx.Logger().Info("error scanning repository", "repo", gitDir, "error", err)
|
||||
continue
|
||||
@@ -281,11 +278,10 @@ func (s *Source) scanDirs(ctx context.Context, chunksChan chan *sources.Chunk) e
|
||||
return nil
|
||||
}
|
||||
|
||||
func RepoFromPath(path string, isBare bool) (*git.Repository, error) {
|
||||
options := &git.PlainOpenOptions{}
|
||||
if !isBare {
|
||||
options.DetectDotGit = true
|
||||
options.EnableDotGitCommonDir = true
|
||||
func RepoFromPath(path string) (*git.Repository, error) {
|
||||
options := &git.PlainOpenOptions{
|
||||
DetectDotGit: true,
|
||||
EnableDotGitCommonDir: true,
|
||||
}
|
||||
return git.PlainOpenWithOptions(path, options)
|
||||
}
|
||||
@@ -397,7 +393,7 @@ func (s *Git) ScanCommits(ctx context.Context, repo *git.Repository, path string
|
||||
return err
|
||||
}
|
||||
|
||||
commitChan, err := gitparse.NewParser().RepoPath(ctx, path, scanOptions.HeadHash, scanOptions.BaseHash == "", scanOptions.ExcludeGlobs, scanOptions.Bare)
|
||||
commitChan, err := gitparse.NewParser().RepoPath(ctx, path, scanOptions.HeadHash, scanOptions.BaseHash == "", scanOptions.ExcludeGlobs)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -625,10 +621,8 @@ func (s *Git) ScanRepo(ctx context.Context, repo *git.Repository, repoPath strin
|
||||
if err := s.ScanCommits(ctx, repo, repoPath, scanOptions, chunksChan); err != nil {
|
||||
return err
|
||||
}
|
||||
if !scanOptions.Bare {
|
||||
if err := s.ScanStaged(ctx, repo, repoPath, scanOptions, chunksChan); err != nil {
|
||||
ctx.Logger().V(1).Info("error scanning unstaged changes", "error", err)
|
||||
}
|
||||
if err := s.ScanStaged(ctx, repo, repoPath, scanOptions, chunksChan); err != nil {
|
||||
ctx.Logger().V(1).Info("error scanning unstaged changes", "error", err)
|
||||
}
|
||||
|
||||
// We're logging time, but the repoPath is usually a dynamically generated folder in /tmp
|
||||
@@ -640,7 +634,7 @@ func (s *Git) ScanRepo(ctx context.Context, repo *git.Repository, repoPath strin
|
||||
}
|
||||
|
||||
scanTime := time.Now().Unix() - start
|
||||
ctx.Logger().V(1).Info("scanning git repo complete", "repo", repoUrl, "path", repoPath, "time_seconds", scanTime, "commits_scanned", atomic.LoadUint64(&s.metrics.commitsScanned))
|
||||
ctx.Logger().V(1).Info("scanning git repo complete", "repo", repoUrl, "path", repoPath, "time_seconds", scanTime)
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -930,19 +924,15 @@ func handleBinary(ctx context.Context, repo *git.Repository, chunksChan chan *so
|
||||
}
|
||||
reader.Stop()
|
||||
|
||||
chunkReader := sources.NewChunkReader()
|
||||
chunkResChan := chunkReader(ctx, reader)
|
||||
for data := range chunkResChan {
|
||||
chunk := *chunkSkel
|
||||
chunk.Data = data.Bytes()
|
||||
if err := data.Error(); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := common.CancellableWrite(ctx, chunksChan, &chunk); err != nil {
|
||||
return err
|
||||
}
|
||||
chunkData, err := io.ReadAll(reader)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
chunk := *chunkSkel
|
||||
chunk.Data = chunkData
|
||||
chunksChan <- &chunk
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
|
||||
@@ -10,7 +10,6 @@ type ScanOptions struct {
|
||||
BaseHash string // When scanning a git.Log, this is the oldest/first commit.
|
||||
HeadHash string
|
||||
MaxDepth int64
|
||||
Bare bool
|
||||
ExcludeGlobs []string
|
||||
LogOptions *git.LogOptions
|
||||
}
|
||||
@@ -53,12 +52,6 @@ func ScanOptionLogOptions(logOptions *git.LogOptions) ScanOption {
|
||||
}
|
||||
}
|
||||
|
||||
func ScanOptionBare(bare bool) ScanOption {
|
||||
return func(scanOptions *ScanOptions) {
|
||||
scanOptions.Bare = bare
|
||||
}
|
||||
}
|
||||
|
||||
func NewScanOptions(options ...ScanOption) *ScanOptions {
|
||||
scanOptions := &ScanOptions{
|
||||
Filter: common.FilterEmpty(),
|
||||
|
||||
@@ -7,6 +7,7 @@ import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
)
|
||||
|
||||
@@ -102,14 +103,10 @@ type JobProgress struct {
|
||||
|
||||
// JobProgressMetrics tracks the metrics of a job.
|
||||
type JobProgressMetrics struct {
|
||||
StartTime time.Time
|
||||
EndTime time.Time
|
||||
// Total number of units found by the Source.
|
||||
TotalUnits uint64
|
||||
// Total number of units that have finished chunking.
|
||||
FinishedUnits uint64
|
||||
// Total number of chunks produced. This metric updates before the
|
||||
// chunk is sent on the output channel.
|
||||
StartTime time.Time
|
||||
EndTime time.Time
|
||||
TotalUnits uint64
|
||||
FinishedUnits uint64
|
||||
TotalChunks uint64
|
||||
Errors []error
|
||||
DoneEnumerating bool
|
||||
@@ -166,15 +163,11 @@ func (jp *JobProgress) Finish() {
|
||||
}
|
||||
func (jp *JobProgress) Done() <-chan struct{} { return jp.ctx.Done() }
|
||||
func (jp *JobProgress) ReportUnit(unit SourceUnit) {
|
||||
jp.metricsLock.Lock()
|
||||
jp.metrics.TotalUnits++
|
||||
jp.metricsLock.Unlock()
|
||||
atomic.AddUint64(&jp.metrics.TotalUnits, 1)
|
||||
jp.executeHooks(func(hook JobProgressHook) { hook.ReportUnit(jp.Ref(), unit) })
|
||||
}
|
||||
func (jp *JobProgress) ReportChunk(unit SourceUnit, chunk *Chunk) {
|
||||
jp.metricsLock.Lock()
|
||||
jp.metrics.TotalChunks++
|
||||
jp.metricsLock.Unlock()
|
||||
atomic.AddUint64(&jp.metrics.TotalChunks, 1)
|
||||
jp.executeHooks(func(hook JobProgressHook) { hook.ReportChunk(jp.Ref(), unit, chunk) })
|
||||
}
|
||||
func (jp *JobProgress) StartUnitChunking(unit SourceUnit, start time.Time) {
|
||||
@@ -183,9 +176,7 @@ func (jp *JobProgress) StartUnitChunking(unit SourceUnit, start time.Time) {
|
||||
}
|
||||
func (jp *JobProgress) EndUnitChunking(unit SourceUnit, end time.Time) {
|
||||
// TODO: Record time.
|
||||
jp.metricsLock.Lock()
|
||||
jp.metrics.FinishedUnits++
|
||||
jp.metricsLock.Unlock()
|
||||
atomic.AddUint64(&jp.metrics.FinishedUnits, 1)
|
||||
jp.executeHooks(func(hook JobProgressHook) { hook.EndUnitChunking(jp.Ref(), unit, end) })
|
||||
}
|
||||
func (jp *JobProgress) StartEnumerating(start time.Time) {
|
||||
@@ -205,12 +196,7 @@ func (jp *JobProgress) EndEnumerating(end time.Time) {
|
||||
func (jp *JobProgress) Snapshot() JobProgressMetrics {
|
||||
jp.metricsLock.Lock()
|
||||
defer jp.metricsLock.Unlock()
|
||||
|
||||
metrics := jp.metrics
|
||||
metrics.Errors = make([]error, len(metrics.Errors))
|
||||
copy(metrics.Errors, jp.metrics.Errors)
|
||||
|
||||
return metrics
|
||||
return jp.metrics
|
||||
}
|
||||
|
||||
// ReportError adds a non-nil error to the aggregate of errors
|
||||
|
||||
+11
-15
@@ -2,6 +2,7 @@ package s3
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
@@ -228,19 +229,19 @@ func (s *Source) pageChunker(ctx context.Context, client *s3.S3, chunksChan chan
|
||||
// ignore large files
|
||||
if *obj.Size > s.maxObjectSize {
|
||||
s.log.V(3).Info("Skipping %d byte file (over maxObjectSize limit)", "object", *obj.Key)
|
||||
continue
|
||||
return
|
||||
}
|
||||
|
||||
// file empty file
|
||||
if *obj.Size == 0 {
|
||||
s.log.V(5).Info("Skipping 0 byte file", "object", *obj.Key)
|
||||
continue
|
||||
return
|
||||
}
|
||||
|
||||
// skip incompatible extensions
|
||||
if common.SkipFile(*obj.Key) {
|
||||
s.log.V(5).Info("Skipping file with incompatible extension", "object", *obj.Key)
|
||||
continue
|
||||
return
|
||||
}
|
||||
|
||||
s.jobPool.Go(func() error {
|
||||
@@ -335,21 +336,16 @@ func (s *Source) pageChunker(ctx context.Context, client *s3.S3, chunksChan chan
|
||||
reader.Stop()
|
||||
|
||||
chunk := *chunkSkel
|
||||
chunkReader := sources.NewChunkReader()
|
||||
chunkResChan := chunkReader(ctx, reader)
|
||||
for data := range chunkResChan {
|
||||
chunk.Data = data.Bytes()
|
||||
if err := data.Error(); err != nil {
|
||||
s.log.Error(err, "error reading chunk.")
|
||||
continue
|
||||
}
|
||||
if err := common.CancellableWrite(ctx, chunksChan, &chunk); err != nil {
|
||||
return err
|
||||
}
|
||||
chunkData, err := io.ReadAll(reader)
|
||||
if err != nil {
|
||||
s.log.Error(err, "Could not read file data.")
|
||||
return nil
|
||||
}
|
||||
|
||||
atomic.AddUint64(objectCount, 1)
|
||||
s.log.V(5).Info("S3 object scanned.", "object_count", objectCount, "page_number", pageNumber)
|
||||
chunk.Data = chunkData
|
||||
chunksChan <- &chunk
|
||||
|
||||
nErr, ok = errorCount.Load(prefix)
|
||||
if !ok {
|
||||
nErr = 0
|
||||
|
||||
@@ -21,18 +21,11 @@ type handle int64
|
||||
// initialized Source.
|
||||
type SourceInitFunc func(ctx context.Context, sourceID int64, jobID int64) (Source, error)
|
||||
|
||||
// sourceInfo is an aggregate struct to store source information provided on
|
||||
// initialization.
|
||||
type sourceInfo struct {
|
||||
initFunc SourceInitFunc
|
||||
name string
|
||||
}
|
||||
|
||||
type SourceManager struct {
|
||||
api apiClient
|
||||
hooks []JobProgressHook
|
||||
// Map of handle to source initializer.
|
||||
handles map[handle]sourceInfo
|
||||
handles map[handle]SourceInitFunc
|
||||
handlesLock sync.Mutex
|
||||
// Pool limiting the amount of concurrent sources running.
|
||||
pool errgroup.Group
|
||||
@@ -65,8 +58,8 @@ func WithReportHook(hook JobProgressHook) func(*SourceManager) {
|
||||
}
|
||||
}
|
||||
|
||||
// WithConcurrentSources limits the concurrent number of sources a manager can run.
|
||||
func WithConcurrentSources(concurrency int) func(*SourceManager) {
|
||||
// WithConcurrency limits the concurrent number of sources a manager can run.
|
||||
func WithConcurrency(concurrency int) func(*SourceManager) {
|
||||
return func(mgr *SourceManager) { mgr.pool.SetLimit(concurrency) }
|
||||
}
|
||||
|
||||
@@ -92,7 +85,7 @@ func NewManager(opts ...func(*SourceManager)) *SourceManager {
|
||||
mgr := SourceManager{
|
||||
// Default to the headless API. Can be overwritten by the WithAPI option.
|
||||
api: &headlessAPI{},
|
||||
handles: make(map[handle]sourceInfo),
|
||||
handles: make(map[handle]SourceInitFunc),
|
||||
outputChunks: make(chan *Chunk),
|
||||
}
|
||||
for _, opt := range opts {
|
||||
@@ -117,22 +110,19 @@ func (s *SourceManager) Enroll(ctx context.Context, name string, kind sourcespb.
|
||||
// TODO: smartly handle this?
|
||||
return 0, fmt.Errorf("handle ID '%d' already in use", handleID)
|
||||
}
|
||||
s.handles[handleID] = sourceInfo{
|
||||
initFunc: f,
|
||||
name: name,
|
||||
}
|
||||
s.handles[handleID] = f
|
||||
return handleID, nil
|
||||
}
|
||||
|
||||
// Run blocks until a resource is available to run the source, then
|
||||
// synchronously runs it. The first fatal error, if any, will be returned.
|
||||
func (s *SourceManager) Run(ctx context.Context, handle handle) (JobProgressRef, error) {
|
||||
progress, err := s.asyncRun(ctx, handle)
|
||||
report, err := s.asyncRun(ctx, handle)
|
||||
if err != nil {
|
||||
return progress, err
|
||||
return report, err
|
||||
}
|
||||
<-progress.Done()
|
||||
return progress, progress.Snapshot().FatalError()
|
||||
<-report.Done()
|
||||
return report, report.Snapshot().FatalError()
|
||||
}
|
||||
|
||||
// ScheduleRun blocks until a resource is available to run the source, then
|
||||
@@ -143,30 +133,26 @@ func (s *SourceManager) ScheduleRun(ctx context.Context, handle handle) (JobProg
|
||||
}
|
||||
|
||||
// asyncRun is a helper method to asynchronously run the Source. It calls out
|
||||
// to the API to get a job ID for this run, creates a JobProgress object, then
|
||||
// waits for an available goroutine to asynchronously run it.
|
||||
// to the API to get a job ID for this run, creates a report, then waits for an
|
||||
// available goroutine to asynchronously run it.
|
||||
func (s *SourceManager) asyncRun(ctx context.Context, handle handle) (JobProgressRef, error) {
|
||||
// Do preflight checks before waiting on the pool.
|
||||
if err := s.preflightChecks(ctx, handle); err != nil {
|
||||
return JobProgressRef{}, err
|
||||
}
|
||||
// Get a Job ID.
|
||||
ctx = context.WithValue(ctx, "source_id", int64(handle))
|
||||
jobID, err := s.api.GetJobID(ctx, int64(handle))
|
||||
if err != nil {
|
||||
return JobProgressRef{SourceID: int64(handle)}, err
|
||||
}
|
||||
// Create a JobProgress object for tracking progress.
|
||||
progress := NewJobProgress(int64(handle), jobID, WithHooks(s.hooks...))
|
||||
// Start a report for this job.
|
||||
report := NewJobProgress(int64(handle), jobID, WithHooks(s.hooks...))
|
||||
s.pool.Go(func() error {
|
||||
ctx := context.WithValues(ctx,
|
||||
"job_id", jobID,
|
||||
"source_manager_worker_id", common.RandomID(5),
|
||||
)
|
||||
defer common.Recover(ctx)
|
||||
return s.run(ctx, handle, jobID, progress)
|
||||
_ = s.run(ctx, handle, jobID, report)
|
||||
return nil
|
||||
})
|
||||
return progress.Ref(), nil
|
||||
return report.Ref(), nil
|
||||
}
|
||||
|
||||
// Chunks returns the read only channel of all the chunks produced by all of
|
||||
@@ -179,16 +165,17 @@ func (s *SourceManager) Chunks() <-chan *Chunk {
|
||||
// returned by Chunks(). The manager should not be reused after calling this
|
||||
// method. This current implementation is not thread safe and should only be
|
||||
// called by one thread.
|
||||
func (s *SourceManager) Wait() error {
|
||||
func (s *SourceManager) Wait() {
|
||||
// Check if the manager has been Waited.
|
||||
if s.done {
|
||||
return s.pool.Wait()
|
||||
return
|
||||
}
|
||||
defer close(s.outputChunks)
|
||||
defer func() { s.done = true }()
|
||||
|
||||
// Return the first error returned by run.
|
||||
return s.pool.Wait()
|
||||
// We are only using the errgroup for limiting concurrency.
|
||||
// TODO: Maybe switch to using a semaphore.Weighted.
|
||||
_ = s.pool.Wait()
|
||||
}
|
||||
|
||||
// preflightChecks is a helper method to check the Manager or the context isn't
|
||||
@@ -199,7 +186,7 @@ func (s *SourceManager) preflightChecks(ctx context.Context, handle handle) erro
|
||||
return fmt.Errorf("manager is done")
|
||||
}
|
||||
// Check the handle is valid.
|
||||
if _, ok := s.getSourceInfo(handle); !ok {
|
||||
if _, ok := s.getInitFunc(handle); !ok {
|
||||
return fmt.Errorf("unrecognized handle")
|
||||
}
|
||||
return ctx.Err()
|
||||
@@ -214,22 +201,18 @@ func (s *SourceManager) run(ctx context.Context, handle handle, jobID int64, rep
|
||||
defer func() { report.End(time.Now()) }()
|
||||
|
||||
// Initialize the source.
|
||||
sourceInfo, ok := s.getSourceInfo(handle)
|
||||
initFunc, ok := s.getInitFunc(handle)
|
||||
if !ok {
|
||||
// Shouldn't happen due to preflight checks.
|
||||
err := fmt.Errorf("unrecognized handle")
|
||||
report.ReportError(Fatal{err})
|
||||
return Fatal{err}
|
||||
}
|
||||
source, err := sourceInfo.initFunc(ctx, int64(handle), jobID)
|
||||
source, err := initFunc(ctx, int64(handle), jobID)
|
||||
if err != nil {
|
||||
report.ReportError(Fatal{err})
|
||||
return Fatal{err}
|
||||
}
|
||||
ctx = context.WithValues(ctx,
|
||||
"source_type", source.Type().String(),
|
||||
"source_name", sourceInfo.name,
|
||||
)
|
||||
// Check for the preferred method of tracking source units.
|
||||
if enumChunker, ok := source.(SourceUnitEnumChunker); ok && s.useSourceUnits {
|
||||
return s.runWithUnits(ctx, handle, enumChunker, report)
|
||||
@@ -241,7 +224,7 @@ func (s *SourceManager) run(ctx context.Context, handle handle, jobID int64, rep
|
||||
// job reporting.
|
||||
func (s *SourceManager) runWithoutUnits(ctx context.Context, handle handle, source Source, report *JobProgress) error {
|
||||
// Introspect on the chunks we get from the Chunks method.
|
||||
ch := make(chan *Chunk, 1)
|
||||
ch := make(chan *Chunk)
|
||||
var wg sync.WaitGroup
|
||||
// Consume chunks and export chunks.
|
||||
wg.Add(1)
|
||||
@@ -249,7 +232,7 @@ func (s *SourceManager) runWithoutUnits(ctx context.Context, handle handle, sour
|
||||
defer wg.Done()
|
||||
for chunk := range ch {
|
||||
report.ReportChunk(nil, chunk)
|
||||
s.outputChunks <- chunk
|
||||
_ = common.CancellableWrite(ctx, s.outputChunks, chunk)
|
||||
}
|
||||
}()
|
||||
// Don't return from this function until the goroutine has finished
|
||||
@@ -270,7 +253,7 @@ func (s *SourceManager) runWithoutUnits(ctx context.Context, handle handle, sour
|
||||
// scanned and any errors encountered.
|
||||
func (s *SourceManager) runWithUnits(ctx context.Context, handle handle, source SourceUnitEnumChunker, report *JobProgress) error {
|
||||
unitReporter := &mgrUnitReporter{
|
||||
unitCh: make(chan SourceUnit, 1),
|
||||
unitCh: make(chan SourceUnit),
|
||||
report: report,
|
||||
}
|
||||
// Create a function that will save the first error encountered (if
|
||||
@@ -288,7 +271,6 @@ func (s *SourceManager) runWithUnits(ctx context.Context, handle handle, source
|
||||
report.StartEnumerating(time.Now())
|
||||
defer func() { report.EndEnumerating(time.Now()) }()
|
||||
defer close(unitReporter.unitCh)
|
||||
ctx.Logger().V(2).Info("enumerating source")
|
||||
if err := source.Enumerate(ctx, unitReporter); err != nil {
|
||||
report.ReportError(Fatal{err})
|
||||
catchFirstFatal(Fatal{err})
|
||||
@@ -305,7 +287,7 @@ func (s *SourceManager) runWithUnits(ctx context.Context, handle handle, source
|
||||
unit := unit
|
||||
chunkReporter := &mgrChunkReporter{
|
||||
unit: unit,
|
||||
chunkCh: make(chan *Chunk, 1),
|
||||
chunkCh: make(chan *Chunk),
|
||||
report: report,
|
||||
}
|
||||
// Consume units and produce chunks.
|
||||
@@ -313,8 +295,6 @@ func (s *SourceManager) runWithUnits(ctx context.Context, handle handle, source
|
||||
report.StartUnitChunking(unit, time.Now())
|
||||
// TODO: Catch panics and add to report.
|
||||
defer close(chunkReporter.chunkCh)
|
||||
ctx := context.WithValue(ctx, "unit", unit.SourceUnitID())
|
||||
ctx.Logger().V(3).Info("chunking unit")
|
||||
if err := source.ChunkUnit(ctx, unit, chunkReporter); err != nil {
|
||||
report.ReportError(Fatal{err})
|
||||
catchFirstFatal(Fatal{err})
|
||||
@@ -327,7 +307,8 @@ func (s *SourceManager) runWithUnits(ctx context.Context, handle handle, source
|
||||
defer wg.Done()
|
||||
defer func() { report.EndUnitChunking(unit, time.Now()) }()
|
||||
for chunk := range chunkReporter.chunkCh {
|
||||
s.outputChunks <- chunk
|
||||
report.ReportChunk(chunkReporter.unit, chunk)
|
||||
_ = common.CancellableWrite(ctx, s.outputChunks, chunk)
|
||||
}
|
||||
}()
|
||||
}
|
||||
@@ -340,9 +321,9 @@ func (s *SourceManager) runWithUnits(ctx context.Context, handle handle, source
|
||||
}
|
||||
}
|
||||
|
||||
// getSourceInfo is a helper method for safe concurrent access to the
|
||||
// getInitFunc is a helper method for safe concurrent access to the
|
||||
// map[handle]SourceInitFunc map.
|
||||
func (s *SourceManager) getSourceInfo(handle handle) (sourceInfo, bool) {
|
||||
func (s *SourceManager) getInitFunc(handle handle) (SourceInitFunc, bool) {
|
||||
s.handlesLock.Lock()
|
||||
defer s.handlesLock.Unlock()
|
||||
f, ok := s.handles[handle]
|
||||
@@ -371,7 +352,6 @@ type mgrUnitReporter struct {
|
||||
}
|
||||
|
||||
func (s *mgrUnitReporter) UnitOk(ctx context.Context, unit SourceUnit) error {
|
||||
s.report.ReportUnit(unit)
|
||||
return common.CancellableWrite(ctx, s.unitCh, unit)
|
||||
}
|
||||
|
||||
@@ -388,7 +368,6 @@ type mgrChunkReporter struct {
|
||||
}
|
||||
|
||||
func (s *mgrChunkReporter) ChunkOk(ctx context.Context, chunk Chunk) error {
|
||||
s.report.ReportChunk(s.unit, &chunk)
|
||||
return common.CancellableWrite(ctx, s.chunkCh, &chunk)
|
||||
}
|
||||
|
||||
|
||||
@@ -4,8 +4,6 @@ import (
|
||||
"fmt"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/context"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
|
||||
"google.golang.org/protobuf/types/known/anypb"
|
||||
@@ -41,14 +39,10 @@ type counterChunker struct {
|
||||
count int
|
||||
}
|
||||
|
||||
func (c *counterChunker) Chunks(ctx context.Context, ch chan *Chunk) error {
|
||||
func (c *counterChunker) Chunks(_ context.Context, ch chan *Chunk) error {
|
||||
for i := 0; i < c.count; i++ {
|
||||
select {
|
||||
case ch <- &Chunk{Data: []byte{c.chunkCounter}}:
|
||||
c.chunkCounter++
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
}
|
||||
ch <- &Chunk{Data: []byte{c.chunkCounter}}
|
||||
c.chunkCounter++
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -104,50 +98,71 @@ func tryRead(ch <-chan *Chunk) (*Chunk, error) {
|
||||
func TestSourceManagerRun(t *testing.T) {
|
||||
mgr := NewManager(WithBufferedOutput(8))
|
||||
handle, err := enrollDummy(mgr, &counterChunker{count: 1})
|
||||
assert.NoError(t, err)
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error enrolling source: %v", err)
|
||||
}
|
||||
for i := 0; i < 3; i++ {
|
||||
_, err = mgr.Run(context.Background(), handle)
|
||||
assert.NoError(t, err)
|
||||
if _, err := mgr.Run(context.Background(), handle); err != nil {
|
||||
t.Fatalf("unexpected error running source: %v", err)
|
||||
}
|
||||
chunk, err := tryRead(mgr.Chunks())
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, []byte{byte(i)}, chunk.Data)
|
||||
if err != nil {
|
||||
t.Fatalf("reading chunk failed: %v", err)
|
||||
}
|
||||
if chunk.Data[0] != byte(i) {
|
||||
t.Fatalf("unexpected chunk value, wanted %v, got: %v", chunk.Data[0], i)
|
||||
}
|
||||
|
||||
// The Chunks channel should be empty now.
|
||||
_, err = tryRead(mgr.Chunks())
|
||||
assert.Error(t, err)
|
||||
if chunk, err := tryRead(mgr.Chunks()); err == nil {
|
||||
t.Fatalf("unexpected chunk found: %+v", chunk)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestSourceManagerWait(t *testing.T) {
|
||||
mgr := NewManager()
|
||||
handle, err := enrollDummy(mgr, &counterChunker{count: 1})
|
||||
assert.NoError(t, err)
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error enrolling source: %v", err)
|
||||
}
|
||||
// Asynchronously run the source.
|
||||
_, err = mgr.ScheduleRun(context.Background(), handle)
|
||||
assert.NoError(t, err)
|
||||
if _, err := mgr.ScheduleRun(context.Background(), handle); err != nil {
|
||||
t.Fatalf("unexpected error scheduling run: %v", err)
|
||||
}
|
||||
// Read the 1 chunk we're expecting so Waiting completes.
|
||||
<-mgr.Chunks()
|
||||
// Wait for all resources to complete.
|
||||
assert.NoError(t, mgr.Wait())
|
||||
mgr.Wait()
|
||||
// Enroll and run should return an error now.
|
||||
_, err = enrollDummy(mgr, &counterChunker{count: 1})
|
||||
assert.Error(t, err)
|
||||
_, err = mgr.ScheduleRun(context.Background(), handle)
|
||||
assert.Error(t, err)
|
||||
if _, err := enrollDummy(mgr, &counterChunker{count: 1}); err == nil {
|
||||
t.Fatalf("expected enroll to fail")
|
||||
}
|
||||
if _, err := mgr.ScheduleRun(context.Background(), handle); err == nil {
|
||||
t.Fatalf("expected scheduling run to fail")
|
||||
}
|
||||
}
|
||||
|
||||
func TestSourceManagerError(t *testing.T) {
|
||||
mgr := NewManager()
|
||||
handle, err := enrollDummy(mgr, errorChunker{fmt.Errorf("oops")})
|
||||
assert.NoError(t, err)
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error enrolling source: %v", err)
|
||||
}
|
||||
// A synchronous run should fail.
|
||||
_, err = mgr.Run(context.Background(), handle)
|
||||
assert.Error(t, err)
|
||||
if _, err := mgr.Run(context.Background(), handle); err == nil {
|
||||
t.Fatalf("expected run to fail")
|
||||
}
|
||||
// Scheduling a run should not fail, but the error should surface in
|
||||
// Wait().
|
||||
ref, err := mgr.ScheduleRun(context.Background(), handle)
|
||||
assert.NoError(t, err)
|
||||
assert.Error(t, mgr.Wait())
|
||||
assert.Error(t, ref.Snapshot().FatalError())
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error scheduling run: %v", err)
|
||||
}
|
||||
<-ref.Done()
|
||||
if err := ref.Snapshot().FatalError(); err == nil {
|
||||
t.Fatalf("expected wait to fail")
|
||||
}
|
||||
}
|
||||
|
||||
func TestSourceManagerReport(t *testing.T) {
|
||||
@@ -158,90 +173,19 @@ func TestSourceManagerReport(t *testing.T) {
|
||||
} {
|
||||
mgr := NewManager(opts...)
|
||||
handle, err := enrollDummy(mgr, &counterChunker{count: 4})
|
||||
assert.NoError(t, err)
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error enrolling source: %v", err)
|
||||
}
|
||||
// Synchronously run the source.
|
||||
ref, err := mgr.Run(context.Background(), handle)
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, 0, len(ref.Snapshot().Errors))
|
||||
assert.Equal(t, uint64(4), ref.Snapshot().TotalChunks)
|
||||
}
|
||||
}
|
||||
|
||||
type unitChunk struct {
|
||||
unit string
|
||||
output string
|
||||
err string
|
||||
}
|
||||
|
||||
type unitChunker struct{ steps []unitChunk }
|
||||
|
||||
func (c *unitChunker) Chunks(ctx context.Context, ch chan *Chunk) error {
|
||||
for _, step := range c.steps {
|
||||
if step.err != "" {
|
||||
continue
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error running source: %v", err)
|
||||
}
|
||||
if err := common.CancellableWrite(ctx, ch, &Chunk{Data: []byte(step.output)}); err != nil {
|
||||
return err
|
||||
if errs := ref.Snapshot().Errors; len(errs) > 0 {
|
||||
t.Fatalf("unexpected errors in report: %v", errs)
|
||||
}
|
||||
if count := ref.Snapshot().TotalChunks; count != 4 {
|
||||
t.Fatalf("expected report to have 4 chunks, got: %d", count)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
func (c *unitChunker) Enumerate(ctx context.Context, rep UnitReporter) error {
|
||||
for _, step := range c.steps {
|
||||
if err := rep.UnitOk(ctx, CommonSourceUnit{step.unit}); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
func (c *unitChunker) ChunkUnit(ctx context.Context, unit SourceUnit, rep ChunkReporter) error {
|
||||
for _, step := range c.steps {
|
||||
if unit.SourceUnitID() != step.unit {
|
||||
continue
|
||||
}
|
||||
if step.err != "" {
|
||||
if err := rep.ChunkErr(ctx, fmt.Errorf(step.err)); err != nil {
|
||||
return err
|
||||
}
|
||||
continue
|
||||
}
|
||||
if err := rep.ChunkOk(ctx, Chunk{Data: []byte(step.output)}); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func TestSourceManagerNonFatalError(t *testing.T) {
|
||||
input := []unitChunk{
|
||||
{unit: "one", output: "bar"},
|
||||
{unit: "two", err: "oh no"},
|
||||
{unit: "three", err: "not again"},
|
||||
}
|
||||
mgr := NewManager(WithBufferedOutput(8), WithSourceUnits())
|
||||
handle, err := enrollDummy(mgr, &unitChunker{input})
|
||||
assert.NoError(t, err)
|
||||
ref, err := mgr.Run(context.Background(), handle)
|
||||
assert.NoError(t, err)
|
||||
report := ref.Snapshot()
|
||||
assert.Equal(t, len(input), int(report.TotalUnits))
|
||||
assert.Equal(t, len(input), int(report.FinishedUnits))
|
||||
assert.Equal(t, 1, int(report.TotalChunks))
|
||||
assert.Equal(t, 2, len(report.Errors))
|
||||
assert.True(t, report.DoneEnumerating)
|
||||
}
|
||||
|
||||
func TestSourceManagerContextCancelled(t *testing.T) {
|
||||
mgr := NewManager(WithBufferedOutput(8))
|
||||
handle, err := enrollDummy(mgr, &counterChunker{count: 100})
|
||||
assert.NoError(t, err)
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
ref, err := mgr.ScheduleRun(ctx, handle)
|
||||
assert.NoError(t, err)
|
||||
|
||||
cancel()
|
||||
<-ref.Done()
|
||||
report := ref.Snapshot()
|
||||
assert.Error(t, report.FatalError())
|
||||
}
|
||||
|
||||
@@ -141,8 +141,6 @@ type GitConfig struct {
|
||||
BaseRef string
|
||||
// MaxDepth is the maximum depth to scan the source.
|
||||
MaxDepth int
|
||||
// Bare is an indicator to handle bare repositories properly.
|
||||
Bare bool
|
||||
// Filter is the filter to use to scan the source.
|
||||
Filter *common.Filter
|
||||
// ExcludeGlobs is a list of globs to exclude from the scan.
|
||||
|
||||
@@ -1,24 +0,0 @@
|
||||
package common
|
||||
|
||||
import (
|
||||
"github.com/aymanbagabas/go-osc52"
|
||||
zone "github.com/lrstanley/bubblezone"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/tui/keymap"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/tui/styles"
|
||||
)
|
||||
|
||||
// Common is a struct all components should embed.
|
||||
type Common struct {
|
||||
Copy *osc52.Output
|
||||
Styles *styles.Styles
|
||||
KeyMap *keymap.KeyMap
|
||||
Width int
|
||||
Height int
|
||||
Zone *zone.Manager
|
||||
}
|
||||
|
||||
// SetSize sets the width and height of the common struct.
|
||||
func (c *Common) SetSize(width, height int) {
|
||||
c.Width = width
|
||||
c.Height = height
|
||||
}
|
||||
@@ -1,13 +0,0 @@
|
||||
package common
|
||||
|
||||
import (
|
||||
"github.com/charmbracelet/bubbles/help"
|
||||
tea "github.com/charmbracelet/bubbletea"
|
||||
)
|
||||
|
||||
// Component represents a Bubble Tea model that implements a SetSize function.
|
||||
type Component interface {
|
||||
tea.Model
|
||||
help.KeyMap
|
||||
SetSize(width, height int)
|
||||
}
|
||||
@@ -1,13 +0,0 @@
|
||||
package common
|
||||
|
||||
import tea "github.com/charmbracelet/bubbletea"
|
||||
|
||||
// ErrorMsg is a Bubble Tea message that represents an error.
|
||||
type ErrorMsg error
|
||||
|
||||
// ErrorCmd returns an ErrorMsg from error.
|
||||
func ErrorCmd(err error) tea.Cmd {
|
||||
return func() tea.Msg {
|
||||
return ErrorMsg(err)
|
||||
}
|
||||
}
|
||||
@@ -1,27 +0,0 @@
|
||||
package common
|
||||
|
||||
import (
|
||||
"github.com/charmbracelet/glamour"
|
||||
gansi "github.com/charmbracelet/glamour/ansi"
|
||||
)
|
||||
|
||||
func strptr(s string) *string {
|
||||
return &s
|
||||
}
|
||||
|
||||
// StyleConfig returns the default Glamour style configuration.
|
||||
func StyleConfig() gansi.StyleConfig {
|
||||
noColor := strptr("")
|
||||
s := glamour.DarkStyleConfig
|
||||
s.H1.BackgroundColor = noColor
|
||||
s.H1.Prefix = "# "
|
||||
s.H1.Suffix = ""
|
||||
s.H1.Color = strptr("39")
|
||||
s.Document.StylePrimitive.Color = noColor
|
||||
s.CodeBlock.Chroma.Text.Color = noColor
|
||||
s.CodeBlock.Chroma.Name.Color = noColor
|
||||
// This fixes an issue with the default style config. For example
|
||||
// highlighting empty spaces with red in Dockerfile type.
|
||||
s.CodeBlock.Chroma.Error.BackgroundColor = noColor
|
||||
return s
|
||||
}
|
||||
@@ -1,27 +0,0 @@
|
||||
package common
|
||||
|
||||
import (
|
||||
"strings"
|
||||
|
||||
"github.com/muesli/reflow/truncate"
|
||||
)
|
||||
|
||||
// TruncateString is a convenient wrapper around truncate.TruncateString.
|
||||
func TruncateString(s string, max int) string {
|
||||
if max < 0 {
|
||||
max = 0
|
||||
}
|
||||
return truncate.StringWithTail(s, uint(max), "…")
|
||||
}
|
||||
|
||||
func SummarizeSource(keys []string, inputs map[string]string, labels map[string]string) string {
|
||||
summary := strings.Builder{}
|
||||
for _, key := range keys {
|
||||
if inputs[key] != "" {
|
||||
summary.WriteString("\t" + labels[key] + ": " + inputs[key] + "\n")
|
||||
}
|
||||
}
|
||||
|
||||
summary.WriteString("\n")
|
||||
return summary.String()
|
||||
}
|
||||
@@ -1,96 +0,0 @@
|
||||
package footer
|
||||
|
||||
import (
|
||||
"github.com/charmbracelet/bubbles/help"
|
||||
"github.com/charmbracelet/bubbles/key"
|
||||
tea "github.com/charmbracelet/bubbletea"
|
||||
"github.com/charmbracelet/lipgloss"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/tui/common"
|
||||
)
|
||||
|
||||
// ToggleFooterMsg is a message sent to show/hide the footer.
|
||||
type ToggleFooterMsg struct{}
|
||||
|
||||
// Footer is a Bubble Tea model that displays help and other info.
|
||||
type Footer struct {
|
||||
common common.Common
|
||||
help help.Model
|
||||
keymap help.KeyMap
|
||||
}
|
||||
|
||||
// New creates a new Footer.
|
||||
func New(c common.Common, keymap help.KeyMap) *Footer {
|
||||
h := help.New()
|
||||
h.Styles.ShortKey = c.Styles.HelpKey
|
||||
h.Styles.ShortDesc = c.Styles.HelpValue
|
||||
h.Styles.FullKey = c.Styles.HelpKey
|
||||
h.Styles.FullDesc = c.Styles.HelpValue
|
||||
f := &Footer{
|
||||
common: c,
|
||||
help: h,
|
||||
keymap: keymap,
|
||||
}
|
||||
f.SetSize(c.Width, c.Height)
|
||||
return f
|
||||
}
|
||||
|
||||
// SetSize implements common.Component.
|
||||
func (f *Footer) SetSize(width, height int) {
|
||||
f.common.SetSize(width, height)
|
||||
f.help.Width = width -
|
||||
f.common.Styles.Footer.GetHorizontalFrameSize()
|
||||
}
|
||||
|
||||
// Init implements tea.Model.
|
||||
func (f *Footer) Init() tea.Cmd {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Update implements tea.Model.
|
||||
func (f *Footer) Update(msg tea.Msg) (tea.Model, tea.Cmd) {
|
||||
return f, nil
|
||||
}
|
||||
|
||||
// View implements tea.Model.
|
||||
func (f *Footer) View() string {
|
||||
if f.keymap == nil {
|
||||
return ""
|
||||
}
|
||||
s := f.common.Styles.Footer.Copy().
|
||||
Width(f.common.Width)
|
||||
helpView := f.help.View(f.keymap)
|
||||
return f.common.Zone.Mark(
|
||||
"footer",
|
||||
s.Render(helpView),
|
||||
)
|
||||
}
|
||||
|
||||
// ShortHelp returns the short help key bindings.
|
||||
func (f *Footer) ShortHelp() []key.Binding {
|
||||
return f.keymap.ShortHelp()
|
||||
}
|
||||
|
||||
// FullHelp returns the full help key bindings.
|
||||
func (f *Footer) FullHelp() [][]key.Binding {
|
||||
return f.keymap.FullHelp()
|
||||
}
|
||||
|
||||
// ShowAll returns whether the full help is shown.
|
||||
func (f *Footer) ShowAll() bool {
|
||||
return f.help.ShowAll
|
||||
}
|
||||
|
||||
// SetShowAll sets whether the full help is shown.
|
||||
func (f *Footer) SetShowAll(show bool) {
|
||||
f.help.ShowAll = show
|
||||
}
|
||||
|
||||
// Height returns the height of the footer.
|
||||
func (f *Footer) Height() int {
|
||||
return lipgloss.Height(f.View())
|
||||
}
|
||||
|
||||
// ToggleFooterCmd sends a ToggleFooterMsg to show/hide the help footer.
|
||||
func ToggleFooterCmd() tea.Msg {
|
||||
return ToggleFooterMsg{}
|
||||
}
|
||||
@@ -1,38 +0,0 @@
|
||||
package formfield
|
||||
|
||||
import (
|
||||
"strings"
|
||||
|
||||
tea "github.com/charmbracelet/bubbletea"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/tui/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/tui/styles"
|
||||
)
|
||||
|
||||
type FormField struct {
|
||||
Label string
|
||||
Required bool
|
||||
Help string
|
||||
Component tea.Model
|
||||
}
|
||||
|
||||
func NewFormField(common common.Common) *FormField {
|
||||
return &FormField{}
|
||||
}
|
||||
|
||||
func (field *FormField) ViewLabel() string {
|
||||
var label strings.Builder
|
||||
if field.Required {
|
||||
label.WriteString(styles.BoldTextStyle.Render(field.Label) + "*\n")
|
||||
} else {
|
||||
label.WriteString(styles.BoldTextStyle.Render(field.Label) + "\n")
|
||||
}
|
||||
|
||||
return label.String()
|
||||
}
|
||||
|
||||
func (field *FormField) ViewHelp() string {
|
||||
var help strings.Builder
|
||||
help.WriteString(styles.HintTextStyle.Render(field.Help) + "\n")
|
||||
|
||||
return help.String()
|
||||
}
|
||||
@@ -1,42 +0,0 @@
|
||||
package header
|
||||
|
||||
import (
|
||||
"strings"
|
||||
|
||||
tea "github.com/charmbracelet/bubbletea"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/tui/common"
|
||||
)
|
||||
|
||||
// Header represents a header component.
|
||||
type Header struct {
|
||||
common common.Common
|
||||
text string
|
||||
}
|
||||
|
||||
// New creates a new header component.
|
||||
func New(c common.Common, text string) *Header {
|
||||
return &Header{
|
||||
common: c,
|
||||
text: text,
|
||||
}
|
||||
}
|
||||
|
||||
// SetSize implements common.Component.
|
||||
func (h *Header) SetSize(width, height int) {
|
||||
h.common.SetSize(width, height)
|
||||
}
|
||||
|
||||
// Init implements tea.Model.
|
||||
func (h *Header) Init() tea.Cmd {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Update implements tea.Model.
|
||||
func (h *Header) Update(msg tea.Msg) (tea.Model, tea.Cmd) {
|
||||
return h, nil
|
||||
}
|
||||
|
||||
// View implements tea.Model.
|
||||
func (h *Header) View() string {
|
||||
return h.common.Styles.ServerName.Render(strings.TrimSpace(h.text))
|
||||
}
|
||||
@@ -1,237 +0,0 @@
|
||||
package selector
|
||||
|
||||
import (
|
||||
"github.com/charmbracelet/bubbles/key"
|
||||
"github.com/charmbracelet/bubbles/list"
|
||||
tea "github.com/charmbracelet/bubbletea"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/tui/common"
|
||||
)
|
||||
|
||||
// Selector is a list of items that can be selected.
|
||||
type Selector struct {
|
||||
list.Model
|
||||
common common.Common
|
||||
active int
|
||||
filterState list.FilterState
|
||||
}
|
||||
|
||||
// IdentifiableItem is an item that can be identified by a string. Implements
|
||||
// list.DefaultItem.
|
||||
type IdentifiableItem interface {
|
||||
list.DefaultItem
|
||||
ID() string
|
||||
}
|
||||
|
||||
// ItemDelegate is a wrapper around list.ItemDelegate.
|
||||
type ItemDelegate interface {
|
||||
list.ItemDelegate
|
||||
}
|
||||
|
||||
// SelectMsg is a message that is sent when an item is selected.
|
||||
type SelectMsg struct{ IdentifiableItem }
|
||||
|
||||
// ActiveMsg is a message that is sent when an item is active but not selected.
|
||||
type ActiveMsg struct{ IdentifiableItem }
|
||||
|
||||
// New creates a new selector.
|
||||
func New(common common.Common, items []IdentifiableItem, delegate ItemDelegate) *Selector {
|
||||
itms := make([]list.Item, len(items))
|
||||
for i, item := range items {
|
||||
itms[i] = item
|
||||
}
|
||||
l := list.New(itms, delegate, common.Width, common.Height)
|
||||
s := &Selector{
|
||||
Model: l,
|
||||
common: common,
|
||||
}
|
||||
s.SetSize(common.Width, common.Height)
|
||||
return s
|
||||
}
|
||||
|
||||
// PerPage returns the number of items per page.
|
||||
func (s *Selector) PerPage() int {
|
||||
return s.Model.Paginator.PerPage
|
||||
}
|
||||
|
||||
// SetPage sets the current page.
|
||||
func (s *Selector) SetPage(page int) {
|
||||
s.Model.Paginator.Page = page
|
||||
}
|
||||
|
||||
// Page returns the current page.
|
||||
func (s *Selector) Page() int {
|
||||
return s.Model.Paginator.Page
|
||||
}
|
||||
|
||||
// TotalPages returns the total number of pages.
|
||||
func (s *Selector) TotalPages() int {
|
||||
return s.Model.Paginator.TotalPages
|
||||
}
|
||||
|
||||
// Select selects the item at the given index.
|
||||
func (s *Selector) Select(index int) {
|
||||
s.Model.Select(index)
|
||||
}
|
||||
|
||||
// SetShowTitle sets the show title flag.
|
||||
func (s *Selector) SetShowTitle(show bool) {
|
||||
s.Model.SetShowTitle(show)
|
||||
}
|
||||
|
||||
// SetShowHelp sets the show help flag.
|
||||
func (s *Selector) SetShowHelp(show bool) {
|
||||
s.Model.SetShowHelp(show)
|
||||
}
|
||||
|
||||
// SetShowStatusBar sets the show status bar flag.
|
||||
func (s *Selector) SetShowStatusBar(show bool) {
|
||||
s.Model.SetShowStatusBar(show)
|
||||
}
|
||||
|
||||
// DisableQuitKeybindings disables the quit keybindings.
|
||||
func (s *Selector) DisableQuitKeybindings() {
|
||||
s.Model.DisableQuitKeybindings()
|
||||
}
|
||||
|
||||
// SetShowFilter sets the show filter flag.
|
||||
func (s *Selector) SetShowFilter(show bool) {
|
||||
s.Model.SetShowFilter(show)
|
||||
}
|
||||
|
||||
// SetShowPagination sets the show pagination flag.
|
||||
func (s *Selector) SetShowPagination(show bool) {
|
||||
s.Model.SetShowPagination(show)
|
||||
}
|
||||
|
||||
// SetFilteringEnabled sets the filtering enabled flag.
|
||||
func (s *Selector) SetFilteringEnabled(enabled bool) {
|
||||
s.Model.SetFilteringEnabled(enabled)
|
||||
}
|
||||
|
||||
// SetSize implements common.Component.
|
||||
func (s *Selector) SetSize(width, height int) {
|
||||
s.common.SetSize(width, height)
|
||||
s.Model.SetSize(width, height)
|
||||
}
|
||||
|
||||
// SetItems sets the items in the selector.
|
||||
func (s *Selector) SetItems(items []IdentifiableItem) tea.Cmd {
|
||||
its := make([]list.Item, len(items))
|
||||
for i, item := range items {
|
||||
its[i] = item
|
||||
}
|
||||
return s.Model.SetItems(its)
|
||||
}
|
||||
|
||||
// Index returns the index of the selected item.
|
||||
func (s *Selector) Index() int {
|
||||
return s.Model.Index()
|
||||
}
|
||||
|
||||
// Init implements tea.Model.
|
||||
func (s *Selector) Init() tea.Cmd {
|
||||
return s.activeCmd
|
||||
}
|
||||
|
||||
// Update implements tea.Model.
|
||||
func (s *Selector) Update(msg tea.Msg) (tea.Model, tea.Cmd) {
|
||||
cmds := make([]tea.Cmd, 0)
|
||||
switch msg := msg.(type) {
|
||||
case tea.MouseMsg:
|
||||
switch msg.Type {
|
||||
case tea.MouseWheelUp:
|
||||
s.Model.CursorUp()
|
||||
case tea.MouseWheelDown:
|
||||
s.Model.CursorDown()
|
||||
case tea.MouseLeft:
|
||||
curIdx := s.Model.Index()
|
||||
for i, item := range s.Model.Items() {
|
||||
item, _ := item.(IdentifiableItem)
|
||||
// Check each item to see if it's in bounds.
|
||||
if item != nil && s.common.Zone.Get(item.ID()).InBounds(msg) {
|
||||
if i == curIdx {
|
||||
cmds = append(cmds, s.selectCmd)
|
||||
} else {
|
||||
s.Model.Select(i)
|
||||
}
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
case tea.KeyMsg:
|
||||
filterState := s.Model.FilterState()
|
||||
switch {
|
||||
case key.Matches(msg, s.common.KeyMap.Help):
|
||||
if filterState == list.Filtering {
|
||||
return s, tea.Batch(cmds...)
|
||||
}
|
||||
case key.Matches(msg, s.common.KeyMap.Select):
|
||||
if filterState != list.Filtering {
|
||||
cmds = append(cmds, s.selectCmd)
|
||||
}
|
||||
}
|
||||
case list.FilterMatchesMsg:
|
||||
cmds = append(cmds, s.activeFilterCmd)
|
||||
}
|
||||
m, cmd := s.Model.Update(msg)
|
||||
s.Model = m
|
||||
if cmd != nil {
|
||||
cmds = append(cmds, cmd)
|
||||
}
|
||||
// Track filter state and update active item when filter state changes.
|
||||
filterState := s.Model.FilterState()
|
||||
if s.filterState != filterState {
|
||||
cmds = append(cmds, s.activeFilterCmd)
|
||||
}
|
||||
s.filterState = filterState
|
||||
// Send ActiveMsg when index change.
|
||||
if s.active != s.Model.Index() {
|
||||
cmds = append(cmds, s.activeCmd)
|
||||
}
|
||||
s.active = s.Model.Index()
|
||||
return s, tea.Batch(cmds...)
|
||||
}
|
||||
|
||||
// View implements tea.Model.
|
||||
func (s *Selector) View() string {
|
||||
return s.Model.View()
|
||||
}
|
||||
|
||||
// SelectItem is a command that selects the currently active item.
|
||||
func (s *Selector) SelectItem() tea.Msg {
|
||||
return s.selectCmd()
|
||||
}
|
||||
|
||||
func (s *Selector) selectCmd() tea.Msg {
|
||||
item := s.Model.SelectedItem()
|
||||
i, ok := item.(IdentifiableItem)
|
||||
if !ok {
|
||||
return SelectMsg{}
|
||||
}
|
||||
return SelectMsg{i}
|
||||
}
|
||||
|
||||
func (s *Selector) activeCmd() tea.Msg {
|
||||
item := s.Model.SelectedItem()
|
||||
i, ok := item.(IdentifiableItem)
|
||||
if !ok {
|
||||
return ActiveMsg{}
|
||||
}
|
||||
return ActiveMsg{i}
|
||||
}
|
||||
|
||||
func (s *Selector) activeFilterCmd() tea.Msg {
|
||||
// Here we use VisibleItems because when list.FilterMatchesMsg is sent,
|
||||
// VisibleItems is the only way to get the list of filtered items. The list
|
||||
// bubble should export something like list.FilterMatchesMsg.Items().
|
||||
items := s.Model.VisibleItems()
|
||||
if len(items) == 0 {
|
||||
return nil
|
||||
}
|
||||
item := items[0]
|
||||
i, ok := item.(IdentifiableItem)
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
return ActiveMsg{i}
|
||||
}
|
||||
@@ -1,88 +0,0 @@
|
||||
package statusbar
|
||||
|
||||
import (
|
||||
tea "github.com/charmbracelet/bubbletea"
|
||||
"github.com/charmbracelet/lipgloss"
|
||||
"github.com/muesli/reflow/truncate"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/tui/common"
|
||||
)
|
||||
|
||||
// StatusBarMsg is a message sent to the status bar.
|
||||
type StatusBarMsg struct {
|
||||
Key string
|
||||
Value string
|
||||
Info string
|
||||
Branch string
|
||||
}
|
||||
|
||||
// StatusBar is a status bar model.
|
||||
type StatusBar struct {
|
||||
common common.Common
|
||||
msg StatusBarMsg
|
||||
}
|
||||
|
||||
// Model is an interface that supports setting the status bar information.
|
||||
type Model interface {
|
||||
StatusBarValue() string
|
||||
StatusBarInfo() string
|
||||
}
|
||||
|
||||
// New creates a new status bar component.
|
||||
func New(c common.Common) *StatusBar {
|
||||
s := &StatusBar{
|
||||
common: c,
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
// SetSize implements common.Component.
|
||||
func (s *StatusBar) SetSize(width, height int) {
|
||||
s.common.Width = width
|
||||
s.common.Height = height
|
||||
}
|
||||
|
||||
// Init implements tea.Model.
|
||||
func (s *StatusBar) Init() tea.Cmd {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Update implements tea.Model.
|
||||
func (s *StatusBar) Update(msg tea.Msg) (tea.Model, tea.Cmd) {
|
||||
switch msg := msg.(type) {
|
||||
case StatusBarMsg:
|
||||
s.msg = msg
|
||||
}
|
||||
return s, nil
|
||||
}
|
||||
|
||||
// View implements tea.Model.
|
||||
func (s *StatusBar) View() string {
|
||||
st := s.common.Styles
|
||||
w := lipgloss.Width
|
||||
help := s.common.Zone.Mark(
|
||||
"repo-help",
|
||||
st.StatusBarHelp.Render("? Help"),
|
||||
)
|
||||
key := st.StatusBarKey.Render(s.msg.Key)
|
||||
info := ""
|
||||
if s.msg.Info != "" {
|
||||
info = st.StatusBarInfo.Render(s.msg.Info)
|
||||
}
|
||||
branch := st.StatusBarBranch.Render(s.msg.Branch)
|
||||
maxWidth := s.common.Width - w(key) - w(info) - w(branch) - w(help)
|
||||
v := truncate.StringWithTail(s.msg.Value, uint(maxWidth-st.StatusBarValue.GetHorizontalFrameSize()), "…")
|
||||
value := st.StatusBarValue.
|
||||
Width(maxWidth).
|
||||
Render(v)
|
||||
|
||||
return lipgloss.NewStyle().MaxWidth(s.common.Width).
|
||||
Render(
|
||||
lipgloss.JoinHorizontal(lipgloss.Top,
|
||||
key,
|
||||
value,
|
||||
info,
|
||||
branch,
|
||||
help,
|
||||
),
|
||||
)
|
||||
}
|
||||
@@ -1,122 +0,0 @@
|
||||
package tabs
|
||||
|
||||
import (
|
||||
"strings"
|
||||
|
||||
tea "github.com/charmbracelet/bubbletea"
|
||||
"github.com/charmbracelet/lipgloss"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/tui/common"
|
||||
)
|
||||
|
||||
// SelectTabMsg is a message that contains the index of the tab to select.
|
||||
type SelectTabMsg int
|
||||
|
||||
// ActiveTabMsg is a message that contains the index of the current active tab.
|
||||
type ActiveTabMsg int
|
||||
|
||||
// Tabs is bubbletea component that displays a list of tabs.
|
||||
type Tabs struct {
|
||||
common common.Common
|
||||
tabs []string
|
||||
activeTab int
|
||||
TabSeparator lipgloss.Style
|
||||
TabInactive lipgloss.Style
|
||||
TabActive lipgloss.Style
|
||||
TabDot lipgloss.Style
|
||||
UseDot bool
|
||||
}
|
||||
|
||||
// New creates a new Tabs component.
|
||||
func New(c common.Common, tabs []string) *Tabs {
|
||||
r := &Tabs{
|
||||
common: c,
|
||||
tabs: tabs,
|
||||
activeTab: 0,
|
||||
TabSeparator: c.Styles.TabSeparator,
|
||||
TabInactive: c.Styles.TabInactive,
|
||||
TabActive: c.Styles.TabActive,
|
||||
}
|
||||
return r
|
||||
}
|
||||
|
||||
// SetSize implements common.Component.
|
||||
func (t *Tabs) SetSize(width, height int) {
|
||||
t.common.SetSize(width, height)
|
||||
}
|
||||
|
||||
// Init implements tea.Model.
|
||||
func (t *Tabs) Init() tea.Cmd {
|
||||
t.activeTab = 0
|
||||
return nil
|
||||
}
|
||||
|
||||
// Update implements tea.Model.
|
||||
func (t *Tabs) Update(msg tea.Msg) (tea.Model, tea.Cmd) {
|
||||
cmds := make([]tea.Cmd, 0)
|
||||
switch msg := msg.(type) {
|
||||
case tea.KeyMsg:
|
||||
switch msg.String() {
|
||||
case "tab":
|
||||
t.activeTab = (t.activeTab + 1) % len(t.tabs)
|
||||
cmds = append(cmds, t.activeTabCmd)
|
||||
case "shift+tab":
|
||||
t.activeTab = (t.activeTab - 1 + len(t.tabs)) % len(t.tabs)
|
||||
cmds = append(cmds, t.activeTabCmd)
|
||||
}
|
||||
case tea.MouseMsg:
|
||||
if msg.Type == tea.MouseLeft {
|
||||
for i, tab := range t.tabs {
|
||||
if t.common.Zone.Get(tab).InBounds(msg) {
|
||||
t.activeTab = i
|
||||
cmds = append(cmds, t.activeTabCmd)
|
||||
}
|
||||
}
|
||||
}
|
||||
case SelectTabMsg:
|
||||
tab := int(msg)
|
||||
if tab >= 0 && tab < len(t.tabs) {
|
||||
t.activeTab = int(msg)
|
||||
}
|
||||
}
|
||||
return t, tea.Batch(cmds...)
|
||||
}
|
||||
|
||||
// View implements tea.Model.
|
||||
func (t *Tabs) View() string {
|
||||
s := strings.Builder{}
|
||||
sep := t.TabSeparator
|
||||
for i, tab := range t.tabs {
|
||||
style := t.TabInactive.Copy()
|
||||
prefix := " "
|
||||
if i == t.activeTab {
|
||||
style = t.TabActive.Copy()
|
||||
prefix = t.TabDot.Render("• ")
|
||||
}
|
||||
if t.UseDot {
|
||||
s.WriteString(prefix)
|
||||
}
|
||||
s.WriteString(
|
||||
t.common.Zone.Mark(
|
||||
tab,
|
||||
style.Render(tab),
|
||||
),
|
||||
)
|
||||
if i != len(t.tabs)-1 {
|
||||
s.WriteString(sep.String())
|
||||
}
|
||||
}
|
||||
return lipgloss.NewStyle().
|
||||
MaxWidth(t.common.Width).
|
||||
Render(s.String())
|
||||
}
|
||||
|
||||
func (t *Tabs) activeTabCmd() tea.Msg {
|
||||
return ActiveTabMsg(t.activeTab)
|
||||
}
|
||||
|
||||
// SelectTabCmd is a bubbletea command that selects the tab at the given index.
|
||||
func SelectTabCmd(tab int) tea.Cmd {
|
||||
return func() tea.Msg {
|
||||
return SelectTabMsg(tab)
|
||||
}
|
||||
}
|
||||
@@ -1,51 +0,0 @@
|
||||
package textinput
|
||||
|
||||
import (
|
||||
"github.com/charmbracelet/bubbles/textinput"
|
||||
tea "github.com/charmbracelet/bubbletea"
|
||||
)
|
||||
|
||||
type (
|
||||
errMsg error
|
||||
)
|
||||
|
||||
type TextInput struct {
|
||||
textInput textinput.Model
|
||||
err error
|
||||
}
|
||||
|
||||
func New(placeholder string) TextInput {
|
||||
ti := textinput.New()
|
||||
ti.Placeholder = placeholder
|
||||
ti.Focus()
|
||||
ti.CharLimit = 156
|
||||
ti.Width = 60
|
||||
|
||||
return TextInput{
|
||||
textInput: ti,
|
||||
err: nil,
|
||||
}
|
||||
}
|
||||
|
||||
func (m TextInput) Init() tea.Cmd {
|
||||
return textinput.Blink
|
||||
}
|
||||
|
||||
func (m TextInput) Update(msg tea.Msg) (tea.Model, tea.Cmd) {
|
||||
var cmd tea.Cmd
|
||||
|
||||
switch msg := msg.(type) {
|
||||
|
||||
// We handle errors just like any other message
|
||||
case errMsg:
|
||||
m.err = msg
|
||||
return m, nil
|
||||
}
|
||||
|
||||
m.textInput, cmd = m.textInput.Update(msg)
|
||||
return m, cmd
|
||||
}
|
||||
|
||||
func (m TextInput) View() string {
|
||||
return m.textInput.View()
|
||||
}
|
||||
@@ -1,229 +0,0 @@
|
||||
package textinputs
|
||||
|
||||
// from https://github.com/charmbracelet/bubbletea/blob/master/examples/textinputs/main.go
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"github.com/charmbracelet/bubbles/textinput"
|
||||
tea "github.com/charmbracelet/bubbletea"
|
||||
"github.com/charmbracelet/lipgloss"
|
||||
)
|
||||
|
||||
var (
|
||||
focusedStyle = lipgloss.NewStyle().Foreground(lipgloss.Color("205"))
|
||||
blurredStyle = lipgloss.NewStyle().Foreground(lipgloss.Color("240"))
|
||||
noStyle = lipgloss.NewStyle()
|
||||
helpStyle = blurredStyle.Copy()
|
||||
// cursorStyle = focusedStyle.Copy()
|
||||
// cursorModeHelpStyle = lipgloss.NewStyle().Foreground(lipgloss.Color("244"))
|
||||
|
||||
focusedButton = focusedStyle.Copy().Render("[ Next ]")
|
||||
blurredButton = fmt.Sprintf("[ %s ]", blurredStyle.Render("Next"))
|
||||
focusedSkipButton = lipgloss.NewStyle().Foreground(lipgloss.Color("205")).Render("[ Run with defaults ]")
|
||||
blurredSkipButton = fmt.Sprintf("[ %s ]", lipgloss.NewStyle().Foreground(lipgloss.Color("240")).Render("Run with defaults"))
|
||||
)
|
||||
|
||||
// SelectNextMsg used for emitting events when the 'Next' button is selected.
|
||||
type SelectNextMsg int
|
||||
|
||||
// SelectSkipMsg used for emitting events when the 'Skip' button is selected.
|
||||
type SelectSkipMsg int
|
||||
|
||||
type Model struct {
|
||||
focusIndex int
|
||||
inputs []textinput.Model
|
||||
configs []InputConfig
|
||||
// cursorMode cursor.Mode
|
||||
skipButton bool
|
||||
}
|
||||
|
||||
type InputConfig struct {
|
||||
Label string
|
||||
Key string
|
||||
Help string
|
||||
Required bool
|
||||
Placeholder string
|
||||
}
|
||||
|
||||
func (m Model) GetInputs() map[string]string {
|
||||
inputs := make(map[string]string)
|
||||
|
||||
for i, input := range m.inputs {
|
||||
inputs[m.configs[i].Key] = input.Value()
|
||||
}
|
||||
|
||||
return inputs
|
||||
}
|
||||
|
||||
func (m Model) GetLabels() map[string]string {
|
||||
labels := make(map[string]string)
|
||||
|
||||
for _, config := range m.configs {
|
||||
labels[config.Key] = config.Label
|
||||
}
|
||||
|
||||
return labels
|
||||
}
|
||||
|
||||
func New(config []InputConfig) Model {
|
||||
m := Model{
|
||||
inputs: make([]textinput.Model, len(config)),
|
||||
}
|
||||
|
||||
for i, conf := range config {
|
||||
input := textinput.New()
|
||||
input.Placeholder = conf.Placeholder
|
||||
|
||||
if i == 0 {
|
||||
input.Focus()
|
||||
input.TextStyle = focusedStyle
|
||||
input.PromptStyle = focusedStyle
|
||||
}
|
||||
|
||||
m.inputs[i] = input
|
||||
}
|
||||
|
||||
m.configs = config
|
||||
return m
|
||||
}
|
||||
|
||||
func (m Model) Init() tea.Cmd {
|
||||
return textinput.Blink
|
||||
}
|
||||
|
||||
func (m Model) Update(msg tea.Msg) (tea.Model, tea.Cmd) {
|
||||
switch msg := msg.(type) {
|
||||
case tea.KeyMsg:
|
||||
switch msg.String() {
|
||||
// Set focus to next input
|
||||
case "enter", "up", "down":
|
||||
s := msg.String()
|
||||
|
||||
// Did the user press enter while the submit or skip button was focused?
|
||||
// If so, emit the appropriate command.
|
||||
if s == "enter" && m.focusIndex == len(m.inputs) {
|
||||
return m, func() tea.Msg { return SelectNextMsg(0) }
|
||||
} else if s == "enter" && m.focusIndex == -1 {
|
||||
return m, func() tea.Msg { return SelectSkipMsg(0) }
|
||||
}
|
||||
|
||||
// Cycle indexes
|
||||
if s == "up" {
|
||||
m.focusIndex--
|
||||
} else {
|
||||
m.focusIndex++
|
||||
}
|
||||
|
||||
if m.focusIndex > len(m.inputs) {
|
||||
m.focusIndex = 0
|
||||
} else if !m.skipButton && m.focusIndex < 0 {
|
||||
m.focusIndex = len(m.inputs)
|
||||
} else if m.skipButton && m.focusIndex < -1 {
|
||||
m.focusIndex = len(m.inputs)
|
||||
}
|
||||
|
||||
cmds := make([]tea.Cmd, len(m.inputs))
|
||||
for i := 0; i < len(m.inputs); i++ {
|
||||
if i == m.focusIndex {
|
||||
// Set focused state
|
||||
cmds[i] = m.focusInput(i)
|
||||
continue
|
||||
}
|
||||
// Remove focused state
|
||||
m.unfocusInput(i)
|
||||
}
|
||||
|
||||
return m, tea.Batch(cmds...)
|
||||
}
|
||||
}
|
||||
|
||||
// Handle character input and blinking
|
||||
cmd := m.updateInputs(msg)
|
||||
|
||||
return m, cmd
|
||||
}
|
||||
|
||||
func (m *Model) updateInputs(msg tea.Msg) tea.Cmd {
|
||||
cmds := make([]tea.Cmd, len(m.inputs))
|
||||
|
||||
// Only text inputs with Focus() set will respond, so it's safe to simply
|
||||
// update all of them here without any further logic.
|
||||
for i := range m.inputs {
|
||||
m.inputs[i], cmds[i] = m.inputs[i].Update(msg)
|
||||
}
|
||||
|
||||
return tea.Batch(cmds...)
|
||||
}
|
||||
|
||||
func (m Model) View() string {
|
||||
var b strings.Builder
|
||||
|
||||
if m.skipButton {
|
||||
button := &blurredSkipButton
|
||||
if m.focusIndex == -1 {
|
||||
button = &focusedSkipButton
|
||||
}
|
||||
fmt.Fprintf(&b, "%s\n\n\n", *button)
|
||||
}
|
||||
|
||||
for i := range m.inputs {
|
||||
if m.configs[i].Label != "" {
|
||||
b.WriteString(m.GetLabel(m.configs[i]))
|
||||
}
|
||||
|
||||
b.WriteString(m.inputs[i].View())
|
||||
b.WriteRune('\n')
|
||||
if i < len(m.inputs)-1 {
|
||||
b.WriteRune('\n')
|
||||
}
|
||||
}
|
||||
|
||||
button := &blurredButton
|
||||
if m.focusIndex == len(m.inputs) {
|
||||
button = &focusedButton
|
||||
}
|
||||
fmt.Fprintf(&b, "\n\n%s\n\n", *button)
|
||||
|
||||
return b.String()
|
||||
}
|
||||
|
||||
func (m Model) GetLabel(c InputConfig) string {
|
||||
var label strings.Builder
|
||||
|
||||
label.WriteString(c.Label)
|
||||
if c.Required {
|
||||
label.WriteString("*")
|
||||
}
|
||||
|
||||
if len(c.Help) > 0 {
|
||||
label.WriteString("\n" + helpStyle.Render(c.Help))
|
||||
}
|
||||
|
||||
label.WriteString("\n")
|
||||
return label.String()
|
||||
}
|
||||
|
||||
func (m Model) SetSkip(skip bool) Model {
|
||||
m.skipButton = skip
|
||||
if m.skipButton {
|
||||
if len(m.inputs) > 0 {
|
||||
m.unfocusInput(0)
|
||||
}
|
||||
m.focusIndex = -1
|
||||
}
|
||||
return m
|
||||
}
|
||||
|
||||
func (m *Model) unfocusInput(index int) {
|
||||
m.inputs[index].Blur()
|
||||
m.inputs[index].PromptStyle = noStyle
|
||||
m.inputs[index].TextStyle = noStyle
|
||||
}
|
||||
|
||||
func (m *Model) focusInput(index int) tea.Cmd {
|
||||
m.inputs[index].PromptStyle = focusedStyle
|
||||
m.inputs[index].TextStyle = focusedStyle
|
||||
return m.inputs[index].Focus()
|
||||
}
|
||||
@@ -1,97 +0,0 @@
|
||||
package viewport
|
||||
|
||||
import (
|
||||
"github.com/charmbracelet/bubbles/viewport"
|
||||
tea "github.com/charmbracelet/bubbletea"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/tui/common"
|
||||
)
|
||||
|
||||
// Viewport represents a viewport component.
|
||||
type Viewport struct {
|
||||
common common.Common
|
||||
*viewport.Model
|
||||
}
|
||||
|
||||
// New returns a new Viewport.
|
||||
func New(c common.Common) *Viewport {
|
||||
vp := viewport.New(c.Width, c.Height)
|
||||
vp.MouseWheelEnabled = true
|
||||
return &Viewport{
|
||||
common: c,
|
||||
Model: &vp,
|
||||
}
|
||||
}
|
||||
|
||||
// SetSize implements common.Component.
|
||||
func (v *Viewport) SetSize(width, height int) {
|
||||
v.common.SetSize(width, height)
|
||||
v.Model.Width = width
|
||||
v.Model.Height = height
|
||||
}
|
||||
|
||||
// Init implements tea.Model.
|
||||
func (v *Viewport) Init() tea.Cmd {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Update implements tea.Model.
|
||||
func (v *Viewport) Update(msg tea.Msg) (tea.Model, tea.Cmd) {
|
||||
vp, cmd := v.Model.Update(msg)
|
||||
v.Model = &vp
|
||||
return v, cmd
|
||||
}
|
||||
|
||||
// View implements tea.Model.
|
||||
func (v *Viewport) View() string {
|
||||
return v.Model.View()
|
||||
}
|
||||
|
||||
// SetContent sets the viewport's content.
|
||||
func (v *Viewport) SetContent(content string) {
|
||||
v.Model.SetContent(content)
|
||||
}
|
||||
|
||||
// GotoTop moves the viewport to the top of the log.
|
||||
func (v *Viewport) GotoTop() {
|
||||
v.Model.GotoTop()
|
||||
}
|
||||
|
||||
// GotoBottom moves the viewport to the bottom of the log.
|
||||
func (v *Viewport) GotoBottom() {
|
||||
v.Model.GotoBottom()
|
||||
}
|
||||
|
||||
// HalfViewDown moves the viewport down by half the viewport height.
|
||||
func (v *Viewport) HalfViewDown() {
|
||||
v.Model.HalfViewDown()
|
||||
}
|
||||
|
||||
// HalfViewUp moves the viewport up by half the viewport height.
|
||||
func (v *Viewport) HalfViewUp() {
|
||||
v.Model.HalfViewUp()
|
||||
}
|
||||
|
||||
// ViewUp moves the viewport up by a page.
|
||||
func (v *Viewport) ViewUp() []string {
|
||||
return v.Model.ViewUp()
|
||||
}
|
||||
|
||||
// ViewDown moves the viewport down by a page.
|
||||
func (v *Viewport) ViewDown() []string {
|
||||
return v.Model.ViewDown()
|
||||
}
|
||||
|
||||
// LineUp moves the viewport up by the given number of lines.
|
||||
func (v *Viewport) LineUp(n int) []string {
|
||||
return v.Model.LineUp(n)
|
||||
}
|
||||
|
||||
// LineDown moves the viewport down by the given number of lines.
|
||||
func (v *Viewport) LineDown(n int) []string {
|
||||
return v.Model.LineDown(n)
|
||||
}
|
||||
|
||||
// ScrollPercent returns the viewport's scroll percentage.
|
||||
func (v *Viewport) ScrollPercent() float64 {
|
||||
return v.Model.ScrollPercent()
|
||||
}
|
||||
@@ -1,217 +0,0 @@
|
||||
package keymap
|
||||
|
||||
import "github.com/charmbracelet/bubbles/key"
|
||||
|
||||
// KeyMap is a map of key bindings for the UI.
|
||||
type KeyMap struct {
|
||||
Quit key.Binding
|
||||
CmdQuit key.Binding
|
||||
Up key.Binding
|
||||
Down key.Binding
|
||||
UpDown key.Binding
|
||||
LeftRight key.Binding
|
||||
Arrows key.Binding
|
||||
Select key.Binding
|
||||
Section key.Binding
|
||||
Back key.Binding
|
||||
PrevPage key.Binding
|
||||
NextPage key.Binding
|
||||
Help key.Binding
|
||||
|
||||
SelectItem key.Binding
|
||||
BackItem key.Binding
|
||||
|
||||
Copy key.Binding
|
||||
}
|
||||
|
||||
// DefaultKeyMap returns the default key map.
|
||||
func DefaultKeyMap() *KeyMap {
|
||||
km := new(KeyMap)
|
||||
|
||||
km.Quit = key.NewBinding(
|
||||
key.WithKeys(
|
||||
"ctrl+c",
|
||||
),
|
||||
key.WithHelp(
|
||||
"ctrl+c",
|
||||
"quit",
|
||||
),
|
||||
)
|
||||
|
||||
km.CmdQuit = key.NewBinding(
|
||||
key.WithKeys(
|
||||
"q",
|
||||
"ctrl+c",
|
||||
),
|
||||
key.WithHelp(
|
||||
"q",
|
||||
"quit",
|
||||
),
|
||||
)
|
||||
|
||||
km.Up = key.NewBinding(
|
||||
key.WithKeys(
|
||||
"up",
|
||||
"k",
|
||||
),
|
||||
key.WithHelp(
|
||||
"↑/k",
|
||||
"up",
|
||||
),
|
||||
)
|
||||
|
||||
km.Down = key.NewBinding(
|
||||
key.WithKeys(
|
||||
"down",
|
||||
"j",
|
||||
),
|
||||
key.WithHelp(
|
||||
"↓/j",
|
||||
"down",
|
||||
),
|
||||
)
|
||||
|
||||
km.UpDown = key.NewBinding(
|
||||
key.WithKeys(
|
||||
"up",
|
||||
"down",
|
||||
"k",
|
||||
"j",
|
||||
),
|
||||
key.WithHelp(
|
||||
"↑↓",
|
||||
"navigate",
|
||||
),
|
||||
)
|
||||
|
||||
km.LeftRight = key.NewBinding(
|
||||
key.WithKeys(
|
||||
"left",
|
||||
"h",
|
||||
"right",
|
||||
"l",
|
||||
),
|
||||
key.WithHelp(
|
||||
"←→",
|
||||
"navigate",
|
||||
),
|
||||
)
|
||||
|
||||
km.Arrows = key.NewBinding(
|
||||
key.WithKeys(
|
||||
"up",
|
||||
"right",
|
||||
"down",
|
||||
"left",
|
||||
"k",
|
||||
"j",
|
||||
"h",
|
||||
"l",
|
||||
),
|
||||
key.WithHelp(
|
||||
"↑←↓→",
|
||||
"navigate",
|
||||
),
|
||||
)
|
||||
|
||||
km.Select = key.NewBinding(
|
||||
key.WithKeys(
|
||||
"enter",
|
||||
),
|
||||
key.WithHelp(
|
||||
"enter",
|
||||
"select",
|
||||
),
|
||||
)
|
||||
|
||||
km.Section = key.NewBinding(
|
||||
key.WithKeys(
|
||||
"tab",
|
||||
"shift+tab",
|
||||
),
|
||||
key.WithHelp(
|
||||
"tab",
|
||||
"section",
|
||||
),
|
||||
)
|
||||
|
||||
km.Back = key.NewBinding(
|
||||
key.WithKeys(
|
||||
"esc",
|
||||
),
|
||||
key.WithHelp(
|
||||
"esc",
|
||||
"back",
|
||||
),
|
||||
)
|
||||
|
||||
km.PrevPage = key.NewBinding(
|
||||
key.WithKeys(
|
||||
"pgup",
|
||||
"b",
|
||||
"u",
|
||||
),
|
||||
key.WithHelp(
|
||||
"pgup",
|
||||
"prev page",
|
||||
),
|
||||
)
|
||||
|
||||
km.NextPage = key.NewBinding(
|
||||
key.WithKeys(
|
||||
"pgdown",
|
||||
"f",
|
||||
"d",
|
||||
),
|
||||
key.WithHelp(
|
||||
"pgdn",
|
||||
"next page",
|
||||
),
|
||||
)
|
||||
|
||||
km.Help = key.NewBinding(
|
||||
key.WithKeys(
|
||||
"?",
|
||||
),
|
||||
key.WithHelp(
|
||||
"?",
|
||||
"toggle help",
|
||||
),
|
||||
)
|
||||
|
||||
km.SelectItem = key.NewBinding(
|
||||
key.WithKeys(
|
||||
"l",
|
||||
"right",
|
||||
),
|
||||
key.WithHelp(
|
||||
"→",
|
||||
"select",
|
||||
),
|
||||
)
|
||||
|
||||
km.BackItem = key.NewBinding(
|
||||
key.WithKeys(
|
||||
"h",
|
||||
"left",
|
||||
"backspace",
|
||||
),
|
||||
key.WithHelp(
|
||||
"←",
|
||||
"back",
|
||||
),
|
||||
)
|
||||
|
||||
km.Copy = key.NewBinding(
|
||||
key.WithKeys(
|
||||
"c",
|
||||
"ctrl+c",
|
||||
),
|
||||
key.WithHelp(
|
||||
"c",
|
||||
"copy text",
|
||||
),
|
||||
)
|
||||
|
||||
return km
|
||||
}
|
||||
@@ -1,60 +0,0 @@
|
||||
package contact_enterprise
|
||||
|
||||
import (
|
||||
"strings"
|
||||
|
||||
"github.com/charmbracelet/bubbles/key"
|
||||
tea "github.com/charmbracelet/bubbletea"
|
||||
"github.com/charmbracelet/lipgloss"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/tui/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/tui/styles"
|
||||
)
|
||||
|
||||
type ContactEnterprise struct {
|
||||
common.Common
|
||||
viewed bool
|
||||
}
|
||||
|
||||
var (
|
||||
linkStyle = lipgloss.NewStyle().Foreground(
|
||||
lipgloss.Color("28")) // green
|
||||
)
|
||||
|
||||
func New(c common.Common) *ContactEnterprise {
|
||||
return &ContactEnterprise{
|
||||
Common: c,
|
||||
viewed: false,
|
||||
}
|
||||
}
|
||||
|
||||
func (m *ContactEnterprise) Init() tea.Cmd {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *ContactEnterprise) Update(msg tea.Msg) (tea.Model, tea.Cmd) {
|
||||
if m.viewed {
|
||||
return m, tea.Quit
|
||||
}
|
||||
|
||||
return m, func() tea.Msg { return nil }
|
||||
}
|
||||
|
||||
func (m *ContactEnterprise) View() string {
|
||||
|
||||
s := strings.Builder{}
|
||||
s.WriteString("Interested in Trufflehog enterprise?\n")
|
||||
s.WriteString(linkStyle.Render("🔗 https://trufflesecurity.com/contact"))
|
||||
|
||||
m.viewed = true
|
||||
return styles.AppStyle.Render(s.String())
|
||||
}
|
||||
|
||||
func (m *ContactEnterprise) ShortHelp() []key.Binding {
|
||||
// TODO: actually return something
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *ContactEnterprise) FullHelp() [][]key.Binding {
|
||||
// TODO: actually return something
|
||||
return nil
|
||||
}
|
||||
@@ -1,23 +0,0 @@
|
||||
package source_configure
|
||||
|
||||
type Item struct {
|
||||
title string
|
||||
description string
|
||||
}
|
||||
|
||||
func (i Item) ID() string { return i.title }
|
||||
|
||||
func (i Item) Title() string {
|
||||
return i.title
|
||||
}
|
||||
func (i Item) Description() string {
|
||||
return i.description
|
||||
}
|
||||
|
||||
func (i Item) SetDescription(d string) Item {
|
||||
i.description = d
|
||||
return i
|
||||
}
|
||||
|
||||
// We shouldn't be filtering for these list items.
|
||||
func (i Item) FilterValue() string { return "" }
|
||||
@@ -1,118 +0,0 @@
|
||||
package source_configure
|
||||
|
||||
import (
|
||||
"strings"
|
||||
|
||||
"github.com/charmbracelet/bubbles/key"
|
||||
"github.com/charmbracelet/bubbles/list"
|
||||
tea "github.com/charmbracelet/bubbletea"
|
||||
"github.com/charmbracelet/lipgloss"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/tui/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/tui/styles"
|
||||
)
|
||||
|
||||
type SetArgsMsg string
|
||||
|
||||
type RunComponent struct {
|
||||
common.Common
|
||||
parent *SourceConfigure
|
||||
reviewList list.Model
|
||||
reviewListItems []list.Item
|
||||
}
|
||||
|
||||
func NewRunComponent(common common.Common, parent *SourceConfigure) *RunComponent {
|
||||
// Make list of SourceItems.
|
||||
listItems := []list.Item{
|
||||
Item{title: "🔎 Source configuration"},
|
||||
Item{title: "🐽 TruffleHog configuration"},
|
||||
Item{title: "💸 Sales pitch", description: "\tContinuous monitoring, state tracking, remediations, and more\n\t🔗 https://trufflesecurity.com/trufflehog"},
|
||||
}
|
||||
|
||||
// Setup list
|
||||
delegate := list.NewDefaultDelegate()
|
||||
delegate.Styles.SelectedTitle.Foreground(lipgloss.Color("white"))
|
||||
delegate.Styles.SelectedDesc.Foreground(lipgloss.Color("white"))
|
||||
delegate.SetHeight(3)
|
||||
|
||||
reviewList := list.New(listItems, delegate, common.Width, common.Height)
|
||||
|
||||
reviewList.SetShowTitle(false)
|
||||
reviewList.SetShowStatusBar(false)
|
||||
reviewList.SetFilteringEnabled(false)
|
||||
|
||||
return &RunComponent{
|
||||
Common: common,
|
||||
parent: parent,
|
||||
reviewList: reviewList,
|
||||
reviewListItems: listItems,
|
||||
}
|
||||
}
|
||||
|
||||
func (m *RunComponent) Init() tea.Cmd {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *RunComponent) Update(msg tea.Msg) (tea.Model, tea.Cmd) {
|
||||
switch msg := msg.(type) {
|
||||
case tea.WindowSizeMsg:
|
||||
h, v := styles.AppStyle.GetFrameSize()
|
||||
m.reviewList.SetSize(msg.Width-h, msg.Height/2-v)
|
||||
case tea.KeyMsg:
|
||||
if msg.Type == tea.KeyEnter {
|
||||
command := m.parent.sourceFields.Cmd()
|
||||
if m.parent.truffleFields.Cmd() != "" {
|
||||
command += " " + m.parent.truffleFields.Cmd()
|
||||
}
|
||||
cmd := func() tea.Msg { return SetArgsMsg(command) }
|
||||
return m, cmd
|
||||
}
|
||||
}
|
||||
if len(m.reviewListItems) > 0 && m.parent != nil && m.parent.sourceFields != nil {
|
||||
m.reviewListItems[0] = m.reviewListItems[0].(Item).SetDescription(m.parent.sourceFields.Summary())
|
||||
m.reviewListItems[1] = m.reviewListItems[1].(Item).SetDescription(m.parent.truffleFields.Summary())
|
||||
}
|
||||
var cmd tea.Cmd
|
||||
m.reviewList, cmd = m.reviewList.Update(msg)
|
||||
return m, tea.Batch(cmd)
|
||||
}
|
||||
|
||||
func (m *RunComponent) View() string {
|
||||
var view strings.Builder
|
||||
|
||||
view.WriteString("\n🔎 Source configuration\n")
|
||||
view.WriteString(m.parent.sourceFields.Summary())
|
||||
|
||||
view.WriteString("\n🐽 TruffleHog configuration\n")
|
||||
view.WriteString(m.parent.truffleFields.Summary())
|
||||
|
||||
view.WriteString("\n💸 Sales pitch\n")
|
||||
view.WriteString("\tContinuous monitoring, state tracking, remediations, and more\n")
|
||||
view.WriteString("\t🔗 https://trufflesecurity.com/trufflehog\n\n")
|
||||
|
||||
view.WriteString(styles.BoldTextStyle.Render("\n\n🐷 Run TruffleHog for "+m.parent.configTabSource) + " 🐷\n\n")
|
||||
|
||||
view.WriteString("Generated TruffleHog command\n")
|
||||
view.WriteString(styles.HintTextStyle.Render("Save this if you want to run it again later!") + "\n")
|
||||
|
||||
command := m.parent.sourceFields.Cmd()
|
||||
if m.parent.truffleFields.Cmd() != "" {
|
||||
command += " " + m.parent.truffleFields.Cmd()
|
||||
}
|
||||
view.WriteString(styles.CodeTextStyle.Render(command))
|
||||
|
||||
focusedStyle := lipgloss.NewStyle().Foreground(lipgloss.Color("205"))
|
||||
view.WriteString("\n\n" + focusedStyle.Render("[ Run TruffleHog ]") + "\n\n")
|
||||
|
||||
// view.WriteString(m.reviewList.View())
|
||||
return view.String()
|
||||
}
|
||||
|
||||
func (m *RunComponent) ShortHelp() []key.Binding {
|
||||
// TODO: actually return something
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *RunComponent) FullHelp() [][]key.Binding {
|
||||
// TODO: actually return something
|
||||
return nil
|
||||
}
|
||||
@@ -1,71 +0,0 @@
|
||||
package source_configure
|
||||
|
||||
import (
|
||||
"strings"
|
||||
|
||||
"github.com/charmbracelet/bubbles/key"
|
||||
tea "github.com/charmbracelet/bubbletea"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/tui/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/tui/sources"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/tui/styles"
|
||||
)
|
||||
|
||||
type SourceComponent struct {
|
||||
common.Common
|
||||
parent *SourceConfigure
|
||||
form tea.Model
|
||||
}
|
||||
|
||||
func NewSourceComponent(common common.Common, parent *SourceConfigure) *SourceComponent {
|
||||
return &SourceComponent{
|
||||
Common: common,
|
||||
parent: parent,
|
||||
}
|
||||
}
|
||||
|
||||
func (m *SourceComponent) SetForm(form tea.Model) {
|
||||
m.form = form
|
||||
}
|
||||
|
||||
func (m *SourceComponent) Init() tea.Cmd {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *SourceComponent) Update(msg tea.Msg) (tea.Model, tea.Cmd) {
|
||||
// TODO: Add a focus variable.
|
||||
if m.form != nil {
|
||||
model, cmd := m.form.Update(msg)
|
||||
m.form = model
|
||||
return m, cmd
|
||||
}
|
||||
return m, nil
|
||||
}
|
||||
|
||||
func (m *SourceComponent) View() string {
|
||||
var view strings.Builder
|
||||
|
||||
view.WriteString(styles.BoldTextStyle.Render("\nConfiguring "+styles.PrimaryTextStyle.Render(m.parent.configTabSource)) + "\n")
|
||||
|
||||
view.WriteString(styles.HintTextStyle.Render("* required field") + "\n\n")
|
||||
|
||||
sourceNote := sources.GetSourceNotes(m.parent.configTabSource)
|
||||
if len(sourceNote) > 0 {
|
||||
view.WriteString("⭐ " + sourceNote + " ⭐\n\n")
|
||||
}
|
||||
|
||||
if m.form != nil {
|
||||
view.WriteString(m.form.View())
|
||||
view.WriteString("\n")
|
||||
}
|
||||
return view.String()
|
||||
}
|
||||
|
||||
func (m *SourceComponent) ShortHelp() []key.Binding {
|
||||
// TODO: actually return something
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *SourceComponent) FullHelp() [][]key.Binding {
|
||||
// TODO: actually return something
|
||||
return nil
|
||||
}
|
||||
@@ -1,139 +0,0 @@
|
||||
package source_configure
|
||||
|
||||
import (
|
||||
"github.com/charmbracelet/bubbles/key"
|
||||
tea "github.com/charmbracelet/bubbletea"
|
||||
"github.com/charmbracelet/lipgloss"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/tui/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/tui/components/tabs"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/tui/components/textinputs"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/tui/sources"
|
||||
)
|
||||
|
||||
type SetSourceMsg struct {
|
||||
Source string
|
||||
}
|
||||
|
||||
type tab int
|
||||
|
||||
const (
|
||||
configTab tab = iota
|
||||
truffleConfigTab
|
||||
runTab
|
||||
)
|
||||
|
||||
func (t tab) String() string {
|
||||
return []string{
|
||||
"1. Source Configuration",
|
||||
"2. TruffleHog Configuration",
|
||||
"3. Run",
|
||||
}[t]
|
||||
}
|
||||
|
||||
type SourceConfigure struct {
|
||||
common.Common
|
||||
activeTab tab
|
||||
tabs *tabs.Tabs
|
||||
configTabSource string
|
||||
tabComponents []common.Component
|
||||
sourceFields sources.CmdModel
|
||||
truffleFields sources.CmdModel
|
||||
}
|
||||
|
||||
func (m SourceConfigure) Init() tea.Cmd {
|
||||
return m.tabs.Init()
|
||||
}
|
||||
|
||||
func New(c common.Common) *SourceConfigure {
|
||||
conf := SourceConfigure{Common: c, truffleFields: GetTrufflehogConfiguration()}
|
||||
conf.tabs = tabs.New(c, []string{configTab.String(), truffleConfigTab.String(), runTab.String()})
|
||||
|
||||
conf.tabComponents = []common.Component{
|
||||
configTab: NewSourceComponent(c, &conf),
|
||||
truffleConfigTab: NewTrufflehogComponent(c, &conf),
|
||||
runTab: NewRunComponent(c, &conf),
|
||||
}
|
||||
return &conf
|
||||
}
|
||||
|
||||
func (m *SourceConfigure) Update(msg tea.Msg) (tea.Model, tea.Cmd) {
|
||||
var cmds []tea.Cmd
|
||||
|
||||
switch msg := msg.(type) {
|
||||
case tea.WindowSizeMsg:
|
||||
for i := range m.tabComponents {
|
||||
model, cmd := m.tabComponents[i].Update(msg)
|
||||
m.tabComponents[i] = model.(common.Component)
|
||||
cmds = append(cmds, cmd)
|
||||
}
|
||||
|
||||
case tabs.ActiveTabMsg:
|
||||
m.activeTab = tab(msg)
|
||||
t, cmd := m.tabs.Update(msg)
|
||||
m.tabs = t.(*tabs.Tabs)
|
||||
|
||||
if cmd != nil {
|
||||
cmds = append(cmds, cmd)
|
||||
}
|
||||
case tabs.SelectTabMsg:
|
||||
m.activeTab = tab(msg)
|
||||
t, cmd := m.tabs.Update(msg)
|
||||
m.tabs = t.(*tabs.Tabs)
|
||||
|
||||
if cmd != nil {
|
||||
cmds = append(cmds, cmd)
|
||||
}
|
||||
case tea.KeyMsg:
|
||||
t, cmd := m.tabs.Update(msg)
|
||||
m.tabs = t.(*tabs.Tabs)
|
||||
if cmd != nil {
|
||||
cmds = append(cmds, cmd)
|
||||
}
|
||||
case SetSourceMsg:
|
||||
m.configTabSource = msg.Source
|
||||
// TODO: Use actual messages or something?
|
||||
m.tabComponents[truffleConfigTab].(*TrufflehogComponent).SetForm(m.truffleFields)
|
||||
fields := sources.GetSourceFields(m.configTabSource)
|
||||
|
||||
if fields != nil {
|
||||
m.sourceFields = fields
|
||||
m.tabComponents[configTab].(*SourceComponent).SetForm(fields)
|
||||
}
|
||||
|
||||
case textinputs.SelectNextMsg, textinputs.SelectSkipMsg:
|
||||
if m.activeTab < runTab {
|
||||
m.activeTab++
|
||||
}
|
||||
t, cmd := m.tabs.Update(tabs.SelectTabMsg(int(m.activeTab)))
|
||||
m.tabs = t.(*tabs.Tabs)
|
||||
|
||||
if cmd != nil {
|
||||
cmds = append(cmds, cmd)
|
||||
}
|
||||
}
|
||||
|
||||
tab, cmd := m.tabComponents[m.activeTab].Update(msg)
|
||||
m.tabComponents[m.activeTab] = tab.(common.Component)
|
||||
if cmd != nil {
|
||||
cmds = append(cmds, cmd)
|
||||
}
|
||||
|
||||
return m, tea.Batch(cmds...)
|
||||
}
|
||||
|
||||
func (m *SourceConfigure) View() string {
|
||||
return lipgloss.JoinVertical(lipgloss.Top,
|
||||
m.tabs.View(),
|
||||
m.tabComponents[m.activeTab].View(),
|
||||
)
|
||||
}
|
||||
|
||||
func (m *SourceConfigure) ShortHelp() []key.Binding {
|
||||
// TODO: actually return something
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *SourceConfigure) FullHelp() [][]key.Binding {
|
||||
// TODO: actually return something
|
||||
return nil
|
||||
}
|
||||
@@ -1,66 +0,0 @@
|
||||
package source_configure
|
||||
|
||||
import (
|
||||
"strings"
|
||||
|
||||
"github.com/charmbracelet/bubbles/key"
|
||||
tea "github.com/charmbracelet/bubbletea"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/tui/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/tui/styles"
|
||||
)
|
||||
|
||||
type TrufflehogComponent struct {
|
||||
common.Common
|
||||
parent *SourceConfigure
|
||||
form tea.Model
|
||||
}
|
||||
|
||||
func NewTrufflehogComponent(common common.Common, parent *SourceConfigure) *TrufflehogComponent {
|
||||
return &TrufflehogComponent{
|
||||
Common: common,
|
||||
parent: parent,
|
||||
}
|
||||
}
|
||||
|
||||
func (m *TrufflehogComponent) SetForm(form tea.Model) {
|
||||
m.form = form
|
||||
}
|
||||
|
||||
func (m *TrufflehogComponent) Init() tea.Cmd {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *TrufflehogComponent) Update(msg tea.Msg) (tea.Model, tea.Cmd) {
|
||||
// TODO: Add a focus variable.
|
||||
if m.form != nil {
|
||||
model, cmd := m.form.Update(msg)
|
||||
m.form = model
|
||||
|
||||
return m, cmd
|
||||
}
|
||||
return m, nil
|
||||
}
|
||||
|
||||
func (m *TrufflehogComponent) View() string {
|
||||
var view strings.Builder
|
||||
|
||||
view.WriteString(styles.BoldTextStyle.Render("\nConfiguring "+styles.PrimaryTextStyle.Render("TruffleHog")) + "\n")
|
||||
view.WriteString(styles.HintTextStyle.Render("You can skip this completely and run with defaults") + "\n\n")
|
||||
|
||||
if m.form != nil {
|
||||
view.WriteString(m.form.View())
|
||||
view.WriteString("\n")
|
||||
}
|
||||
|
||||
return view.String()
|
||||
}
|
||||
|
||||
func (m *TrufflehogComponent) ShortHelp() []key.Binding {
|
||||
// TODO: actually return something
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *TrufflehogComponent) FullHelp() [][]key.Binding {
|
||||
// TODO: actually return something
|
||||
return nil
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user