Compare commits
2 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 Miccah Castorina
|
c0752eae2c | ||
|
Miccah Castorina
|
c6597969eb |
@@ -6,7 +6,11 @@ labels: bug, needs triage
|
||||
assignees: trufflesecurity/product-eng
|
||||
---
|
||||
|
||||
Please review the [Community Note](https://github.com/trufflesecurity/trufflehog/blob/main/.github/community_note.md) before submitting
|
||||
### Community Note
|
||||
|
||||
* Please vote on this issue by adding a 👍 [reaction](https://blog.github.com/2016-03-10-add-reactions-to-pull-requests-issues-and-comments/) to the original issue to help the community and maintainers prioritize this request
|
||||
* Please do not leave "+1" or other comments that do not add relevant new information or questions, they generate extra noise for issue followers and do not help prioritize the request
|
||||
* If you are interested in working on this issue or have submitted a pull request, please leave a comment
|
||||
|
||||
### TruffleHog Version
|
||||
<!--- Please run `trufflehog --version` to show the version. If you are not running the latest version, please upgrade because your issue may have already been fixed. --->
|
||||
|
||||
@@ -6,27 +6,33 @@ labels: enhancement, needs triage
|
||||
assignees: trufflesecurity/product-eng
|
||||
---
|
||||
|
||||
Please review the [Community Note](https://github.com/trufflesecurity/trufflehog/blob/main/.github/community_note.md) before submitting
|
||||
### Community Note
|
||||
|
||||
* Please vote on this issue by adding a 👍 [reaction](https://blog.github.com/2016-03-10-add-reactions-to-pull-requests-issues-and-comments/) to the original issue to help the community and maintainers prioritize this request
|
||||
* Please do not leave "+1" or other comments that do not add relevant new information or questions, they generate extra noise for issue followers and do not help prioritize the request
|
||||
* If you are interested in working on this issue or have submitted a pull request, please leave a comment
|
||||
|
||||
### Description
|
||||
|
||||
## Description
|
||||
<!--- Please leave a helpful description of the feature request here. --->
|
||||
|
||||
### Preferred Solution
|
||||
<!--- A clear and concise description of what you want to happen. What
|
||||
information may be required and what would be the preferred way to provide it?
|
||||
## Problem to be Addressed
|
||||
<!--- A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] --->
|
||||
|
||||
## Description of the Preferred Solution
|
||||
<!--- A clear and concise description of what you want to happen. What
|
||||
information may be required and what would be the preferred way to provide it?
|
||||
What should the output include? --->
|
||||
|
||||
### Additional Context
|
||||
## Additional Context
|
||||
<!--- Add any other context or screenshots about the feature request here. --->
|
||||
|
||||
#### References
|
||||
### References
|
||||
|
||||
<!---
|
||||
Information about referencing Github Issues: https://help.github.com/articles/basic-writing-and-formatting-syntax/#referencing-issues-and-pull-requests
|
||||
|
||||
Are there any other GitHub issues (open or closed) or pull requests that should be linked here? Vendor blog posts or documentation? For example:
|
||||
|
||||
* #0000
|
||||
--->
|
||||
|
||||
|
||||
* #0000
|
||||
|
||||
@@ -2,11 +2,3 @@
|
||||
Please create an issue to collect feedback prior to feature additions. Please also reference that issue in any PRs.
|
||||
If possible try to keep PRs scoped to one feature, and add tests for new features.
|
||||
-->
|
||||
|
||||
### Description:
|
||||
Explain the purpose of the PR.
|
||||
|
||||
### Checklist:
|
||||
* [ ] Tests passing (`make test-community`)?
|
||||
* [ ] Lint passing (`make lint` this requires [golangci-lint](https://golangci-lint.run/usage/install/#local-installation))?
|
||||
|
||||
|
||||
@@ -1,7 +0,0 @@
|
||||
## Community Note
|
||||
|
||||
Please vote on this issue by adding a 👍 reaction to the original issue to help the community and maintainers prioritize this request.
|
||||
|
||||
Please do not leave "+1" or other comments that do not add relevant new information or questions, they generate extra noise for issue followers and do not help prioritize the request.
|
||||
|
||||
If you are interested in working on this issue or have submitted a pull request, please leave a comment.
|
||||
@@ -17,7 +17,7 @@ jobs:
|
||||
steps:
|
||||
- uses: actions/setup-go@v4
|
||||
with:
|
||||
go-version: '1.21'
|
||||
go-version: '1.20'
|
||||
- uses: actions/checkout@v3
|
||||
- name: golangci-lint
|
||||
uses: golangci/golangci-lint-action@v3
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
name: Performance Test
|
||||
name: Performance Test
|
||||
|
||||
on: [pull_request]
|
||||
|
||||
@@ -7,13 +7,13 @@ jobs:
|
||||
# skip if PR is from a fork.
|
||||
# TODO: this could probabaly be refactored a bit so that it runs on forks
|
||||
if: ${{ ! github.event.pull_request.head.repo.fork }}
|
||||
|
||||
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Install Go
|
||||
uses: actions/setup-go@v4
|
||||
with:
|
||||
go-version: '1.21'
|
||||
go-version: '1.20'
|
||||
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v3
|
||||
|
||||
@@ -35,7 +35,7 @@ jobs:
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v4
|
||||
with:
|
||||
go-version: '1.21'
|
||||
go-version: '1.20'
|
||||
- name: Run GoReleaser
|
||||
uses: goreleaser/goreleaser-action@v4
|
||||
with:
|
||||
|
||||
@@ -16,7 +16,7 @@ jobs:
|
||||
- name: Install Go
|
||||
uses: actions/setup-go@v4
|
||||
with:
|
||||
go-version: '1.21'
|
||||
go-version: '1.20'
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v3
|
||||
with:
|
||||
|
||||
@@ -11,7 +11,7 @@ jobs:
|
||||
- name: Install Go
|
||||
uses: actions/setup-go@v4
|
||||
with:
|
||||
go-version: '1.21'
|
||||
go-version: '1.20'
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v3
|
||||
- name: Smoke
|
||||
|
||||
@@ -18,7 +18,7 @@ jobs:
|
||||
steps:
|
||||
- uses: actions/setup-go@v4
|
||||
with:
|
||||
go-version: '1.21'
|
||||
go-version: '1.20'
|
||||
- uses: actions/checkout@v3
|
||||
- name: Run Snifftest
|
||||
run: make snifftest
|
||||
run: make snifftest
|
||||
@@ -21,7 +21,7 @@ jobs:
|
||||
- name: Install Go
|
||||
uses: actions/setup-go@v4
|
||||
with:
|
||||
go-version: '1.21'
|
||||
go-version: '1.20'
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v3
|
||||
- id: 'auth'
|
||||
@@ -42,7 +42,7 @@ jobs:
|
||||
- name: Install Go
|
||||
uses: actions/setup-go@v4
|
||||
with:
|
||||
go-version: '1.21'
|
||||
go-version: '1.20'
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v3
|
||||
- id: 'auth'
|
||||
@@ -52,8 +52,8 @@ jobs:
|
||||
service_account: 'github-ci-external@trufflehog-testing.iam.gserviceaccount.com'
|
||||
- name: Test
|
||||
run: make test-detectors
|
||||
continue-on-error: true
|
||||
test-community:
|
||||
continue-on-error: true
|
||||
test-fork:
|
||||
if: ${{ github.event.pull_request.head.repo.fork }}
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
@@ -63,8 +63,8 @@ jobs:
|
||||
- name: Install Go
|
||||
uses: actions/setup-go@v4
|
||||
with:
|
||||
go-version: '1.21'
|
||||
go-version: '1.20'
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v3
|
||||
- name: Test
|
||||
run: make test-community
|
||||
run: make test-forks
|
||||
|
||||
@@ -39,7 +39,7 @@ test-race:
|
||||
test-detectors:
|
||||
CGO_ENABLED=0 go test -tags=detectors -timeout=5m $(shell go list ./... | grep pkg/detectors)
|
||||
|
||||
test-community:
|
||||
test-forks:
|
||||
CGO_ENABLED=0 go test -timeout=5m $(shell go list ./... | grep -v /vendor/ | grep -v pkg/detectors | grep -v pkg/sources)
|
||||
|
||||
bench:
|
||||
|
||||
@@ -8,6 +8,7 @@
|
||||
|
||||
<div align="center">
|
||||
|
||||
[](https://github.com/trufflesecurity/trufflehog/actions)
|
||||
[](https://goreportcard.com/report/github.com/trufflesecurity/trufflehog/v3)
|
||||
[](/LICENSE)
|
||||
[](/pkg/detectors)
|
||||
@@ -263,9 +264,9 @@ jobs:
|
||||
extra_args: --debug --only-verified
|
||||
```
|
||||
|
||||
# Pre-commit Hook
|
||||
# Precommit Hook
|
||||
|
||||
Trufflehog can be used in a pre-commit hook to prevent credentials from leaking before they ever leave your computer.
|
||||
Trufflehog can be used in a precommit hook to prevent credentials from leaking before they ever leave your computer.
|
||||
An example `.pre-commit-config.yaml` is provided (see [pre-commit.com](https://pre-commit.com/) for installation).
|
||||
|
||||
```yaml
|
||||
|
||||
@@ -4,9 +4,6 @@ go 1.20
|
||||
|
||||
replace github.com/jpillora/overseer => github.com/trufflesecurity/overseer v1.1.7-custom5
|
||||
|
||||
// go wants to pull in the latest but after v10.2.5 has a breaking API change
|
||||
replace github.com/couchbase/gocbcore/v10 => github.com/couchbase/gocbcore/v10 v10.2.4
|
||||
|
||||
require (
|
||||
cloud.google.com/go/secretmanager v1.11.1
|
||||
cloud.google.com/go/storage v1.31.0
|
||||
@@ -14,15 +11,10 @@ require (
|
||||
github.com/BobuSumisu/aho-corasick v1.0.3
|
||||
github.com/TheZeroSlave/zapsentry v1.17.0
|
||||
github.com/aws/aws-sdk-go v1.44.83
|
||||
github.com/aymanbagabas/go-osc52 v1.2.1
|
||||
github.com/bill-rich/disk-buffer-reader v0.1.7
|
||||
github.com/bill-rich/go-syslog v0.0.0-20220413021637-49edb52a574c
|
||||
github.com/bitfinexcom/bitfinex-api-go v0.0.0-20210608095005-9e0b26f200fb
|
||||
github.com/bradleyfalzon/ghinstallation/v2 v2.6.0
|
||||
github.com/charmbracelet/bubbles v0.16.1
|
||||
github.com/charmbracelet/bubbletea v0.24.1
|
||||
github.com/charmbracelet/glamour v0.6.0
|
||||
github.com/charmbracelet/lipgloss v0.7.1
|
||||
github.com/bradleyfalzon/ghinstallation/v2 v2.4.0
|
||||
github.com/couchbase/gocb/v2 v2.6.3
|
||||
github.com/crewjam/rfc5424 v0.1.0
|
||||
github.com/denisenkom/go-mssqldb v0.12.3
|
||||
@@ -31,7 +23,7 @@ require (
|
||||
github.com/felixge/fgprof v0.9.3
|
||||
github.com/getsentry/sentry-go v0.22.0
|
||||
github.com/go-errors/errors v1.4.2
|
||||
github.com/go-git/go-git/v5 v5.8.1
|
||||
github.com/go-git/go-git/v5 v5.8.0
|
||||
github.com/go-ldap/ldap/v3 v3.4.5
|
||||
github.com/go-logr/logr v1.2.4
|
||||
github.com/go-logr/zapr v1.2.4
|
||||
@@ -40,22 +32,17 @@ require (
|
||||
github.com/gobwas/glob v0.2.3
|
||||
github.com/golang-jwt/jwt v3.2.2+incompatible
|
||||
github.com/google/go-cmp v0.5.9
|
||||
github.com/google/go-containerregistry v0.15.2
|
||||
github.com/google/go-containerregistry v0.14.0
|
||||
github.com/google/go-github/v42 v42.0.0
|
||||
github.com/googleapis/gax-go/v2 v2.12.0
|
||||
github.com/h2non/filetype v1.1.3
|
||||
github.com/hashicorp/go-retryablehttp v0.7.4
|
||||
github.com/hashicorp/golang-lru v0.5.1
|
||||
github.com/jlaffaye/ftp v0.2.0
|
||||
github.com/joho/godotenv v1.5.1
|
||||
github.com/jpillora/overseer v1.1.6
|
||||
github.com/kylelemons/godebug v1.1.0
|
||||
github.com/lib/pq v1.10.9
|
||||
github.com/lrstanley/bubblezone v0.0.0-20221222153816-e95291e2243e
|
||||
github.com/mattn/go-isatty v0.0.18
|
||||
github.com/mattn/go-sqlite3 v1.14.17
|
||||
github.com/mholt/archiver/v4 v4.0.0-alpha.8
|
||||
github.com/muesli/reflow v0.3.0
|
||||
github.com/patrickmn/go-cache v2.1.0+incompatible
|
||||
github.com/paulbellamy/ratecounter v0.2.0
|
||||
github.com/pkg/errors v0.9.1
|
||||
@@ -64,9 +51,8 @@ require (
|
||||
github.com/sergi/go-diff v1.3.1
|
||||
github.com/stretchr/testify v1.8.4
|
||||
github.com/tailscale/depaware v0.0.0-20210622194025-720c4b409502
|
||||
github.com/xanzy/go-gitlab v0.88.0
|
||||
github.com/xanzy/go-gitlab v0.86.0
|
||||
go.mongodb.org/mongo-driver v1.12.0
|
||||
go.uber.org/mock v0.2.0
|
||||
go.uber.org/zap v1.24.0
|
||||
golang.org/x/crypto v0.11.0
|
||||
golang.org/x/exp v0.0.0-20221018205818-5c77f4b2bbd7
|
||||
@@ -85,7 +71,6 @@ require (
|
||||
cloud.google.com/go/compute v1.20.1 // indirect
|
||||
cloud.google.com/go/compute/metadata v0.2.3 // indirect
|
||||
cloud.google.com/go/iam v1.1.0 // indirect
|
||||
dario.cat/mergo v1.0.0 // indirect
|
||||
github.com/Azure/go-autorest v14.2.0+incompatible // indirect
|
||||
github.com/Azure/go-autorest/autorest v0.11.24 // indirect
|
||||
github.com/Azure/go-autorest/autorest/adal v0.9.18 // indirect
|
||||
@@ -95,15 +80,11 @@ require (
|
||||
github.com/Azure/go-autorest/tracing v0.6.0 // indirect
|
||||
github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 // indirect
|
||||
github.com/Microsoft/go-winio v0.6.1 // indirect
|
||||
github.com/ProtonMail/go-crypto v0.0.0-20230717121422-5aa5874ade95 // indirect
|
||||
github.com/ProtonMail/go-crypto v0.0.0-20230518184743-7afd39499903 // indirect
|
||||
github.com/acomagu/bufpipe v1.0.4 // indirect
|
||||
github.com/alecthomas/chroma v0.10.0 // indirect
|
||||
github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 // indirect
|
||||
github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 // indirect
|
||||
github.com/andybalholm/brotli v1.0.5 // indirect
|
||||
github.com/atotto/clipboard v0.1.4 // indirect
|
||||
github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect
|
||||
github.com/aymerick/douceur v0.2.0 // indirect
|
||||
github.com/benbjohnson/clock v1.1.0 // indirect
|
||||
github.com/beorn7/perks v1.0.1 // indirect
|
||||
github.com/bodgit/plumbing v1.2.0 // indirect
|
||||
@@ -112,12 +93,10 @@ require (
|
||||
github.com/cespare/xxhash/v2 v2.2.0 // indirect
|
||||
github.com/cloudflare/circl v1.3.3 // indirect
|
||||
github.com/connesc/cipherio v0.2.1 // indirect
|
||||
github.com/containerd/console v1.0.4-0.20230313162750-1ae8d489ac81 // indirect
|
||||
github.com/containerd/stargz-snapshotter/estargz v0.14.3 // indirect
|
||||
github.com/couchbase/gocbcore/v10 v10.2.4 // indirect
|
||||
github.com/davecgh/go-spew v1.1.1 // indirect
|
||||
github.com/dimchansky/utfbom v1.1.1 // indirect
|
||||
github.com/dlclark/regexp2 v1.4.0 // indirect
|
||||
github.com/docker/cli v23.0.5+incompatible // indirect
|
||||
github.com/docker/distribution v2.8.2+incompatible // indirect
|
||||
github.com/docker/docker v23.0.5+incompatible // indirect
|
||||
@@ -134,36 +113,29 @@ require (
|
||||
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
|
||||
github.com/golang/protobuf v1.5.3 // indirect
|
||||
github.com/golang/snappy v0.0.4 // indirect
|
||||
github.com/google/go-github/v53 v53.2.0 // indirect
|
||||
github.com/google/go-github/v52 v52.0.0 // indirect
|
||||
github.com/google/go-querystring v1.1.0 // indirect
|
||||
github.com/google/pprof v0.0.0-20211214055906-6f57359322fd // indirect
|
||||
github.com/google/s2a-go v0.1.4 // indirect
|
||||
github.com/google/uuid v1.3.0 // indirect
|
||||
github.com/googleapis/enterprise-certificate-proxy v0.2.5 // indirect
|
||||
github.com/gorilla/css v1.0.0 // indirect
|
||||
github.com/h2non/parth v0.0.0-20190131123155-b4df798d6542 // indirect
|
||||
github.com/hashicorp/errwrap v1.0.0 // indirect
|
||||
github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
|
||||
github.com/hashicorp/go-multierror v1.1.1 // indirect
|
||||
github.com/imdario/mergo v0.3.15 // indirect
|
||||
github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect
|
||||
github.com/jmespath/go-jmespath v0.4.0 // indirect
|
||||
github.com/jpillora/s3 v1.1.4 // indirect
|
||||
github.com/kevinburke/ssh_config v1.2.0 // indirect
|
||||
github.com/klauspost/compress v1.16.5 // indirect
|
||||
github.com/klauspost/pgzip v1.2.5 // indirect
|
||||
github.com/lucasb-eyer/go-colorful v1.2.0 // indirect
|
||||
github.com/mattn/go-colorable v0.1.13 // indirect
|
||||
github.com/mattn/go-localereader v0.0.1 // indirect
|
||||
github.com/mattn/go-runewidth v0.0.14 // indirect
|
||||
github.com/mattn/go-isatty v0.0.17 // indirect
|
||||
github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
|
||||
github.com/microcosm-cc/bluemonday v1.0.23 // indirect
|
||||
github.com/mitchellh/go-homedir v1.1.0 // indirect
|
||||
github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe // indirect
|
||||
github.com/muesli/ansi v0.0.0-20211031195517-c9f0611b6c70 // indirect
|
||||
github.com/muesli/cancelreader v0.2.2 // indirect
|
||||
github.com/muesli/termenv v0.15.1 // indirect
|
||||
github.com/nwaples/rardecode/v2 v2.0.0-beta.2 // indirect
|
||||
github.com/olekukonko/tablewriter v0.0.5 // indirect
|
||||
github.com/onsi/ginkgo v1.16.5 // indirect
|
||||
github.com/onsi/gomega v1.23.0 // indirect
|
||||
github.com/opencontainers/go-digest v1.0.0 // indirect
|
||||
@@ -175,11 +147,9 @@ require (
|
||||
github.com/prometheus/client_model v0.3.0 // indirect
|
||||
github.com/prometheus/common v0.42.0 // indirect
|
||||
github.com/prometheus/procfs v0.10.1 // indirect
|
||||
github.com/rivo/uniseg v0.4.2 // indirect
|
||||
github.com/rogpeppe/go-internal v1.10.0 // indirect
|
||||
github.com/sahilm/fuzzy v0.1.0 // indirect
|
||||
github.com/sirupsen/logrus v1.9.0 // indirect
|
||||
github.com/skeema/knownhosts v1.2.0 // indirect
|
||||
github.com/skeema/knownhosts v1.1.1 // indirect
|
||||
github.com/therootcompany/xz v1.0.1 // indirect
|
||||
github.com/ulikunitz/xz v0.5.10 // indirect
|
||||
github.com/vbatts/tar-split v0.11.3 // indirect
|
||||
@@ -188,8 +158,6 @@ require (
|
||||
github.com/xdg-go/scram v1.1.2 // indirect
|
||||
github.com/xdg-go/stringprep v1.0.4 // indirect
|
||||
github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d // indirect
|
||||
github.com/yuin/goldmark v1.5.2 // indirect
|
||||
github.com/yuin/goldmark-emoji v1.0.1 // indirect
|
||||
github.com/yusufpapurcu/wmi v1.2.2 // indirect
|
||||
go.opencensus.io v0.24.0 // indirect
|
||||
go.uber.org/atomic v1.7.0 // indirect
|
||||
@@ -198,7 +166,6 @@ require (
|
||||
golang.org/x/mod v0.11.0 // indirect
|
||||
golang.org/x/net v0.12.0 // indirect
|
||||
golang.org/x/sys v0.10.0 // indirect
|
||||
golang.org/x/term v0.10.0 // indirect
|
||||
golang.org/x/time v0.3.0 // indirect
|
||||
golang.org/x/tools v0.10.0 // indirect
|
||||
golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect
|
||||
|
||||
@@ -27,8 +27,6 @@ cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiy
|
||||
cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos=
|
||||
cloud.google.com/go/storage v1.31.0 h1:+S3LjjEN2zZ+L5hOwj4+1OkGCsLVe0NzpXKQ1pSdTCI=
|
||||
cloud.google.com/go/storage v1.31.0/go.mod h1:81ams1PrhW16L4kF7qg+4mTq7SRs5HsbDTM0bWvrwJ0=
|
||||
dario.cat/mergo v1.0.0 h1:AGCNq9Evsj31mOgNPcLyXc+4PNABt905YmuqPYYpBWk=
|
||||
dario.cat/mergo v1.0.0/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk=
|
||||
dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/azcore v0.19.0/go.mod h1:h6H6c8enJmmocHUbLiiGY6sx7f9i+X3m1CHdd5c6Rdw=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/azidentity v0.11.0/go.mod h1:HcM1YX14R7CJcghJGOYCgdezslRSVzqwLf/q+4Y2r/0=
|
||||
@@ -62,14 +60,12 @@ github.com/Microsoft/go-winio v0.5.2/go.mod h1:WpS1mjBmmwHBEWmogvA2mj8546UReBk4v
|
||||
github.com/Microsoft/go-winio v0.6.1 h1:9/kr64B9VUZrLm5YYwbGtUJnMgqWVOdUAXu6Migciow=
|
||||
github.com/Microsoft/go-winio v0.6.1/go.mod h1:LRdKpFKfdobln8UmuiYcKPot9D2v6svN5+sAH+4kjUM=
|
||||
github.com/ProtonMail/go-crypto v0.0.0-20230217124315-7d5c6f04bbb8/go.mod h1:I0gYDMZ6Z5GRU7l58bNFSkPTFN6Yl12dsUlAZ8xy98g=
|
||||
github.com/ProtonMail/go-crypto v0.0.0-20230717121422-5aa5874ade95 h1:KLq8BE0KwCL+mmXnjLWEAOYO+2l2AE4YMmqG1ZpZHBs=
|
||||
github.com/ProtonMail/go-crypto v0.0.0-20230717121422-5aa5874ade95/go.mod h1:EjAoLdwvbIOoOQr3ihjnSoLZRtE8azugULFRteWMNc0=
|
||||
github.com/ProtonMail/go-crypto v0.0.0-20230518184743-7afd39499903 h1:ZK3C5DtzV2nVAQTx5S5jQvMeDqWtD1By5mOoyY/xJek=
|
||||
github.com/ProtonMail/go-crypto v0.0.0-20230518184743-7afd39499903/go.mod h1:8TI4H3IbrackdNgv+92dI+rhpCaLqM0IfpgCgenFvRE=
|
||||
github.com/TheZeroSlave/zapsentry v1.17.0 h1:RIQCG89U7vWWZVmmCxeUz/g32WEcAYXUrXHoMlbSDmc=
|
||||
github.com/TheZeroSlave/zapsentry v1.17.0/go.mod h1:D1YMfSuu6xnkhwFXxrronesmsiyDhIqo+86I3Ok+r64=
|
||||
github.com/acomagu/bufpipe v1.0.4 h1:e3H4WUzM3npvo5uv95QuJM3cQspFNtFBzvJ2oNjKIDQ=
|
||||
github.com/acomagu/bufpipe v1.0.4/go.mod h1:mxdxdup/WdsKVreO5GpW4+M/1CE2sMG4jeGJ2sYmHc4=
|
||||
github.com/alecthomas/chroma v0.10.0 h1:7XDcGkCQopCNKjZHfYrNLraA+M7e0fMiJ/Mfikbfjek=
|
||||
github.com/alecthomas/chroma v0.10.0/go.mod h1:jtJATyUxlIORhUOFNA9NZDWGAQ8wpxQQqNSB4rjA/1s=
|
||||
github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 h1:JYp7IbQjafoB+tBA3gMyHYHrpOtNuDiK/uB5uXxq5wM=
|
||||
github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
|
||||
github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 h1:s6gZFSlWYmbqAuRjVTiNNhvNRfY2Wxp9nhfyel4rklc=
|
||||
@@ -81,17 +77,8 @@ github.com/andybalholm/brotli v1.0.5/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHG
|
||||
github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be h1:9AeTilPcZAjCFIImctFaOjnTIavg87rW78vTPkQqLI8=
|
||||
github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY=
|
||||
github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio=
|
||||
github.com/atotto/clipboard v0.1.4 h1:EH0zSVneZPSuFR11BlR9YppQTVDbh5+16AmcJi4g1z4=
|
||||
github.com/atotto/clipboard v0.1.4/go.mod h1:ZY9tmq7sm5xIbd9bOK4onWV4S6X0u6GY7Vn0Yu86PYI=
|
||||
github.com/aws/aws-sdk-go v1.44.83 h1:7+Rtc2Eio6EKUNoZeMV/IVxzVrY5oBQcNPtCcgIHYJA=
|
||||
github.com/aws/aws-sdk-go v1.44.83/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo=
|
||||
github.com/aymanbagabas/go-osc52 v1.0.3/go.mod h1:zT8H+Rk4VSabYN90pWyugflM3ZhpTZNC7cASDfUCdT4=
|
||||
github.com/aymanbagabas/go-osc52 v1.2.1 h1:q2sWUyDcozPLcLabEMd+a+7Ea2DitxZVN9hTxab9L4E=
|
||||
github.com/aymanbagabas/go-osc52 v1.2.1/go.mod h1:zT8H+Rk4VSabYN90pWyugflM3ZhpTZNC7cASDfUCdT4=
|
||||
github.com/aymanbagabas/go-osc52/v2 v2.0.1 h1:HwpRHbFMcZLEVr42D4p7XBqjyuxQH5SMiErDT4WkJ2k=
|
||||
github.com/aymanbagabas/go-osc52/v2 v2.0.1/go.mod h1:uYgXzlJ7ZpABp8OJ+exZzJJhRNQ2ASbcXHWsFqH8hp8=
|
||||
github.com/aymerick/douceur v0.2.0 h1:Mv+mAeH1Q+n9Fr+oyamOlAkUNPWPlA8PPGR0QAaYuPk=
|
||||
github.com/aymerick/douceur v0.2.0/go.mod h1:wlT5vV2O3h55X9m7iVYN0TBM0NH/MmbLnd30/FjWUq4=
|
||||
github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8=
|
||||
github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA=
|
||||
github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
|
||||
@@ -108,22 +95,13 @@ github.com/bodgit/sevenzip v1.3.0 h1:1ljgELgtHqvgIp8W8kgeEGHIWP4ch3xGI8uOBZgLVKY
|
||||
github.com/bodgit/sevenzip v1.3.0/go.mod h1:omwNcgZTEooWM8gA/IJ2Nk/+ZQ94+GsytRzOJJ8FBlM=
|
||||
github.com/bodgit/windows v1.0.0 h1:rLQ/XjsleZvx4fR1tB/UxQrK+SJ2OFHzfPjLWWOhDIA=
|
||||
github.com/bodgit/windows v1.0.0/go.mod h1:a6JLwrB4KrTR5hBpp8FI9/9W9jJfeQ2h4XDXU74ZCdM=
|
||||
github.com/bradleyfalzon/ghinstallation/v2 v2.6.0 h1:IRY7Xy588KylkoycsUhFpW7cdGpy5Y5BPsz4IfuJtGk=
|
||||
github.com/bradleyfalzon/ghinstallation/v2 v2.6.0/go.mod h1:oQ3etOwN3TRH4EwgW5/7MxSVMGlMlzG/O8TU7eYdoSk=
|
||||
github.com/bradleyfalzon/ghinstallation/v2 v2.4.0 h1:zYSzkoIwekCQAr6GT6KxISLt4YRS6kd4/ixfzMN+7yc=
|
||||
github.com/bradleyfalzon/ghinstallation/v2 v2.4.0/go.mod h1:4MwZLSgBJJgg4i3nJwZJ95AMooSqN8fJDmegLVn9Q2U=
|
||||
github.com/bwesterb/go-ristretto v1.2.0/go.mod h1:fUIoIZaG73pV5biE2Blr2xEzDoMj7NFEuV9ekS419A0=
|
||||
github.com/bwesterb/go-ristretto v1.2.3/go.mod h1:fUIoIZaG73pV5biE2Blr2xEzDoMj7NFEuV9ekS419A0=
|
||||
github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
|
||||
github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
|
||||
github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44=
|
||||
github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
|
||||
github.com/charmbracelet/bubbles v0.16.1 h1:6uzpAAaT9ZqKssntbvZMlksWHruQLNxg49H5WdeuYSY=
|
||||
github.com/charmbracelet/bubbles v0.16.1/go.mod h1:2QCp9LFlEsBQMvIYERr7Ww2H2bA7xen1idUDIzm/+Xc=
|
||||
github.com/charmbracelet/bubbletea v0.24.1 h1:LpdYfnu+Qc6XtvMz6d/6rRY71yttHTP5HtrjMgWvixc=
|
||||
github.com/charmbracelet/bubbletea v0.24.1/go.mod h1:rK3g/2+T8vOSEkNHvtq40umJpeVYDn6bLaqbgzhL/hg=
|
||||
github.com/charmbracelet/glamour v0.6.0 h1:wi8fse3Y7nfcabbbDuwolqTqMQPMnVPeZhDM273bISc=
|
||||
github.com/charmbracelet/glamour v0.6.0/go.mod h1:taqWV4swIMMbWALc0m7AfE9JkPSU8om2538k9ITBxOc=
|
||||
github.com/charmbracelet/lipgloss v0.7.1 h1:17WMwi7N1b1rVWOjMT+rCh7sQkvDU75B2hbZpc5Kc1E=
|
||||
github.com/charmbracelet/lipgloss v0.7.1/go.mod h1:yG0k3giv8Qj8edTCbbg6AlQ5e8KNWpFujkNawKNhE2c=
|
||||
github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
|
||||
github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
|
||||
github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
|
||||
@@ -139,12 +117,11 @@ github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWH
|
||||
github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
|
||||
github.com/connesc/cipherio v0.2.1 h1:FGtpTPMbKNNWByNrr9aEBtaJtXjqOzkIXNYJp6OEycw=
|
||||
github.com/connesc/cipherio v0.2.1/go.mod h1:ukY0MWJDFnJEbXMQtOcn2VmTpRfzcTz4OoVrWGGJZcA=
|
||||
github.com/containerd/console v1.0.4-0.20230313162750-1ae8d489ac81 h1:q2hJAaP1k2wIvVRd/hEHD7lacgqrCPS+k8g1MndzfWY=
|
||||
github.com/containerd/console v1.0.4-0.20230313162750-1ae8d489ac81/go.mod h1:YynlIjWYF8myEu6sdkwKIvGQq+cOckRm6So2avqoYAk=
|
||||
github.com/containerd/stargz-snapshotter/estargz v0.14.3 h1:OqlDCK3ZVUO6C3B/5FSkDwbkEETK84kQgEeFwDC+62k=
|
||||
github.com/containerd/stargz-snapshotter/estargz v0.14.3/go.mod h1:KY//uOCIkSuNAHhJogcZtrNHdKrA99/FCCRjE3HD36o=
|
||||
github.com/couchbase/gocb/v2 v2.6.3 h1:5RsMo+RRfK0mVxHLAfpBz3/tHlgXZb1WBNItLk9Ab+c=
|
||||
github.com/couchbase/gocb/v2 v2.6.3/go.mod h1:yF5F6BHTZ/ZowhEuZbySbXrlI4rHd1TIhm5azOaMbJU=
|
||||
github.com/couchbase/gocbcore/v10 v10.2.3/go.mod h1:lYQIIk+tzoMcwtwU5GzPbDdqEkwkH3isI2rkSpfL0oM=
|
||||
github.com/couchbase/gocbcore/v10 v10.2.4 h1:TDTQ1mSBUw9eajuV71ZDJcLRbmfGcsvZV9SMeUDAVGc=
|
||||
github.com/couchbase/gocbcore/v10 v10.2.4/go.mod h1:lYQIIk+tzoMcwtwU5GzPbDdqEkwkH3isI2rkSpfL0oM=
|
||||
github.com/couchbaselabs/gocaves/client v0.0.0-20230307083111-cc3960c624b1/go.mod h1:AVekAZwIY2stsJOMWLAS/0uA/+qdp7pjO8EHnl61QkY=
|
||||
@@ -161,8 +138,6 @@ github.com/denisenkom/go-mssqldb v0.12.3 h1:pBSGx9Tq67pBOTLmxNuirNTeB8Vjmf886Kx+
|
||||
github.com/denisenkom/go-mssqldb v0.12.3/go.mod h1:k0mtMFOnU+AihqFxPMiF05rtiDrorD1Vrm1KEz5hxDo=
|
||||
github.com/dimchansky/utfbom v1.1.1 h1:vV6w1AhK4VMnhBno/TPVCoK9U/LP0PkLCS9tbxHdi/U=
|
||||
github.com/dimchansky/utfbom v1.1.1/go.mod h1:SxdoEBH5qIqFocHMyGOXVAybYJdr71b1Q/j0mACtrfE=
|
||||
github.com/dlclark/regexp2 v1.4.0 h1:F1rxgk7p4uKjwIQxBs9oAXe5CqrXlCduYEJvrF4u93E=
|
||||
github.com/dlclark/regexp2 v1.4.0/go.mod h1:2pZnwuY/m+8K6iRw6wQdMtk+rH5tNGR1i55kozfMjCc=
|
||||
github.com/dnaeon/go-vcr v1.2.0/go.mod h1:R4UdLID7HZT3taECzJs4YgbbH6PIGXB6W/sc5OLb6RQ=
|
||||
github.com/docker/cli v23.0.5+incompatible h1:ufWmAOuD3Vmr7JP2G5K3cyuNC4YZWiAsuDEvFVVDafE=
|
||||
github.com/docker/cli v23.0.5+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8=
|
||||
@@ -206,8 +181,8 @@ github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376/go.mod h1:an3vInlBmS
|
||||
github.com/go-git/go-billy/v5 v5.4.1 h1:Uwp5tDRkPr+l/TnbHOQzp+tmJfLceOlbVucgpTz8ix4=
|
||||
github.com/go-git/go-billy/v5 v5.4.1/go.mod h1:vjbugF6Fz7JIflbVpl1hJsGjSHNltrSw45YK/ukIvQg=
|
||||
github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20230305113008-0c11038e723f h1:Pz0DHeFij3XFhoBRGUDPzSJ+w2UcK5/0JvF8DRI58r8=
|
||||
github.com/go-git/go-git/v5 v5.8.1 h1:Zo79E4p7TRk0xoRgMq0RShiTHGKcKI4+DI6BfJc/Q+A=
|
||||
github.com/go-git/go-git/v5 v5.8.1/go.mod h1:FHFuoD6yGz5OSKEBK+aWN9Oah0q54Jxl0abmj6GnqAo=
|
||||
github.com/go-git/go-git/v5 v5.8.0 h1:Rc543s6Tyq+YcyPwZRvU4jzZGM8rB/wWu94TnTIYALQ=
|
||||
github.com/go-git/go-git/v5 v5.8.0/go.mod h1:coJHKEOk5kUClpsNlXrUvPrDxY3w3gjHvhcZd8Fodw8=
|
||||
github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
|
||||
github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
|
||||
github.com/go-ldap/ldap/v3 v3.4.5 h1:ekEKmaDrpvR2yf5Nc/DClsGG9lAmdDixe44mLzlW5r8=
|
||||
@@ -283,12 +258,12 @@ github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/
|
||||
github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
|
||||
github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
|
||||
github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
|
||||
github.com/google/go-containerregistry v0.15.2 h1:MMkSh+tjSdnmJZO7ljvEqV1DjfekB6VUEAZgy3a+TQE=
|
||||
github.com/google/go-containerregistry v0.15.2/go.mod h1:wWK+LnOv4jXMM23IT/F1wdYftGWGr47Is8CG+pmHK1Q=
|
||||
github.com/google/go-containerregistry v0.14.0 h1:z58vMqHxuwvAsVwvKEkmVBz2TlgBgH5k6koEXBtlYkw=
|
||||
github.com/google/go-containerregistry v0.14.0/go.mod h1:aiJ2fp/SXvkWgmYHioXnbMdlgB8eXiiYOY55gfN91Wk=
|
||||
github.com/google/go-github/v42 v42.0.0 h1:YNT0FwjPrEysRkLIiKuEfSvBPCGKphW5aS5PxwaoLec=
|
||||
github.com/google/go-github/v42 v42.0.0/go.mod h1:jgg/jvyI0YlDOM1/ps6XYh04HNQ3vKf0CVko62/EhRg=
|
||||
github.com/google/go-github/v53 v53.2.0 h1:wvz3FyF53v4BK+AsnvCmeNhf8AkTaeh2SoYu/XUvTtI=
|
||||
github.com/google/go-github/v53 v53.2.0/go.mod h1:XhFRObz+m/l+UCm9b7KSIC3lT3NWSXGt7mOsAWEloao=
|
||||
github.com/google/go-github/v52 v52.0.0 h1:uyGWOY+jMQ8GVGSX8dkSwCzlehU3WfdxQ7GweO/JP7M=
|
||||
github.com/google/go-github/v52 v52.0.0/go.mod h1:WJV6VEEUPuMo5pXqqa2ZCZEdbQqua4zAk2MZTIo+m+4=
|
||||
github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8=
|
||||
github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU=
|
||||
github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPgecwXBIDzw5no=
|
||||
@@ -311,12 +286,8 @@ github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+
|
||||
github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
|
||||
github.com/googleapis/gax-go/v2 v2.12.0 h1:A+gCJKdRfqXkr+BIRGtZLibNXf0m1f9E4HG56etFpas=
|
||||
github.com/googleapis/gax-go/v2 v2.12.0/go.mod h1:y+aIqrI5eb1YGMVJfuV3185Ts/D7qKpsEkdD5+I6QGU=
|
||||
github.com/gorilla/css v1.0.0 h1:BQqNyPTi50JCFMTw/b67hByjMVXZRwGha6wxVGkeihY=
|
||||
github.com/gorilla/css v1.0.0/go.mod h1:Dn721qIggHpt4+EFCcTLTU/vk5ySda2ReITrtgBl60c=
|
||||
github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
|
||||
github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw=
|
||||
github.com/h2non/filetype v1.1.3 h1:FKkx9QbD7HR/zjK1Ia5XiBsq9zdLi5Kf3zGyFTAFkGg=
|
||||
github.com/h2non/filetype v1.1.3/go.mod h1:319b3zT68BvV+WRj7cwy856M2ehB3HqNOt6sy1HndBY=
|
||||
github.com/h2non/parth v0.0.0-20190131123155-b4df798d6542 h1:2VTzZjLZBgl62/EtslCrtky5vbi9dd7HrQPQIx6wqiw=
|
||||
github.com/h2non/parth v0.0.0-20190131123155-b4df798d6542/go.mod h1:Ow0tF8D4Kplbc8s8sSb3V2oUCygFHVp8gC3Dn6U4MNI=
|
||||
github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA=
|
||||
@@ -330,11 +301,12 @@ github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9
|
||||
github.com/hashicorp/go-retryablehttp v0.7.4 h1:ZQgVdpTdAL7WpMIwLzCfbalOcSUdkDZnpUv3/+BxzFA=
|
||||
github.com/hashicorp/go-retryablehttp v0.7.4/go.mod h1:Jy/gPYAdjqffZ/yFGCFV2doI5wjtH1ewM9u8iYVjtX8=
|
||||
github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
|
||||
github.com/hashicorp/golang-lru v0.5.1 h1:0hERBMJE1eitiLkihrMvRVBYAkpHzc/J3QdDN+dAcgU=
|
||||
github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
|
||||
github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
|
||||
github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
|
||||
github.com/ianlancetaylor/demangle v0.0.0-20210905161508-09a460cdf81d/go.mod h1:aYm2/VgdVmcIU8iMfdMvDMsRAQjcfZSKFby6HOFvi/w=
|
||||
github.com/imdario/mergo v0.3.15 h1:M8XP7IuFNsqUx6VPK2P9OSmsYsI/YFaGil0uD21V3dM=
|
||||
github.com/imdario/mergo v0.3.15/go.mod h1:WBLT9ZmE3lPoWsEzCh9LPo3TiwVN+ZKEjmz+hD27ysY=
|
||||
github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOlocH6Fxy8MmwDt+yVQYULKfN0RoTN8A=
|
||||
github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i6rXxKeerYnT8Nvf0QmHCRC1n8sfWVwXF2Frvo=
|
||||
github.com/jlaffaye/ftp v0.2.0 h1:lXNvW7cBu7R/68bknOX3MrRIIqZ61zELs1P2RAiA3lg=
|
||||
@@ -370,46 +342,24 @@ github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0
|
||||
github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
|
||||
github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw=
|
||||
github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
|
||||
github.com/lrstanley/bubblezone v0.0.0-20221222153816-e95291e2243e h1:XoxHx8K6ZKoMtjzWOMDuM69LCdjDDsTOtTfWGrT/fns=
|
||||
github.com/lrstanley/bubblezone v0.0.0-20221222153816-e95291e2243e/go.mod h1:v5lEwWaguF1o2MW/ucO0ZIA/IZymdBYJJ+2cMRLE7LU=
|
||||
github.com/lucasb-eyer/go-colorful v1.2.0 h1:1nnpGOrhyZZuNyfu1QjKiUICQ74+3FNCN69Aj6K7nkY=
|
||||
github.com/lucasb-eyer/go-colorful v1.2.0/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i7ruzyGqttikkLy0=
|
||||
github.com/matryer/is v1.2.0 h1:92UTHpy8CDwaJ08GqLDzhhuixiBUUD1p3AU6PHddz4A=
|
||||
github.com/matryer/is v1.2.0/go.mod h1:2fLPjFQM9rhQ15aVEtbuwhJinnOqrmgXPNdZsdwlWXA=
|
||||
github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=
|
||||
github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg=
|
||||
github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
|
||||
github.com/mattn/go-isatty v0.0.18 h1:DOKFKCQ7FNG2L1rbrmstDN4QVRdS89Nkh85u68Uwp98=
|
||||
github.com/mattn/go-isatty v0.0.18/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
|
||||
github.com/mattn/go-localereader v0.0.1 h1:ygSAOl7ZXTx4RdPYinUpg6W99U8jWvWi9Ye2JC/oIi4=
|
||||
github.com/mattn/go-localereader v0.0.1/go.mod h1:8fBrzywKY7BI3czFoHkuzRoWE9C+EiG4R1k4Cjx5p88=
|
||||
github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI=
|
||||
github.com/mattn/go-runewidth v0.0.12/go.mod h1:RAqKPSqVFrSLVXbA8x7dzmKdmGzieGRCM46jaSJTDAk=
|
||||
github.com/mattn/go-runewidth v0.0.14 h1:+xnbZSEeDbOIg5/mE6JF0w6n9duR1l3/WmbinWVwUuU=
|
||||
github.com/mattn/go-runewidth v0.0.14/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
|
||||
github.com/mattn/go-isatty v0.0.17 h1:BTarxUcIeDqL27Mc+vyvdWYSL28zpIhv3RoTdsLMPng=
|
||||
github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
|
||||
github.com/mattn/go-sqlite3 v1.14.17 h1:mCRHCLDUBXgpKAqIKsaAaAsrAlbkeomtRFKXh2L6YIM=
|
||||
github.com/mattn/go-sqlite3 v1.14.17/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg=
|
||||
github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo=
|
||||
github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4=
|
||||
github.com/mholt/archiver/v4 v4.0.0-alpha.8 h1:tRGQuDVPh66WCOelqe6LIGh0gwmfwxUrSSDunscGsRM=
|
||||
github.com/mholt/archiver/v4 v4.0.0-alpha.8/go.mod h1:5f7FUYGXdJWUjESffJaYR4R60VhnHxb2X3T1teMyv5A=
|
||||
github.com/microcosm-cc/bluemonday v1.0.21/go.mod h1:ytNkv4RrDrLJ2pqlsSI46O6IVXmZOBBD4SaJyDwwTkM=
|
||||
github.com/microcosm-cc/bluemonday v1.0.23 h1:SMZe2IGa0NuHvnVNAZ+6B38gsTbi5e4sViiWJyDDqFY=
|
||||
github.com/microcosm-cc/bluemonday v1.0.23/go.mod h1:mN70sk7UkkF8TUr2IGBpNN0jAgStuPzlK76QuruE/z4=
|
||||
github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y=
|
||||
github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
|
||||
github.com/modocache/gover v0.0.0-20171022184752-b58185e213c5/go.mod h1:caMODM3PzxT8aQXRPkAt8xlV/e7d7w8GM5g0fa5F0D8=
|
||||
github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe h1:iruDEfMl2E6fbMZ9s0scYfZQ84/6SPL6zC8ACM2oIL0=
|
||||
github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc=
|
||||
github.com/muesli/ansi v0.0.0-20211031195517-c9f0611b6c70 h1:kMlmsLSbjkikxQJ1IPwaM+7LJ9ltFu/fi8CRzvSnQmA=
|
||||
github.com/muesli/ansi v0.0.0-20211031195517-c9f0611b6c70/go.mod h1:fQuZ0gauxyBcmsdE3ZT4NasjaRdxmbCS0jRHsrWu3Ho=
|
||||
github.com/muesli/cancelreader v0.2.2 h1:3I4Kt4BQjOR54NavqnDogx/MIoWBFa0StPA8ELUXHmA=
|
||||
github.com/muesli/cancelreader v0.2.2/go.mod h1:3XuTXfFS2VjM+HTLZY9Ak0l6eUKfijIfMUZ4EgX0QYo=
|
||||
github.com/muesli/reflow v0.3.0 h1:IFsN6K9NfGtjeggFP+68I4chLZV2yIKsXJFNZ+eWh6s=
|
||||
github.com/muesli/reflow v0.3.0/go.mod h1:pbwTDkVPibjO2kyvBQRBxTWEEGDGq0FlB1BIKtnHY/8=
|
||||
github.com/muesli/termenv v0.13.0/go.mod h1:sP1+uffeLaEYpyOTb8pLCUctGcGLnoFjSn4YJK5e2bc=
|
||||
github.com/muesli/termenv v0.15.1 h1:UzuTb/+hhlBugQz28rpzey4ZuKcZ03MeKsoG7IJZIxs=
|
||||
github.com/muesli/termenv v0.15.1/go.mod h1:HeAQPTzpfs016yGtA4g00CsdYnVLJvxsS4ANqrZs2sQ=
|
||||
github.com/nbio/st v0.0.0-20140626010706-e9e8d9816f32 h1:W6apQkHrMkS0Muv8G/TipAy/FJl/rCYT0+EuS8+Z0z4=
|
||||
github.com/nbio/st v0.0.0-20140626010706-e9e8d9816f32/go.mod h1:9wM+0iRr9ahx58uYLpLIr5fm8diHn0JbqRycJi6w0Ms=
|
||||
github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
|
||||
@@ -418,8 +368,6 @@ github.com/nwaples/rardecode/v2 v2.0.0-beta.2/go.mod h1:yntwv/HfMc/Hbvtq9I19D1n5
|
||||
github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A=
|
||||
github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE=
|
||||
github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU=
|
||||
github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec=
|
||||
github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY=
|
||||
github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
|
||||
github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk=
|
||||
github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE=
|
||||
@@ -461,26 +409,20 @@ github.com/prometheus/procfs v0.10.1 h1:kYK1Va/YMlutzCGazswoHKo//tZVlFpKYh+Pymzi
|
||||
github.com/prometheus/procfs v0.10.1/go.mod h1:nwNm2aOCAYw8uTR/9bWRREkZFxAUcWzPHWJq+XBB/FM=
|
||||
github.com/rabbitmq/amqp091-go v1.8.1 h1:RejT1SBUim5doqcL6s7iN6SBmsQqyTgXb1xMlH0h1hA=
|
||||
github.com/rabbitmq/amqp091-go v1.8.1/go.mod h1:+jPrT9iY2eLjRaMSRHUhc3z14E/l85kv/f+6luSD3pc=
|
||||
github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
|
||||
github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
|
||||
github.com/rivo/uniseg v0.4.2 h1:YwD0ulJSJytLpiaWua0sBDusfsCZohxjxzVTYjwxfV8=
|
||||
github.com/rivo/uniseg v0.4.2/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
|
||||
github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ=
|
||||
github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
|
||||
github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ=
|
||||
github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog=
|
||||
github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
|
||||
github.com/rwcarlsen/goexif v0.0.0-20190401172101-9e8deecbddbd/go.mod h1:hPqNNc0+uJM6H+SuU8sEs5K5IQeKccPqeSjfgcKGgPk=
|
||||
github.com/sahilm/fuzzy v0.1.0 h1:FzWGaw2Opqyu+794ZQ9SYifWv2EIXpwP4q8dY1kDAwI=
|
||||
github.com/sahilm/fuzzy v0.1.0/go.mod h1:VFvziUEIMCrT6A6tw2RFIXPXXmzXbOsSHF0DOI8ZK9Y=
|
||||
github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
|
||||
github.com/sergi/go-diff v1.3.1 h1:xkr+Oxo4BOQKmkn/B9eMK0g5Kg/983T9DqqPHwYqD+8=
|
||||
github.com/sergi/go-diff v1.3.1/go.mod h1:aMJSSKb2lpPvRNec0+w3fl7LP9IOFzdc9Pa4NFbPK1I=
|
||||
github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
|
||||
github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0=
|
||||
github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
|
||||
github.com/skeema/knownhosts v1.2.0 h1:h9r9cf0+u7wSE+M183ZtMGgOJKiL96brpaz5ekfJCpM=
|
||||
github.com/skeema/knownhosts v1.2.0/go.mod h1:g4fPeYpque7P0xefxtGzV81ihjC8sX2IqpAoNkjxbMo=
|
||||
github.com/skeema/knownhosts v1.1.1 h1:MTk78x9FPgDFVFkDLTrsnnfCJl7g1C/nnKvePgrIngE=
|
||||
github.com/skeema/knownhosts v1.1.1/go.mod h1:g4fPeYpque7P0xefxtGzV81ihjC8sX2IqpAoNkjxbMo=
|
||||
github.com/smartystreets/assertions v1.0.1 h1:voD4ITNjPL5jjBfgR/r8fPIIBrliWrWHeiJApdr3r4w=
|
||||
github.com/smartystreets/assertions v1.0.1/go.mod h1:kHHU4qYBaI3q23Pp3VPrmWhuIUrLW/7eUrw0BU5VaoM=
|
||||
github.com/smartystreets/gunit v1.1.3 h1:32x+htJCu3aMswhPw3teoJ+PnWPONqdNgaGs6Qt8ZaU=
|
||||
@@ -513,8 +455,8 @@ github.com/ulikunitz/xz v0.5.10/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0o
|
||||
github.com/urfave/cli v1.22.12/go.mod h1:sSBEIC79qR6OvcmsD4U3KABeOTxDqQtdDnaFuUN30b8=
|
||||
github.com/vbatts/tar-split v0.11.3 h1:hLFqsOLQ1SsppQNTMpkpPXClLDfC2A3Zgy9OUU+RVck=
|
||||
github.com/vbatts/tar-split v0.11.3/go.mod h1:9QlHN18E+fEH7RdG+QAJJcuya3rqT7eXSTY7wGrAokY=
|
||||
github.com/xanzy/go-gitlab v0.88.0 h1:9GHBrxyCUNZZNuAsbJ1NbEH6XAYsKyTn6NfE0wYO5SY=
|
||||
github.com/xanzy/go-gitlab v0.88.0/go.mod h1:5ryv+MnpZStBH8I/77HuQBsMbBGANtVpLWC15qOjWAw=
|
||||
github.com/xanzy/go-gitlab v0.86.0 h1:jR8V9cK9jXRQDb46KOB20NCF3ksY09luaG0IfXE6p7w=
|
||||
github.com/xanzy/go-gitlab v0.86.0/go.mod h1:5ryv+MnpZStBH8I/77HuQBsMbBGANtVpLWC15qOjWAw=
|
||||
github.com/xanzy/ssh-agent v0.3.3 h1:+/15pJfg/RsTxqYcX6fHqOXZwwMP+2VyYWJeWM2qQFM=
|
||||
github.com/xanzy/ssh-agent v0.3.3/go.mod h1:6dzNDKs0J9rVPHPhaGCukekBHKqfl+L3KghI1Bc68Uw=
|
||||
github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c=
|
||||
@@ -528,10 +470,6 @@ github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7Jul
|
||||
github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
|
||||
github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
|
||||
github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
|
||||
github.com/yuin/goldmark v1.5.2 h1:ALmeCk/px5FSm1MAcFBAsVKZjDuMVj8Tm7FFIlMJnqU=
|
||||
github.com/yuin/goldmark v1.5.2/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
|
||||
github.com/yuin/goldmark-emoji v1.0.1 h1:ctuWEyzGBwiucEqxzwe0SOYDXPAucOrE9NQC18Wa1os=
|
||||
github.com/yuin/goldmark-emoji v1.0.1/go.mod h1:2w1E6FEWLcDQkoTE+7HU6QF1F6SLlNGjRIBbIZQFqkQ=
|
||||
github.com/yusufpapurcu/wmi v1.2.2 h1:KBNDSne4vP5mbSWnJbO+51IMOXJB67QiYCSBrubbPRg=
|
||||
github.com/yusufpapurcu/wmi v1.2.2/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0=
|
||||
go.mongodb.org/mongo-driver v1.12.0 h1:aPx33jmn/rQuJXPQLZQ8NtfPQG8CaqgLThFtqRb0PiE=
|
||||
@@ -548,8 +486,6 @@ go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
|
||||
go.uber.org/goleak v1.1.11/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ=
|
||||
go.uber.org/goleak v1.2.1 h1:NBol2c7O1ZokfZ0LEU9K6Whx/KnwvepVetCUhtKja4A=
|
||||
go.uber.org/goleak v1.2.1/go.mod h1:qlT2yGI9QafXHhZZLxlSuNsMw3FFLxBr+tBRlmO1xH4=
|
||||
go.uber.org/mock v0.2.0 h1:TaP3xedm7JaAgScZO7tlvlKrqT0p7I6OsdGB5YNSMDU=
|
||||
go.uber.org/mock v0.2.0/go.mod h1:J0y0rp9L3xiff1+ZBfKxlC1fz2+aO16tw0tsDOixfuM=
|
||||
go.uber.org/multierr v1.6.0 h1:y6IPFStTAIT5Ytl7/XYmHvzXQ7S3g/IeZW9hyZ5thw4=
|
||||
go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU=
|
||||
go.uber.org/zap v1.24.0 h1:FiJd5l1UOLj0wCgbSE0rwwXHzEdAZS6hiiSnxJN/D60=
|
||||
@@ -566,7 +502,6 @@ golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5y
|
||||
golang.org/x/crypto v0.0.0-20211215153901-e495a2d5b3d3/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
|
||||
golang.org/x/crypto v0.0.0-20220314234659-1baeb1ce4c0b/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
|
||||
golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
|
||||
golang.org/x/crypto v0.3.1-0.20221117191849-2c476679df9a/go.mod h1:hebNnKkNXi2UzZN1eVRvBB7co0a+JxK6XbPiWVs/3J4=
|
||||
golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU=
|
||||
golang.org/x/crypto v0.11.0 h1:6Ewdq3tDic1mg5xRO4milcWCfMVQhI4NkqWWvqejpuA=
|
||||
golang.org/x/crypto v0.11.0/go.mod h1:xgJhtzW8F9jGdVFWZESrid1U1bjeNy4zgy5cRr/CIio=
|
||||
@@ -629,11 +564,9 @@ golang.org/x/net v0.0.0-20210610132358-84b48f89b13b/go.mod h1:9nx3DQGgdP8bBQD5qx
|
||||
golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
|
||||
golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
|
||||
golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
|
||||
golang.org/x/net v0.0.0-20221002022538-bcab6841153b/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk=
|
||||
golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY=
|
||||
golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
|
||||
golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc=
|
||||
golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg=
|
||||
golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns=
|
||||
golang.org/x/net v0.12.0 h1:cfawfvKITfUsFCeJIHJrbSxpeu/E81khclypR0GVT50=
|
||||
golang.org/x/net v0.12.0/go.mod h1:zEVYFnQC7m/vmpQFELhcD1EWkZlX69l4oqgmer6hfKA=
|
||||
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
|
||||
@@ -641,7 +574,7 @@ golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4Iltr
|
||||
golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
|
||||
golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
|
||||
golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
|
||||
golang.org/x/oauth2 v0.8.0/go.mod h1:yr7u4HXZRm1R1kBWqr/xKNqewf0plRYoB7sla+BCIXE=
|
||||
golang.org/x/oauth2 v0.7.0/go.mod h1:hPLQkd9LyjfXTiRohC/41GhcFqxisoUQ99sCUOHO9x4=
|
||||
golang.org/x/oauth2 v0.10.0 h1:zHCpF2Khkwy4mMB4bv0U37YtJdTGW8jI0glAApi0Kh8=
|
||||
golang.org/x/oauth2 v0.10.0/go.mod h1:kTpgurOux7LqtuxjuyZa4Gj2gdezIt/jQtGnNFfypQI=
|
||||
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
@@ -690,25 +623,20 @@ golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBc
|
||||
golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220906165534-d0df966e6959/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.10.0 h1:SqMFp9UcQJZa+pmYuAKjd9xq1f0j5rLcDIk0mj4qAsA=
|
||||
golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
|
||||
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
|
||||
golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc=
|
||||
golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
|
||||
golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U=
|
||||
golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo=
|
||||
golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY=
|
||||
golang.org/x/term v0.10.0 h1:3R7pNqamzBraeqj/Tj8qt1aQ2HpmlC+Cx/qL/7hn4/c=
|
||||
golang.org/x/term v0.10.0/go.mod h1:lpqdcUyK/oCiQxvxVrppt5ggO2KCZ5QblwqPnfZ6d5o=
|
||||
golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
@@ -717,7 +645,6 @@ golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
|
||||
golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
|
||||
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
|
||||
golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ=
|
||||
golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
|
||||
golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
|
||||
golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
|
||||
golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
|
||||
|
||||
@@ -9,11 +9,11 @@ import (
|
||||
"strconv"
|
||||
"strings"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"github.com/felixge/fgprof"
|
||||
"github.com/go-logr/logr"
|
||||
"github.com/jpillora/overseer"
|
||||
"github.com/mattn/go-isatty"
|
||||
"google.golang.org/protobuf/types/known/anypb"
|
||||
"gopkg.in/alecthomas/kingpin.v2"
|
||||
|
||||
@@ -29,7 +29,6 @@ import (
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/sources/git"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/tui"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/updater"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/version"
|
||||
)
|
||||
@@ -40,7 +39,6 @@ var (
|
||||
debug = cli.Flag("debug", "Run in debug mode.").Bool()
|
||||
trace = cli.Flag("trace", "Run in trace mode.").Bool()
|
||||
profile = cli.Flag("profile", "Enables profiling and sets a pprof and fgprof server on :18066.").Bool()
|
||||
localDev = cli.Flag("local-dev", "Hidden feature to disable overseer for local dev.").Hidden().Bool()
|
||||
jsonOut = cli.Flag("json", "Output in JSON format.").Short('j').Bool()
|
||||
jsonLegacy = cli.Flag("json-legacy", "Use the pre-v3.0 JSON format. Only works with git, gitlab, and github sources.").Bool()
|
||||
gitHubActionsFormat = cli.Flag("github-actions", "Output in GitHub Actions format.").Bool()
|
||||
@@ -68,7 +66,6 @@ var (
|
||||
gitScanSinceCommit = gitScan.Flag("since-commit", "Commit to start scan from.").String()
|
||||
gitScanBranch = gitScan.Flag("branch", "Branch to scan.").String()
|
||||
gitScanMaxDepth = gitScan.Flag("max-depth", "Maximum depth of commits to scan.").Int()
|
||||
gitScanBare = gitScan.Flag("bare", "Scan bare repository (e.g. useful while using in pre-receive hooks)").Bool()
|
||||
_ = gitScan.Flag("allow", "No-op flag for backwards compat.").Bool()
|
||||
_ = gitScan.Flag("entropy", "No-op flag for backwards compat.").Bool()
|
||||
_ = gitScan.Flag("regex", "No-op flag for backwards compat.").Bool()
|
||||
@@ -146,18 +143,6 @@ func init() {
|
||||
}
|
||||
|
||||
cli.Version("trufflehog " + version.BuildVersion)
|
||||
|
||||
if len(os.Args) <= 1 && isatty.IsTerminal(os.Stdout.Fd()) {
|
||||
args := tui.Run()
|
||||
if len(args) == 0 {
|
||||
os.Exit(0)
|
||||
}
|
||||
|
||||
// Overwrite the Args slice so overseer works properly.
|
||||
os.Args = os.Args[:1]
|
||||
os.Args = append(os.Args, args...)
|
||||
}
|
||||
|
||||
cmd = kingpin.MustParse(cli.Parse(os.Args[1:]))
|
||||
|
||||
switch {
|
||||
@@ -177,12 +162,6 @@ func main() {
|
||||
logger, sync := log.New("trufflehog", logFormat(os.Stderr))
|
||||
// make it the default logger for contexts
|
||||
context.SetDefaultLogger(logger)
|
||||
|
||||
if *localDev {
|
||||
run(overseer.State{})
|
||||
os.Exit(0)
|
||||
}
|
||||
|
||||
defer func() { _ = sync() }()
|
||||
logFatal := logFatalFunc(logger)
|
||||
|
||||
@@ -337,21 +316,8 @@ func run(state overseer.State) {
|
||||
return true
|
||||
}
|
||||
|
||||
// Set how the engine will print its results.
|
||||
var printer engine.Printer
|
||||
switch {
|
||||
case *jsonLegacy:
|
||||
printer = new(output.LegacyJSONPrinter)
|
||||
case *jsonOut:
|
||||
printer = new(output.JSONPrinter)
|
||||
case *gitHubActionsFormat:
|
||||
printer = new(output.GitHubActionsPrinter)
|
||||
default:
|
||||
printer = new(output.PlainPrinter)
|
||||
}
|
||||
|
||||
e, err := engine.Start(ctx,
|
||||
engine.WithConcurrency(uint8(*concurrency)),
|
||||
e := engine.Start(ctx,
|
||||
engine.WithConcurrency(*concurrency),
|
||||
engine.WithDecoders(decoders.DefaultDecoders()...),
|
||||
engine.WithDetectors(!*noVerification, engine.DefaultDetectors()...),
|
||||
engine.WithDetectors(!*noVerification, conf.Detectors...),
|
||||
@@ -359,13 +325,7 @@ func run(state overseer.State) {
|
||||
engine.WithFilterDetectors(excludeFilter),
|
||||
engine.WithFilterDetectors(endpointCustomizer),
|
||||
engine.WithFilterUnverified(*filterUnverified),
|
||||
engine.WithOnlyVerified(*onlyVerified),
|
||||
engine.WithPrintAvgDetectorTime(*printAvgDetectorTime),
|
||||
engine.WithPrinter(printer),
|
||||
)
|
||||
if err != nil {
|
||||
logFatal(err, "error initializing engine")
|
||||
}
|
||||
|
||||
var repoPath string
|
||||
var remote bool
|
||||
@@ -392,7 +352,6 @@ func run(state overseer.State) {
|
||||
HeadRef: *gitScanBranch,
|
||||
BaseRef: *gitScanSinceCommit,
|
||||
MaxDepth: *gitScanMaxDepth,
|
||||
Bare: *gitScanBare,
|
||||
Filter: filter,
|
||||
ExcludeGlobs: excludedGlobs,
|
||||
}
|
||||
@@ -516,49 +475,52 @@ func run(state overseer.State) {
|
||||
logFatal(err, "Failed to scan Docker.")
|
||||
}
|
||||
}
|
||||
// asynchronously wait for scanning to finish and cleanup
|
||||
go e.Finish(ctx, logFatal)
|
||||
|
||||
if !*jsonLegacy && !*jsonOut {
|
||||
fmt.Fprintf(os.Stderr, "🐷🔑🐷 TruffleHog. Unearth your secrets. 🐷🔑🐷\n\n")
|
||||
}
|
||||
|
||||
// Wait for all workers to finish.
|
||||
if err = e.Finish(ctx); err != nil {
|
||||
logFatal(err, "engine failed to finish execution")
|
||||
}
|
||||
// NOTE: this loop will terminate when the results channel is closed in
|
||||
// e.Finish()
|
||||
foundResults := false
|
||||
for r := range e.ResultsChan() {
|
||||
if *onlyVerified && !r.Verified {
|
||||
continue
|
||||
}
|
||||
foundResults = true
|
||||
|
||||
metrics := e.GetMetrics()
|
||||
// Print results.
|
||||
logger.Info("finished scanning",
|
||||
"chunks", metrics.ChunksScanned,
|
||||
"bytes", metrics.BytesScanned,
|
||||
"verified_secrets", metrics.VerifiedSecretsFound,
|
||||
"unverified_secrets", metrics.UnverifiedSecretsFound,
|
||||
"scan_duration", metrics.ScanDuration.String(),
|
||||
var err error
|
||||
switch {
|
||||
case *jsonLegacy:
|
||||
err = output.PrintLegacyJSON(ctx, &r)
|
||||
case *jsonOut:
|
||||
err = output.PrintJSON(&r)
|
||||
case *gitHubActionsFormat:
|
||||
err = output.PrintGitHubActionsOutput(&r)
|
||||
default:
|
||||
err = output.PrintPlainOutput(&r)
|
||||
}
|
||||
if err != nil {
|
||||
logFatal(err, "error printing results")
|
||||
}
|
||||
}
|
||||
logger.V(2).Info("finished scanning",
|
||||
"chunks", e.ChunksScanned(),
|
||||
"bytes", e.BytesScanned(),
|
||||
)
|
||||
|
||||
if *printAvgDetectorTime {
|
||||
printAverageDetectorTime(e)
|
||||
}
|
||||
|
||||
if e.HasFoundResults() && *fail {
|
||||
if foundResults && *fail {
|
||||
logger.V(2).Info("exiting with code 183 because results were found")
|
||||
os.Exit(183)
|
||||
}
|
||||
}
|
||||
|
||||
// logFatalFunc returns a log.Fatal style function. Calling the returned
|
||||
// function will terminate the program without cleanup.
|
||||
func logFatalFunc(logger logr.Logger) func(error, string, ...any) {
|
||||
return func(err error, message string, keyAndVals ...any) {
|
||||
logger.Error(err, message, keyAndVals...)
|
||||
if err != nil {
|
||||
os.Exit(1)
|
||||
return
|
||||
}
|
||||
os.Exit(0)
|
||||
}
|
||||
}
|
||||
|
||||
func commaSeparatedToSlice(s []string) []string {
|
||||
var result []string
|
||||
for _, items := range s {
|
||||
@@ -575,8 +537,26 @@ func commaSeparatedToSlice(s []string) []string {
|
||||
|
||||
func printAverageDetectorTime(e *engine.Engine) {
|
||||
fmt.Fprintln(os.Stderr, "Average detector time is the measurement of average time spent on each detector when results are returned.")
|
||||
for detectorName, duration := range e.GetDetectorsMetrics() {
|
||||
fmt.Fprintf(os.Stderr, "%s: %s\n", detectorName, duration)
|
||||
for detectorName, durations := range e.DetectorAvgTime() {
|
||||
var total time.Duration
|
||||
for _, d := range durations {
|
||||
total += d
|
||||
}
|
||||
avgDuration := total / time.Duration(len(durations))
|
||||
fmt.Fprintf(os.Stderr, "%s: %s\n", detectorName, avgDuration)
|
||||
}
|
||||
}
|
||||
|
||||
// logFatalFunc returns a log.Fatal style function. Calling the returned
|
||||
// function will terminate the program without cleanup.
|
||||
func logFatalFunc(logger logr.Logger) func(error, string, ...any) {
|
||||
return func(err error, message string, keyAndVals ...any) {
|
||||
logger.Error(err, message, keyAndVals...)
|
||||
if err != nil {
|
||||
os.Exit(1)
|
||||
return
|
||||
}
|
||||
os.Exit(0)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -16,14 +16,9 @@ func IsDone(ctx context.Context) bool {
|
||||
// write would succeed, either operation will be performed randomly.
|
||||
func CancellableWrite[T any](ctx context.Context, ch chan<- T, item T) error {
|
||||
select {
|
||||
case <-ctx.Done(): // priority to context cancellation
|
||||
case ch <- item:
|
||||
return nil
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
default:
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
case ch <- item:
|
||||
return nil
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
+2
-2
@@ -100,14 +100,14 @@ func NewCustomTransport(T http.RoundTripper) *CustomTransport {
|
||||
return &CustomTransport{T}
|
||||
}
|
||||
|
||||
func ConstantResponseHttpClient(statusCode int, body string) *http.Client {
|
||||
func ConstantStatusHttpClient(statusCode int) *http.Client {
|
||||
return &http.Client{
|
||||
Timeout: DefaultResponseTimeout,
|
||||
Transport: FakeTransport{
|
||||
CreateResponse: func(req *http.Request) (*http.Response, error) {
|
||||
return &http.Response{
|
||||
Request: req,
|
||||
Body: io.NopCloser(strings.NewReader(body)),
|
||||
Body: io.NopCloser(strings.NewReader("")),
|
||||
StatusCode: statusCode,
|
||||
}, nil
|
||||
},
|
||||
|
||||
@@ -3,9 +3,7 @@ package common
|
||||
import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
"crypto/rand"
|
||||
"io"
|
||||
"math/big"
|
||||
"strings"
|
||||
)
|
||||
|
||||
@@ -51,16 +49,3 @@ func ResponseContainsSubstring(reader io.ReadCloser, target string) (bool, error
|
||||
}
|
||||
return false, nil
|
||||
}
|
||||
|
||||
var letters = []rune("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789")
|
||||
|
||||
// RandomID returns a random string of the given length.
|
||||
func RandomID(length int) string {
|
||||
b := make([]rune, length)
|
||||
for i := range b {
|
||||
randInt, _ := rand.Int(rand.Reader, big.NewInt(int64(len(letters))))
|
||||
b[i] = letters[randInt.Int64()]
|
||||
}
|
||||
|
||||
return string(b)
|
||||
}
|
||||
|
||||
+44
-16
@@ -2,24 +2,19 @@ package context
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"fmt"
|
||||
"runtime/debug"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/go-logr/logr"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/log"
|
||||
)
|
||||
|
||||
var (
|
||||
// defaultLogger can be set via SetDefaultLogger.
|
||||
// It is initialized to write to stderr. To disable, you can call
|
||||
// SetDefaultLogger with logr.Discard().
|
||||
defaultLogger logr.Logger
|
||||
defaultLogger logr.Logger = logr.Discard()
|
||||
)
|
||||
|
||||
func init() {
|
||||
defaultLogger, _ = log.New("context", log.WithConsoleSink(os.Stderr))
|
||||
}
|
||||
|
||||
// Context wraps context.Context and includes an additional Logger() method.
|
||||
type Context interface {
|
||||
context.Context
|
||||
@@ -34,7 +29,9 @@ type CancelFunc = context.CancelFunc
|
||||
type logCtx struct {
|
||||
// Embed context.Context to get all methods for free.
|
||||
context.Context
|
||||
log logr.Logger
|
||||
log logr.Logger
|
||||
err *error
|
||||
errLock *sync.Mutex
|
||||
}
|
||||
|
||||
// Logger returns a structured logger.
|
||||
@@ -42,6 +39,13 @@ func (l logCtx) Logger() logr.Logger {
|
||||
return l.log
|
||||
}
|
||||
|
||||
func (l logCtx) Err() error {
|
||||
if l.err != nil && *l.err != nil {
|
||||
return *l.err
|
||||
}
|
||||
return l.Context.Err()
|
||||
}
|
||||
|
||||
// Background returns context.Background with a default logger.
|
||||
func Background() Context {
|
||||
return logCtx{
|
||||
@@ -65,7 +69,7 @@ func WithCancel(parent Context) (Context, context.CancelFunc) {
|
||||
log: parent.Logger(),
|
||||
Context: ctx,
|
||||
}
|
||||
return lCtx, cancel
|
||||
return captureCancelCallstack(lCtx, cancel)
|
||||
}
|
||||
|
||||
// WithDeadline returns context.WithDeadline with the log object propagated and
|
||||
@@ -76,7 +80,7 @@ func WithDeadline(parent Context, d time.Time) (Context, context.CancelFunc) {
|
||||
log: parent.Logger().WithValues("deadline", d),
|
||||
Context: ctx,
|
||||
}
|
||||
return lCtx, cancel
|
||||
return captureCancelCallstack(lCtx, cancel)
|
||||
}
|
||||
|
||||
// WithTimeout returns context.WithTimeout with the log object propagated and
|
||||
@@ -87,7 +91,7 @@ func WithTimeout(parent Context, timeout time.Duration) (Context, context.Cancel
|
||||
log: parent.Logger().WithValues("timeout", timeout),
|
||||
Context: ctx,
|
||||
}
|
||||
return lCtx, cancel
|
||||
return captureCancelCallstack(lCtx, cancel)
|
||||
}
|
||||
|
||||
// WithValue returns context.WithValue with the log object propagated and
|
||||
@@ -132,9 +136,33 @@ func AddLogger(parent context.Context) Context {
|
||||
}
|
||||
|
||||
// SetupDefaultLogger sets the package-level global default logger that will be
|
||||
// used for Background and TODO contexts. On startup, the default logger will
|
||||
// be configured to output logs to stderr. Use logr.Discard() to disable all
|
||||
// logs from Contexts.
|
||||
// used for Background and TODO contexts.
|
||||
func SetDefaultLogger(l logr.Logger) {
|
||||
defaultLogger = l
|
||||
}
|
||||
|
||||
// captureCancelCallstack is a helper function to capture the callstack where
|
||||
// the cancel function was first called.
|
||||
func captureCancelCallstack(ctx logCtx, f context.CancelFunc) (Context, context.CancelFunc) {
|
||||
if ctx.err == nil {
|
||||
var err error
|
||||
ctx.err = &err
|
||||
ctx.errLock = &sync.Mutex{}
|
||||
}
|
||||
return ctx, func() {
|
||||
ctx.errLock.Lock()
|
||||
defer ctx.errLock.Unlock()
|
||||
// We must check Err() before calling f() since f() sets the error.
|
||||
// If there's already an error, do nothing special.
|
||||
if ctx.Err() != nil {
|
||||
f()
|
||||
return
|
||||
}
|
||||
f()
|
||||
// Set the error with the stacktrace if the err pointer is non-nil.
|
||||
*ctx.err = fmt.Errorf(
|
||||
"%w (canceled at %v\n%s)",
|
||||
ctx.Err(), time.Now(), string(debug.Stack()),
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -158,7 +158,7 @@ func TestWithValues(t *testing.T) {
|
||||
assert.NotContains(t, logs[6], `what does this do?`)
|
||||
}
|
||||
|
||||
func TestDefaultLogger(t *testing.T) {
|
||||
func TestDiscardLogger(t *testing.T) {
|
||||
var panicked bool
|
||||
defer func() {
|
||||
if r := recover(); r != nil {
|
||||
@@ -170,10 +170,51 @@ func TestDefaultLogger(t *testing.T) {
|
||||
ctx.Logger().Info("this shouldn't panic")
|
||||
}
|
||||
|
||||
func TestRace(t *testing.T) {
|
||||
ctx, cancel := WithCancel(Background())
|
||||
go cancel()
|
||||
go func() { _ = ctx.Err() }()
|
||||
func TestErrCallstack(t *testing.T) {
|
||||
c, cancel := WithCancel(Background())
|
||||
ctx := c.(logCtx)
|
||||
cancel()
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
assert.Contains(t, ctx.Err().Error(), "TestErrCallstack")
|
||||
case <-time.After(1 * time.Second):
|
||||
assert.Fail(t, "context should be done")
|
||||
}
|
||||
}
|
||||
|
||||
func TestErrCallstackTimeout(t *testing.T) {
|
||||
ctx, cancel := WithTimeout(Background(), 10*time.Millisecond)
|
||||
defer cancel()
|
||||
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
// Deadline exceeded errors will not have a callstack from the cancel
|
||||
// function.
|
||||
assert.NotContains(t, ctx.Err().Error(), "TestErrCallstackTimeout")
|
||||
case <-time.After(1 * time.Second):
|
||||
assert.Fail(t, "context should be done")
|
||||
}
|
||||
}
|
||||
|
||||
func TestErrCallstackTimeoutCancel(t *testing.T) {
|
||||
ctx, cancel := WithTimeout(Background(), 10*time.Millisecond)
|
||||
|
||||
var err error
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
err = ctx.Err()
|
||||
case <-time.After(1 * time.Second):
|
||||
assert.Fail(t, "context should be done")
|
||||
}
|
||||
|
||||
// Calling cancel after deadline exceeded should not overwrite the original
|
||||
// error.
|
||||
cancel()
|
||||
assert.Equal(t, err, ctx.Err())
|
||||
}
|
||||
|
||||
func TestRace(t *testing.T) {
|
||||
_, cancel := WithCancel(Background())
|
||||
go cancel()
|
||||
cancel()
|
||||
_ = ctx.Err()
|
||||
}
|
||||
|
||||
@@ -62,21 +62,21 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
defer res.Body.Close()
|
||||
if res.StatusCode >= 200 && res.StatusCode < 300 {
|
||||
s1.Verified = true
|
||||
} else if res.StatusCode == 401 {
|
||||
// The secret is determinately not verified (nothing to do)
|
||||
} else {
|
||||
s1.VerificationError = fmt.Errorf("unexpected HTTP response status %d", res.StatusCode)
|
||||
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
|
||||
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
|
||||
continue
|
||||
}
|
||||
|
||||
if res.StatusCode != 401 {
|
||||
s1.VerificationError = fmt.Errorf("request to %v returned unexpected status %d", res.Request.URL, res.StatusCode)
|
||||
}
|
||||
}
|
||||
} else {
|
||||
s1.VerificationError = err
|
||||
}
|
||||
}
|
||||
|
||||
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
|
||||
if !s1.Verified && detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
|
||||
continue
|
||||
}
|
||||
|
||||
results = append(results, s1)
|
||||
}
|
||||
|
||||
|
||||
@@ -6,11 +6,10 @@ package alchemy
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"github.com/google/go-cmp/cmp"
|
||||
"github.com/google/go-cmp/cmp/cmpopts"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/kylelemons/godebug/pretty"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
@@ -105,7 +104,7 @@ func TestAlchemy_FromChunk(t *testing.T) {
|
||||
},
|
||||
{
|
||||
name: "found, verified but unexpected api surface",
|
||||
s: Scanner{client: common.ConstantResponseHttpClient(404, "")},
|
||||
s: Scanner{client: common.ConstantStatusHttpClient(404)},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a alchemy secret %s within", secret)),
|
||||
@@ -132,12 +131,13 @@ func TestAlchemy_FromChunk(t *testing.T) {
|
||||
if len(got[i].Raw) == 0 {
|
||||
t.Fatalf("no raw secret present: \n %+v", got[i])
|
||||
}
|
||||
got[i].Raw = nil
|
||||
if (got[i].VerificationError != nil) != tt.wantVerificationErr {
|
||||
t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError)
|
||||
t.Fatalf("verification error = %v, wantVerificationError %v", got[i].VerificationError, tt.wantVerificationErr)
|
||||
}
|
||||
got[i].VerificationError = nil
|
||||
}
|
||||
ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "VerificationError")
|
||||
if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {
|
||||
if diff := pretty.Compare(got, tt.want); diff != "" {
|
||||
t.Errorf("Alchemy.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
|
||||
}
|
||||
})
|
||||
|
||||
@@ -18,8 +18,7 @@ import (
|
||||
)
|
||||
|
||||
type scanner struct {
|
||||
verificationClient *http.Client
|
||||
skipIDs map[string]struct{}
|
||||
skipIDs map[string]struct{}
|
||||
}
|
||||
|
||||
func New(opts ...func(*scanner)) *scanner {
|
||||
@@ -49,7 +48,7 @@ func WithSkipIDs(skipIDs []string) func(*scanner) {
|
||||
var _ detectors.Detector = (*scanner)(nil)
|
||||
|
||||
var (
|
||||
defaultVerificationClient = common.SaneHttpClient()
|
||||
client = common.SaneHttpClient()
|
||||
|
||||
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
|
||||
// Key types are from this list https://docs.aws.amazon.com/IAM/latest/UserGuide/reference_identifiers.html#identifiers-unique-ids
|
||||
@@ -124,9 +123,8 @@ func (s scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
host := "sts.amazonaws.com"
|
||||
region := "us-east-1"
|
||||
endpoint := "https://sts.amazonaws.com"
|
||||
now := time.Now().UTC()
|
||||
datestamp := now.Format("20060102")
|
||||
amzDate := now.Format("20060102T150405Z0700")
|
||||
datestamp := time.Now().UTC().Format("20060102")
|
||||
amzDate := time.Now().UTC().Format("20060102T150405Z0700")
|
||||
|
||||
req, err := http.NewRequestWithContext(ctx, method, endpoint, nil)
|
||||
if err != nil {
|
||||
@@ -173,10 +171,6 @@ func (s scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
req.Header.Add("Content-type", "application/x-www-form-urlencoded; charset=utf-8")
|
||||
req.URL.RawQuery = params.Encode()
|
||||
|
||||
client := s.verificationClient
|
||||
if client == nil {
|
||||
client = defaultVerificationClient
|
||||
}
|
||||
res, err := client.Do(req)
|
||||
if err == nil {
|
||||
|
||||
@@ -200,25 +194,7 @@ func (s scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
continue
|
||||
}
|
||||
|
||||
if res.StatusCode == 403 {
|
||||
var body awsErrorResponseBody
|
||||
err = json.NewDecoder(res.Body).Decode(&body)
|
||||
res.Body.Close()
|
||||
if err == nil {
|
||||
// All instances of the code I've seen in the wild are PascalCased but this check is
|
||||
// case-insensitive out of an abundance of caution
|
||||
if strings.EqualFold(body.Error.Code, "InvalidClientTokenId") {
|
||||
// determinate failure - nothing to do
|
||||
} else {
|
||||
// We see a surprising number of false-negative signature mismatch errors here
|
||||
// (The official SDK somehow elicits even more than just making the request
|
||||
// ourselves)
|
||||
s1.VerificationError = fmt.Errorf("request to %v returned status %d with an unexpected reason (%s: %s)", res.Request.URL, res.StatusCode, body.Error.Code, body.Error.Message)
|
||||
}
|
||||
} else {
|
||||
s1.VerificationError = fmt.Errorf("couldn't parse the sts response body (%v)", err)
|
||||
}
|
||||
} else {
|
||||
if res.StatusCode != 403 {
|
||||
s1.VerificationError = fmt.Errorf("request to %v returned unexpected status %d", res.Request.URL, res.StatusCode)
|
||||
}
|
||||
}
|
||||
@@ -269,15 +245,6 @@ func awsCustomCleanResults(results []detectors.Result) []detectors.Result {
|
||||
return out
|
||||
}
|
||||
|
||||
type awsError struct {
|
||||
Code string `json:"Code"`
|
||||
Message string `json:"Message"`
|
||||
}
|
||||
|
||||
type awsErrorResponseBody struct {
|
||||
Error awsError `json:"Error"`
|
||||
}
|
||||
|
||||
type identityRes struct {
|
||||
GetCallerIdentityResponse struct {
|
||||
GetCallerIdentityResult struct {
|
||||
|
||||
@@ -17,8 +17,6 @@ import (
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
|
||||
)
|
||||
|
||||
var unverifiedSecretClient = common.ConstantResponseHttpClient(403, `{"Error": {"Code": "InvalidClientTokenId"} }`)
|
||||
|
||||
func TestAWS_FromChunk(t *testing.T) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
|
||||
defer cancel()
|
||||
@@ -71,7 +69,7 @@ func TestAWS_FromChunk(t *testing.T) {
|
||||
},
|
||||
{
|
||||
name: "found, unverified",
|
||||
s: scanner{verificationClient: unverifiedSecretClient},
|
||||
s: scanner{},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a aws secret %s within aws %s but not valid", inactiveSecret, id)), // the secret would satisfy the regex but not pass validation
|
||||
@@ -165,7 +163,7 @@ func TestAWS_FromChunk(t *testing.T) {
|
||||
},
|
||||
{
|
||||
name: "found, unverified, with leading +",
|
||||
s: scanner{verificationClient: unverifiedSecretClient},
|
||||
s: scanner{},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a aws secret %s within aws %s but not valid", "+HaNv9cTwheDKGJaws/+BMF2GgybQgBWdhcOOdfF", id)), // the secret would satisfy the regex but not pass validation
|
||||
@@ -196,45 +194,9 @@ func TestAWS_FromChunk(t *testing.T) {
|
||||
},
|
||||
{
|
||||
name: "found, would be verified if not for http timeout",
|
||||
s: scanner{verificationClient: common.SaneHttpClientTimeOut(1 * time.Microsecond)},
|
||||
s: scanner{},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a aws secret %s within aws %s", secret, id)),
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_AWS,
|
||||
Verified: false,
|
||||
Redacted: "AKIASP2TPHJSQH3FJRUX",
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
wantVerificationError: true,
|
||||
},
|
||||
{
|
||||
name: "found, unverified due to unexpected http response status",
|
||||
s: scanner{verificationClient: common.ConstantResponseHttpClient(500, "internal server error")},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a aws secret %s within aws %s", secret, id)),
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_AWS,
|
||||
Verified: false,
|
||||
Redacted: "AKIASP2TPHJSQH3FJRUX",
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
wantVerificationError: true,
|
||||
},
|
||||
{
|
||||
name: "found, unverified due to unexpected 403 response reason",
|
||||
s: scanner{verificationClient: common.ConstantResponseHttpClient(403, `{"Error": {"Code": "SignatureDoesNotMatch"} }`)},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
ctx: timeoutContext(1 * time.Microsecond),
|
||||
data: []byte(fmt.Sprintf("You can find a aws secret %s within aws %s", secret, id)),
|
||||
verify: true,
|
||||
},
|
||||
@@ -262,7 +224,7 @@ func TestAWS_FromChunk(t *testing.T) {
|
||||
t.Fatalf("no raw secret present: \n %+v", got[i])
|
||||
}
|
||||
if (got[i].VerificationError != nil) != tt.wantVerificationError {
|
||||
t.Fatalf("wantVerificationError %v, verification error = %v", tt.wantVerificationError, got[i].VerificationError)
|
||||
t.Fatalf("verification error = %v, wantVerificationError %v", got[i].VerificationError, tt.wantVerificationError)
|
||||
}
|
||||
}
|
||||
ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "RawV2", "Raw", "VerificationError")
|
||||
@@ -287,3 +249,8 @@ func BenchmarkFromData(benchmark *testing.B) {
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func timeoutContext(timeout time.Duration) context.Context {
|
||||
c, _ := context.WithTimeout(context.Background(), timeout)
|
||||
return c
|
||||
}
|
||||
|
||||
@@ -20,7 +20,7 @@ var (
|
||||
client = common.SaneHttpClient()
|
||||
|
||||
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
|
||||
appPat = regexp.MustCompile(detectors.PrefixRegex([]string{"datadog", "dd"}) + `\b([a-zA-Z-0-9]{40})\b`)
|
||||
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"datadog", "dd"}) + `\b([a-zA-Z-0-9]{40})\b`)
|
||||
apiPat = regexp.MustCompile(detectors.PrefixRegex([]string{"datadog", "dd"}) + `\b([a-zA-Z-0-9]{32})\b`)
|
||||
)
|
||||
|
||||
@@ -34,28 +34,24 @@ func (s Scanner) Keywords() []string {
|
||||
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
|
||||
dataStr := string(data)
|
||||
|
||||
appMatches := appPat.FindAllStringSubmatch(dataStr, -1)
|
||||
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
|
||||
apiMatches := apiPat.FindAllStringSubmatch(dataStr, -1)
|
||||
|
||||
for _, apiMatch := range apiMatches {
|
||||
if len(apiMatch) != 2 {
|
||||
for _, match := range matches {
|
||||
if len(match) != 2 {
|
||||
continue
|
||||
}
|
||||
resApiMatch := strings.TrimSpace(apiMatch[1])
|
||||
appIncluded := false
|
||||
for _, appMatch := range appMatches {
|
||||
if len(appMatch) != 2 {
|
||||
resMatch := strings.TrimSpace(match[1])
|
||||
for _, apimatch := range apiMatches {
|
||||
if len(apimatch) != 2 {
|
||||
continue
|
||||
}
|
||||
resAppMatch := strings.TrimSpace(appMatch[1])
|
||||
resApiMatch := strings.TrimSpace(apimatch[1])
|
||||
|
||||
s1 := detectors.Result{
|
||||
DetectorType: detectorspb.DetectorType_DatadogToken,
|
||||
Raw: []byte(resAppMatch),
|
||||
RawV2: []byte(resAppMatch + resApiMatch),
|
||||
ExtraData: map[string]string{
|
||||
"Type": "Application+APIKey",
|
||||
},
|
||||
Raw: []byte(resMatch),
|
||||
RawV2: []byte(resMatch + resApiMatch),
|
||||
}
|
||||
|
||||
if verify {
|
||||
@@ -66,7 +62,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
}
|
||||
req.Header.Add("Content-Type", "application/json")
|
||||
req.Header.Add("DD-API-KEY", resApiMatch)
|
||||
req.Header.Add("DD-APPLICATION-KEY", resAppMatch)
|
||||
req.Header.Add("DD-APPLICATION-KEY", resMatch)
|
||||
res, err := client.Do(req)
|
||||
if err == nil {
|
||||
defer res.Body.Close()
|
||||
@@ -74,49 +70,16 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
s1.Verified = true
|
||||
} else {
|
||||
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
|
||||
if detectors.IsKnownFalsePositive(resApiMatch, detectors.DefaultFalsePositives, true) {
|
||||
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
|
||||
continue
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
appIncluded = true
|
||||
|
||||
results = append(results, s1)
|
||||
}
|
||||
|
||||
if !appIncluded {
|
||||
s1 := detectors.Result{
|
||||
DetectorType: detectorspb.DetectorType_DatadogToken,
|
||||
Raw: []byte(resApiMatch),
|
||||
RawV2: []byte(resApiMatch),
|
||||
ExtraData: map[string]string{
|
||||
"Type": "APIKeyOnly",
|
||||
},
|
||||
}
|
||||
|
||||
if verify {
|
||||
|
||||
req, err := http.NewRequestWithContext(ctx, "GET", "https://api.datadoghq.com/api/v1/validate", nil)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
req.Header.Add("Content-Type", "application/json")
|
||||
req.Header.Add("DD-API-KEY", resApiMatch)
|
||||
res, err := client.Do(req)
|
||||
if err == nil {
|
||||
defer res.Body.Close()
|
||||
if res.StatusCode >= 200 && res.StatusCode < 300 {
|
||||
s1.Verified = true
|
||||
} else {
|
||||
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
|
||||
if detectors.IsKnownFalsePositive(resApiMatch, detectors.DefaultFalsePositives, true) {
|
||||
continue
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
results = append(results, s1)
|
||||
}
|
||||
}
|
||||
|
||||
return results, nil
|
||||
|
||||
@@ -19,11 +19,11 @@ import (
|
||||
func TestDatadogToken_FromChunk(t *testing.T) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
|
||||
defer cancel()
|
||||
testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")
|
||||
testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
|
||||
if err != nil {
|
||||
t.Fatalf("could not get test secrets from GCP: %s", err)
|
||||
}
|
||||
apiKey := testSecrets.MustGetField("DATADOGTOKEN_TOKEN")
|
||||
secret := testSecrets.MustGetField("DATADOGTOKEN_TOKEN")
|
||||
appKey := testSecrets.MustGetField("DATADOGTOKEN_APPKEY")
|
||||
inactiveAppKey := testSecrets.MustGetField("DATADOGTOKEN_INACTIVE")
|
||||
|
||||
@@ -44,16 +44,13 @@ func TestDatadogToken_FromChunk(t *testing.T) {
|
||||
s: Scanner{},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a datadogtoken secret %s within datadog %s", appKey, apiKey)),
|
||||
data: []byte(fmt.Sprintf("You can find a datadogtoken secret %s within datadog %s", appKey, secret)),
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_DatadogToken,
|
||||
Verified: true,
|
||||
ExtraData: map[string]string{
|
||||
"Type": "Application+APIKey",
|
||||
},
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
@@ -63,35 +60,13 @@ func TestDatadogToken_FromChunk(t *testing.T) {
|
||||
s: Scanner{},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a datadogtoken secret %s within but datadog %s not valid", inactiveAppKey, apiKey)), // the secret would satisfy the regex but not pass validation
|
||||
data: []byte(fmt.Sprintf("You can find a datadogtoken secret %s within but datadog %s not valid", inactiveAppKey, secret)), // the secret would satisfy the regex but not pass validation
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_DatadogToken,
|
||||
Verified: false,
|
||||
ExtraData: map[string]string{
|
||||
"Type": "Application+APIKey",
|
||||
},
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "api key found, verified",
|
||||
s: Scanner{},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a datadogtoken secret %s", apiKey)), // the secret would satisfy the regex but not pass validation
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_DatadogToken,
|
||||
Verified: true,
|
||||
ExtraData: map[string]string{
|
||||
"Type": "APIKeyOnly",
|
||||
},
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
@@ -121,7 +96,6 @@ func TestDatadogToken_FromChunk(t *testing.T) {
|
||||
t.Fatalf("no raw secret present: \n %+v", got[i])
|
||||
}
|
||||
got[i].Raw = nil
|
||||
got[i].RawV2 = nil
|
||||
}
|
||||
if diff := pretty.Compare(got, tt.want); diff != "" {
|
||||
t.Errorf("DatadogToken.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
|
||||
|
||||
+12
-35
@@ -2,8 +2,6 @@ package ftp
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"net/textproto"
|
||||
"net/url"
|
||||
"regexp"
|
||||
"strings"
|
||||
@@ -15,17 +13,7 @@ import (
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
|
||||
)
|
||||
|
||||
const (
|
||||
// https://datatracker.ietf.org/doc/html/rfc959
|
||||
ftpNotLoggedIn = 530
|
||||
|
||||
defaultVerificationTimeout = 5 * time.Second
|
||||
)
|
||||
|
||||
type Scanner struct {
|
||||
// Verification timeout. Defaults to 5 seconds if unset.
|
||||
verificationTimeout time.Duration
|
||||
}
|
||||
type Scanner struct{}
|
||||
|
||||
// Ensure the Scanner satisfies the interface at compile time.
|
||||
var _ detectors.Detector = (*Scanner)(nil)
|
||||
@@ -70,59 +58,48 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
rawURL.Path = ""
|
||||
redact := strings.TrimSpace(strings.Replace(rawURL.String(), password, "********", -1))
|
||||
|
||||
r := detectors.Result{
|
||||
s := detectors.Result{
|
||||
DetectorType: detectorspb.DetectorType_FTP,
|
||||
Raw: []byte(rawURL.String()),
|
||||
Redacted: redact,
|
||||
}
|
||||
|
||||
if verify {
|
||||
timeout := s.verificationTimeout
|
||||
if timeout == 0 {
|
||||
timeout = defaultVerificationTimeout
|
||||
}
|
||||
verificationErr := verifyFTP(timeout, parsedURL)
|
||||
r.Verified = verificationErr == nil
|
||||
if !isErrDeterminate(verificationErr) {
|
||||
r.VerificationError = verificationErr
|
||||
}
|
||||
s.Verified = verifyFTP(ctx, parsedURL)
|
||||
}
|
||||
|
||||
if !r.Verified {
|
||||
if !s.Verified {
|
||||
// Skip unverified findings where the password starts with a `$` - it's almost certainly a variable.
|
||||
if strings.HasPrefix(password, "$") {
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
if detectors.IsKnownFalsePositive(string(r.Raw), []detectors.FalsePositive{"@ftp.freebsd.org"}, false) {
|
||||
if detectors.IsKnownFalsePositive(string(s.Raw), []detectors.FalsePositive{"@ftp.freebsd.org"}, false) {
|
||||
continue
|
||||
}
|
||||
|
||||
results = append(results, r)
|
||||
results = append(results, s)
|
||||
}
|
||||
|
||||
return results, nil
|
||||
}
|
||||
|
||||
func isErrDeterminate(e error) bool {
|
||||
ftpErr := &textproto.Error{}
|
||||
return errors.As(e, &ftpErr) && ftpErr.Code == ftpNotLoggedIn
|
||||
}
|
||||
|
||||
func verifyFTP(timeout time.Duration, u *url.URL) error {
|
||||
func verifyFTP(ctx context.Context, u *url.URL) bool {
|
||||
host := u.Host
|
||||
if !strings.Contains(host, ":") {
|
||||
host = host + ":21"
|
||||
}
|
||||
|
||||
c, err := ftp.Dial(host, ftp.DialWithTimeout(timeout))
|
||||
c, err := ftp.Dial(host, ftp.DialWithTimeout(5*time.Second))
|
||||
if err != nil {
|
||||
return err
|
||||
return false
|
||||
}
|
||||
|
||||
password, _ := u.User.Password()
|
||||
return c.Login(u.User.Username(), password)
|
||||
err = c.Login(u.User.Username(), password)
|
||||
|
||||
return err == nil
|
||||
}
|
||||
|
||||
func (s Scanner) Type() detectorspb.DetectorType {
|
||||
|
||||
@@ -5,11 +5,9 @@ package ftp
|
||||
|
||||
import (
|
||||
"context"
|
||||
"github.com/google/go-cmp/cmp"
|
||||
"github.com/google/go-cmp/cmp/cmpopts"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/kylelemons/godebug/pretty"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
|
||||
)
|
||||
@@ -21,12 +19,11 @@ func TestFTP_FromChunk(t *testing.T) {
|
||||
verify bool
|
||||
}
|
||||
tests := []struct {
|
||||
name string
|
||||
s Scanner
|
||||
args args
|
||||
want []detectors.Result
|
||||
wantErr bool
|
||||
wantVerificationErr bool
|
||||
name string
|
||||
s Scanner
|
||||
args args
|
||||
want []detectors.Result
|
||||
wantErr bool
|
||||
}{
|
||||
{
|
||||
name: "bad scheme",
|
||||
@@ -51,7 +48,7 @@ func TestFTP_FromChunk(t *testing.T) {
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_FTP,
|
||||
Verified: true,
|
||||
Redacted: "ftp://dlpuser:********@ftp.dlptest.com",
|
||||
Redacted: "ftp://dlpuser:*************************@ftp.dlptest.com",
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
@@ -69,49 +66,11 @@ func TestFTP_FromChunk(t *testing.T) {
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_FTP,
|
||||
Verified: false,
|
||||
Redacted: "ftp://dlpuser:********@ftp.dlptest.com",
|
||||
Redacted: "ftp://dlpuser:*******@ftp.dlptest.com",
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "bad host",
|
||||
s: Scanner{},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
// https://dlptest.com/ftp-test/
|
||||
data: []byte("ftp://dlpuser:rNrKYTX9g7z3RgJRmxWuGHbeu@ftp.dlptest.com.badhost"),
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_FTP,
|
||||
Verified: false,
|
||||
Redacted: "ftp://dlpuser:********@ftp.dlptest.com.badhost",
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
wantVerificationErr: true,
|
||||
},
|
||||
{
|
||||
name: "timeout",
|
||||
s: Scanner{verificationTimeout: 1 * time.Microsecond},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
// https://dlptest.com/ftp-test/
|
||||
data: []byte("ftp://dlpuser:rNrKYTX9g7z3RgJRmxWuGHbeu@ftp.dlptest.com"),
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_FTP,
|
||||
Verified: false,
|
||||
Redacted: "ftp://dlpuser:********@ftp.dlptest.com",
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
wantVerificationErr: true,
|
||||
},
|
||||
{
|
||||
name: "blocked FP",
|
||||
s: Scanner{},
|
||||
@@ -125,7 +84,8 @@ func TestFTP_FromChunk(t *testing.T) {
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
|
||||
s := Scanner{}
|
||||
got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
|
||||
if (err != nil) != tt.wantErr {
|
||||
t.Errorf("URI.FromData() error = %v, wantErr %v", err, tt.wantErr)
|
||||
return
|
||||
@@ -135,14 +95,9 @@ func TestFTP_FromChunk(t *testing.T) {
|
||||
// }
|
||||
for i := range got {
|
||||
got[i].Raw = nil
|
||||
|
||||
if (got[i].VerificationError != nil) != tt.wantVerificationErr {
|
||||
t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError)
|
||||
}
|
||||
}
|
||||
opts := cmpopts.IgnoreFields(detectors.Result{}, "VerificationError")
|
||||
if diff := cmp.Diff(got, tt.want, opts); diff != "" {
|
||||
t.Errorf("FTP.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
|
||||
if diff := pretty.Compare(got, tt.want); diff != "" {
|
||||
t.Errorf("URI.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
@@ -1,81 +0,0 @@
|
||||
package github_oauth2
|
||||
|
||||
import (
|
||||
"context"
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
|
||||
"golang.org/x/oauth2/clientcredentials"
|
||||
"golang.org/x/oauth2/github"
|
||||
)
|
||||
|
||||
type Scanner struct{ detectors.EndpointSetter }
|
||||
|
||||
// Ensure the Scanner satisfies the interfaces at compile time.
|
||||
var _ detectors.Detector = (*Scanner)(nil)
|
||||
|
||||
var (
|
||||
// Oauth2 client ID and secret
|
||||
oauth2ClientIDPat = regexp.MustCompile(detectors.PrefixRegex([]string{"github"}) + `\b([a-f0-9]{20})\b`)
|
||||
oauth2ClientSecretPat = regexp.MustCompile(detectors.PrefixRegex([]string{"github"}) + `\b([a-f0-9]{40})\b`)
|
||||
)
|
||||
|
||||
const (
|
||||
githubBadVerificationCodeError = "bad_verification_code"
|
||||
)
|
||||
|
||||
// Keywords are used for efficiently pre-filtering chunks.
|
||||
// Use identifiers in the secret preferably, or the provider name.
|
||||
func (s Scanner) Keywords() []string {
|
||||
return []string{"github"}
|
||||
}
|
||||
|
||||
// FromData will find and optionally verify GitHub secrets in a given set of bytes.
|
||||
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
|
||||
dataStr := string(data)
|
||||
|
||||
// Oauth2 client ID and secret
|
||||
oauth2ClientIDMatches := oauth2ClientIDPat.FindAllStringSubmatch(dataStr, -1)
|
||||
oauth2ClientSecretMatches := oauth2ClientSecretPat.FindAllStringSubmatch(dataStr, -1)
|
||||
|
||||
for _, idMatch := range oauth2ClientIDMatches {
|
||||
if len(idMatch) != 2 {
|
||||
continue
|
||||
}
|
||||
for _, secretMatch := range oauth2ClientSecretMatches {
|
||||
if len(secretMatch) != 2 {
|
||||
continue
|
||||
}
|
||||
|
||||
s1 := detectors.Result{
|
||||
DetectorType: detectorspb.DetectorType_GitHubOauth2,
|
||||
Raw: []byte(idMatch[1]),
|
||||
RawV2: []byte(idMatch[1] + secretMatch[1]),
|
||||
}
|
||||
|
||||
config := &clientcredentials.Config{
|
||||
ClientID: idMatch[1],
|
||||
ClientSecret: secretMatch[1],
|
||||
TokenURL: github.Endpoint.TokenURL,
|
||||
}
|
||||
_, err := config.Token(ctx)
|
||||
if err != nil && strings.Contains(err.Error(), githubBadVerificationCodeError) {
|
||||
s1.Verified = true
|
||||
}
|
||||
|
||||
if !s1.Verified && detectors.IsKnownFalsePositive(string(s1.Raw), detectors.DefaultFalsePositives, true) {
|
||||
continue
|
||||
}
|
||||
|
||||
results = append(results, s1)
|
||||
}
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
func (s Scanner) Type() detectorspb.DetectorType {
|
||||
return detectorspb.DetectorType_GitHubOauth2
|
||||
}
|
||||
@@ -12,11 +12,7 @@ import (
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
|
||||
)
|
||||
|
||||
type Scanner struct {
|
||||
client *http.Client
|
||||
|
||||
detectors.EndpointSetter
|
||||
}
|
||||
type Scanner struct{ detectors.EndpointSetter }
|
||||
|
||||
// Ensure the Scanner satisfies the interfaces at compile time.
|
||||
var _ detectors.Detector = (*Scanner)(nil)
|
||||
@@ -27,8 +23,7 @@ func (Scanner) Version() int { return 1 }
|
||||
func (Scanner) DefaultEndpoint() string { return "https://gitlab.com" }
|
||||
|
||||
var (
|
||||
defaultClient = common.SaneHttpClient()
|
||||
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"gitlab"}) + `\b((?:glpat|)[a-zA-Z0-9\-=_]{20,22})\b`)
|
||||
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"gitlab"}) + `\b((?:glpat|)[a-zA-Z0-9\-=_]{20,22})\b`)
|
||||
)
|
||||
|
||||
// Keywords are used for efficiently pre-filtering chunks.
|
||||
@@ -64,10 +59,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
// one of these scopes has access to, so we just check an example endpoint for each scope. If any
|
||||
// of them contain data, we know we have a valid key, but if they all fail, we don't
|
||||
|
||||
client := s.client
|
||||
if client == nil {
|
||||
client = defaultClient
|
||||
}
|
||||
client := common.SaneHttpClient()
|
||||
for _, baseURL := range s.Endpoints(s.DefaultEndpoint()) {
|
||||
// test `read_user` scope
|
||||
req, err := http.NewRequestWithContext(ctx, "GET", baseURL+"/api/v4/user", nil)
|
||||
@@ -82,19 +74,9 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
// 200 means good key and has `read_user` scope
|
||||
// 403 means good key but not the right scope
|
||||
// 401 is bad key
|
||||
switch res.StatusCode {
|
||||
case http.StatusOK:
|
||||
if res.StatusCode == http.StatusOK || res.StatusCode == http.StatusForbidden {
|
||||
secret.Verified = true
|
||||
case http.StatusForbidden:
|
||||
// Good key but not the right scope
|
||||
secret.Verified = true
|
||||
case http.StatusUnauthorized:
|
||||
// Nothing to do; zero values are the ones we want
|
||||
default:
|
||||
secret.VerificationError = fmt.Errorf("unexpected HTTP response status %d", res.StatusCode)
|
||||
}
|
||||
} else {
|
||||
secret.VerificationError = err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -6,11 +6,11 @@ package gitlab
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"github.com/google/go-cmp/cmp"
|
||||
"github.com/google/go-cmp/cmp/cmpopts"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/kylelemons/godebug/pretty"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
|
||||
@@ -31,12 +31,11 @@ func TestGitlab_FromChunk(t *testing.T) {
|
||||
verify bool
|
||||
}
|
||||
tests := []struct {
|
||||
name string
|
||||
s Scanner
|
||||
args args
|
||||
want []detectors.Result
|
||||
wantErr bool
|
||||
wantVerificationErr bool
|
||||
name string
|
||||
s Scanner
|
||||
args args
|
||||
want []detectors.Result
|
||||
wantErr bool
|
||||
}{
|
||||
{
|
||||
name: "found",
|
||||
@@ -86,56 +85,6 @@ func TestGitlab_FromChunk(t *testing.T) {
|
||||
},
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "found, would be verified but for timeout",
|
||||
s: Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a gitlab super secret %s within", secret)),
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_Gitlab,
|
||||
Verified: false,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
wantVerificationErr: true,
|
||||
},
|
||||
{
|
||||
name: "found and valid but unexpected api response",
|
||||
s: Scanner{client: common.ConstantResponseHttpClient(500, "")},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a gitlab super secret %s within", secret)),
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_Gitlab,
|
||||
Verified: false,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
wantVerificationErr: true,
|
||||
},
|
||||
{
|
||||
name: "found, good key but wrong scope",
|
||||
s: Scanner{client: common.ConstantResponseHttpClient(403, "")},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a gitlab super secret %s within", secret)),
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_Gitlab,
|
||||
Verified: true,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "not found",
|
||||
s: Scanner{},
|
||||
@@ -150,7 +99,8 @@ func TestGitlab_FromChunk(t *testing.T) {
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
|
||||
s := Scanner{}
|
||||
got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
|
||||
if (err != nil) != tt.wantErr {
|
||||
t.Errorf("Gitlab.FromData() error = %v, wantErr %v", err, tt.wantErr)
|
||||
return
|
||||
@@ -159,12 +109,9 @@ func TestGitlab_FromChunk(t *testing.T) {
|
||||
if len(got[i].Raw) == 0 {
|
||||
t.Fatal("no raw secret present")
|
||||
}
|
||||
if (got[i].VerificationError != nil) != tt.wantVerificationErr {
|
||||
t.Fatalf(" wantVerificationError = %v, verification error = %v,", tt.wantVerificationErr, got[i].VerificationError)
|
||||
}
|
||||
got[i].Raw = nil
|
||||
}
|
||||
opts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "VerificationError")
|
||||
if diff := cmp.Diff(got, tt.want, opts); diff != "" {
|
||||
if diff := pretty.Compare(got, tt.want); diff != "" {
|
||||
t.Errorf("Gitlab.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
|
||||
}
|
||||
})
|
||||
|
||||
@@ -1,91 +0,0 @@
|
||||
package huggingface
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
|
||||
)
|
||||
|
||||
type Scanner struct {
|
||||
client *http.Client
|
||||
}
|
||||
|
||||
// Ensure the Scanner satisfies the interface at compile time.
|
||||
var _ detectors.Detector = (*Scanner)(nil)
|
||||
|
||||
var (
|
||||
defaultClient = common.SaneHttpClient()
|
||||
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
|
||||
keyPat = regexp.MustCompile(`\bhf_[a-zA-Z]{34}\b`)
|
||||
)
|
||||
|
||||
// Keywords are used for efficiently pre-filtering chunks.
|
||||
// Use identifiers in the secret preferably, or the provider name.
|
||||
func (s Scanner) Keywords() []string {
|
||||
return []string{"huggingface", "hugging_face", "hf"} // Huggingface docs occasionally use "hf" instead of "huggingface"
|
||||
}
|
||||
|
||||
// FromData will find and optionally verify Huggingface secrets in a given set of bytes.
|
||||
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
|
||||
dataStr := string(data)
|
||||
|
||||
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
|
||||
|
||||
for _, match := range matches {
|
||||
if len(match) != 1 {
|
||||
continue
|
||||
}
|
||||
resMatch := strings.TrimSpace(match[0])
|
||||
|
||||
s1 := detectors.Result{
|
||||
DetectorType: detectorspb.DetectorType_HuggingFace,
|
||||
Raw: []byte(resMatch),
|
||||
}
|
||||
|
||||
if verify {
|
||||
client := s.client
|
||||
if client == nil {
|
||||
client = defaultClient
|
||||
}
|
||||
req, err := http.NewRequestWithContext(ctx, "GET", "https://huggingface.co/api/whoami-v2", nil)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
|
||||
req.Header.Set("Authorization", "Bearer "+resMatch)
|
||||
|
||||
res, err := client.Do(req)
|
||||
if err == nil {
|
||||
defer res.Body.Close()
|
||||
if res.StatusCode >= 200 && res.StatusCode < 300 {
|
||||
s1.Verified = true
|
||||
} else if res.StatusCode == 401 {
|
||||
// The secret is determinately not verified (nothing to do)
|
||||
} else {
|
||||
s1.VerificationError = fmt.Errorf("unexpected HTTP response status %d", res.StatusCode)
|
||||
}
|
||||
} else {
|
||||
s1.VerificationError = err
|
||||
}
|
||||
}
|
||||
|
||||
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
|
||||
if !s1.Verified && detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
|
||||
continue
|
||||
}
|
||||
|
||||
results = append(results, s1)
|
||||
}
|
||||
|
||||
return results, nil
|
||||
}
|
||||
|
||||
func (s Scanner) Type() detectorspb.DetectorType {
|
||||
return detectorspb.DetectorType_HuggingFace
|
||||
}
|
||||
@@ -1,160 +0,0 @@
|
||||
//go:build detectors
|
||||
// +build detectors
|
||||
|
||||
package huggingface
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"github.com/google/go-cmp/cmp"
|
||||
"github.com/google/go-cmp/cmp/cmpopts"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
|
||||
)
|
||||
|
||||
func TestHuggingface_FromChunk(t *testing.T) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
|
||||
defer cancel()
|
||||
testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")
|
||||
if err != nil {
|
||||
t.Fatalf("could not get test secrets from GCP: %s", err)
|
||||
}
|
||||
secret := testSecrets.MustGetField("HUGGINGFACE")
|
||||
inactiveSecret := testSecrets.MustGetField("HUGGINGFACE_INACTIVE")
|
||||
|
||||
type args struct {
|
||||
ctx context.Context
|
||||
data []byte
|
||||
verify bool
|
||||
}
|
||||
tests := []struct {
|
||||
name string
|
||||
s Scanner
|
||||
args args
|
||||
want []detectors.Result
|
||||
wantErr bool
|
||||
wantVerificationErr bool
|
||||
}{
|
||||
{
|
||||
name: "found, verified",
|
||||
s: Scanner{},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a huggingface secret %s within", secret)),
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_HuggingFace,
|
||||
Verified: true,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
wantVerificationErr: false,
|
||||
},
|
||||
{
|
||||
name: "found, unverified",
|
||||
s: Scanner{},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a huggingface secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_HuggingFace,
|
||||
Verified: false,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
wantVerificationErr: false,
|
||||
},
|
||||
{
|
||||
name: "not found",
|
||||
s: Scanner{},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte("You cannot find the secret within"),
|
||||
verify: true,
|
||||
},
|
||||
want: nil,
|
||||
wantErr: false,
|
||||
wantVerificationErr: false,
|
||||
},
|
||||
{
|
||||
name: "found, would be verified if not for timeout",
|
||||
s: Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a huggingface secret %s within", secret)),
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_HuggingFace,
|
||||
Verified: false,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
wantVerificationErr: true,
|
||||
},
|
||||
{
|
||||
name: "found, verified but unexpected api surface",
|
||||
s: Scanner{client: common.ConstantResponseHttpClient(404, "")},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a huggingface secret %s within", secret)),
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_HuggingFace,
|
||||
Verified: false,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
wantVerificationErr: true,
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
|
||||
if (err != nil) != tt.wantErr {
|
||||
t.Errorf("Huggingface.FromData() error = %v, wantErr %v", err, tt.wantErr)
|
||||
return
|
||||
}
|
||||
for i := range got {
|
||||
if len(got[i].Raw) == 0 {
|
||||
t.Fatalf("no raw secret present: \n %+v", got[i])
|
||||
}
|
||||
if (got[i].VerificationError != nil) != tt.wantVerificationErr {
|
||||
t.Fatalf(" wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError)
|
||||
}
|
||||
}
|
||||
ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "VerificationError")
|
||||
if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {
|
||||
t.Errorf("Huggingface.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkFromData(benchmark *testing.B) {
|
||||
ctx := context.Background()
|
||||
s := Scanner{}
|
||||
for name, data := range detectors.MustGetBenchmarkData() {
|
||||
benchmark.Run(name, func(b *testing.B) {
|
||||
for n := 0; n < b.N; n++ {
|
||||
_, err := s.FromData(ctx, false, data)
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
+26
-46
@@ -3,8 +3,6 @@ package ldap
|
||||
import (
|
||||
"context"
|
||||
"crypto/tls"
|
||||
"fmt"
|
||||
"net"
|
||||
"net/url"
|
||||
"regexp"
|
||||
"strings"
|
||||
@@ -65,11 +63,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
}
|
||||
|
||||
if verify {
|
||||
verificationErr := verifyLDAP(username[1], password[1], ldapURL)
|
||||
s1.Verified = verificationErr == nil
|
||||
if !isErrDeterminate(verificationErr) {
|
||||
s1.VerificationError = verificationErr
|
||||
}
|
||||
s1.Verified = verifyLDAP(ctx, username[1], password[1], ldapURL)
|
||||
}
|
||||
|
||||
results = append(results, s1)
|
||||
@@ -95,12 +89,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
}
|
||||
|
||||
if verify {
|
||||
verificationError := verifyLDAP(username, password, ldapURL)
|
||||
|
||||
s1.Verified = verificationError == nil
|
||||
if !isErrDeterminate(verificationError) {
|
||||
s1.VerificationError = verificationError
|
||||
}
|
||||
s1.Verified = verifyLDAP(ctx, username, password, ldapURL)
|
||||
}
|
||||
|
||||
results = append(results, s1)
|
||||
@@ -109,19 +98,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
return results, nil
|
||||
}
|
||||
|
||||
func isErrDeterminate(err error) bool {
|
||||
switch e := err.(type) {
|
||||
case *ldap.Error:
|
||||
switch e.Err.(type) {
|
||||
case *net.OpError:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
return true
|
||||
}
|
||||
|
||||
func verifyLDAP(username, password string, ldapURL *url.URL) error {
|
||||
func verifyLDAP(ctx context.Context, username, password string, ldapURL *url.URL) bool {
|
||||
// Tests with non-TLS, TLS, and STARTTLS
|
||||
|
||||
ldap.DefaultTimeout = 5 * time.Second
|
||||
@@ -132,35 +109,38 @@ func verifyLDAP(username, password string, ldapURL *url.URL) error {
|
||||
case "ldap":
|
||||
// Non-TLS dial
|
||||
l, err := ldap.DialURL(uri)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer l.Close()
|
||||
// Non-TLS verify
|
||||
err = l.Bind(username, password)
|
||||
if err == nil {
|
||||
return nil
|
||||
}
|
||||
defer l.Close()
|
||||
// Non-TLS verify
|
||||
err = l.Bind(username, password)
|
||||
if err == nil {
|
||||
return true
|
||||
}
|
||||
|
||||
// STARTTLS
|
||||
err = l.StartTLS(&tls.Config{InsecureSkipVerify: true})
|
||||
if err != nil {
|
||||
return err
|
||||
// STARTTLS
|
||||
err = l.StartTLS(&tls.Config{InsecureSkipVerify: true})
|
||||
if err == nil {
|
||||
// STARTTLS verify
|
||||
err = l.Bind(username, password)
|
||||
if err == nil {
|
||||
return true
|
||||
}
|
||||
}
|
||||
}
|
||||
// STARTTLS verify
|
||||
return l.Bind(username, password)
|
||||
case "ldaps":
|
||||
// TLS dial
|
||||
l, err := ldap.DialTLS("tcp", uri, &tls.Config{InsecureSkipVerify: true})
|
||||
if err != nil {
|
||||
return err
|
||||
if err == nil {
|
||||
defer l.Close()
|
||||
// TLS verify
|
||||
err = l.Bind(username, password)
|
||||
if err == nil {
|
||||
return true
|
||||
}
|
||||
}
|
||||
defer l.Close()
|
||||
// TLS verify
|
||||
return l.Bind(username, password)
|
||||
}
|
||||
|
||||
return fmt.Errorf("unknown ldap scheme %q", ldapURL.Scheme)
|
||||
return false
|
||||
}
|
||||
|
||||
func (s Scanner) Type() detectorspb.DetectorType {
|
||||
|
||||
@@ -7,14 +7,13 @@ import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"github.com/google/go-cmp/cmp"
|
||||
"github.com/google/go-cmp/cmp/cmpopts"
|
||||
"os"
|
||||
"os/exec"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/kylelemons/godebug/pretty"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
|
||||
@@ -61,12 +60,11 @@ func TestLdap_Integration_FromChunk(t *testing.T) {
|
||||
verify bool
|
||||
}
|
||||
tests := []struct {
|
||||
name string
|
||||
s Scanner
|
||||
args args
|
||||
want []detectors.Result
|
||||
wantErr bool
|
||||
wantVerificationErr bool
|
||||
name string
|
||||
s Scanner
|
||||
args args
|
||||
want []detectors.Result
|
||||
wantErr bool
|
||||
}{
|
||||
{
|
||||
name: "found with URI and separate user+password usage, verified",
|
||||
@@ -154,26 +152,6 @@ func TestLdap_Integration_FromChunk(t *testing.T) {
|
||||
},
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "inaccessible host",
|
||||
s: Scanner{},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(`
|
||||
ldap://badhost:1389
|
||||
binddn="cn=admin,dc=example,dc=org"
|
||||
pass="P@55w0rd"`),
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_LDAP,
|
||||
Verified: false,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
wantVerificationErr: true,
|
||||
},
|
||||
{
|
||||
name: "not found",
|
||||
s: Scanner{},
|
||||
@@ -198,12 +176,9 @@ func TestLdap_Integration_FromChunk(t *testing.T) {
|
||||
if len(got[i].Raw) == 0 {
|
||||
t.Fatalf("no raw secret present: \n %+v", got[i])
|
||||
}
|
||||
if (got[i].VerificationError != nil) != tt.wantVerificationErr {
|
||||
t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError)
|
||||
}
|
||||
got[i].Raw = nil
|
||||
}
|
||||
ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "VerificationError")
|
||||
if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {
|
||||
if diff := pretty.Compare(got, tt.want); diff != "" {
|
||||
t.Errorf("Ldap.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
|
||||
}
|
||||
})
|
||||
@@ -230,7 +205,7 @@ func startOpenLDAP() error {
|
||||
return nil
|
||||
case <-time.After(30 * time.Second):
|
||||
stopOpenLDAP()
|
||||
return errors.New("timeout waiting for ldap service to be ready")
|
||||
return errors.New("timeout waiting for postgres database to be ready")
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -2,8 +2,6 @@ package mongodb
|
||||
|
||||
import (
|
||||
"context"
|
||||
"go.mongodb.org/mongo-driver/x/mongo/driver/auth"
|
||||
"go.mongodb.org/mongo-driver/x/mongo/driver/topology"
|
||||
"regexp"
|
||||
"strings"
|
||||
"time"
|
||||
@@ -16,17 +14,14 @@ import (
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
|
||||
)
|
||||
|
||||
type Scanner struct {
|
||||
timeout time.Duration // Zero value means "default timeout"
|
||||
}
|
||||
type Scanner struct{}
|
||||
|
||||
// Ensure the Scanner satisfies the interface at compile time.
|
||||
var _ detectors.Detector = (*Scanner)(nil)
|
||||
|
||||
var (
|
||||
defaultTimeout = 2 * time.Second
|
||||
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
|
||||
keyPat = regexp.MustCompile(`\b(mongodb(\+srv)?://[\S]{3,50}:([\S]{3,88})@[-.%\w\/:]+)\b`)
|
||||
keyPat = regexp.MustCompile(`\b(mongodb(\+srv)?://[\S]{3,50}:([\S]{3,50})@[-.%\w\/:]+)\b`)
|
||||
// TODO: Add support for sharded cluster, replica set and Atlas Deployment.
|
||||
)
|
||||
|
||||
@@ -51,15 +46,23 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
}
|
||||
|
||||
if verify {
|
||||
timeout := s.timeout
|
||||
if timeout == 0 {
|
||||
timeout = defaultTimeout
|
||||
}
|
||||
err := verifyUri(resMatch, timeout)
|
||||
s1.Verified = err == nil
|
||||
if !isErrDeterminate(err) {
|
||||
s1.VerificationError = err
|
||||
}
|
||||
func() {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
|
||||
defer cancel()
|
||||
client, err := mongo.Connect(ctx, options.Client().ApplyURI(resMatch))
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
defer func() {
|
||||
if err := client.Disconnect(ctx); err != nil {
|
||||
return
|
||||
}
|
||||
}()
|
||||
if err := client.Ping(ctx, readpref.Primary()); err != nil {
|
||||
return
|
||||
}
|
||||
s1.Verified = true
|
||||
}()
|
||||
}
|
||||
results = append(results, s1)
|
||||
}
|
||||
@@ -67,33 +70,6 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
return results, nil
|
||||
}
|
||||
|
||||
func isErrDeterminate(err error) bool {
|
||||
switch e := err.(type) {
|
||||
case topology.ConnectionError:
|
||||
switch e.Unwrap().(type) {
|
||||
case *auth.Error:
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
func verifyUri(uri string, timeout time.Duration) error {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), timeout)
|
||||
defer cancel()
|
||||
client, err := mongo.Connect(ctx, options.Client().ApplyURI(uri))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer func() {
|
||||
_ = client.Disconnect(ctx)
|
||||
}()
|
||||
return client.Ping(ctx, readpref.Primary())
|
||||
}
|
||||
|
||||
func (s Scanner) Type() detectorspb.DetectorType {
|
||||
return detectorspb.DetectorType_MongoDB
|
||||
}
|
||||
|
||||
@@ -6,7 +6,6 @@ package mongodb
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
@@ -35,12 +34,11 @@ func TestMongoDB_FromChunk(t *testing.T) {
|
||||
verify bool
|
||||
}
|
||||
tests := []struct {
|
||||
name string
|
||||
s Scanner
|
||||
args args
|
||||
want []detectors.Result
|
||||
wantErr bool
|
||||
wantVerificationErr bool
|
||||
name string
|
||||
s Scanner
|
||||
args args
|
||||
want []detectors.Result
|
||||
wantErr bool
|
||||
}{
|
||||
{
|
||||
name: "found, verified",
|
||||
@@ -74,40 +72,6 @@ func TestMongoDB_FromChunk(t *testing.T) {
|
||||
},
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "found, would be verified but for connection timeout",
|
||||
s: Scanner{timeout: 1 * time.Microsecond},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a mongodb secret %s within", secret)),
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_MongoDB,
|
||||
Verified: false,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
wantVerificationErr: true,
|
||||
},
|
||||
{
|
||||
name: "found, bad host",
|
||||
s: Scanner{},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a mongodb secret %s within", strings.ReplaceAll(secret, ".mongodb.net", ".mongodb.net.bad"))),
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_MongoDB,
|
||||
Verified: false,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
wantVerificationErr: true,
|
||||
},
|
||||
{
|
||||
name: "not found",
|
||||
s: Scanner{},
|
||||
@@ -122,7 +86,8 @@ func TestMongoDB_FromChunk(t *testing.T) {
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
|
||||
s := Scanner{}
|
||||
got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
|
||||
if (err != nil) != tt.wantErr {
|
||||
t.Errorf("MongoDB.FromData() error = %v, wantErr %v", err, tt.wantErr)
|
||||
return
|
||||
@@ -132,11 +97,8 @@ func TestMongoDB_FromChunk(t *testing.T) {
|
||||
t.Fatalf("no raw secret present: \n %+v", got[i])
|
||||
}
|
||||
got[i].Raw = nil
|
||||
if (got[i].VerificationError != nil) != tt.wantVerificationErr {
|
||||
t.Fatalf("wantVerificationErr = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError)
|
||||
}
|
||||
}
|
||||
ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "RawV2", "VerificationError")
|
||||
ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "RawV2")
|
||||
if diff := cmp.Diff(tt.want, got, ignoreOpts); diff != "" {
|
||||
t.Errorf("MongoDB.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
|
||||
}
|
||||
|
||||
@@ -52,7 +52,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
}
|
||||
|
||||
if verify {
|
||||
req, err := http.NewRequestWithContext(ctx, "GET", "https://api.opsgenie.com/v2/alerts", nil)
|
||||
req, err := http.NewRequestWithContext(ctx, "GET", "https://api.opsgenie.com/v2/users", nil)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
|
||||
@@ -22,8 +22,8 @@ var (
|
||||
client = common.SaneHttpClient()
|
||||
|
||||
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
|
||||
idPat = regexp.MustCompile(`\b([A-Za-z0-9_\.]{7}-[A-Za-z0-9_\.]{72}|[A-Za-z0-9_\.]{5}-[A-Za-z0-9_\.]{38})\b`)
|
||||
keyPat = regexp.MustCompile(`\b([A-Za-z0-9_\.\-]{44,80})\b`)
|
||||
idPat = regexp.MustCompile(`\b([A-Za-z0-9_\.]{7}-[A-Za-z0-9_\.]{72})\b`)
|
||||
keyPat = regexp.MustCompile(`\b([A-Za-z0-9_\.]{69}-[A-Za-z0-9_\.]{10})\b`)
|
||||
)
|
||||
|
||||
// Keywords are used for efficiently pre-filtering chunks.
|
||||
@@ -39,24 +39,24 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
|
||||
idmatches := idPat.FindAllStringSubmatch(dataStr, -1)
|
||||
|
||||
for _, idMatch := range idmatches {
|
||||
if len(idMatch) != 2 {
|
||||
for _, match := range matches {
|
||||
if len(match) != 2 {
|
||||
continue
|
||||
}
|
||||
resIDMatch := strings.TrimSpace(idMatch[1])
|
||||
for _, secretMatch := range matches {
|
||||
if len(secretMatch) != 2 {
|
||||
resMatch := strings.TrimSpace(match[1])
|
||||
for _, idMatch := range idmatches {
|
||||
if len(idMatch) != 2 {
|
||||
continue
|
||||
}
|
||||
resSecretMatch := strings.TrimSpace(secretMatch[1])
|
||||
residMatch := strings.TrimSpace(idMatch[1])
|
||||
|
||||
s1 := detectors.Result{
|
||||
DetectorType: detectorspb.DetectorType_PaypalOauth,
|
||||
Raw: []byte(resSecretMatch),
|
||||
Raw: []byte(resMatch),
|
||||
}
|
||||
|
||||
if verify {
|
||||
data := fmt.Sprintf("%s:%s", resIDMatch, resSecretMatch)
|
||||
data := fmt.Sprintf("%s:%s", residMatch, resMatch)
|
||||
encoded := b64.StdEncoding.EncodeToString([]byte(data))
|
||||
payload := strings.NewReader("grant_type=client_credentials")
|
||||
req, err := http.NewRequestWithContext(ctx, "POST", "https://api-m.sandbox.paypal.com/v1/oauth2/token", payload)
|
||||
@@ -74,7 +74,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
s1.Verified = true
|
||||
} else {
|
||||
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
|
||||
if detectors.IsKnownFalsePositive(resIDMatch, detectors.DefaultFalsePositives, true) {
|
||||
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
@@ -19,7 +19,7 @@ import (
|
||||
func TestPaypalOauth_FromChunk(t *testing.T) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
|
||||
defer cancel()
|
||||
testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")
|
||||
testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
|
||||
if err != nil {
|
||||
t.Fatalf("could not get test secrets from GCP: %s", err)
|
||||
}
|
||||
@@ -27,9 +27,6 @@ func TestPaypalOauth_FromChunk(t *testing.T) {
|
||||
inactiveSecret := testSecrets.MustGetField("PAYPALOAUTH_SECRET_INACTIVE")
|
||||
id := testSecrets.MustGetField("PAYPALOAUTH_CLIENTID")
|
||||
|
||||
newId := testSecrets.MustGetField("PAYPALOAUTH_NEW_INACTIVE_CLIENTID")
|
||||
newSecret := testSecrets.MustGetField("PAYPALOAUTH_NEW_INACTIVE_SECRET")
|
||||
|
||||
type args struct {
|
||||
ctx context.Context
|
||||
data []byte
|
||||
@@ -55,10 +52,6 @@ func TestPaypalOauth_FromChunk(t *testing.T) {
|
||||
DetectorType: detectorspb.DetectorType_PaypalOauth,
|
||||
Verified: true,
|
||||
},
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_PaypalOauth,
|
||||
Verified: false,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
},
|
||||
@@ -75,30 +68,6 @@ func TestPaypalOauth_FromChunk(t *testing.T) {
|
||||
DetectorType: detectorspb.DetectorType_PaypalOauth,
|
||||
Verified: false,
|
||||
},
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_PaypalOauth,
|
||||
Verified: false,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "new format, unverified",
|
||||
s: Scanner{},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a paypaloauth secret %s within %s but not valid", newSecret, newId)), // the secret would satisfy the regex but not pass validation
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_PaypalOauth,
|
||||
Verified: false,
|
||||
},
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_PaypalOauth,
|
||||
Verified: false,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
},
|
||||
|
||||
@@ -2,7 +2,6 @@ package pubnubpublishkey
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"strings"
|
||||
@@ -12,19 +11,17 @@ import (
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
|
||||
)
|
||||
|
||||
type Scanner struct {
|
||||
client *http.Client
|
||||
}
|
||||
type Scanner struct{}
|
||||
|
||||
// Ensure the Scanner satisfies the interface at compile time.
|
||||
var _ detectors.Detector = (*Scanner)(nil)
|
||||
|
||||
var (
|
||||
defaultClient = common.SaneHttpClient()
|
||||
client = common.SaneHttpClient()
|
||||
|
||||
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
|
||||
pubPat = regexp.MustCompile(`\b(pub-c-[0-9a-z]{8}-[0-9a-z]{4}-[a-z0-9]{4}-[a-z0-9]{4}-[a-z0-9]{12})\b`)
|
||||
subPat = regexp.MustCompile(`\b(sub-c-[0-9a-z]{8}-[a-z0-9]{4}-[a-z0-9]{4}-[a-z0-9]{4}-[a-z0-9]{12})\b`)
|
||||
subPat = regexp.MustCompile(`\b(sub-c-[0-9a-z]{8}-[a-z]{4}-[a-z0-9]{4}-[a-z0-9]{4}-[a-z0-9]{12})\b`)
|
||||
)
|
||||
|
||||
// Keywords are used for efficiently pre-filtering chunks.
|
||||
@@ -63,31 +60,19 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
client := s.client
|
||||
if client == nil {
|
||||
client = defaultClient
|
||||
}
|
||||
res, err := client.Do(req)
|
||||
if err == nil {
|
||||
defer res.Body.Close()
|
||||
if res.StatusCode >= 200 && res.StatusCode < 300 {
|
||||
s1.Verified = true
|
||||
} else if res.StatusCode == 400 || res.StatusCode == 403 {
|
||||
// 403 is suggested by the API docs (https://www.pubnub.com/docs/sdks/rest-api/send-signal-to-channel)
|
||||
// 400 is what actually seems to be coming back for invalid credentials
|
||||
// There's nothing to do here, because zero values of Result are what we want
|
||||
} else {
|
||||
s1.VerificationError = fmt.Errorf("unexpected HTTP response status %d", res.StatusCode)
|
||||
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
|
||||
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
|
||||
continue
|
||||
}
|
||||
}
|
||||
} else {
|
||||
s1.VerificationError = err
|
||||
}
|
||||
}
|
||||
// This function will check false positives for common test words, but also it will make sure the key
|
||||
// appears 'random' enough to be a real key.
|
||||
if !s1.Verified && detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
|
||||
continue
|
||||
}
|
||||
|
||||
results = append(results, s1)
|
||||
}
|
||||
|
||||
@@ -6,11 +6,10 @@ package pubnubpublishkey
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"github.com/google/go-cmp/cmp"
|
||||
"github.com/google/go-cmp/cmp/cmpopts"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/kylelemons/godebug/pretty"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
@@ -34,12 +33,11 @@ func TestPubNubPublishKey_FromChunk(t *testing.T) {
|
||||
verify bool
|
||||
}
|
||||
tests := []struct {
|
||||
name string
|
||||
s Scanner
|
||||
args args
|
||||
want []detectors.Result
|
||||
wantErr bool
|
||||
wantVerificationErr bool
|
||||
name string
|
||||
s Scanner
|
||||
args args
|
||||
want []detectors.Result
|
||||
wantErr bool
|
||||
}{
|
||||
{
|
||||
name: "found, verified",
|
||||
@@ -73,40 +71,6 @@ func TestPubNubPublishKey_FromChunk(t *testing.T) {
|
||||
},
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "found, would be verified but for timeout",
|
||||
s: Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a pubnub secret %s within pubnub %s", pub, sub)),
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_PubNubPublishKey,
|
||||
Verified: false,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
wantVerificationErr: true,
|
||||
},
|
||||
{
|
||||
name: "found, valid but unexpected api response",
|
||||
s: Scanner{client: common.ConstantResponseHttpClient(500, "")},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a pubnub secret %s within pubnub %s", pub, sub)),
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_PubNubPublishKey,
|
||||
Verified: false,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
wantVerificationErr: true,
|
||||
},
|
||||
{
|
||||
name: "not found",
|
||||
s: Scanner{},
|
||||
@@ -121,7 +85,8 @@ func TestPubNubPublishKey_FromChunk(t *testing.T) {
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
|
||||
s := Scanner{}
|
||||
got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
|
||||
if (err != nil) != tt.wantErr {
|
||||
t.Errorf("PubNubPublishKey.FromData() error = %v, wantErr %v", err, tt.wantErr)
|
||||
return
|
||||
@@ -130,12 +95,9 @@ func TestPubNubPublishKey_FromChunk(t *testing.T) {
|
||||
if len(got[i].Raw) == 0 {
|
||||
t.Fatalf("no raw secret present: \n %+v", got[i])
|
||||
}
|
||||
if (got[i].VerificationError != nil) != tt.wantVerificationErr {
|
||||
t.Fatalf(" wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError)
|
||||
}
|
||||
got[i].Raw = nil
|
||||
}
|
||||
opts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "VerificationError")
|
||||
if diff := cmp.Diff(got, tt.want, opts); diff != "" {
|
||||
if diff := pretty.Compare(got, tt.want); diff != "" {
|
||||
t.Errorf("PubNubPublishKey.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
|
||||
}
|
||||
})
|
||||
|
||||
@@ -20,7 +20,7 @@ var (
|
||||
client = common.SaneHttpClient()
|
||||
|
||||
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
|
||||
keyPat = regexp.MustCompile(`\b(sub-c-[0-9a-z]{8}-[a-z0-9]{4}-[a-z0-9]{4}-[a-z0-9]{4}-[a-z0-9]{12})\b`)
|
||||
keyPat = regexp.MustCompile(`\b(sub-c-[0-9a-z]{8}-[a-z]{4}-[a-z0-9]{4}-[a-z0-9]{4}-[a-z0-9]{12})\b`)
|
||||
)
|
||||
|
||||
// Keywords are used for efficiently pre-filtering chunks.
|
||||
|
||||
@@ -2,8 +2,6 @@ package shodankey
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"io"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"strings"
|
||||
@@ -50,58 +48,31 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
}
|
||||
|
||||
if verify {
|
||||
s1.Verified = verifyToken(ctx, client, resMatch)
|
||||
if !s1.Verified && detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
|
||||
req, err := http.NewRequestWithContext(ctx, "GET", "https://api.shodan.io/shodan/host/count?key="+resMatch+"&query=port:22&facets=org,os", nil)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
|
||||
res, err := client.Do(req)
|
||||
if err == nil {
|
||||
defer res.Body.Close()
|
||||
if res.StatusCode >= 200 && res.StatusCode < 300 {
|
||||
s1.Verified = true
|
||||
} else {
|
||||
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
|
||||
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
|
||||
continue
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
results = append(results, s1)
|
||||
}
|
||||
|
||||
return results, nil
|
||||
}
|
||||
|
||||
type shodanInfoRes struct {
|
||||
ScanCredits int `json:"scan_credits"`
|
||||
UsageLimits struct {
|
||||
ScanCredits int `json:"scan_credits"`
|
||||
QueryCredits int `json:"query_credits"`
|
||||
MonitoredIps int `json:"monitored_ips"`
|
||||
} `json:"usage_limits"`
|
||||
Plan string `json:"plan"`
|
||||
HTTPS bool `json:"https"`
|
||||
Unlocked bool `json:"unlocked"`
|
||||
QueryCredits int `json:"query_credits"`
|
||||
MonitoredIps int `json:"monitored_ips"`
|
||||
UnlockedLeft int `json:"unlocked_left"`
|
||||
Telnet bool `json:"telnet"`
|
||||
}
|
||||
|
||||
func verifyToken(ctx context.Context, client *http.Client, token string) bool {
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, "https://api.shodan.io/api-info?key="+token, nil)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
|
||||
res, err := client.Do(req)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
defer res.Body.Close()
|
||||
|
||||
if res.StatusCode < 200 || res.StatusCode >= 300 {
|
||||
return false
|
||||
}
|
||||
|
||||
bytes, err := io.ReadAll(res.Body)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
|
||||
var info shodanInfoRes
|
||||
return json.Unmarshal(bytes, &info) == nil
|
||||
}
|
||||
|
||||
func (s Scanner) Type() detectorspb.DetectorType {
|
||||
return detectorspb.DetectorType_ShodanKey
|
||||
}
|
||||
|
||||
@@ -10,9 +10,9 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/kylelemons/godebug/pretty"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
|
||||
)
|
||||
|
||||
|
||||
@@ -20,7 +20,7 @@ var (
|
||||
client = common.SaneHttpClient()
|
||||
|
||||
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
|
||||
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"timezoneapi"}) + `\b([a-zA-Z]{20})\b`)
|
||||
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"timezoneapi"}) + `\b([a-zA-Z0-9]{20})\b`)
|
||||
)
|
||||
|
||||
// Keywords are used for efficiently pre-filtering chunks.
|
||||
@@ -54,12 +54,8 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
|
||||
res, err := client.Do(req)
|
||||
if err == nil {
|
||||
verifiedBodyResponse, err := common.ResponseContainsSubstring(res.Body, "date")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer res.Body.Close()
|
||||
if res.StatusCode >= 200 && res.StatusCode < 300 && verifiedBodyResponse {
|
||||
if res.StatusCode >= 200 && res.StatusCode < 300 {
|
||||
s1.Verified = true
|
||||
} else {
|
||||
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
|
||||
|
||||
@@ -1,7 +1,10 @@
|
||||
package virustotal
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"io"
|
||||
"mime/multipart"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"strings"
|
||||
@@ -47,36 +50,43 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
}
|
||||
|
||||
if verify {
|
||||
s1.Verified = verifyToken(ctx, client, resMatch)
|
||||
if !s1.Verified && detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
|
||||
body := &bytes.Buffer{}
|
||||
writer := multipart.NewWriter(body)
|
||||
fw, err := writer.CreateFormField("url")
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
_, err = io.Copy(fw, strings.NewReader("https://www.amazon.com"))
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
writer.Close()
|
||||
req, err := http.NewRequestWithContext(ctx, "POST", "https://www.virustotal.com/api/v3/urls", bytes.NewReader(body.Bytes()))
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
req.Header.Add("Content-Type", writer.FormDataContentType())
|
||||
req.Header.Add("x-apikey", resMatch)
|
||||
res, err := client.Do(req)
|
||||
if err == nil {
|
||||
defer res.Body.Close()
|
||||
if res.StatusCode >= 200 && res.StatusCode < 300 {
|
||||
s1.Verified = true
|
||||
} else {
|
||||
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
|
||||
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
|
||||
continue
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
results = append(results, s1)
|
||||
}
|
||||
|
||||
return results, nil
|
||||
}
|
||||
|
||||
func verifyToken(ctx context.Context, client *http.Client, token string) bool {
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, "https://www.virustotal.com/api/v3/metadata", nil)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
req.Header.Add("x-apikey", token)
|
||||
|
||||
res, err := client.Do(req)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
defer res.Body.Close()
|
||||
|
||||
if res.StatusCode < 200 || res.StatusCode >= 300 {
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func (s Scanner) Type() detectorspb.DetectorType {
|
||||
return detectorspb.DetectorType_VirusTotal
|
||||
}
|
||||
|
||||
@@ -10,9 +10,9 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/kylelemons/godebug/pretty"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
|
||||
)
|
||||
|
||||
|
||||
+20
-12
@@ -1,14 +1,16 @@
|
||||
package engine
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"runtime"
|
||||
|
||||
"github.com/go-errors/errors"
|
||||
"google.golang.org/protobuf/proto"
|
||||
"google.golang.org/protobuf/types/known/anypb"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/context"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/sources/circleci"
|
||||
)
|
||||
|
||||
@@ -27,17 +29,23 @@ func (e *Engine) ScanCircleCI(ctx context.Context, token string) error {
|
||||
return err
|
||||
}
|
||||
|
||||
handle, err := e.sourceManager.Enroll(ctx, "trufflehog - Circle CI", new(circleci.Source).Type(),
|
||||
func(ctx context.Context, jobID, sourceID int64) (sources.Source, error) {
|
||||
circleSource := circleci.Source{}
|
||||
if err := circleSource.Init(ctx, "trufflehog - Circle CI", jobID, sourceID, true, &conn, runtime.NumCPU()); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &circleSource, nil
|
||||
})
|
||||
circleSource := circleci.Source{}
|
||||
ctx = context.WithValues(ctx,
|
||||
"source_type", circleSource.Type().String(),
|
||||
"source_name", "Circle CI",
|
||||
)
|
||||
err = circleSource.Init(ctx, "trufflehog - Circle CI", 0, int64(sourcespb.SourceType_SOURCE_TYPE_CIRCLECI), true, &conn, runtime.NumCPU())
|
||||
if err != nil {
|
||||
return err
|
||||
return errors.WrapPrefix(err, "failed to init Circle CI source", 0)
|
||||
}
|
||||
_, err = e.sourceManager.ScheduleRun(ctx, handle)
|
||||
return err
|
||||
|
||||
e.sourcesWg.Go(func() error {
|
||||
defer common.RecoverWithExit(ctx)
|
||||
err := circleSource.Chunks(ctx, e.ChunksChan())
|
||||
if err != nil {
|
||||
return fmt.Errorf("error scanning CircleCI: %w", err)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -556,7 +556,6 @@ import (
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/satismeterwritekey"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/saucelabs"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/scalewaykey"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/github_oauth2"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/scalr"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/scrapeowl"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/scraperapi"
|
||||
@@ -1531,7 +1530,6 @@ func DefaultDetectors() []detectors.Detector {
|
||||
dockerhub.Scanner{},
|
||||
couchbase.Scanner{},
|
||||
envoyapikey.Scanner{},
|
||||
github_oauth2.Scanner{},
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
+22
-12
@@ -1,28 +1,38 @@
|
||||
package engine
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"runtime"
|
||||
|
||||
"github.com/go-errors/errors"
|
||||
"google.golang.org/protobuf/types/known/anypb"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/context"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/sources/docker"
|
||||
)
|
||||
|
||||
// ScanDocker scans a given docker connection.
|
||||
func (e *Engine) ScanDocker(ctx context.Context, conn *anypb.Any) error {
|
||||
handle, err := e.sourceManager.Enroll(ctx, "trufflehog - docker", new(docker.Source).Type(),
|
||||
func(ctx context.Context, jobID, sourceID int64) (sources.Source, error) {
|
||||
dockerSource := docker.Source{}
|
||||
if err := dockerSource.Init(ctx, "trufflehog - docker", jobID, sourceID, true, conn, runtime.NumCPU()); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &dockerSource, nil
|
||||
})
|
||||
dockerSource := docker.Source{}
|
||||
|
||||
ctx = context.WithValues(ctx,
|
||||
"source_type", dockerSource.Type().String(),
|
||||
"source_name", "docker",
|
||||
)
|
||||
err := dockerSource.Init(ctx, "trufflehog - docker", 0, int64(sourcespb.SourceType_SOURCE_TYPE_DOCKER), true, conn, runtime.NumCPU())
|
||||
if err != nil {
|
||||
return err
|
||||
return errors.WrapPrefix(err, "could not init docker source", 0)
|
||||
}
|
||||
_, err = e.sourceManager.ScheduleRun(ctx, handle)
|
||||
return err
|
||||
|
||||
e.sourcesWg.Go(func() error {
|
||||
defer common.RecoverWithExit(ctx)
|
||||
err := dockerSource.Chunks(ctx, e.ChunksChan())
|
||||
if err != nil {
|
||||
return fmt.Errorf("error scanning docker image: %w", err)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
return nil
|
||||
}
|
||||
|
||||
+162
-359
@@ -11,7 +11,7 @@ import (
|
||||
"time"
|
||||
|
||||
ahocorasick "github.com/BobuSumisu/aho-corasick"
|
||||
lru "github.com/hashicorp/golang-lru"
|
||||
"golang.org/x/sync/errgroup"
|
||||
"google.golang.org/protobuf/proto"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
@@ -19,80 +19,36 @@ import (
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/context"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/decoders"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/output"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/source_metadatapb"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
|
||||
)
|
||||
|
||||
// Metrics for the scan engine for external consumption.
|
||||
type Metrics struct {
|
||||
BytesScanned uint64
|
||||
ChunksScanned uint64
|
||||
VerifiedSecretsFound uint64
|
||||
UnverifiedSecretsFound uint64
|
||||
AvgDetectorTime map[string]time.Duration
|
||||
|
||||
scanStartTime time.Time
|
||||
ScanDuration time.Duration
|
||||
}
|
||||
|
||||
// runtimeMetrics for the scan engine for internal use by the engine.
|
||||
type runtimeMetrics struct {
|
||||
mu sync.RWMutex
|
||||
Metrics
|
||||
detectorAvgTime sync.Map
|
||||
}
|
||||
|
||||
// Printer is used to format found results and output them to the user. Ex JSON, plain text, etc.
|
||||
// Please note printer implementations SHOULD BE thread safe.
|
||||
type Printer interface {
|
||||
Print(ctx context.Context, r *detectors.ResultWithMetadata) error
|
||||
}
|
||||
|
||||
type Engine struct {
|
||||
// CLI flags.
|
||||
concurrency uint8
|
||||
decoders []decoders.Decoder
|
||||
detectors map[bool][]detectors.Detector
|
||||
concurrency int
|
||||
chunks chan *sources.Chunk
|
||||
results chan detectors.ResultWithMetadata
|
||||
decoders []decoders.Decoder
|
||||
detectors map[bool][]detectors.Detector
|
||||
chunksScanned uint64
|
||||
bytesScanned uint64
|
||||
detectorAvgTime sync.Map
|
||||
sourcesWg *errgroup.Group
|
||||
workersWg sync.WaitGroup
|
||||
// filterUnverified is used to reduce the number of unverified results.
|
||||
// If there are multiple unverified results for the same chunk for the same detector,
|
||||
// only the first one will be kept.
|
||||
filterUnverified bool
|
||||
onlyVerified bool
|
||||
printAvgDetectorTime bool
|
||||
filterUnverified bool
|
||||
|
||||
// prefilter is a ahocorasick struct used for doing efficient string
|
||||
// matching given a set of words (keywords from the rules in the config)
|
||||
prefilter ahocorasick.Trie
|
||||
|
||||
// Engine synchronization primitives.
|
||||
sourceManager *sources.SourceManager
|
||||
results chan detectors.ResultWithMetadata
|
||||
detectableChunksChan chan detectableChunk
|
||||
workersWg sync.WaitGroup
|
||||
wgDetectorWorkers sync.WaitGroup
|
||||
WgNotifier sync.WaitGroup
|
||||
|
||||
// Runtime information.
|
||||
metrics runtimeMetrics
|
||||
// numFoundResults is used to keep track of the number of results found.
|
||||
numFoundResults uint32
|
||||
|
||||
// printer provides a method for formatting and outputting search results.
|
||||
// The specific implementation (e.g., JSON, plain text)
|
||||
// should be set during initialization based on user preference or program requirements.
|
||||
printer Printer
|
||||
|
||||
// dedupeCache is used to deduplicate results by comparing the
|
||||
// detector type, raw result, and source metadata
|
||||
dedupeCache *lru.Cache
|
||||
}
|
||||
|
||||
type EngineOption func(*Engine)
|
||||
|
||||
func WithConcurrency(concurrency uint8) EngineOption {
|
||||
func WithConcurrency(concurrency int) EngineOption {
|
||||
return func(e *Engine) {
|
||||
e.concurrency = concurrency
|
||||
}
|
||||
@@ -127,26 +83,6 @@ func WithFilterUnverified(filter bool) EngineOption {
|
||||
}
|
||||
}
|
||||
|
||||
// WithOnlyVerified sets the onlyVerified flag on the engine. If set to true,
|
||||
// the engine will only print verified results.
|
||||
func WithOnlyVerified(onlyVerified bool) EngineOption {
|
||||
return func(e *Engine) {
|
||||
e.onlyVerified = onlyVerified
|
||||
}
|
||||
}
|
||||
|
||||
// WithPrintAvgDetectorTime sets the printAvgDetectorTime flag on the engine. If set to
|
||||
// true, the engine will print the average time taken by each detector.
|
||||
// This option allows us to measure the time taken for each detector ONLY if
|
||||
// the engine is configured to print the results.
|
||||
// Calculating the average time taken by each detector is an expensive operation
|
||||
// and should be avoided unless specified by the user.
|
||||
func WithPrintAvgDetectorTime(printAvgDetectorTime bool) EngineOption {
|
||||
return func(e *Engine) {
|
||||
e.printAvgDetectorTime = printAvgDetectorTime
|
||||
}
|
||||
}
|
||||
|
||||
// WithFilterDetectors applies a filter to the configured list of detectors. If
|
||||
// the filterFunc returns true, the detector will be included for scanning.
|
||||
// This option applies to the existing list of detectors configured, so the
|
||||
@@ -162,13 +98,6 @@ func WithFilterDetectors(filterFunc func(detectors.Detector) bool) EngineOption
|
||||
}
|
||||
}
|
||||
|
||||
// WithPrinter sets the Printer on the engine.
|
||||
func WithPrinter(printer Printer) EngineOption {
|
||||
return func(e *Engine) {
|
||||
e.printer = printer
|
||||
}
|
||||
}
|
||||
|
||||
func filterDetectors(filterFunc func(detectors.Detector) bool, input []detectors.Detector) []detectors.Detector {
|
||||
var output []detectors.Detector
|
||||
for _, detector := range input {
|
||||
@@ -179,103 +108,12 @@ func filterDetectors(filterFunc func(detectors.Detector) bool, input []detectors
|
||||
return output
|
||||
}
|
||||
|
||||
// HasFoundResults returns true if any results are found.
|
||||
func (e *Engine) HasFoundResults() bool {
|
||||
return atomic.LoadUint32(&e.numFoundResults) > 0
|
||||
}
|
||||
|
||||
// GetMetrics returns a copy of Metrics.
|
||||
// It's safe for concurrent use, and the caller can't modify the original data.
|
||||
func (e *Engine) GetMetrics() Metrics {
|
||||
e.metrics.mu.RLock()
|
||||
defer e.metrics.mu.RUnlock()
|
||||
|
||||
result := e.metrics.Metrics
|
||||
result.AvgDetectorTime = make(map[string]time.Duration, len(e.metrics.AvgDetectorTime))
|
||||
|
||||
for detectorName, durations := range e.DetectorAvgTime() {
|
||||
var total time.Duration
|
||||
for _, d := range durations {
|
||||
total += d
|
||||
}
|
||||
avgDuration := total / time.Duration(len(durations))
|
||||
result.AvgDetectorTime[detectorName] = avgDuration
|
||||
}
|
||||
|
||||
result.ScanDuration = e.metrics.getScanDuration()
|
||||
|
||||
return result
|
||||
}
|
||||
|
||||
// GetDetectorsMetrics returns a copy of the average time taken by each detector.
|
||||
func (e *Engine) GetDetectorsMetrics() map[string]time.Duration {
|
||||
e.metrics.mu.RLock()
|
||||
defer e.metrics.mu.RUnlock()
|
||||
|
||||
result := make(map[string]time.Duration, len(DefaultDetectors()))
|
||||
for detectorName, durations := range e.DetectorAvgTime() {
|
||||
var total time.Duration
|
||||
for _, d := range durations {
|
||||
total += d
|
||||
}
|
||||
avgDuration := total / time.Duration(len(durations))
|
||||
result[detectorName] = avgDuration
|
||||
}
|
||||
|
||||
return result
|
||||
}
|
||||
|
||||
// getScanDuration returns the duration of the scan.
|
||||
// If the scan is still running, it returns the time since the scan started.
|
||||
func (m *Metrics) getScanDuration() time.Duration {
|
||||
if m.ScanDuration == 0 {
|
||||
return time.Since(m.scanStartTime)
|
||||
}
|
||||
|
||||
return m.ScanDuration
|
||||
}
|
||||
|
||||
// DetectorAvgTime returns the average time taken by each detector.
|
||||
func (e *Engine) DetectorAvgTime() map[string][]time.Duration {
|
||||
logger := context.Background().Logger()
|
||||
avgTime := map[string][]time.Duration{}
|
||||
e.metrics.detectorAvgTime.Range(func(k, v interface{}) bool {
|
||||
key, ok := k.(string)
|
||||
if !ok {
|
||||
logger.Info("expected detectorAvgTime key to be a string")
|
||||
return true
|
||||
}
|
||||
|
||||
value, ok := v.([]time.Duration)
|
||||
if !ok {
|
||||
logger.Info("expected detectorAvgTime value to be []time.Duration")
|
||||
return true
|
||||
}
|
||||
avgTime[key] = value
|
||||
return true
|
||||
})
|
||||
return avgTime
|
||||
}
|
||||
|
||||
// Start the engine with options.
|
||||
func Start(ctx context.Context, options ...EngineOption) (*Engine, error) {
|
||||
const (
|
||||
defaultChannelBuffer = 1
|
||||
// TODO (ahrav): Determine the optimal cache size.
|
||||
cacheSize = 512 // number of entries in the LRU cache
|
||||
)
|
||||
|
||||
cache, err := lru.New(cacheSize)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to initialize LRU cache: %w", err)
|
||||
}
|
||||
|
||||
func Start(ctx context.Context, options ...EngineOption) *Engine {
|
||||
e := &Engine{
|
||||
detectableChunksChan: make(chan detectableChunk, defaultChannelBuffer),
|
||||
results: make(chan detectors.ResultWithMetadata, defaultChannelBuffer),
|
||||
dedupeCache: cache,
|
||||
printer: new(output.PlainPrinter), // default printer
|
||||
metrics: runtimeMetrics{Metrics: Metrics{scanStartTime: time.Now()}},
|
||||
chunks: make(chan *sources.Chunk),
|
||||
results: make(chan detectors.ResultWithMetadata),
|
||||
detectorAvgTime: sync.Map{},
|
||||
sourcesWg: &errgroup.Group{},
|
||||
}
|
||||
|
||||
for _, option := range options {
|
||||
@@ -286,15 +124,12 @@ func Start(ctx context.Context, options ...EngineOption) (*Engine, error) {
|
||||
if e.concurrency == 0 {
|
||||
numCPU := runtime.NumCPU()
|
||||
ctx.Logger().Info("No concurrency specified, defaulting to max", "cpu", numCPU)
|
||||
e.concurrency = uint8(numCPU)
|
||||
e.concurrency = numCPU
|
||||
}
|
||||
ctx.Logger().V(3).Info("engine started", "workers", e.concurrency)
|
||||
ctx.Logger().V(2).Info("engine started", "workers", e.concurrency)
|
||||
|
||||
// Create SourceManager.
|
||||
e.sourceManager = sources.NewManager(
|
||||
sources.WithConcurrentSources(int(e.concurrency)),
|
||||
sources.WithConcurrentUnits(int(e.concurrency)),
|
||||
)
|
||||
// Limit number of concurrent goroutines dedicated to chunking a source.
|
||||
e.sourcesWg.SetLimit(e.concurrency)
|
||||
|
||||
if len(e.decoders) == 0 {
|
||||
e.decoders = decoders.DefaultDecoders()
|
||||
@@ -321,8 +156,8 @@ func Start(ctx context.Context, options ...EngineOption) (*Engine, error) {
|
||||
}
|
||||
e.prefilter = *ahocorasick.NewTrieBuilder().AddStrings(keywords).Build()
|
||||
|
||||
ctx.Logger().V(3).Info("loaded decoders", "count", len(e.decoders))
|
||||
ctx.Logger().V(3).Info("loaded detectors",
|
||||
ctx.Logger().Info("loaded decoders", "count", len(e.decoders))
|
||||
ctx.Logger().Info("loaded detectors",
|
||||
"total", len(e.detectors[true])+len(e.detectors[false]),
|
||||
"verification_enabled", len(e.detectors[true]),
|
||||
"verification_disabled", len(e.detectors[false]),
|
||||
@@ -343,93 +178,102 @@ func Start(ctx context.Context, options ...EngineOption) (*Engine, error) {
|
||||
}
|
||||
}
|
||||
|
||||
ctx.Logger().V(2).Info("starting scanner workers", "count", e.concurrency)
|
||||
// Run the Secret scanner workers and Notifier pipelines.
|
||||
for worker := uint64(0); worker < uint64(e.concurrency); worker++ {
|
||||
// Start the workers.
|
||||
for i := 0; i < e.concurrency; i++ {
|
||||
e.workersWg.Add(1)
|
||||
go func() {
|
||||
ctx := context.WithValue(ctx, "secret_worker_id", common.RandomID(5))
|
||||
defer common.Recover(ctx)
|
||||
defer e.workersWg.Done()
|
||||
e.detectorWorker(ctx)
|
||||
}()
|
||||
}
|
||||
|
||||
const detectorWorkerMultiplier = 50
|
||||
ctx.Logger().V(2).Info("starting detector workers", "count", e.concurrency*detectorWorkerMultiplier)
|
||||
for worker := uint64(0); worker < uint64(e.concurrency*detectorWorkerMultiplier); worker++ {
|
||||
e.wgDetectorWorkers.Add(1)
|
||||
go func() {
|
||||
ctx := context.WithValue(ctx, "detector_worker_id", common.RandomID(5))
|
||||
defer common.Recover(ctx)
|
||||
defer e.wgDetectorWorkers.Done()
|
||||
e.detectChunks(ctx)
|
||||
}()
|
||||
}
|
||||
|
||||
// We want 1/4th of the notifier workers as the number of scanner workers.
|
||||
const notifierWorkerRatio = 4
|
||||
maxNotifierWorkers := 1
|
||||
if numWorkers := e.concurrency / notifierWorkerRatio; numWorkers > 0 {
|
||||
maxNotifierWorkers = int(numWorkers)
|
||||
}
|
||||
ctx.Logger().V(2).Info("starting notifier workers", "count", maxNotifierWorkers)
|
||||
for worker := 0; worker < maxNotifierWorkers; worker++ {
|
||||
e.WgNotifier.Add(1)
|
||||
go func() {
|
||||
ctx := context.WithValue(ctx, "notifier_worker_id", common.RandomID(5))
|
||||
defer common.Recover(ctx)
|
||||
defer e.WgNotifier.Done()
|
||||
e.notifyResults(ctx)
|
||||
}()
|
||||
}
|
||||
|
||||
return e, nil
|
||||
return e
|
||||
}
|
||||
|
||||
// Finish waits for running sources to complete and workers to finish scanning
|
||||
// chunks before closing their respective channels. Once Finish is called, no
|
||||
// more sources may be scanned by the engine.
|
||||
func (e *Engine) Finish(ctx context.Context) error {
|
||||
func (e *Engine) Finish(ctx context.Context, logFunc func(error, string, ...any)) {
|
||||
defer common.RecoverWithExit(ctx)
|
||||
// Wait for the sources to finish putting chunks onto the chunks channel.
|
||||
err := e.sourceManager.Wait()
|
||||
// wait for the sources to finish putting chunks onto the chunks channel
|
||||
sourceErr := e.sourcesWg.Wait()
|
||||
if sourceErr != nil {
|
||||
logFunc(sourceErr, "error occurred while collecting chunks")
|
||||
}
|
||||
|
||||
e.workersWg.Wait() // Wait for the workers to finish scanning chunks.
|
||||
close(e.detectableChunksChan)
|
||||
e.wgDetectorWorkers.Wait() // Wait for the detector workers to finish detecting chunks.
|
||||
close(e.chunks)
|
||||
// wait for the workers to finish processing all of the chunks and putting
|
||||
// results onto the results channel
|
||||
e.workersWg.Wait()
|
||||
|
||||
close(e.results) // Detector workers are done, close the results channel and call it a day.
|
||||
e.WgNotifier.Wait() // Wait for the notifier workers to finish notifying results.
|
||||
|
||||
e.metrics.ScanDuration = time.Since(e.metrics.scanStartTime)
|
||||
|
||||
return err
|
||||
// TODO: re-evaluate whether this is needed and investigate why if so
|
||||
//
|
||||
// not entirely sure why results don't get processed without this pause
|
||||
// since we've put all results on the channel at this point.
|
||||
time.Sleep(time.Second)
|
||||
close(e.results)
|
||||
}
|
||||
|
||||
func (e *Engine) ChunksChan() <-chan *sources.Chunk {
|
||||
return e.sourceManager.Chunks()
|
||||
func (e *Engine) ChunksChan() chan *sources.Chunk {
|
||||
return e.chunks
|
||||
}
|
||||
|
||||
func (e *Engine) ResultsChan() chan detectors.ResultWithMetadata {
|
||||
return e.results
|
||||
}
|
||||
|
||||
// detectableChunk is a decoded chunk that is ready to be scanned by its detector.
|
||||
type detectableChunk struct {
|
||||
detector detectors.Detector
|
||||
chunk sources.Chunk
|
||||
decoder detectorspb.DecoderType
|
||||
wgDoneFn func()
|
||||
func (e *Engine) ChunksScanned() uint64 {
|
||||
return e.chunksScanned
|
||||
}
|
||||
|
||||
func (e *Engine) BytesScanned() uint64 {
|
||||
return e.bytesScanned
|
||||
}
|
||||
|
||||
func (e *Engine) dedupeAndSend(chunkResults []detectors.ResultWithMetadata) {
|
||||
dedupeMap := make(map[string]struct{})
|
||||
for _, result := range chunkResults {
|
||||
// dedupe by comparing the detector type, raw result, and source metadata
|
||||
// NOTE: in order for the PLAIN decoder to maintain precedence, make sure UTF8 is the first decoder in the
|
||||
// default decoders list
|
||||
key := fmt.Sprintf("%s%s%s%+v", result.DetectorType.String(), result.Raw, result.RawV2, result.SourceMetadata)
|
||||
if _, ok := dedupeMap[key]; ok {
|
||||
continue
|
||||
}
|
||||
dedupeMap[key] = struct{}{}
|
||||
e.results <- result
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
func (e *Engine) DetectorAvgTime() map[string][]time.Duration {
|
||||
logger := context.Background().Logger()
|
||||
avgTime := map[string][]time.Duration{}
|
||||
e.detectorAvgTime.Range(func(k, v interface{}) bool {
|
||||
key, ok := k.(string)
|
||||
if !ok {
|
||||
logger.Info("expected DetectorAvgTime key to be a string")
|
||||
return true
|
||||
}
|
||||
|
||||
value, ok := v.([]time.Duration)
|
||||
if !ok {
|
||||
logger.Info("expected DetectorAvgTime value to be []time.Duration")
|
||||
return true
|
||||
}
|
||||
avgTime[key] = value
|
||||
return true
|
||||
})
|
||||
return avgTime
|
||||
}
|
||||
|
||||
func (e *Engine) detectorWorker(ctx context.Context) {
|
||||
var wgDetect sync.WaitGroup
|
||||
|
||||
for originalChunk := range e.ChunksChan() {
|
||||
for originalChunk := range e.chunks {
|
||||
for chunk := range sources.Chunker(originalChunk) {
|
||||
var chunkResults []detectors.ResultWithMetadata
|
||||
matchedKeywords := make(map[string]struct{})
|
||||
atomic.AddUint64(&e.metrics.BytesScanned, uint64(len(chunk.Data)))
|
||||
atomic.AddUint64(&e.bytesScanned, uint64(len(chunk.Data)))
|
||||
for _, decoder := range e.decoders {
|
||||
var decoderType detectorspb.DecoderType
|
||||
switch decoder.(type) {
|
||||
@@ -469,110 +313,72 @@ func (e *Engine) detectorWorker(ctx context.Context) {
|
||||
continue
|
||||
}
|
||||
|
||||
decoded.Verify = verify
|
||||
wgDetect.Add(1)
|
||||
e.detectableChunksChan <- detectableChunk{
|
||||
chunk: *decoded,
|
||||
detector: detector,
|
||||
decoder: decoderType,
|
||||
wgDoneFn: wgDetect.Done,
|
||||
start := time.Now()
|
||||
|
||||
results, err := func() ([]detectors.Result, error) {
|
||||
ctx, cancel := context.WithTimeout(ctx, time.Second*10)
|
||||
defer cancel()
|
||||
defer common.Recover(ctx)
|
||||
return detector.FromData(ctx, verify, decoded.Data)
|
||||
}()
|
||||
if err != nil {
|
||||
ctx.Logger().Error(err, "could not scan chunk",
|
||||
"source_type", decoded.SourceType.String(),
|
||||
"metadata", decoded.SourceMetadata,
|
||||
)
|
||||
continue
|
||||
}
|
||||
|
||||
if e.filterUnverified {
|
||||
results = detectors.CleanResults(results)
|
||||
}
|
||||
for _, result := range results {
|
||||
resultChunk := chunk
|
||||
ignoreLinePresent := false
|
||||
if SupportsLineNumbers(chunk.SourceType) {
|
||||
copyChunk := *chunk
|
||||
copyMetaDataClone := proto.Clone(chunk.SourceMetadata)
|
||||
if copyMetaData, ok := copyMetaDataClone.(*source_metadatapb.MetaData); ok {
|
||||
copyChunk.SourceMetadata = copyMetaData
|
||||
}
|
||||
fragStart, mdLine := FragmentFirstLine(©Chunk)
|
||||
ignoreLinePresent = SetResultLineNumber(©Chunk, &result, fragStart, mdLine)
|
||||
resultChunk = ©Chunk
|
||||
}
|
||||
if ignoreLinePresent {
|
||||
continue
|
||||
}
|
||||
result.DecoderType = decoderType
|
||||
if result.ExtraData == nil && chunk.HandleMetadata != nil {
|
||||
result.ExtraData = chunk.HandleMetadata
|
||||
} else {
|
||||
for k, v := range chunk.HandleMetadata {
|
||||
// TODO: Check key collisions.
|
||||
result.ExtraData[k] = v
|
||||
}
|
||||
}
|
||||
chunkResults = append(chunkResults, detectors.CopyMetadata(resultChunk, result))
|
||||
}
|
||||
if len(results) > 0 {
|
||||
elapsed := time.Since(start)
|
||||
detectorName := results[0].DetectorType.String()
|
||||
avgTimeI, ok := e.detectorAvgTime.Load(detectorName)
|
||||
var avgTime []time.Duration
|
||||
if ok {
|
||||
avgTime, ok = avgTimeI.([]time.Duration)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
}
|
||||
avgTime = append(avgTime, elapsed)
|
||||
e.detectorAvgTime.Store(detectorName, avgTime)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
e.dedupeAndSend(chunkResults)
|
||||
}
|
||||
atomic.AddUint64(&e.metrics.ChunksScanned, 1)
|
||||
}
|
||||
wgDetect.Wait()
|
||||
}
|
||||
|
||||
func (e *Engine) detectChunks(ctx context.Context) {
|
||||
for data := range e.detectableChunksChan {
|
||||
e.detectChunk(ctx, data)
|
||||
}
|
||||
}
|
||||
|
||||
func (e *Engine) detectChunk(ctx context.Context, data detectableChunk) {
|
||||
var start time.Time
|
||||
if e.printAvgDetectorTime {
|
||||
start = time.Now()
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(ctx, time.Second*10)
|
||||
defer common.Recover(ctx)
|
||||
defer cancel()
|
||||
|
||||
results, err := data.detector.FromData(ctx, data.chunk.Verify, data.chunk.Data)
|
||||
if err != nil {
|
||||
ctx.Logger().Error(err, "error scanning chunk")
|
||||
}
|
||||
if e.printAvgDetectorTime && len(results) > 0 {
|
||||
elapsed := time.Since(start)
|
||||
detectorName := results[0].DetectorType.String()
|
||||
avgTimeI, ok := e.metrics.detectorAvgTime.Load(detectorName)
|
||||
var avgTime []time.Duration
|
||||
if ok {
|
||||
avgTime, ok = avgTimeI.([]time.Duration)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
}
|
||||
avgTime = append(avgTime, elapsed)
|
||||
e.metrics.detectorAvgTime.Store(detectorName, avgTime)
|
||||
}
|
||||
|
||||
if e.filterUnverified {
|
||||
results = detectors.CleanResults(results)
|
||||
}
|
||||
|
||||
for _, res := range results {
|
||||
e.processResult(data, res)
|
||||
}
|
||||
data.wgDoneFn()
|
||||
}
|
||||
|
||||
func (e *Engine) processResult(data detectableChunk, res detectors.Result) {
|
||||
ignoreLinePresent := false
|
||||
if SupportsLineNumbers(data.chunk.SourceType) {
|
||||
copyChunk := data.chunk
|
||||
copyMetaDataClone := proto.Clone(data.chunk.SourceMetadata)
|
||||
if copyMetaData, ok := copyMetaDataClone.(*source_metadatapb.MetaData); ok {
|
||||
copyChunk.SourceMetadata = copyMetaData
|
||||
}
|
||||
fragStart, mdLine := FragmentFirstLine(©Chunk)
|
||||
ignoreLinePresent = SetResultLineNumber(©Chunk, &res, fragStart, mdLine)
|
||||
data.chunk = copyChunk
|
||||
}
|
||||
if ignoreLinePresent {
|
||||
return
|
||||
}
|
||||
|
||||
secret := detectors.CopyMetadata(&data.chunk, res)
|
||||
secret.DecoderType = data.decoder
|
||||
e.results <- secret
|
||||
}
|
||||
|
||||
func (e *Engine) notifyResults(ctx context.Context) {
|
||||
for r := range e.ResultsChan() {
|
||||
if e.onlyVerified && !r.Verified {
|
||||
continue
|
||||
}
|
||||
atomic.AddUint32(&e.numFoundResults, 1)
|
||||
|
||||
key := fmt.Sprintf("%s%s%s%+v", r.DetectorType.String(), r.Raw, r.RawV2, r.SourceMetadata)
|
||||
if _, ok := e.dedupeCache.Get(key); ok {
|
||||
continue
|
||||
}
|
||||
e.dedupeCache.Add(key, struct{}{})
|
||||
|
||||
if r.Verified {
|
||||
atomic.AddUint64(&e.metrics.VerifiedSecretsFound, 1)
|
||||
} else {
|
||||
atomic.AddUint64(&e.metrics.UnverifiedSecretsFound, 1)
|
||||
}
|
||||
|
||||
if err := e.printer.Print(ctx, &r); err != nil {
|
||||
ctx.Logger().Error(err, "error printing result")
|
||||
}
|
||||
atomic.AddUint64(&e.chunksScanned, 1)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -595,20 +401,17 @@ func SupportsLineNumbers(sourceType sourcespb.SourceType) bool {
|
||||
|
||||
// FragmentLineOffset sets the line number for a provided source chunk with a given detector result.
|
||||
func FragmentLineOffset(chunk *sources.Chunk, result *detectors.Result) (int64, bool) {
|
||||
before, after, found := bytes.Cut(chunk.Data, result.Raw)
|
||||
if !found {
|
||||
return 0, false
|
||||
lines := bytes.Split(chunk.Data, []byte("\n"))
|
||||
for i, line := range lines {
|
||||
if bytes.Contains(line, result.Raw) {
|
||||
// if the line contains the ignore tag, we should ignore the result
|
||||
if bytes.Contains(line, []byte(ignoreTag)) {
|
||||
return int64(i), true
|
||||
}
|
||||
return int64(i), false
|
||||
}
|
||||
}
|
||||
lineNumber := int64(bytes.Count(before, []byte("\n")))
|
||||
// If the line contains the ignore tag, we should ignore the result.
|
||||
endLine := bytes.Index(after, []byte("\n"))
|
||||
if endLine == -1 {
|
||||
endLine = len(after)
|
||||
}
|
||||
if bytes.Contains(after[:endLine], []byte(ignoreTag)) {
|
||||
return lineNumber, true
|
||||
}
|
||||
return lineNumber, false
|
||||
return 0, false
|
||||
}
|
||||
|
||||
// FragmentFirstLine returns the first line number of a fragment along with a pointer to the value to update in the
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
package engine
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"testing"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/decoders"
|
||||
@@ -51,39 +50,6 @@ func TestFragmentLineOffset(t *testing.T) {
|
||||
expectedLine: 4,
|
||||
ignore: false,
|
||||
},
|
||||
{
|
||||
name: "match on consecutive lines",
|
||||
chunk: &sources.Chunk{
|
||||
Data: []byte("line1\nline2\ntrufflehog:ignore\nline4\nsecret\nhere\nline6"),
|
||||
},
|
||||
result: &detectors.Result{
|
||||
Raw: []byte("secret\nhere"),
|
||||
},
|
||||
expectedLine: 4,
|
||||
ignore: false,
|
||||
},
|
||||
{
|
||||
name: "ignore on last consecutive lines",
|
||||
chunk: &sources.Chunk{
|
||||
Data: []byte("line1\nline2\nline3\nsecret\nhere // trufflehog:ignore\nline5"),
|
||||
},
|
||||
result: &detectors.Result{
|
||||
Raw: []byte("secret\nhere"),
|
||||
},
|
||||
expectedLine: 3,
|
||||
ignore: true,
|
||||
},
|
||||
{
|
||||
name: "ignore on last line",
|
||||
chunk: &sources.Chunk{
|
||||
Data: []byte("line1\nline2\nline3\nsecret here // trufflehog:ignore"),
|
||||
},
|
||||
result: &detectors.Result{
|
||||
Raw: []byte("secret here"),
|
||||
},
|
||||
expectedLine: 3,
|
||||
ignore: true,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
@@ -99,43 +65,6 @@ func TestFragmentLineOffset(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func setupFragmentLineOffsetBench(totalLines, needleLine int) (*sources.Chunk, *detectors.Result) {
|
||||
data := make([]byte, 0, 4096)
|
||||
needle := []byte("needle")
|
||||
for i := 0; i < totalLines; i++ {
|
||||
if i != needleLine {
|
||||
data = append(data, []byte(fmt.Sprintf("line%d\n", i))...)
|
||||
continue
|
||||
}
|
||||
data = append(data, needle...)
|
||||
data = append(data, '\n')
|
||||
}
|
||||
chunk := &sources.Chunk{Data: data}
|
||||
result := &detectors.Result{Raw: needle}
|
||||
return chunk, result
|
||||
}
|
||||
|
||||
func BenchmarkFragmentLineOffsetStart(b *testing.B) {
|
||||
chunk, result := setupFragmentLineOffsetBench(512, 2)
|
||||
for i := 0; i < b.N; i++ {
|
||||
_, _ = FragmentLineOffset(chunk, result)
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkFragmentLineOffsetMiddle(b *testing.B) {
|
||||
chunk, result := setupFragmentLineOffsetBench(512, 256)
|
||||
for i := 0; i < b.N; i++ {
|
||||
_, _ = FragmentLineOffset(chunk, result)
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkFragmentLineOffsetEnd(b *testing.B) {
|
||||
chunk, result := setupFragmentLineOffsetBench(512, 510)
|
||||
for i := 0; i < b.N; i++ {
|
||||
_, _ = FragmentLineOffset(chunk, result)
|
||||
}
|
||||
}
|
||||
|
||||
// Test to make sure that DefaultDecoders always returns the UTF8 decoder first.
|
||||
// Technically a decoder test but we want this to run and fail in CI
|
||||
func TestDefaultDecoders(t *testing.T) {
|
||||
|
||||
+36
-12
@@ -1,11 +1,14 @@
|
||||
package engine
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"runtime"
|
||||
|
||||
"github.com/go-errors/errors"
|
||||
"google.golang.org/protobuf/proto"
|
||||
"google.golang.org/protobuf/types/known/anypb"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/context"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
|
||||
@@ -24,18 +27,39 @@ func (e *Engine) ScanFileSystem(ctx context.Context, c sources.FilesystemConfig)
|
||||
return err
|
||||
}
|
||||
|
||||
handle, err := e.sourceManager.Enroll(ctx, "trufflehog - filesystem", new(filesystem.Source).Type(),
|
||||
func(ctx context.Context, jobID, sourceID int64) (sources.Source, error) {
|
||||
fileSystemSource := filesystem.Source{}
|
||||
fileSystemSource.WithFilter(c.Filter)
|
||||
if err := fileSystemSource.Init(ctx, "trufflehog - filesystem", jobID, sourceID, true, &conn, runtime.NumCPU()); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &fileSystemSource, nil
|
||||
})
|
||||
fileSystemSource := filesystem.Source{}
|
||||
fileSystemSource.WithFilter(c.Filter)
|
||||
|
||||
// TODO: This is how it would be used; everything after these comments would be removed.
|
||||
// handle, err := e.sourceManager.Enroll(ctx, "trufflehog - filesystem", fileSystemSource.Type(),
|
||||
// func(ctx context.Context, jobID, sourceID int64) (sources.Source, error) {
|
||||
// fileSystemSource := filesystem.Source{}
|
||||
// fileSystemSource.WithFilter(c.Filter)
|
||||
// if err := fileSystemSource.Init(ctx, "trufflehog - filesystem", jobID, sourceID, true, &conn, runtime.NumCPU()); err != nil {
|
||||
// return nil, err
|
||||
// }
|
||||
// return &fileSystemSource, nil
|
||||
// })
|
||||
// if err != nil {
|
||||
// return err
|
||||
// }
|
||||
// return e.sourceManager.ScheduleRun(ctx, handle)
|
||||
|
||||
ctx = context.WithValues(ctx,
|
||||
"source_type", fileSystemSource.Type().String(),
|
||||
"source_name", "filesystem",
|
||||
)
|
||||
err = fileSystemSource.Init(ctx, "trufflehog - filesystem", 0, int64(sourcespb.SourceType_SOURCE_TYPE_FILESYSTEM), true, &conn, runtime.NumCPU())
|
||||
if err != nil {
|
||||
return err
|
||||
return errors.WrapPrefix(err, "could not init filesystem source", 0)
|
||||
}
|
||||
_, err = e.sourceManager.ScheduleRun(ctx, handle)
|
||||
return err
|
||||
e.sourcesWg.Go(func() error {
|
||||
defer common.RecoverWithExit(ctx)
|
||||
err := fileSystemSource.Chunks(ctx, e.ChunksChan())
|
||||
if err != nil {
|
||||
return fmt.Errorf("error scanning filesystem: %w", err)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
return nil
|
||||
}
|
||||
|
||||
+17
-12
@@ -6,6 +6,7 @@ import (
|
||||
"google.golang.org/protobuf/proto"
|
||||
"google.golang.org/protobuf/types/known/anypb"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/context"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
|
||||
@@ -44,19 +45,23 @@ func (e *Engine) ScanGCS(ctx context.Context, c sources.GCSConfig) error {
|
||||
return fmt.Errorf("failed to marshal GCS connection: %w", err)
|
||||
}
|
||||
|
||||
handle, err := e.sourceManager.Enroll(ctx, "trufflehog - gcs", new(gcs.Source).Type(),
|
||||
func(ctx context.Context, jobID, sourceID int64) (sources.Source, error) {
|
||||
gcsSource := gcs.Source{}
|
||||
if err := gcsSource.Init(ctx, "trufflehog - gcs", jobID, sourceID, true, &conn, int(c.Concurrency)); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &gcsSource, nil
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
source := gcs.Source{}
|
||||
ctx = context.WithValues(ctx,
|
||||
"source_type", source.Type().String(),
|
||||
"source_name", "gcs",
|
||||
)
|
||||
if err = source.Init(ctx, "trufflehog - GCS", 0, 0, true, &conn, int(c.Concurrency)); err != nil {
|
||||
return fmt.Errorf("failed to initialize GCS source: %w", err)
|
||||
}
|
||||
_, err = e.sourceManager.ScheduleRun(ctx, handle)
|
||||
return err
|
||||
|
||||
e.sourcesWg.Go(func() error {
|
||||
defer common.RecoverWithExit(ctx)
|
||||
if err := source.Chunks(ctx, e.ChunksChan()); err != nil {
|
||||
return fmt.Errorf("could not scan GCS: %w", err)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
return nil
|
||||
}
|
||||
|
||||
func isAuthValid(ctx context.Context, c sources.GCSConfig, connection *sourcespb.GCS) bool {
|
||||
|
||||
@@ -4,8 +4,6 @@ import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/context"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/decoders"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
|
||||
@@ -61,13 +59,11 @@ func TestScanGCS(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(context.TODO())
|
||||
defer cancel()
|
||||
|
||||
e, err := Start(ctx,
|
||||
e := Start(ctx,
|
||||
WithConcurrency(1),
|
||||
WithDecoders(decoders.DefaultDecoders()...),
|
||||
WithDetectors(false, DefaultDetectors()...),
|
||||
)
|
||||
assert.Nil(t, err)
|
||||
|
||||
go func() {
|
||||
resultCount := 0
|
||||
for range e.ResultsChan() {
|
||||
@@ -75,15 +71,15 @@ func TestScanGCS(t *testing.T) {
|
||||
}
|
||||
}()
|
||||
|
||||
err = e.ScanGCS(ctx, test.gcsConfig)
|
||||
err := e.ScanGCS(ctx, test.gcsConfig)
|
||||
if err != nil && !test.wantErr && !strings.Contains(err.Error(), "googleapi: Error 400: Bad Request") {
|
||||
t.Errorf("ScanGCS() got: %v, want: %v", err, nil)
|
||||
return
|
||||
}
|
||||
if err := e.Finish(ctx); err != nil && !test.wantErr && !strings.Contains(err.Error(), "googleapi: Error 400: Bad Request") {
|
||||
t.Errorf("Finish() got: %v, want: %v", err, nil)
|
||||
return
|
||||
logFatalFunc := func(_ error, _ string, _ ...any) {
|
||||
t.Fatalf("error logging function should not have been called")
|
||||
}
|
||||
e.Finish(ctx, logFatalFunc)
|
||||
|
||||
if err == nil && test.wantErr {
|
||||
t.Errorf("ScanGCS() got: %v, want: %v", err, "error")
|
||||
|
||||
+40
-33
@@ -1,13 +1,14 @@
|
||||
package engine
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"runtime"
|
||||
|
||||
gogit "github.com/go-git/go-git/v5"
|
||||
"google.golang.org/protobuf/proto"
|
||||
"google.golang.org/protobuf/types/known/anypb"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/context"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/source_metadatapb"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/sources/git"
|
||||
@@ -21,6 +22,16 @@ func (e *Engine) ScanGit(ctx context.Context, c sources.GitConfig) error {
|
||||
git.ScanOptionLogOptions(logOptions),
|
||||
}
|
||||
|
||||
options := &gogit.PlainOpenOptions{
|
||||
DetectDotGit: true,
|
||||
EnableDotGitCommonDir: true,
|
||||
}
|
||||
|
||||
repo, err := gogit.PlainOpenWithOptions(c.RepoPath, options)
|
||||
if err != nil {
|
||||
return fmt.Errorf("could not open repo: %s: %w", c.RepoPath, err)
|
||||
}
|
||||
|
||||
if c.MaxDepth != 0 {
|
||||
opts = append(opts, git.ScanOptionMaxDepth(int64(c.MaxDepth)))
|
||||
}
|
||||
@@ -33,39 +44,35 @@ func (e *Engine) ScanGit(ctx context.Context, c sources.GitConfig) error {
|
||||
if c.ExcludeGlobs != nil {
|
||||
opts = append(opts, git.ScanOptionExcludeGlobs(c.ExcludeGlobs))
|
||||
}
|
||||
if c.Bare {
|
||||
opts = append(opts, git.ScanOptionBare(c.Bare))
|
||||
}
|
||||
scanOptions := git.NewScanOptions(opts...)
|
||||
|
||||
connection := &sourcespb.Git{
|
||||
// Using Directories here allows us to not pass any
|
||||
// authentication. Also by this point, the c.RepoPath should
|
||||
// still have been prepared and downloaded to a temporary
|
||||
// directory if it was a URL.
|
||||
Directories: []string{c.RepoPath},
|
||||
}
|
||||
var conn anypb.Any
|
||||
if err := anypb.MarshalFrom(&conn, connection, proto.MarshalOptions{}); err != nil {
|
||||
ctx.Logger().Error(err, "failed to marshal git connection")
|
||||
return err
|
||||
}
|
||||
|
||||
handle, err := e.sourceManager.Enroll(ctx, "trufflehog - git", new(git.Source).Type(),
|
||||
func(ctx context.Context, jobID, sourceID int64) (sources.Source, error) {
|
||||
gitSource := git.Source{}
|
||||
if err := gitSource.Init(ctx, "trufflehog - git", jobID, sourceID, true, &conn, runtime.NumCPU()); err != nil {
|
||||
return nil, err
|
||||
gitSource := git.NewGit(sourcespb.SourceType_SOURCE_TYPE_GIT, 0, 0, "trufflehog - git", true, runtime.NumCPU(),
|
||||
func(file, email, commit, timestamp, repository string, line int64) *source_metadatapb.MetaData {
|
||||
return &source_metadatapb.MetaData{
|
||||
Data: &source_metadatapb.MetaData_Git{
|
||||
Git: &source_metadatapb.Git{
|
||||
Commit: commit,
|
||||
File: file,
|
||||
Email: email,
|
||||
Repository: repository,
|
||||
Timestamp: timestamp,
|
||||
Line: line,
|
||||
},
|
||||
},
|
||||
}
|
||||
gitSource.WithScanOptions(scanOptions)
|
||||
// Don't try to clean up the provided directory. That's handled by the
|
||||
// caller of ScanGit.
|
||||
gitSource.WithPreserveTempDirs(true)
|
||||
return &gitSource, nil
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = e.sourceManager.ScheduleRun(ctx, handle)
|
||||
return err
|
||||
|
||||
ctx = context.WithValues(ctx,
|
||||
"source_type", sourcespb.SourceType_SOURCE_TYPE_GIT.String(),
|
||||
"source_name", "git",
|
||||
)
|
||||
e.sourcesWg.Go(func() error {
|
||||
defer common.RecoverWithExit(ctx)
|
||||
err := gitSource.ScanRepo(ctx, repo, c.RepoPath, scanOptions, e.ChunksChan())
|
||||
if err != nil {
|
||||
return fmt.Errorf("could not scan repo: %w", err)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
return nil
|
||||
}
|
||||
|
||||
+18
-19
@@ -4,8 +4,6 @@ import (
|
||||
"os"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/context"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/decoders"
|
||||
@@ -21,13 +19,6 @@ type expResult struct {
|
||||
Verified bool
|
||||
}
|
||||
|
||||
type discardPrinter struct{}
|
||||
|
||||
func (p *discardPrinter) Print(context.Context, *detectors.ResultWithMetadata) error {
|
||||
// This method intentionally does nothing.
|
||||
return nil
|
||||
}
|
||||
|
||||
func TestGitEngine(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
repoUrl := "https://github.com/dustin-decker/secretsandstuff.git"
|
||||
@@ -65,13 +56,14 @@ func TestGitEngine(t *testing.T) {
|
||||
},
|
||||
} {
|
||||
t.Run(tName, func(t *testing.T) {
|
||||
e, err := Start(ctx,
|
||||
e := Start(ctx,
|
||||
WithConcurrency(1),
|
||||
WithDecoders(decoders.DefaultDecoders()...),
|
||||
WithDetectors(true, DefaultDetectors()...),
|
||||
WithPrinter(new(discardPrinter)),
|
||||
)
|
||||
assert.Nil(t, err)
|
||||
// Make the channels buffered so Finish returns.
|
||||
e.chunks = make(chan *sources.Chunk, 10)
|
||||
e.results = make(chan detectors.ResultWithMetadata, 10)
|
||||
|
||||
cfg := sources.GitConfig{
|
||||
RepoPath: path,
|
||||
@@ -84,8 +76,12 @@ func TestGitEngine(t *testing.T) {
|
||||
return
|
||||
}
|
||||
|
||||
logFatalFunc := func(_ error, _ string, _ ...any) {
|
||||
t.Fatalf("error logging function should not have been called")
|
||||
}
|
||||
// Wait for all the chunks to be processed.
|
||||
assert.Nil(t, e.Finish(ctx))
|
||||
e.Finish(ctx, logFatalFunc)
|
||||
resultCount := 0
|
||||
for result := range e.ResultsChan() {
|
||||
switch meta := result.SourceMetadata.GetData().(type) {
|
||||
case *source_metadatapb.MetaData_Git:
|
||||
@@ -99,10 +95,12 @@ func TestGitEngine(t *testing.T) {
|
||||
t.Errorf("%s: unexpected verification. Got: %v, Expected: %v", tName, result.Verified, tTest.expected[meta.Git.Commit].Verified)
|
||||
}
|
||||
}
|
||||
resultCount++
|
||||
|
||||
}
|
||||
metrics := e.GetMetrics()
|
||||
assert.Equal(t, len(tTest.expected), int(metrics.VerifiedSecretsFound)+int(metrics.UnverifiedSecretsFound))
|
||||
if resultCount != len(tTest.expected) {
|
||||
t.Errorf("%s: unexpected number of results. Got: %d, Expected: %d", tName, resultCount, len(tTest.expected))
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -119,13 +117,11 @@ func BenchmarkGitEngine(b *testing.B) {
|
||||
ctx, cancel := context.WithCancel(ctx)
|
||||
defer cancel()
|
||||
|
||||
e, err := Start(ctx,
|
||||
e := Start(ctx,
|
||||
WithConcurrency(1),
|
||||
WithDecoders(decoders.DefaultDecoders()...),
|
||||
WithDetectors(false, DefaultDetectors()...),
|
||||
)
|
||||
assert.Nil(b, err)
|
||||
|
||||
go func() {
|
||||
resultCount := 0
|
||||
for range e.ResultsChan() {
|
||||
@@ -144,5 +140,8 @@ func BenchmarkGitEngine(b *testing.B) {
|
||||
return
|
||||
}
|
||||
}
|
||||
assert.Nil(b, e.Finish(ctx))
|
||||
logFatalFunc := func(_ error, _ string, _ ...any) {
|
||||
b.Fatalf("error logging function should not have been called")
|
||||
}
|
||||
e.Finish(ctx, logFatalFunc)
|
||||
}
|
||||
|
||||
+25
-14
@@ -1,10 +1,13 @@
|
||||
package engine
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
gogit "github.com/go-git/go-git/v5"
|
||||
"google.golang.org/protobuf/proto"
|
||||
"google.golang.org/protobuf/types/known/anypb"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/context"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
|
||||
@@ -14,6 +17,8 @@ import (
|
||||
|
||||
// ScanGitHub scans Github with the provided options.
|
||||
func (e *Engine) ScanGitHub(ctx context.Context, c sources.GithubConfig) error {
|
||||
source := github.Source{}
|
||||
|
||||
connection := sourcespb.GitHub{
|
||||
Endpoint: c.Endpoint,
|
||||
Organizations: c.Orgs,
|
||||
@@ -37,25 +42,31 @@ func (e *Engine) ScanGitHub(ctx context.Context, c sources.GithubConfig) error {
|
||||
return err
|
||||
}
|
||||
|
||||
ctx = context.WithValues(ctx,
|
||||
"source_type", source.Type().String(),
|
||||
"source_name", "github",
|
||||
)
|
||||
err = source.Init(ctx, "trufflehog - github", 0, 0, false, &conn, c.Concurrency)
|
||||
if err != nil {
|
||||
ctx.Logger().Error(err, "failed to initialize github source")
|
||||
return err
|
||||
}
|
||||
|
||||
logOptions := &gogit.LogOptions{}
|
||||
opts := []git.ScanOption{
|
||||
git.ScanOptionFilter(c.Filter),
|
||||
git.ScanOptionLogOptions(logOptions),
|
||||
}
|
||||
scanOptions := git.NewScanOptions(opts...)
|
||||
source.WithScanOptions(scanOptions)
|
||||
|
||||
handle, err := e.sourceManager.Enroll(ctx, "trufflehog - github", new(github.Source).Type(),
|
||||
func(ctx context.Context, jobID, sourceID int64) (sources.Source, error) {
|
||||
githubSource := github.Source{}
|
||||
if err := githubSource.Init(ctx, "trufflehog - github", jobID, sourceID, true, &conn, c.Concurrency); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
githubSource.WithScanOptions(scanOptions)
|
||||
return &githubSource, nil
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = e.sourceManager.ScheduleRun(ctx, handle)
|
||||
return err
|
||||
e.sourcesWg.Go(func() error {
|
||||
defer common.RecoverWithExit(ctx)
|
||||
err := source.Chunks(ctx, e.ChunksChan())
|
||||
if err != nil {
|
||||
return fmt.Errorf("could not scan github: %w", err)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
return nil
|
||||
}
|
||||
|
||||
+20
-12
@@ -4,10 +4,12 @@ import (
|
||||
"fmt"
|
||||
"runtime"
|
||||
|
||||
"github.com/go-errors/errors"
|
||||
gogit "github.com/go-git/go-git/v5"
|
||||
"google.golang.org/protobuf/proto"
|
||||
"google.golang.org/protobuf/types/known/anypb"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/context"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
|
||||
@@ -50,18 +52,24 @@ func (e *Engine) ScanGitLab(ctx context.Context, c sources.GitlabConfig) error {
|
||||
return err
|
||||
}
|
||||
|
||||
handle, err := e.sourceManager.Enroll(ctx, "trufflehog - gitlab", new(gitlab.Source).Type(),
|
||||
func(ctx context.Context, jobID, sourceID int64) (sources.Source, error) {
|
||||
gitlabSource := gitlab.Source{}
|
||||
if err := gitlabSource.Init(ctx, "trufflehog - gitlab", jobID, sourceID, true, &conn, runtime.NumCPU()); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
gitlabSource.WithScanOptions(scanOptions)
|
||||
return &gitlabSource, nil
|
||||
})
|
||||
gitlabSource := gitlab.Source{}
|
||||
ctx = context.WithValues(ctx,
|
||||
"source_type", gitlabSource.Type().String(),
|
||||
"source_name", "gitlab",
|
||||
)
|
||||
err = gitlabSource.Init(ctx, "trufflehog - gitlab", 0, int64(sourcespb.SourceType_SOURCE_TYPE_GITLAB), true, &conn, runtime.NumCPU())
|
||||
if err != nil {
|
||||
return err
|
||||
return errors.WrapPrefix(err, "could not init GitLab source", 0)
|
||||
}
|
||||
_, err = e.sourceManager.ScheduleRun(ctx, handle)
|
||||
return err
|
||||
gitlabSource.WithScanOptions(scanOptions)
|
||||
|
||||
e.sourcesWg.Go(func() error {
|
||||
defer common.RecoverWithExit(ctx)
|
||||
err := gitlabSource.Chunks(ctx, e.ChunksChan())
|
||||
if err != nil {
|
||||
return fmt.Errorf("error scanning GitLab: %w", err)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
return nil
|
||||
}
|
||||
|
||||
+19
-11
@@ -4,9 +4,11 @@ import (
|
||||
"fmt"
|
||||
"runtime"
|
||||
|
||||
"github.com/go-errors/errors"
|
||||
"google.golang.org/protobuf/proto"
|
||||
"google.golang.org/protobuf/types/known/anypb"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/context"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/credentialspb"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
|
||||
@@ -53,17 +55,23 @@ func (e *Engine) ScanS3(ctx context.Context, c sources.S3Config) error {
|
||||
return err
|
||||
}
|
||||
|
||||
handle, err := e.sourceManager.Enroll(ctx, "trufflehog - s3", new(s3.Source).Type(),
|
||||
func(ctx context.Context, jobID, sourceID int64) (sources.Source, error) {
|
||||
s3Source := s3.Source{}
|
||||
if err := s3Source.Init(ctx, "trufflehog - s3", jobID, sourceID, true, &conn, runtime.NumCPU()); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &s3Source, nil
|
||||
})
|
||||
s3Source := s3.Source{}
|
||||
ctx = context.WithValues(ctx,
|
||||
"source_type", s3Source.Type().String(),
|
||||
"source_name", "s3",
|
||||
)
|
||||
err = s3Source.Init(ctx, "trufflehog - s3", 0, int64(sourcespb.SourceType_SOURCE_TYPE_S3), true, &conn, runtime.NumCPU())
|
||||
if err != nil {
|
||||
return err
|
||||
return errors.WrapPrefix(err, "failed to init S3 source", 0)
|
||||
}
|
||||
_, err = e.sourceManager.ScheduleRun(ctx, handle)
|
||||
return err
|
||||
|
||||
e.sourcesWg.Go(func() error {
|
||||
defer common.RecoverWithExit(ctx)
|
||||
err := s3Source.Chunks(ctx, e.ChunksChan())
|
||||
if err != nil {
|
||||
return fmt.Errorf("error scanning S3: %w", err)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
return nil
|
||||
}
|
||||
|
||||
+20
-12
@@ -1,12 +1,14 @@
|
||||
package engine
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
|
||||
"github.com/go-errors/errors"
|
||||
"google.golang.org/protobuf/proto"
|
||||
"google.golang.org/protobuf/types/known/anypb"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/context"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
|
||||
@@ -40,19 +42,25 @@ func (e *Engine) ScanSyslog(ctx context.Context, c sources.SyslogConfig) error {
|
||||
if err != nil {
|
||||
return errors.WrapPrefix(err, "error unmarshalling connection", 0)
|
||||
}
|
||||
|
||||
handle, err := e.sourceManager.Enroll(ctx, "trufflehog - syslog", new(syslog.Source).Type(),
|
||||
func(ctx context.Context, jobID, sourceID int64) (sources.Source, error) {
|
||||
syslogSource := syslog.Source{}
|
||||
if err := syslogSource.Init(ctx, "trufflehog - syslog", jobID, sourceID, true, &conn, c.Concurrency); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
syslogSource.InjectConnection(connection)
|
||||
return &syslogSource, nil
|
||||
})
|
||||
source := syslog.Source{}
|
||||
ctx = context.WithValues(ctx,
|
||||
"source_type", source.Type().String(),
|
||||
"source_name", "syslog",
|
||||
)
|
||||
err = source.Init(ctx, "trufflehog - syslog", 0, 0, false, &conn, c.Concurrency)
|
||||
source.InjectConnection(connection)
|
||||
if err != nil {
|
||||
ctx.Logger().Error(err, "failed to initialize syslog source")
|
||||
return err
|
||||
}
|
||||
_, err = e.sourceManager.ScheduleRun(ctx, handle)
|
||||
return err
|
||||
|
||||
e.sourcesWg.Go(func() error {
|
||||
defer common.RecoverWithExit(ctx)
|
||||
err := source.Chunks(ctx, e.ChunksChan())
|
||||
if err != nil {
|
||||
return fmt.Errorf("could not scan syslog: %w", err)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
return nil
|
||||
}
|
||||
|
||||
+17
-43
@@ -5,7 +5,6 @@ import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
@@ -57,7 +56,6 @@ type ParseState int
|
||||
const (
|
||||
Initial ParseState = iota
|
||||
CommitLine
|
||||
MergeLine
|
||||
AuthorLine
|
||||
DateLine
|
||||
MessageStartLine
|
||||
@@ -78,7 +76,6 @@ func (state ParseState) String() string {
|
||||
return [...]string{
|
||||
"Initial",
|
||||
"CommitLine",
|
||||
"MergeLine",
|
||||
"AuthorLine",
|
||||
"DateLine",
|
||||
"MessageStartLine",
|
||||
@@ -161,7 +158,7 @@ func (c1 *Commit) Equal(c2 *Commit) bool {
|
||||
}
|
||||
|
||||
// RepoPath parses the output of the `git log` command for the `source` path.
|
||||
func (c *Parser) RepoPath(ctx context.Context, source string, head string, abbreviatedLog bool, excludedGlobs []string, isBare bool) (chan Commit, error) {
|
||||
func (c *Parser) RepoPath(ctx context.Context, source string, head string, abbreviatedLog bool, excludedGlobs []string) (chan Commit, error) {
|
||||
args := []string{"-C", source, "log", "-p", "--full-history", "--date=format:%a %b %d %H:%M:%S %Y %z"}
|
||||
if abbreviatedLog {
|
||||
args = append(args, "--diff-filter=AM")
|
||||
@@ -178,21 +175,7 @@ func (c *Parser) RepoPath(ctx context.Context, source string, head string, abbre
|
||||
cmd := exec.Command("git", args...)
|
||||
absPath, err := filepath.Abs(source)
|
||||
if err == nil {
|
||||
if !isBare {
|
||||
cmd.Env = append(cmd.Env, "GIT_DIR="+filepath.Join(absPath, ".git"))
|
||||
} else {
|
||||
cmd.Env = append(cmd.Env,
|
||||
"GIT_DIR="+absPath,
|
||||
)
|
||||
// We need those variables to handle incoming commits
|
||||
// while using trufflehog in pre-receive hooks
|
||||
if dir := os.Getenv("GIT_OBJECT_DIRECTORY"); dir != "" {
|
||||
cmd.Env = append(cmd.Env, "GIT_OBJECT_DIRECTORY="+dir)
|
||||
}
|
||||
if dir := os.Getenv("GIT_ALTERNATE_OBJECT_DIRECTORIES"); dir != "" {
|
||||
cmd.Env = append(cmd.Env, "GIT_ALTERNATE_OBJECT_DIRECTORIES="+dir)
|
||||
}
|
||||
}
|
||||
cmd.Env = append(cmd.Env, fmt.Sprintf("GIT_DIR=%s", filepath.Join(absPath, ".git")))
|
||||
}
|
||||
|
||||
return c.executeCommand(ctx, cmd, false)
|
||||
@@ -252,7 +235,7 @@ func (c *Parser) FromReader(ctx context.Context, stdOut io.Reader, commitChan ch
|
||||
outReader := bufio.NewReader(stdOut)
|
||||
var (
|
||||
currentCommit *Commit
|
||||
currentDiff Diff
|
||||
currentDiff *Diff
|
||||
|
||||
totalLogSize int
|
||||
)
|
||||
@@ -275,8 +258,8 @@ func (c *Parser) FromReader(ctx context.Context, stdOut io.Reader, commitChan ch
|
||||
latestState = CommitLine
|
||||
|
||||
// If there is a currentDiff, add it to currentCommit.
|
||||
if currentDiff.Content.Len() > 0 {
|
||||
currentCommit.Diffs = append(currentCommit.Diffs, currentDiff)
|
||||
if currentDiff != nil && currentDiff.Content.Len() > 0 {
|
||||
currentCommit.Diffs = append(currentCommit.Diffs, *currentDiff)
|
||||
currentCommit.Size += currentDiff.Content.Len()
|
||||
}
|
||||
// If there is a currentCommit, send it to the channel.
|
||||
@@ -285,7 +268,7 @@ func (c *Parser) FromReader(ctx context.Context, stdOut io.Reader, commitChan ch
|
||||
totalLogSize += currentCommit.Size
|
||||
}
|
||||
// Create a new currentDiff and currentCommit
|
||||
currentDiff = Diff{}
|
||||
currentDiff = &Diff{}
|
||||
currentCommit = &Commit{
|
||||
Message: strings.Builder{},
|
||||
}
|
||||
@@ -293,8 +276,6 @@ func (c *Parser) FromReader(ctx context.Context, stdOut io.Reader, commitChan ch
|
||||
if len(line) >= 47 {
|
||||
currentCommit.Hash = string(line[7:47])
|
||||
}
|
||||
case isMergeLine(isStaged, latestState, line):
|
||||
latestState = MergeLine
|
||||
case isAuthorLine(isStaged, latestState, line):
|
||||
latestState = AuthorLine
|
||||
|
||||
@@ -324,8 +305,8 @@ func (c *Parser) FromReader(ctx context.Context, stdOut io.Reader, commitChan ch
|
||||
if currentCommit == nil {
|
||||
currentCommit = &Commit{}
|
||||
}
|
||||
if currentDiff.Content.Len() > 0 {
|
||||
currentCommit.Diffs = append(currentCommit.Diffs, currentDiff)
|
||||
if currentDiff != nil && currentDiff.Content.Len() > 0 {
|
||||
currentCommit.Diffs = append(currentCommit.Diffs, *currentDiff)
|
||||
// If the currentDiff is over 1GB, drop it into the channel so it isn't held in memory waiting for more commits.
|
||||
totalSize := 0
|
||||
for _, diff := range currentCommit.Diffs {
|
||||
@@ -346,7 +327,7 @@ func (c *Parser) FromReader(ctx context.Context, stdOut io.Reader, commitChan ch
|
||||
currentCommit.Message.WriteString(oldCommit.Message.String())
|
||||
}
|
||||
}
|
||||
currentDiff = Diff{}
|
||||
currentDiff = &Diff{}
|
||||
case isModeLine(isStaged, latestState, line):
|
||||
latestState = ModeLine
|
||||
// NoOp
|
||||
@@ -369,13 +350,16 @@ func (c *Parser) FromReader(ctx context.Context, stdOut io.Reader, commitChan ch
|
||||
case isHunkLineNumberLine(isStaged, latestState, line):
|
||||
latestState = HunkLineNumberLine
|
||||
|
||||
if currentDiff.Content.Len() > 0 {
|
||||
currentCommit.Diffs = append(currentCommit.Diffs, currentDiff)
|
||||
// TODO: Is it still necessary to check whether the currentDiff is nil?
|
||||
if currentDiff != nil && currentDiff.Content.Len() > 0 {
|
||||
currentCommit.Diffs = append(currentCommit.Diffs, *currentDiff)
|
||||
}
|
||||
currentDiff = Diff{
|
||||
newDiff := &Diff{
|
||||
PathB: currentDiff.PathB,
|
||||
}
|
||||
|
||||
currentDiff = newDiff
|
||||
|
||||
words := bytes.Split(line, []byte(" "))
|
||||
if len(words) >= 3 {
|
||||
startSlice := bytes.Split(words[2], []byte(","))
|
||||
@@ -439,21 +423,11 @@ func (c *Parser) FromReader(ctx context.Context, stdOut io.Reader, commitChan ch
|
||||
break
|
||||
}
|
||||
}
|
||||
cleanupParse(currentCommit, ¤tDiff, commitChan, &totalLogSize)
|
||||
cleanupParse(currentCommit, currentDiff, commitChan, &totalLogSize)
|
||||
|
||||
ctx.Logger().V(2).Info("finished parsing git log.", "total_log_size", totalLogSize)
|
||||
}
|
||||
|
||||
func isMergeLine(isStaged bool, latestState ParseState, line []byte) bool {
|
||||
if isStaged || latestState != CommitLine {
|
||||
return false
|
||||
}
|
||||
if len(line) > 6 && bytes.Equal(line[:6], []byte("Merge:")) {
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// commit 7a95bbf0199e280a0e42dbb1d1a3f56cdd0f6e05
|
||||
func isCommitLine(isStaged bool, latestState ParseState, line []byte) bool {
|
||||
if isStaged || !(latestState == Initial ||
|
||||
@@ -476,7 +450,7 @@ func isCommitLine(isStaged bool, latestState ParseState, line []byte) bool {
|
||||
|
||||
// Author: Bill Rich <bill.rich@trufflesec.com>
|
||||
func isAuthorLine(isStaged bool, latestState ParseState, line []byte) bool {
|
||||
if isStaged || !(latestState == CommitLine || latestState == MergeLine) {
|
||||
if isStaged || !(latestState == CommitLine) {
|
||||
return false
|
||||
}
|
||||
if len(line) > 8 && bytes.Equal(line[:7], []byte("Author:")) {
|
||||
|
||||
@@ -67,25 +67,6 @@ func TestLineChecks(t *testing.T) {
|
||||
},
|
||||
function: isCommitLine,
|
||||
},
|
||||
"mergeLine": {
|
||||
passes: []testCaseLine{
|
||||
{
|
||||
CommitLine,
|
||||
[]byte("Merge: f21a95535a2 ed08d10bcf5"),
|
||||
},
|
||||
},
|
||||
fails: []testCaseLine{
|
||||
{
|
||||
DateLine,
|
||||
[]byte(" Merge pull request #34511 from cescoffier/duplicated-context-doc"),
|
||||
},
|
||||
{
|
||||
CommitLine,
|
||||
[]byte("notcorrect"),
|
||||
},
|
||||
},
|
||||
function: isMergeLine,
|
||||
},
|
||||
"authorLine": {
|
||||
passes: []testCaseLine{
|
||||
{
|
||||
|
||||
+4
-12
@@ -7,20 +7,12 @@ import (
|
||||
"github.com/pkg/errors"
|
||||
)
|
||||
|
||||
type provider string
|
||||
|
||||
const (
|
||||
providerGithub provider = "Github"
|
||||
providerGitlab provider = "Gitlab"
|
||||
providerBitbucket provider = "Bitbucket"
|
||||
)
|
||||
|
||||
func NormalizeBitbucketRepo(repoURL string) (string, error) {
|
||||
if !strings.HasPrefix(repoURL, "https") {
|
||||
return "", errors.New("Bitbucket requires https repo urls: e.g. https://bitbucket.org/org/repo.git")
|
||||
}
|
||||
|
||||
return NormalizeOrgRepoURL(providerBitbucket, repoURL)
|
||||
return NormalizeOrgRepoURL("Bitbucket", repoURL)
|
||||
}
|
||||
|
||||
func NormalizeGerritProject(project string) (string, error) {
|
||||
@@ -28,7 +20,7 @@ func NormalizeGerritProject(project string) (string, error) {
|
||||
}
|
||||
|
||||
func NormalizeGithubRepo(repoURL string) (string, error) {
|
||||
return NormalizeOrgRepoURL(providerGithub, repoURL)
|
||||
return NormalizeOrgRepoURL("Github", repoURL)
|
||||
}
|
||||
|
||||
func NormalizeGitlabRepo(repoURL string) (string, error) {
|
||||
@@ -36,12 +28,12 @@ func NormalizeGitlabRepo(repoURL string) (string, error) {
|
||||
return "", errors.New("Gitlab requires http/https repo urls: e.g. https://gitlab.com/org/repo.git")
|
||||
}
|
||||
|
||||
return NormalizeOrgRepoURL(providerGitlab, repoURL)
|
||||
return NormalizeOrgRepoURL("Gitlab", repoURL)
|
||||
}
|
||||
|
||||
// NormalizeOrgRepoURL attempts to normalize repos for any provider using the example.com/org/repo style.
|
||||
// e.g. %s, Gitlab and Bitbucket
|
||||
func NormalizeOrgRepoURL(provider provider, repoURL string) (string, error) {
|
||||
func NormalizeOrgRepoURL(provider, repoURL string) (string, error) {
|
||||
if strings.HasSuffix(repoURL, ".git") {
|
||||
return repoURL, nil
|
||||
}
|
||||
|
||||
+10
-10
@@ -8,22 +8,22 @@ import (
|
||||
|
||||
func Test_NormalizeOrgRepoURL(t *testing.T) {
|
||||
tests := map[string]struct {
|
||||
Provider provider
|
||||
Provider string
|
||||
Repo string
|
||||
Out string
|
||||
Err error
|
||||
}{
|
||||
"github is good": {Provider: providerGithub, Repo: "https://github.com/org/repo", Out: "https://github.com/org/repo.git", Err: nil},
|
||||
"gitlab is good": {Provider: providerGitlab, Repo: "https://gitlab.com/org/repo", Out: "https://gitlab.com/org/repo.git", Err: nil},
|
||||
"bitbucket is good": {Provider: providerBitbucket, Repo: "https://bitbucket.com/org/repo", Out: "https://bitbucket.com/org/repo.git", Err: nil},
|
||||
"github is good": {Provider: "Github", Repo: "https://github.com/org/repo", Out: "https://github.com/org/repo.git", Err: nil},
|
||||
"gitlab is good": {Provider: "Gitlab", Repo: "https://gitlab.com/org/repo", Out: "https://gitlab.com/org/repo.git", Err: nil},
|
||||
"bitbucket is good": {Provider: "Bitbucket", Repo: "https://bitbucket.com/org/repo", Out: "https://bitbucket.com/org/repo.git", Err: nil},
|
||||
"example provider is good": {Provider: "example", Repo: "https://example.com/org/repo", Out: "https://example.com/org/repo.git", Err: nil},
|
||||
"example provider problem": {Provider: "example", Repo: "https://example.com/org", Out: "", Err: errors.Errorf("example repo appears to be missing the repo name. Org: %q Repo url: %q", "org", "https://example.com/org")},
|
||||
"no path": {Provider: providerGithub, Repo: "https://github.com", Out: "", Err: errors.Errorf("Github repo appears to be missing the path. Repo url: %q", "https://github.com")},
|
||||
"org but no repo": {Provider: providerGithub, Repo: "https://github.com/org", Out: "", Err: errors.Errorf("Github repo appears to be missing the repo name. Org: %q Repo url: %q", "org", "https://github.com/org")},
|
||||
"org but no repo with slash": {Provider: providerGithub, Repo: "https://github.com/org/", Out: "", Err: errors.Errorf("Github repo appears to be missing the repo name. Org: %q Repo url: %q", "org", "https://github.com/org/")},
|
||||
"two slashes": {Provider: providerGithub, Repo: "https://github.com//", Out: "", Err: errors.Errorf("Github repo appears to be missing the org name. Repo url: %q", "https://github.com//")},
|
||||
"repo with trailing slash": {Provider: providerGithub, Repo: "https://github.com/org/repo/", Out: "", Err: errors.Errorf("Github repo contains a trailing slash. Repo url: %q", "https://github.com/org/repo/")},
|
||||
"too many url path parts": {Provider: providerGithub, Repo: "https://github.com/org/repo/unknown/", Out: "", Err: errors.Errorf("Github repo contains a trailing slash. Repo url: %q", "https://github.com/org/repo/unknown/")},
|
||||
"no path": {Provider: "Github", Repo: "https://github.com", Out: "", Err: errors.Errorf("Github repo appears to be missing the path. Repo url: %q", "https://github.com")},
|
||||
"org but no repo": {Provider: "Github", Repo: "https://github.com/org", Out: "", Err: errors.Errorf("Github repo appears to be missing the repo name. Org: %q Repo url: %q", "org", "https://github.com/org")},
|
||||
"org but no repo with slash": {Provider: "Github", Repo: "https://github.com/org/", Out: "", Err: errors.Errorf("Github repo appears to be missing the repo name. Org: %q Repo url: %q", "org", "https://github.com/org/")},
|
||||
"two slashes": {Provider: "Github", Repo: "https://github.com//", Out: "", Err: errors.Errorf("Github repo appears to be missing the org name. Repo url: %q", "https://github.com//")},
|
||||
"repo with trailing slash": {Provider: "Github", Repo: "https://github.com/org/repo/", Out: "", Err: errors.Errorf("Github repo contains a trailing slash. Repo url: %q", "https://github.com/org/repo/")},
|
||||
"too many url path parts": {Provider: "Github", Repo: "https://github.com/org/repo/unknown/", Out: "", Err: errors.Errorf("Github repo contains a trailing slash. Repo url: %q", "https://github.com/org/repo/unknown/")},
|
||||
}
|
||||
|
||||
for name, tt := range tests {
|
||||
|
||||
+29
-300
@@ -6,17 +6,14 @@ import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/h2non/filetype"
|
||||
"github.com/mholt/archiver/v4"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
logContext "github.com/trufflesecurity/trufflehog/v3/pkg/context"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
|
||||
)
|
||||
|
||||
type ctxKey int
|
||||
@@ -31,18 +28,14 @@ var (
|
||||
maxTimeout = time.Duration(30) * time.Second
|
||||
)
|
||||
|
||||
// Ensure the Archive satisfies the interfaces at compile time.
|
||||
var _ SpecializedHandler = (*Archive)(nil)
|
||||
|
||||
// Archive is a handler for extracting and decompressing archives.
|
||||
type Archive struct {
|
||||
size int
|
||||
currentDepth int
|
||||
size int
|
||||
}
|
||||
|
||||
// New sets a default maximum size and current size counter.
|
||||
func (a *Archive) New() {
|
||||
a.size = 0
|
||||
func (d *Archive) New() {
|
||||
d.size = 0
|
||||
}
|
||||
|
||||
// SetArchiveMaxSize sets the maximum size of the archive.
|
||||
@@ -61,14 +54,14 @@ func SetArchiveMaxTimeout(timeout time.Duration) {
|
||||
}
|
||||
|
||||
// FromFile extracts the files from an archive.
|
||||
func (a *Archive) FromFile(originalCtx context.Context, data io.Reader) chan []byte {
|
||||
archiveChan := make(chan []byte, 512)
|
||||
func (d *Archive) FromFile(originalCtx context.Context, data io.Reader) chan ChunkOpt {
|
||||
archiveChan := make(chan ChunkOpt, 512)
|
||||
go func() {
|
||||
ctx, cancel := context.WithTimeout(originalCtx, maxTimeout)
|
||||
logger := logContext.AddLogger(ctx).Logger()
|
||||
defer cancel()
|
||||
defer close(archiveChan)
|
||||
err := a.openArchive(ctx, 0, data, archiveChan)
|
||||
err := d.openArchive(ctx, nil, data, archiveChan)
|
||||
if err != nil {
|
||||
if errors.Is(err, archiver.ErrNoMatch) {
|
||||
return
|
||||
@@ -80,18 +73,21 @@ func (a *Archive) FromFile(originalCtx context.Context, data io.Reader) chan []b
|
||||
}
|
||||
|
||||
// openArchive takes a reader and extracts the contents up to the maximum depth.
|
||||
func (a *Archive) openArchive(ctx context.Context, depth int, reader io.Reader, archiveChan chan []byte) error {
|
||||
if depth >= maxDepth {
|
||||
func (d *Archive) openArchive(ctx context.Context, depth []string, reader io.Reader, archiveChan chan ChunkOpt) error {
|
||||
if len(depth) >= maxDepth {
|
||||
return fmt.Errorf("max archive depth reached")
|
||||
}
|
||||
format, reader, err := archiver.Identify("", reader)
|
||||
if err != nil {
|
||||
if errors.Is(err, archiver.ErrNoMatch) && depth > 0 {
|
||||
if errors.Is(err, archiver.ErrNoMatch) && len(depth) > 0 {
|
||||
chunkSize := 10 * 1024
|
||||
for {
|
||||
chunk := make([]byte, chunkSize)
|
||||
n, _ := reader.Read(chunk)
|
||||
archiveChan <- chunk
|
||||
data := make([]byte, chunkSize)
|
||||
n, _ := reader.Read(data)
|
||||
archiveChan <- func(chunk *sources.Chunk) {
|
||||
chunk.Data = data
|
||||
chunk.HandleMetadata = map[string]string{"archive path": strings.Join(depth, " > ")}
|
||||
}
|
||||
if n < chunkSize {
|
||||
break
|
||||
}
|
||||
@@ -106,14 +102,14 @@ func (a *Archive) openArchive(ctx context.Context, depth int, reader io.Reader,
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fileBytes, err := a.ReadToMax(ctx, compReader)
|
||||
fileBytes, err := d.ReadToMax(ctx, compReader)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
newReader := bytes.NewReader(fileBytes)
|
||||
return a.openArchive(ctx, depth+1, newReader, archiveChan)
|
||||
return d.openArchive(ctx, append(depth, "decompressed"), newReader, archiveChan)
|
||||
case archiver.Extractor:
|
||||
err := archive.Extract(context.WithValue(ctx, depthKey, depth+1), reader, nil, a.extractorHandler(archiveChan))
|
||||
err := archive.Extract(context.WithValue(ctx, depthKey, depth), reader, nil, d.extractorHandler(archiveChan))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -123,7 +119,7 @@ func (a *Archive) openArchive(ctx context.Context, depth int, reader io.Reader,
|
||||
}
|
||||
|
||||
// IsFiletype returns true if the provided reader is an archive.
|
||||
func (a *Archive) IsFiletype(ctx context.Context, reader io.Reader) (io.Reader, bool) {
|
||||
func (d *Archive) IsFiletype(ctx context.Context, reader io.Reader) (io.Reader, bool) {
|
||||
format, readerB, err := archiver.Identify("", reader)
|
||||
if err != nil {
|
||||
return readerB, false
|
||||
@@ -138,12 +134,12 @@ func (a *Archive) IsFiletype(ctx context.Context, reader io.Reader) (io.Reader,
|
||||
}
|
||||
|
||||
// extractorHandler is applied to each file in an archiver.Extractor file.
|
||||
func (a *Archive) extractorHandler(archiveChan chan []byte) func(context.Context, archiver.File) error {
|
||||
func (d *Archive) extractorHandler(archiveChan chan ChunkOpt) func(context.Context, archiver.File) error {
|
||||
return func(ctx context.Context, f archiver.File) error {
|
||||
logger := logContext.AddLogger(ctx).Logger()
|
||||
logger.V(5).Info("Handling extracted file.", "filename", f.Name())
|
||||
depth := 0
|
||||
if ctxDepth, ok := ctx.Value(depthKey).(int); ok {
|
||||
var depth []string
|
||||
if ctxDepth, ok := ctx.Value(depthKey).([]string); ok {
|
||||
depth = ctxDepth
|
||||
}
|
||||
|
||||
@@ -151,13 +147,13 @@ func (a *Archive) extractorHandler(archiveChan chan []byte) func(context.Context
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fileBytes, err := a.ReadToMax(ctx, fReader)
|
||||
fileBytes, err := d.ReadToMax(ctx, fReader)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fileContent := bytes.NewReader(fileBytes)
|
||||
|
||||
err = a.openArchive(ctx, depth, fileContent, archiveChan)
|
||||
err = d.openArchive(ctx, append(depth, f.Name()), fileContent, archiveChan)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -166,7 +162,7 @@ func (a *Archive) extractorHandler(archiveChan chan []byte) func(context.Context
|
||||
}
|
||||
|
||||
// ReadToMax reads up to the max size.
|
||||
func (a *Archive) ReadToMax(ctx context.Context, reader io.Reader) (data []byte, err error) {
|
||||
func (d *Archive) ReadToMax(ctx context.Context, reader io.Reader) (data []byte, err error) {
|
||||
// Archiver v4 is in alpha and using an experimental version of
|
||||
// rardecode. There is a bug somewhere with rar decoder format 29
|
||||
// that can lead to a panic. An issue is open in rardecode repo
|
||||
@@ -184,7 +180,7 @@ func (a *Archive) ReadToMax(ctx context.Context, reader io.Reader) (data []byte,
|
||||
}
|
||||
}()
|
||||
fileContent := bytes.Buffer{}
|
||||
logger.V(5).Info("Remaining buffer capacity", "bytes", maxSize-a.size)
|
||||
logger.V(5).Info("Remaining buffer capacity", "bytes", maxSize-d.size)
|
||||
for i := 0; i <= maxSize/512; i++ {
|
||||
if common.IsDone(ctx) {
|
||||
return nil, ctx.Err()
|
||||
@@ -194,284 +190,17 @@ func (a *Archive) ReadToMax(ctx context.Context, reader io.Reader) (data []byte,
|
||||
if err != nil && !errors.Is(err, io.EOF) {
|
||||
return []byte{}, err
|
||||
}
|
||||
a.size += bRead
|
||||
d.size += bRead
|
||||
if len(fileChunk) > 0 {
|
||||
fileContent.Write(fileChunk[0:bRead])
|
||||
}
|
||||
if bRead < 512 {
|
||||
return fileContent.Bytes(), nil
|
||||
}
|
||||
if a.size >= maxSize && bRead == 512 {
|
||||
if d.size >= maxSize && bRead == 512 {
|
||||
logger.V(2).Info("Max archive size reached.")
|
||||
return fileContent.Bytes(), nil
|
||||
}
|
||||
}
|
||||
return fileContent.Bytes(), nil
|
||||
}
|
||||
|
||||
const (
|
||||
arMimeType = "application/x-unix-archive"
|
||||
rpmMimeType = "application/x-rpm"
|
||||
)
|
||||
|
||||
// Define a map of mime types to corresponding command-line tools
|
||||
var mimeTools = map[string][]string{
|
||||
arMimeType: {"ar"},
|
||||
rpmMimeType: {"rpm2cpio", "cpio"},
|
||||
}
|
||||
|
||||
// Check if the command-line tool is installed.
|
||||
func isToolInstalled(tool string) bool {
|
||||
_, err := exec.LookPath(tool)
|
||||
return err == nil
|
||||
}
|
||||
|
||||
// Ensure all tools are available for given mime type.
|
||||
func ensureToolsForMimeType(mimeType string) error {
|
||||
tools, exists := mimeTools[mimeType]
|
||||
if !exists {
|
||||
return fmt.Errorf("unsupported mime type")
|
||||
}
|
||||
|
||||
for _, tool := range tools {
|
||||
if !isToolInstalled(tool) {
|
||||
return fmt.Errorf("Required tool " + tool + " is not installed")
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// HandleSpecialized takes a file path and an io.Reader representing the input file,
|
||||
// and processes it based on its extension, such as handling Debian (.deb) and RPM (.rpm) packages.
|
||||
// It returns an io.Reader that can be used to read the processed content of the file,
|
||||
// and an error if any issues occurred during processing.
|
||||
// The caller is responsible for closing the returned reader.
|
||||
func (a *Archive) HandleSpecialized(ctx context.Context, reader io.Reader) (io.Reader, bool, error) {
|
||||
mimeType, reader, err := determineMimeType(reader)
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
|
||||
switch mimeType {
|
||||
case arMimeType: // includes .deb files
|
||||
if err := ensureToolsForMimeType(mimeType); err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
reader, err = a.extractDebContent(ctx, reader)
|
||||
case rpmMimeType:
|
||||
if err := ensureToolsForMimeType(mimeType); err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
reader, err = a.extractRpmContent(ctx, reader)
|
||||
default:
|
||||
return reader, false, nil
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
return nil, false, fmt.Errorf("unable to extract file with MIME type %s: %w", mimeType, err)
|
||||
}
|
||||
return reader, true, nil
|
||||
}
|
||||
|
||||
// extractDebContent takes a .deb file as an io.Reader, extracts its contents
|
||||
// into a temporary directory, and returns a Reader for the extracted data archive.
|
||||
// It handles the extraction process by using the 'ar' command and manages temporary
|
||||
// files and directories for the operation.
|
||||
// The caller is responsible for closing the returned reader.
|
||||
func (a *Archive) extractDebContent(ctx context.Context, file io.Reader) (io.ReadCloser, error) {
|
||||
if a.currentDepth >= maxDepth {
|
||||
return nil, fmt.Errorf("max archive depth reached")
|
||||
}
|
||||
|
||||
tmpEnv, err := a.createTempEnv(ctx, file)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer os.Remove(tmpEnv.tempFileName)
|
||||
defer os.RemoveAll(tmpEnv.extractPath)
|
||||
|
||||
cmd := exec.Command("ar", "x", tmpEnv.tempFile.Name())
|
||||
cmd.Dir = tmpEnv.extractPath
|
||||
if err := executeCommand(cmd); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
handler := func(ctx context.Context, env tempEnv, file string) (string, error) {
|
||||
if strings.HasPrefix(file, "data.tar.") {
|
||||
return file, nil
|
||||
}
|
||||
return a.handleNestedFileMIME(ctx, env, file)
|
||||
}
|
||||
|
||||
dataArchiveName, err := a.handleExtractedFiles(ctx, tmpEnv, handler)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return openDataArchive(tmpEnv.extractPath, dataArchiveName)
|
||||
}
|
||||
|
||||
// extractRpmContent takes an .rpm file as an io.Reader, extracts its contents
|
||||
// into a temporary directory, and returns a Reader for the extracted data archive.
|
||||
// It handles the extraction process by using the 'rpm2cpio' and 'cpio' commands and manages temporary
|
||||
// files and directories for the operation.
|
||||
// The caller is responsible for closing the returned reader.
|
||||
func (a *Archive) extractRpmContent(ctx context.Context, file io.Reader) (io.ReadCloser, error) {
|
||||
if a.currentDepth >= maxDepth {
|
||||
return nil, fmt.Errorf("max archive depth reached")
|
||||
}
|
||||
|
||||
tmpEnv, err := a.createTempEnv(ctx, file)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer os.Remove(tmpEnv.tempFileName)
|
||||
defer os.RemoveAll(tmpEnv.extractPath)
|
||||
|
||||
// Use rpm2cpio to convert the RPM file to a cpio archive and then extract it using cpio command.
|
||||
cmd := exec.Command("sh", "-c", "rpm2cpio "+tmpEnv.tempFile.Name()+" | cpio -id")
|
||||
cmd.Dir = tmpEnv.extractPath
|
||||
if err := executeCommand(cmd); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
handler := func(ctx context.Context, env tempEnv, file string) (string, error) {
|
||||
if strings.HasSuffix(file, ".tar.gz") {
|
||||
return file, nil
|
||||
}
|
||||
return a.handleNestedFileMIME(ctx, env, file)
|
||||
}
|
||||
|
||||
dataArchiveName, err := a.handleExtractedFiles(ctx, tmpEnv, handler)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return openDataArchive(tmpEnv.extractPath, dataArchiveName)
|
||||
}
|
||||
|
||||
func (a *Archive) handleNestedFileMIME(ctx context.Context, tempEnv tempEnv, fileName string) (string, error) {
|
||||
nestedFile, err := os.Open(filepath.Join(tempEnv.extractPath, fileName))
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer nestedFile.Close()
|
||||
|
||||
mimeType, reader, err := determineMimeType(nestedFile)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
switch mimeType {
|
||||
case arMimeType:
|
||||
_, _, err = a.HandleSpecialized(ctx, reader)
|
||||
case rpmMimeType:
|
||||
_, _, err = a.HandleSpecialized(ctx, reader)
|
||||
default:
|
||||
return "", nil
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
return fileName, nil
|
||||
}
|
||||
|
||||
// determineMimeType reads from the provided reader to detect the MIME type.
|
||||
// It returns the detected MIME type and a new reader that includes the read portion.
|
||||
func determineMimeType(reader io.Reader) (string, io.Reader, error) {
|
||||
buffer := make([]byte, 512)
|
||||
n, err := reader.Read(buffer)
|
||||
if err != nil {
|
||||
return "", nil, fmt.Errorf("unable to read file for MIME type detection: %w", err)
|
||||
}
|
||||
|
||||
// Create a new reader that starts with the buffer we just read
|
||||
// and continues with the rest of the original reader.
|
||||
reader = io.MultiReader(bytes.NewReader(buffer[:n]), reader)
|
||||
|
||||
kind, err := filetype.Match(buffer)
|
||||
if err != nil {
|
||||
return "", nil, fmt.Errorf("unable to determine file type: %w", err)
|
||||
}
|
||||
|
||||
return kind.MIME.Value, reader, nil
|
||||
}
|
||||
|
||||
// handleExtractedFiles processes each file in the extracted directory using a provided handler function.
|
||||
// The function iterates through the files, applying the handleFile function to each, and returns the name
|
||||
// of the data archive it finds. This centralizes the logic for handling specialized files such as .deb and .rpm
|
||||
// by using the appropriate handling function passed as an argument. This design allows for flexibility and reuse
|
||||
// of this function across various extraction processes in the package.
|
||||
func (a *Archive) handleExtractedFiles(ctx context.Context, env tempEnv, handleFile func(context.Context, tempEnv, string) (string, error)) (string, error) {
|
||||
extractedFiles, err := os.ReadDir(env.extractPath)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("unable to read extracted directory: %w", err)
|
||||
}
|
||||
|
||||
var dataArchiveName string
|
||||
for _, file := range extractedFiles {
|
||||
name, err := handleFile(ctx, env, file.Name())
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if name != "" {
|
||||
dataArchiveName = name
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
return dataArchiveName, nil
|
||||
}
|
||||
|
||||
type tempEnv struct {
|
||||
tempFile *os.File
|
||||
tempFileName string
|
||||
extractPath string
|
||||
}
|
||||
|
||||
// createTempEnv creates a temporary file and a temporary directory for extracting archives.
|
||||
// The caller is responsible for removing these temporary resources
|
||||
// (both the file and directory) when they are no longer needed.
|
||||
func (a *Archive) createTempEnv(ctx context.Context, file io.Reader) (tempEnv, error) {
|
||||
tempFile, err := os.CreateTemp("", "tmp")
|
||||
if err != nil {
|
||||
return tempEnv{}, fmt.Errorf("unable to create temporary file: %w", err)
|
||||
}
|
||||
|
||||
extractPath, err := os.MkdirTemp("", "tmp_archive")
|
||||
if err != nil {
|
||||
return tempEnv{}, fmt.Errorf("unable to create temporary directory: %w", err)
|
||||
}
|
||||
|
||||
b, err := a.ReadToMax(ctx, file)
|
||||
if err != nil {
|
||||
return tempEnv{}, err
|
||||
}
|
||||
|
||||
if _, err = tempFile.Write(b); err != nil {
|
||||
return tempEnv{}, fmt.Errorf("unable to write to temporary file: %w", err)
|
||||
}
|
||||
|
||||
return tempEnv{tempFile: tempFile, tempFileName: tempFile.Name(), extractPath: extractPath}, nil
|
||||
}
|
||||
|
||||
func executeCommand(cmd *exec.Cmd) error {
|
||||
var stderr bytes.Buffer
|
||||
cmd.Stderr = &stderr
|
||||
if err := cmd.Run(); err != nil {
|
||||
return fmt.Errorf("unable to execute command: %w; error: %s", err, stderr.String())
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func openDataArchive(extractPath string, dataArchiveName string) (io.ReadCloser, error) {
|
||||
dataArchivePath := filepath.Join(extractPath, dataArchiveName)
|
||||
dataFile, err := os.Open(dataArchivePath)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("unable to open file: %w", err)
|
||||
}
|
||||
return dataFile, nil
|
||||
}
|
||||
|
||||
@@ -2,9 +2,7 @@ package handlers
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io"
|
||||
"net/http"
|
||||
"os"
|
||||
"regexp"
|
||||
"strings"
|
||||
"testing"
|
||||
@@ -87,9 +85,11 @@ func TestArchiveHandler(t *testing.T) {
|
||||
count := 0
|
||||
re := regexp.MustCompile(testCase.matchString)
|
||||
matched := false
|
||||
for chunk := range archiveChan {
|
||||
var chunk sources.Chunk
|
||||
for opt := range archiveChan {
|
||||
count++
|
||||
if re.Match(chunk) {
|
||||
opt(&chunk)
|
||||
if re.Match(chunk.Data) {
|
||||
matched = true
|
||||
}
|
||||
}
|
||||
@@ -124,39 +124,3 @@ func TestHandleFile(t *testing.T) {
|
||||
assert.True(t, HandleFile(context.Background(), reader, &sources.Chunk{}, ch))
|
||||
assert.Equal(t, 1, len(ch))
|
||||
}
|
||||
|
||||
func TestExtractDebContent(t *testing.T) {
|
||||
// Open the sample .deb file from the testdata folder.
|
||||
file, err := os.Open("testdata/test.deb")
|
||||
assert.Nil(t, err)
|
||||
defer file.Close()
|
||||
|
||||
ctx := context.Background()
|
||||
a := &Archive{}
|
||||
|
||||
reader, err := a.extractDebContent(ctx, file)
|
||||
assert.Nil(t, err)
|
||||
|
||||
content, err := io.ReadAll(reader)
|
||||
assert.Nil(t, err)
|
||||
expectedLength := 1015582
|
||||
assert.Equal(t, expectedLength, len(string(content)))
|
||||
}
|
||||
|
||||
func TestExtractRPMContent(t *testing.T) {
|
||||
// Open the sample .rpm file from the testdata folder.
|
||||
file, err := os.Open("testdata/test.rpm")
|
||||
assert.Nil(t, err)
|
||||
defer file.Close()
|
||||
|
||||
ctx := context.Background()
|
||||
a := &Archive{}
|
||||
|
||||
reader, err := a.extractRpmContent(ctx, file)
|
||||
assert.Nil(t, err)
|
||||
|
||||
content, err := io.ReadAll(reader)
|
||||
assert.Nil(t, err)
|
||||
expectedLength := 1822720
|
||||
assert.Equal(t, expectedLength, len(string(content)))
|
||||
}
|
||||
|
||||
+16
-34
@@ -13,59 +13,41 @@ func DefaultHandlers() []Handler {
|
||||
}
|
||||
}
|
||||
|
||||
// SpecializedHandler defines the interface for handlers that can process specialized archives.
|
||||
// It includes a method to handle specialized archives and determine if the file is of a special type.
|
||||
type SpecializedHandler interface {
|
||||
// HandleSpecialized examines the provided file reader within the context and determines if it is a specialized archive.
|
||||
// It returns a reader with any necessary modifications, a boolean indicating if the file was specialized,
|
||||
// and an error if something went wrong during processing.
|
||||
HandleSpecialized(context.Context, io.Reader) (io.Reader, bool, error)
|
||||
}
|
||||
type ChunkOpt func(*sources.Chunk)
|
||||
|
||||
type Handler interface {
|
||||
FromFile(context.Context, io.Reader) chan ([]byte)
|
||||
FromFile(context.Context, io.Reader) chan ChunkOpt
|
||||
IsFiletype(context.Context, io.Reader) (io.Reader, bool)
|
||||
New()
|
||||
}
|
||||
|
||||
// HandleFile processes a given file by selecting an appropriate handler from DefaultHandlers.
|
||||
// It first checks if the handler implements SpecializedHandler for any special processing,
|
||||
// then falls back to regular file type handling. If successful, it reads the file in chunks,
|
||||
// packages them in the provided chunk skeleton, and sends them to chunksChan.
|
||||
// The function returns true if processing was successful and false otherwise.
|
||||
// Context is used for cancellation, and the caller is responsible for canceling it if needed.
|
||||
func HandleFile(ctx context.Context, file io.Reader, chunkSkel *sources.Chunk, chunksChan chan *sources.Chunk) bool {
|
||||
func HandleFile(ctx context.Context, file io.Reader, chunkSkel *sources.Chunk, chunksChan chan (*sources.Chunk)) bool {
|
||||
// Find a handler for this file.
|
||||
var handler Handler
|
||||
for _, h := range DefaultHandlers() {
|
||||
h.New()
|
||||
var (
|
||||
isSpecial bool
|
||||
err error
|
||||
)
|
||||
|
||||
// Check if the handler implements SpecializedHandler and process accordingly.
|
||||
if specialHandler, ok := h.(SpecializedHandler); ok {
|
||||
if file, isSpecial, err = specialHandler.HandleSpecialized(ctx, file); isSpecial && err == nil {
|
||||
return handleChunks(ctx, h.FromFile(ctx, file), chunkSkel, chunksChan)
|
||||
}
|
||||
}
|
||||
|
||||
var isType bool
|
||||
if file, isType = h.IsFiletype(ctx, file); isType {
|
||||
return handleChunks(ctx, h.FromFile(ctx, file), chunkSkel, chunksChan)
|
||||
handler = h
|
||||
break
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
if handler == nil {
|
||||
return false
|
||||
}
|
||||
|
||||
func handleChunks(ctx context.Context, handlerChan chan []byte, chunkSkel *sources.Chunk, chunksChan chan *sources.Chunk) bool {
|
||||
// Process the file and read all []byte chunks from handlerChan.
|
||||
handlerChan := handler.FromFile(ctx, file)
|
||||
for {
|
||||
select {
|
||||
case data, open := <-handlerChan:
|
||||
case opt, open := <-handlerChan:
|
||||
if !open {
|
||||
// We finished reading everything from handlerChan.
|
||||
return true
|
||||
}
|
||||
chunk := *chunkSkel
|
||||
chunk.Data = data
|
||||
opt(&chunk)
|
||||
// Send data on chunksChan.
|
||||
select {
|
||||
case chunksChan <- &chunk:
|
||||
case <-ctx.Done():
|
||||
|
||||
Vendored
BIN
Binary file not shown.
Vendored
BIN
Binary file not shown.
@@ -4,19 +4,14 @@ import (
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"sync"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/context"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
|
||||
)
|
||||
|
||||
var dedupeCache = make(map[string]struct{})
|
||||
|
||||
// GitHubActionsPrinter is a printer that prints results in GitHub Actions format.
|
||||
type GitHubActionsPrinter struct{ mu sync.Mutex }
|
||||
|
||||
func (p *GitHubActionsPrinter) Print(_ context.Context, r *detectors.ResultWithMetadata) error {
|
||||
func PrintGitHubActionsOutput(r *detectors.ResultWithMetadata) error {
|
||||
out := gitHubActionsOutputFormat{
|
||||
DetectorType: r.Result.DetectorType.String(),
|
||||
DecoderType: r.Result.DecoderType.String(),
|
||||
@@ -57,8 +52,6 @@ func (p *GitHubActionsPrinter) Print(_ context.Context, r *detectors.ResultWithM
|
||||
}
|
||||
dedupeCache[key] = struct{}{}
|
||||
|
||||
p.mu.Lock()
|
||||
defer p.mu.Unlock()
|
||||
message := fmt.Sprintf("Found %s %s result 🐷🔑\n", verifiedStatus, out.DetectorType)
|
||||
if r.Result.DecoderType != detectorspb.DecoderType_PLAIN {
|
||||
message = fmt.Sprintf("Found %s %s result with %s encoding 🐷🔑\n", verifiedStatus, out.DetectorType, out.DecoderType)
|
||||
|
||||
+1
-9
@@ -3,19 +3,14 @@ package output
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"sync"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/context"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/source_metadatapb"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
|
||||
)
|
||||
|
||||
// JSONPrinter is a printer that prints results in JSON format.
|
||||
type JSONPrinter struct{ mu sync.Mutex }
|
||||
|
||||
func (p *JSONPrinter) Print(_ context.Context, r *detectors.ResultWithMetadata) error {
|
||||
func PrintJSON(r *detectors.ResultWithMetadata) error {
|
||||
v := &struct {
|
||||
// SourceMetadata contains source-specific contextual information.
|
||||
SourceMetadata *source_metadatapb.MetaData
|
||||
@@ -61,9 +56,6 @@ func (p *JSONPrinter) Print(_ context.Context, r *detectors.ResultWithMetadata)
|
||||
if err != nil {
|
||||
return fmt.Errorf("could not marshal result: %w", err)
|
||||
}
|
||||
|
||||
p.mu.Lock()
|
||||
fmt.Println(string(out))
|
||||
p.mu.Unlock()
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -7,7 +7,6 @@ import (
|
||||
"net/url"
|
||||
"os"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
gogit "github.com/go-git/go-git/v5"
|
||||
"github.com/go-git/go-git/v5/plumbing"
|
||||
@@ -20,10 +19,7 @@ import (
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/sources/git"
|
||||
)
|
||||
|
||||
// LegacyJSONPrinter is a printer that prints results in legacy JSON format for backwards compatibility.
|
||||
type LegacyJSONPrinter struct{ mu sync.Mutex }
|
||||
|
||||
func (p *LegacyJSONPrinter) Print(ctx context.Context, r *detectors.ResultWithMetadata) error {
|
||||
func PrintLegacyJSON(ctx context.Context, r *detectors.ResultWithMetadata) error {
|
||||
var repo string
|
||||
switch r.SourceType {
|
||||
case sourcespb.SourceType_SOURCE_TYPE_GIT:
|
||||
@@ -45,7 +41,7 @@ func (p *LegacyJSONPrinter) Print(ctx context.Context, r *detectors.ResultWithMe
|
||||
defer os.RemoveAll(repoPath)
|
||||
}
|
||||
|
||||
legacy, err := convertToLegacyJSON(r, repoPath)
|
||||
legacy, err := ConvertToLegacyJSON(r, repoPath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("could not convert to legacy JSON: %w", err)
|
||||
}
|
||||
@@ -53,14 +49,11 @@ func (p *LegacyJSONPrinter) Print(ctx context.Context, r *detectors.ResultWithMe
|
||||
if err != nil {
|
||||
return fmt.Errorf("could not marshal result: %w", err)
|
||||
}
|
||||
|
||||
p.mu.Lock()
|
||||
fmt.Println(string(out))
|
||||
p.mu.Unlock()
|
||||
return nil
|
||||
}
|
||||
|
||||
func convertToLegacyJSON(r *detectors.ResultWithMetadata, repoPath string) (*LegacyJSONOutput, error) {
|
||||
func ConvertToLegacyJSON(r *detectors.ResultWithMetadata, repoPath string) (*LegacyJSONOutput, error) {
|
||||
var source LegacyJSONCompatibleSource
|
||||
switch r.SourceType {
|
||||
case sourcespb.SourceType_SOURCE_TYPE_GIT:
|
||||
|
||||
+5
-12
@@ -5,13 +5,11 @@ import (
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
"github.com/fatih/color"
|
||||
"golang.org/x/text/cases"
|
||||
"golang.org/x/text/language"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/context"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/source_metadatapb"
|
||||
)
|
||||
@@ -22,10 +20,7 @@ var (
|
||||
whitePrinter = color.New(color.FgWhite)
|
||||
)
|
||||
|
||||
// PlainPrinter is a printer that prints results in plain text format.
|
||||
type PlainPrinter struct{ mu sync.Mutex }
|
||||
|
||||
func (p *PlainPrinter) Print(_ context.Context, r *detectors.ResultWithMetadata) error {
|
||||
func PrintPlainOutput(r *detectors.ResultWithMetadata) error {
|
||||
out := outputFormat{
|
||||
DetectorType: r.Result.DetectorType.String(),
|
||||
DecoderType: r.Result.DecoderType.String(),
|
||||
@@ -40,8 +35,6 @@ func (p *PlainPrinter) Print(_ context.Context, r *detectors.ResultWithMetadata)
|
||||
}
|
||||
|
||||
printer := greenPrinter
|
||||
p.mu.Lock()
|
||||
defer p.mu.Unlock()
|
||||
|
||||
if out.Verified {
|
||||
yellowPrinter.Print("Found verified result 🐷🔑\n")
|
||||
@@ -103,9 +96,9 @@ func structToMap(obj interface{}) (m map[string]map[string]interface{}, err erro
|
||||
}
|
||||
|
||||
type outputFormat struct {
|
||||
DetectorType,
|
||||
DecoderType string
|
||||
Verified bool
|
||||
Raw string
|
||||
DetectorType string
|
||||
DecoderType string
|
||||
Verified bool
|
||||
Raw string
|
||||
*source_metadatapb.MetaData
|
||||
}
|
||||
|
||||
@@ -995,9 +995,6 @@ const (
|
||||
DetectorType_Dockerhub DetectorType = 921
|
||||
DetectorType_TrufflehogEnterprise DetectorType = 922
|
||||
DetectorType_EnvoyApiKey DetectorType = 923
|
||||
DetectorType_GitHubOauth2 DetectorType = 924
|
||||
// 925 reserved for Salesforce
|
||||
DetectorType_HuggingFace DetectorType = 926
|
||||
)
|
||||
|
||||
// Enum value maps for DetectorType.
|
||||
@@ -1923,8 +1920,6 @@ var (
|
||||
921: "Dockerhub",
|
||||
922: "TrufflehogEnterprise",
|
||||
923: "EnvoyApiKey",
|
||||
924: "GitHubOauth2",
|
||||
926: "HuggingFace",
|
||||
}
|
||||
DetectorType_value = map[string]int32{
|
||||
"Alibaba": 0,
|
||||
@@ -2847,8 +2842,6 @@ var (
|
||||
"Dockerhub": 921,
|
||||
"TrufflehogEnterprise": 922,
|
||||
"EnvoyApiKey": 923,
|
||||
"GitHubOauth2": 924,
|
||||
"HuggingFace": 926,
|
||||
}
|
||||
)
|
||||
|
||||
@@ -3227,7 +3220,7 @@ var file_detectors_proto_rawDesc = []byte{
|
||||
0x44, 0x65, 0x63, 0x6f, 0x64, 0x65, 0x72, 0x54, 0x79, 0x70, 0x65, 0x12, 0x0b, 0x0a, 0x07, 0x55,
|
||||
0x4e, 0x4b, 0x4e, 0x4f, 0x57, 0x4e, 0x10, 0x00, 0x12, 0x09, 0x0a, 0x05, 0x50, 0x4c, 0x41, 0x49,
|
||||
0x4e, 0x10, 0x01, 0x12, 0x0a, 0x0a, 0x06, 0x42, 0x41, 0x53, 0x45, 0x36, 0x34, 0x10, 0x02, 0x12,
|
||||
0x09, 0x0a, 0x05, 0x55, 0x54, 0x46, 0x31, 0x36, 0x10, 0x03, 0x2a, 0xeb, 0x73, 0x0a, 0x0c, 0x44,
|
||||
0x09, 0x0a, 0x05, 0x55, 0x54, 0x46, 0x31, 0x36, 0x10, 0x03, 0x2a, 0xc6, 0x73, 0x0a, 0x0c, 0x44,
|
||||
0x65, 0x74, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x54, 0x79, 0x70, 0x65, 0x12, 0x0b, 0x0a, 0x07, 0x41,
|
||||
0x6c, 0x69, 0x62, 0x61, 0x62, 0x61, 0x10, 0x00, 0x12, 0x08, 0x0a, 0x04, 0x41, 0x4d, 0x51, 0x50,
|
||||
0x10, 0x01, 0x12, 0x07, 0x0a, 0x03, 0x41, 0x57, 0x53, 0x10, 0x02, 0x12, 0x09, 0x0a, 0x05, 0x41,
|
||||
@@ -4152,13 +4145,11 @@ var file_detectors_proto_rawDesc = []byte{
|
||||
0x68, 0x75, 0x62, 0x10, 0x99, 0x07, 0x12, 0x19, 0x0a, 0x14, 0x54, 0x72, 0x75, 0x66, 0x66, 0x6c,
|
||||
0x65, 0x68, 0x6f, 0x67, 0x45, 0x6e, 0x74, 0x65, 0x72, 0x70, 0x72, 0x69, 0x73, 0x65, 0x10, 0x9a,
|
||||
0x07, 0x12, 0x10, 0x0a, 0x0b, 0x45, 0x6e, 0x76, 0x6f, 0x79, 0x41, 0x70, 0x69, 0x4b, 0x65, 0x79,
|
||||
0x10, 0x9b, 0x07, 0x12, 0x11, 0x0a, 0x0c, 0x47, 0x69, 0x74, 0x48, 0x75, 0x62, 0x4f, 0x61, 0x75,
|
||||
0x74, 0x68, 0x32, 0x10, 0x9c, 0x07, 0x12, 0x10, 0x0a, 0x0b, 0x48, 0x75, 0x67, 0x67, 0x69, 0x6e,
|
||||
0x67, 0x46, 0x61, 0x63, 0x65, 0x10, 0x9e, 0x07, 0x42, 0x3d, 0x5a, 0x3b, 0x67, 0x69, 0x74, 0x68,
|
||||
0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x74, 0x72, 0x75, 0x66, 0x66, 0x6c, 0x65, 0x73, 0x65,
|
||||
0x63, 0x75, 0x72, 0x69, 0x74, 0x79, 0x2f, 0x74, 0x72, 0x75, 0x66, 0x66, 0x6c, 0x65, 0x68, 0x6f,
|
||||
0x67, 0x2f, 0x76, 0x33, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x70, 0x62, 0x2f, 0x64, 0x65, 0x74, 0x65,
|
||||
0x63, 0x74, 0x6f, 0x72, 0x73, 0x70, 0x62, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
|
||||
0x10, 0x9b, 0x07, 0x42, 0x3d, 0x5a, 0x3b, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f,
|
||||
0x6d, 0x2f, 0x74, 0x72, 0x75, 0x66, 0x66, 0x6c, 0x65, 0x73, 0x65, 0x63, 0x75, 0x72, 0x69, 0x74,
|
||||
0x79, 0x2f, 0x74, 0x72, 0x75, 0x66, 0x66, 0x6c, 0x65, 0x68, 0x6f, 0x67, 0x2f, 0x76, 0x33, 0x2f,
|
||||
0x70, 0x6b, 0x67, 0x2f, 0x70, 0x62, 0x2f, 0x64, 0x65, 0x74, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x73,
|
||||
0x70, 0x62, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
|
||||
}
|
||||
|
||||
var (
|
||||
|
||||
@@ -2294,7 +2294,6 @@ type GoogleDrive struct {
|
||||
Timestamp string `protobuf:"bytes,4,opt,name=timestamp,proto3" json:"timestamp,omitempty"`
|
||||
Shared bool `protobuf:"varint,5,opt,name=shared,proto3" json:"shared,omitempty"`
|
||||
LastModifiedBy string `protobuf:"bytes,6,opt,name=last_modified_by,json=lastModifiedBy,proto3" json:"last_modified_by,omitempty"`
|
||||
Path string `protobuf:"bytes,7,opt,name=path,proto3" json:"path,omitempty"`
|
||||
}
|
||||
|
||||
func (x *GoogleDrive) Reset() {
|
||||
@@ -2371,140 +2370,6 @@ func (x *GoogleDrive) GetLastModifiedBy() string {
|
||||
return ""
|
||||
}
|
||||
|
||||
func (x *GoogleDrive) GetPath() string {
|
||||
if x != nil {
|
||||
return x.Path
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
type AzureRepos struct {
|
||||
state protoimpl.MessageState
|
||||
sizeCache protoimpl.SizeCache
|
||||
unknownFields protoimpl.UnknownFields
|
||||
|
||||
Link string `protobuf:"bytes,1,opt,name=link,proto3" json:"link,omitempty"`
|
||||
Username string `protobuf:"bytes,2,opt,name=username,proto3" json:"username,omitempty"`
|
||||
Repository string `protobuf:"bytes,3,opt,name=repository,proto3" json:"repository,omitempty"`
|
||||
Commit string `protobuf:"bytes,4,opt,name=commit,proto3" json:"commit,omitempty"`
|
||||
Email string `protobuf:"bytes,5,opt,name=email,proto3" json:"email,omitempty"`
|
||||
File string `protobuf:"bytes,6,opt,name=file,proto3" json:"file,omitempty"`
|
||||
Timestamp string `protobuf:"bytes,7,opt,name=timestamp,proto3" json:"timestamp,omitempty"`
|
||||
Line int64 `protobuf:"varint,8,opt,name=line,proto3" json:"line,omitempty"`
|
||||
Visibility Visibility `protobuf:"varint,9,opt,name=visibility,proto3,enum=source_metadata.Visibility" json:"visibility,omitempty"`
|
||||
Project string `protobuf:"bytes,10,opt,name=project,proto3" json:"project,omitempty"`
|
||||
Organization string `protobuf:"bytes,11,opt,name=organization,proto3" json:"organization,omitempty"`
|
||||
}
|
||||
|
||||
func (x *AzureRepos) Reset() {
|
||||
*x = AzureRepos{}
|
||||
if protoimpl.UnsafeEnabled {
|
||||
mi := &file_source_metadata_proto_msgTypes[26]
|
||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
||||
ms.StoreMessageInfo(mi)
|
||||
}
|
||||
}
|
||||
|
||||
func (x *AzureRepos) String() string {
|
||||
return protoimpl.X.MessageStringOf(x)
|
||||
}
|
||||
|
||||
func (*AzureRepos) ProtoMessage() {}
|
||||
|
||||
func (x *AzureRepos) ProtoReflect() protoreflect.Message {
|
||||
mi := &file_source_metadata_proto_msgTypes[26]
|
||||
if protoimpl.UnsafeEnabled && x != nil {
|
||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
||||
if ms.LoadMessageInfo() == nil {
|
||||
ms.StoreMessageInfo(mi)
|
||||
}
|
||||
return ms
|
||||
}
|
||||
return mi.MessageOf(x)
|
||||
}
|
||||
|
||||
// Deprecated: Use AzureRepos.ProtoReflect.Descriptor instead.
|
||||
func (*AzureRepos) Descriptor() ([]byte, []int) {
|
||||
return file_source_metadata_proto_rawDescGZIP(), []int{26}
|
||||
}
|
||||
|
||||
func (x *AzureRepos) GetLink() string {
|
||||
if x != nil {
|
||||
return x.Link
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func (x *AzureRepos) GetUsername() string {
|
||||
if x != nil {
|
||||
return x.Username
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func (x *AzureRepos) GetRepository() string {
|
||||
if x != nil {
|
||||
return x.Repository
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func (x *AzureRepos) GetCommit() string {
|
||||
if x != nil {
|
||||
return x.Commit
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func (x *AzureRepos) GetEmail() string {
|
||||
if x != nil {
|
||||
return x.Email
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func (x *AzureRepos) GetFile() string {
|
||||
if x != nil {
|
||||
return x.File
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func (x *AzureRepos) GetTimestamp() string {
|
||||
if x != nil {
|
||||
return x.Timestamp
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func (x *AzureRepos) GetLine() int64 {
|
||||
if x != nil {
|
||||
return x.Line
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
func (x *AzureRepos) GetVisibility() Visibility {
|
||||
if x != nil {
|
||||
return x.Visibility
|
||||
}
|
||||
return Visibility_public
|
||||
}
|
||||
|
||||
func (x *AzureRepos) GetProject() string {
|
||||
if x != nil {
|
||||
return x.Project
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func (x *AzureRepos) GetOrganization() string {
|
||||
if x != nil {
|
||||
return x.Organization
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
type MetaData struct {
|
||||
state protoimpl.MessageState
|
||||
sizeCache protoimpl.SizeCache
|
||||
@@ -2537,14 +2402,13 @@ type MetaData struct {
|
||||
// *MetaData_PublicEventMonitoring
|
||||
// *MetaData_Sharepoint
|
||||
// *MetaData_GoogleDrive
|
||||
// *MetaData_AzureRepos
|
||||
Data isMetaData_Data `protobuf_oneof:"data"`
|
||||
}
|
||||
|
||||
func (x *MetaData) Reset() {
|
||||
*x = MetaData{}
|
||||
if protoimpl.UnsafeEnabled {
|
||||
mi := &file_source_metadata_proto_msgTypes[27]
|
||||
mi := &file_source_metadata_proto_msgTypes[26]
|
||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
||||
ms.StoreMessageInfo(mi)
|
||||
}
|
||||
@@ -2557,7 +2421,7 @@ func (x *MetaData) String() string {
|
||||
func (*MetaData) ProtoMessage() {}
|
||||
|
||||
func (x *MetaData) ProtoReflect() protoreflect.Message {
|
||||
mi := &file_source_metadata_proto_msgTypes[27]
|
||||
mi := &file_source_metadata_proto_msgTypes[26]
|
||||
if protoimpl.UnsafeEnabled && x != nil {
|
||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
||||
if ms.LoadMessageInfo() == nil {
|
||||
@@ -2570,7 +2434,7 @@ func (x *MetaData) ProtoReflect() protoreflect.Message {
|
||||
|
||||
// Deprecated: Use MetaData.ProtoReflect.Descriptor instead.
|
||||
func (*MetaData) Descriptor() ([]byte, []int) {
|
||||
return file_source_metadata_proto_rawDescGZIP(), []int{27}
|
||||
return file_source_metadata_proto_rawDescGZIP(), []int{26}
|
||||
}
|
||||
|
||||
func (m *MetaData) GetData() isMetaData_Data {
|
||||
@@ -2762,13 +2626,6 @@ func (x *MetaData) GetGoogleDrive() *GoogleDrive {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (x *MetaData) GetAzureRepos() *AzureRepos {
|
||||
if x, ok := x.GetData().(*MetaData_AzureRepos); ok {
|
||||
return x.AzureRepos
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
type isMetaData_Data interface {
|
||||
isMetaData_Data()
|
||||
}
|
||||
@@ -2877,10 +2734,6 @@ type MetaData_GoogleDrive struct {
|
||||
GoogleDrive *GoogleDrive `protobuf:"bytes,26,opt,name=googleDrive,proto3,oneof"`
|
||||
}
|
||||
|
||||
type MetaData_AzureRepos struct {
|
||||
AzureRepos *AzureRepos `protobuf:"bytes,27,opt,name=azureRepos,proto3,oneof"`
|
||||
}
|
||||
|
||||
func (*MetaData_Azure) isMetaData_Data() {}
|
||||
|
||||
func (*MetaData_Bitbucket) isMetaData_Data() {}
|
||||
@@ -2933,8 +2786,6 @@ func (*MetaData_Sharepoint) isMetaData_Data() {}
|
||||
|
||||
func (*MetaData_GoogleDrive) isMetaData_Data() {}
|
||||
|
||||
func (*MetaData_AzureRepos) isMetaData_Data() {}
|
||||
|
||||
var File_source_metadata_proto protoreflect.FileDescriptor
|
||||
|
||||
var file_source_metadata_proto_rawDesc = []byte{
|
||||
@@ -3198,7 +3049,7 @@ var file_source_metadata_proto_rawDesc = []byte{
|
||||
0x73, 0x18, 0x05, 0x20, 0x01, 0x28, 0x03, 0x52, 0x05, 0x76, 0x69, 0x65, 0x77, 0x73, 0x12, 0x14,
|
||||
0x0a, 0x05, 0x64, 0x6f, 0x63, 0x69, 0x64, 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x64,
|
||||
0x6f, 0x63, 0x69, 0x64, 0x12, 0x14, 0x0a, 0x05, 0x65, 0x6d, 0x61, 0x69, 0x6c, 0x18, 0x07, 0x20,
|
||||
0x01, 0x28, 0x09, 0x52, 0x05, 0x65, 0x6d, 0x61, 0x69, 0x6c, 0x22, 0xbf, 0x01, 0x0a, 0x0b, 0x47,
|
||||
0x01, 0x28, 0x09, 0x52, 0x05, 0x65, 0x6d, 0x61, 0x69, 0x6c, 0x22, 0xab, 0x01, 0x0a, 0x0b, 0x47,
|
||||
0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x44, 0x72, 0x69, 0x76, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x66, 0x69,
|
||||
0x6c, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x66, 0x69, 0x6c, 0x65, 0x12, 0x12,
|
||||
0x0a, 0x04, 0x6c, 0x69, 0x6e, 0x6b, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6c, 0x69,
|
||||
@@ -3209,130 +3060,104 @@ var file_source_metadata_proto_rawDesc = []byte{
|
||||
0x18, 0x05, 0x20, 0x01, 0x28, 0x08, 0x52, 0x06, 0x73, 0x68, 0x61, 0x72, 0x65, 0x64, 0x12, 0x28,
|
||||
0x0a, 0x10, 0x6c, 0x61, 0x73, 0x74, 0x5f, 0x6d, 0x6f, 0x64, 0x69, 0x66, 0x69, 0x65, 0x64, 0x5f,
|
||||
0x62, 0x79, 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0e, 0x6c, 0x61, 0x73, 0x74, 0x4d, 0x6f,
|
||||
0x64, 0x69, 0x66, 0x69, 0x65, 0x64, 0x42, 0x79, 0x12, 0x12, 0x0a, 0x04, 0x70, 0x61, 0x74, 0x68,
|
||||
0x18, 0x07, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x70, 0x61, 0x74, 0x68, 0x22, 0xcb, 0x02, 0x0a,
|
||||
0x0a, 0x41, 0x7a, 0x75, 0x72, 0x65, 0x52, 0x65, 0x70, 0x6f, 0x73, 0x12, 0x12, 0x0a, 0x04, 0x6c,
|
||||
0x69, 0x6e, 0x6b, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6c, 0x69, 0x6e, 0x6b, 0x12,
|
||||
0x1a, 0x0a, 0x08, 0x75, 0x73, 0x65, 0x72, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28,
|
||||
0x09, 0x52, 0x08, 0x75, 0x73, 0x65, 0x72, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x1e, 0x0a, 0x0a, 0x72,
|
||||
0x65, 0x70, 0x6f, 0x73, 0x69, 0x74, 0x6f, 0x72, 0x79, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52,
|
||||
0x0a, 0x72, 0x65, 0x70, 0x6f, 0x73, 0x69, 0x74, 0x6f, 0x72, 0x79, 0x12, 0x16, 0x0a, 0x06, 0x63,
|
||||
0x6f, 0x6d, 0x6d, 0x69, 0x74, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x63, 0x6f, 0x6d,
|
||||
0x6d, 0x69, 0x74, 0x12, 0x14, 0x0a, 0x05, 0x65, 0x6d, 0x61, 0x69, 0x6c, 0x18, 0x05, 0x20, 0x01,
|
||||
0x28, 0x09, 0x52, 0x05, 0x65, 0x6d, 0x61, 0x69, 0x6c, 0x12, 0x12, 0x0a, 0x04, 0x66, 0x69, 0x6c,
|
||||
0x65, 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x66, 0x69, 0x6c, 0x65, 0x12, 0x1c, 0x0a,
|
||||
0x09, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x18, 0x07, 0x20, 0x01, 0x28, 0x09,
|
||||
0x52, 0x09, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x12, 0x12, 0x0a, 0x04, 0x6c,
|
||||
0x69, 0x6e, 0x65, 0x18, 0x08, 0x20, 0x01, 0x28, 0x03, 0x52, 0x04, 0x6c, 0x69, 0x6e, 0x65, 0x12,
|
||||
0x3b, 0x0a, 0x0a, 0x76, 0x69, 0x73, 0x69, 0x62, 0x69, 0x6c, 0x69, 0x74, 0x79, 0x18, 0x09, 0x20,
|
||||
0x01, 0x28, 0x0e, 0x32, 0x1b, 0x2e, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f, 0x6d, 0x65, 0x74,
|
||||
0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x56, 0x69, 0x73, 0x69, 0x62, 0x69, 0x6c, 0x69, 0x74, 0x79,
|
||||
0x52, 0x0a, 0x76, 0x69, 0x73, 0x69, 0x62, 0x69, 0x6c, 0x69, 0x74, 0x79, 0x12, 0x18, 0x0a, 0x07,
|
||||
0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x70,
|
||||
0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x12, 0x22, 0x0a, 0x0c, 0x6f, 0x72, 0x67, 0x61, 0x6e, 0x69,
|
||||
0x7a, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x0b, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0c, 0x6f, 0x72,
|
||||
0x67, 0x61, 0x6e, 0x69, 0x7a, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x22, 0xbe, 0x0b, 0x0a, 0x08, 0x4d,
|
||||
0x65, 0x74, 0x61, 0x44, 0x61, 0x74, 0x61, 0x12, 0x2e, 0x0a, 0x05, 0x61, 0x7a, 0x75, 0x72, 0x65,
|
||||
0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x16, 0x2e, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f,
|
||||
0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x41, 0x7a, 0x75, 0x72, 0x65, 0x48, 0x00,
|
||||
0x52, 0x05, 0x61, 0x7a, 0x75, 0x72, 0x65, 0x12, 0x3a, 0x0a, 0x09, 0x62, 0x69, 0x74, 0x62, 0x75,
|
||||
0x63, 0x6b, 0x65, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x73, 0x6f, 0x75,
|
||||
0x72, 0x63, 0x65, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x42, 0x69, 0x74,
|
||||
0x62, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x48, 0x00, 0x52, 0x09, 0x62, 0x69, 0x74, 0x62, 0x75, 0x63,
|
||||
0x6b, 0x65, 0x74, 0x12, 0x37, 0x0a, 0x08, 0x63, 0x69, 0x72, 0x63, 0x6c, 0x65, 0x63, 0x69, 0x18,
|
||||
0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x19, 0x2e, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f, 0x6d,
|
||||
0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x43, 0x69, 0x72, 0x63, 0x6c, 0x65, 0x43, 0x49,
|
||||
0x48, 0x00, 0x52, 0x08, 0x63, 0x69, 0x72, 0x63, 0x6c, 0x65, 0x63, 0x69, 0x12, 0x3d, 0x0a, 0x0a,
|
||||
0x63, 0x6f, 0x6e, 0x66, 0x6c, 0x75, 0x65, 0x6e, 0x63, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b,
|
||||
0x32, 0x1b, 0x2e, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61,
|
||||
0x74, 0x61, 0x2e, 0x43, 0x6f, 0x6e, 0x66, 0x6c, 0x75, 0x65, 0x6e, 0x63, 0x65, 0x48, 0x00, 0x52,
|
||||
0x0a, 0x63, 0x6f, 0x6e, 0x66, 0x6c, 0x75, 0x65, 0x6e, 0x63, 0x65, 0x12, 0x31, 0x0a, 0x06, 0x64,
|
||||
0x6f, 0x63, 0x6b, 0x65, 0x72, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x17, 0x2e, 0x73, 0x6f,
|
||||
0x75, 0x72, 0x63, 0x65, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x44, 0x6f,
|
||||
0x63, 0x6b, 0x65, 0x72, 0x48, 0x00, 0x52, 0x06, 0x64, 0x6f, 0x63, 0x6b, 0x65, 0x72, 0x12, 0x28,
|
||||
0x0a, 0x03, 0x65, 0x63, 0x72, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x73, 0x6f,
|
||||
0x75, 0x72, 0x63, 0x65, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x45, 0x43,
|
||||
0x52, 0x48, 0x00, 0x52, 0x03, 0x65, 0x63, 0x72, 0x12, 0x28, 0x0a, 0x03, 0x67, 0x63, 0x73, 0x18,
|
||||
0x07, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f, 0x6d,
|
||||
0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x47, 0x43, 0x53, 0x48, 0x00, 0x52, 0x03, 0x67,
|
||||
0x63, 0x73, 0x12, 0x31, 0x0a, 0x06, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x18, 0x08, 0x20, 0x01,
|
||||
0x28, 0x0b, 0x32, 0x17, 0x2e, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f, 0x6d, 0x65, 0x74, 0x61,
|
||||
0x64, 0x61, 0x74, 0x61, 0x2e, 0x47, 0x69, 0x74, 0x68, 0x75, 0x62, 0x48, 0x00, 0x52, 0x06, 0x67,
|
||||
0x69, 0x74, 0x68, 0x75, 0x62, 0x12, 0x31, 0x0a, 0x06, 0x67, 0x69, 0x74, 0x6c, 0x61, 0x62, 0x18,
|
||||
0x09, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x17, 0x2e, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f, 0x6d,
|
||||
0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x47, 0x69, 0x74, 0x6c, 0x61, 0x62, 0x48, 0x00,
|
||||
0x52, 0x06, 0x67, 0x69, 0x74, 0x6c, 0x61, 0x62, 0x12, 0x2b, 0x0a, 0x04, 0x6a, 0x69, 0x72, 0x61,
|
||||
0x18, 0x0a, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x15, 0x2e, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f,
|
||||
0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x4a, 0x69, 0x72, 0x61, 0x48, 0x00, 0x52,
|
||||
0x04, 0x6a, 0x69, 0x72, 0x61, 0x12, 0x28, 0x0a, 0x03, 0x6e, 0x70, 0x6d, 0x18, 0x0b, 0x20, 0x01,
|
||||
0x28, 0x0b, 0x32, 0x14, 0x2e, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f, 0x6d, 0x65, 0x74, 0x61,
|
||||
0x64, 0x61, 0x74, 0x61, 0x2e, 0x4e, 0x50, 0x4d, 0x48, 0x00, 0x52, 0x03, 0x6e, 0x70, 0x6d, 0x12,
|
||||
0x2b, 0x0a, 0x04, 0x70, 0x79, 0x70, 0x69, 0x18, 0x0c, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x15, 0x2e,
|
||||
0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e,
|
||||
0x50, 0x79, 0x50, 0x69, 0x48, 0x00, 0x52, 0x04, 0x70, 0x79, 0x70, 0x69, 0x12, 0x25, 0x0a, 0x02,
|
||||
0x73, 0x33, 0x18, 0x0d, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x13, 0x2e, 0x73, 0x6f, 0x75, 0x72, 0x63,
|
||||
0x65, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x53, 0x33, 0x48, 0x00, 0x52,
|
||||
0x02, 0x73, 0x33, 0x12, 0x2e, 0x0a, 0x05, 0x73, 0x6c, 0x61, 0x63, 0x6b, 0x18, 0x0e, 0x20, 0x01,
|
||||
0x28, 0x0b, 0x32, 0x16, 0x2e, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f, 0x6d, 0x65, 0x74, 0x61,
|
||||
0x64, 0x61, 0x74, 0x61, 0x2e, 0x53, 0x6c, 0x61, 0x63, 0x6b, 0x48, 0x00, 0x52, 0x05, 0x73, 0x6c,
|
||||
0x61, 0x63, 0x6b, 0x12, 0x3d, 0x0a, 0x0a, 0x66, 0x69, 0x6c, 0x65, 0x73, 0x79, 0x73, 0x74, 0x65,
|
||||
0x6d, 0x18, 0x0f, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1b, 0x2e, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65,
|
||||
0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x46, 0x69, 0x6c, 0x65, 0x73, 0x79,
|
||||
0x73, 0x74, 0x65, 0x6d, 0x48, 0x00, 0x52, 0x0a, 0x66, 0x69, 0x6c, 0x65, 0x73, 0x79, 0x73, 0x74,
|
||||
0x65, 0x6d, 0x12, 0x28, 0x0a, 0x03, 0x67, 0x69, 0x74, 0x18, 0x10, 0x20, 0x01, 0x28, 0x0b, 0x32,
|
||||
0x14, 0x2e, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74,
|
||||
0x61, 0x2e, 0x47, 0x69, 0x74, 0x48, 0x00, 0x52, 0x03, 0x67, 0x69, 0x74, 0x12, 0x2b, 0x0a, 0x04,
|
||||
0x74, 0x65, 0x73, 0x74, 0x18, 0x11, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x15, 0x2e, 0x73, 0x6f, 0x75,
|
||||
0x72, 0x63, 0x65, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x54, 0x65, 0x73,
|
||||
0x74, 0x48, 0x00, 0x52, 0x04, 0x74, 0x65, 0x73, 0x74, 0x12, 0x3a, 0x0a, 0x09, 0x62, 0x75, 0x69,
|
||||
0x6c, 0x64, 0x6b, 0x69, 0x74, 0x65, 0x18, 0x12, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x73,
|
||||
0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x42,
|
||||
0x75, 0x69, 0x6c, 0x64, 0x6b, 0x69, 0x74, 0x65, 0x48, 0x00, 0x52, 0x09, 0x62, 0x75, 0x69, 0x6c,
|
||||
0x64, 0x6b, 0x69, 0x74, 0x65, 0x12, 0x31, 0x0a, 0x06, 0x67, 0x65, 0x72, 0x72, 0x69, 0x74, 0x18,
|
||||
0x13, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x17, 0x2e, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f, 0x6d,
|
||||
0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x47, 0x65, 0x72, 0x72, 0x69, 0x74, 0x48, 0x00,
|
||||
0x52, 0x06, 0x67, 0x65, 0x72, 0x72, 0x69, 0x74, 0x12, 0x34, 0x0a, 0x07, 0x6a, 0x65, 0x6e, 0x6b,
|
||||
0x69, 0x6e, 0x73, 0x18, 0x14, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x18, 0x2e, 0x73, 0x6f, 0x75, 0x72,
|
||||
0x63, 0x65, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x4a, 0x65, 0x6e, 0x6b,
|
||||
0x69, 0x6e, 0x73, 0x48, 0x00, 0x52, 0x07, 0x6a, 0x65, 0x6e, 0x6b, 0x69, 0x6e, 0x73, 0x12, 0x2e,
|
||||
0x0a, 0x05, 0x74, 0x65, 0x61, 0x6d, 0x73, 0x18, 0x15, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x16, 0x2e,
|
||||
0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e,
|
||||
0x54, 0x65, 0x61, 0x6d, 0x73, 0x48, 0x00, 0x52, 0x05, 0x74, 0x65, 0x61, 0x6d, 0x73, 0x12, 0x40,
|
||||
0x0a, 0x0b, 0x61, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x6f, 0x72, 0x79, 0x18, 0x16, 0x20,
|
||||
0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f, 0x6d, 0x65, 0x74,
|
||||
0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x6f, 0x72,
|
||||
0x79, 0x48, 0x00, 0x52, 0x0b, 0x61, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x6f, 0x72, 0x79,
|
||||
0x12, 0x31, 0x0a, 0x06, 0x73, 0x79, 0x73, 0x6c, 0x6f, 0x67, 0x18, 0x17, 0x20, 0x01, 0x28, 0x0b,
|
||||
0x64, 0x69, 0x66, 0x69, 0x65, 0x64, 0x42, 0x79, 0x22, 0xff, 0x0a, 0x0a, 0x08, 0x4d, 0x65, 0x74,
|
||||
0x61, 0x44, 0x61, 0x74, 0x61, 0x12, 0x2e, 0x0a, 0x05, 0x61, 0x7a, 0x75, 0x72, 0x65, 0x18, 0x01,
|
||||
0x20, 0x01, 0x28, 0x0b, 0x32, 0x16, 0x2e, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f, 0x6d, 0x65,
|
||||
0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x41, 0x7a, 0x75, 0x72, 0x65, 0x48, 0x00, 0x52, 0x05,
|
||||
0x61, 0x7a, 0x75, 0x72, 0x65, 0x12, 0x3a, 0x0a, 0x09, 0x62, 0x69, 0x74, 0x62, 0x75, 0x63, 0x6b,
|
||||
0x65, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x73, 0x6f, 0x75, 0x72, 0x63,
|
||||
0x65, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x42, 0x69, 0x74, 0x62, 0x75,
|
||||
0x63, 0x6b, 0x65, 0x74, 0x48, 0x00, 0x52, 0x09, 0x62, 0x69, 0x74, 0x62, 0x75, 0x63, 0x6b, 0x65,
|
||||
0x74, 0x12, 0x37, 0x0a, 0x08, 0x63, 0x69, 0x72, 0x63, 0x6c, 0x65, 0x63, 0x69, 0x18, 0x03, 0x20,
|
||||
0x01, 0x28, 0x0b, 0x32, 0x19, 0x2e, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f, 0x6d, 0x65, 0x74,
|
||||
0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x43, 0x69, 0x72, 0x63, 0x6c, 0x65, 0x43, 0x49, 0x48, 0x00,
|
||||
0x52, 0x08, 0x63, 0x69, 0x72, 0x63, 0x6c, 0x65, 0x63, 0x69, 0x12, 0x3d, 0x0a, 0x0a, 0x63, 0x6f,
|
||||
0x6e, 0x66, 0x6c, 0x75, 0x65, 0x6e, 0x63, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1b,
|
||||
0x2e, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61,
|
||||
0x2e, 0x43, 0x6f, 0x6e, 0x66, 0x6c, 0x75, 0x65, 0x6e, 0x63, 0x65, 0x48, 0x00, 0x52, 0x0a, 0x63,
|
||||
0x6f, 0x6e, 0x66, 0x6c, 0x75, 0x65, 0x6e, 0x63, 0x65, 0x12, 0x31, 0x0a, 0x06, 0x64, 0x6f, 0x63,
|
||||
0x6b, 0x65, 0x72, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x17, 0x2e, 0x73, 0x6f, 0x75, 0x72,
|
||||
0x63, 0x65, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x44, 0x6f, 0x63, 0x6b,
|
||||
0x65, 0x72, 0x48, 0x00, 0x52, 0x06, 0x64, 0x6f, 0x63, 0x6b, 0x65, 0x72, 0x12, 0x28, 0x0a, 0x03,
|
||||
0x65, 0x63, 0x72, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x73, 0x6f, 0x75, 0x72,
|
||||
0x63, 0x65, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x45, 0x43, 0x52, 0x48,
|
||||
0x00, 0x52, 0x03, 0x65, 0x63, 0x72, 0x12, 0x28, 0x0a, 0x03, 0x67, 0x63, 0x73, 0x18, 0x07, 0x20,
|
||||
0x01, 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f, 0x6d, 0x65, 0x74,
|
||||
0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x47, 0x43, 0x53, 0x48, 0x00, 0x52, 0x03, 0x67, 0x63, 0x73,
|
||||
0x12, 0x31, 0x0a, 0x06, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x18, 0x08, 0x20, 0x01, 0x28, 0x0b,
|
||||
0x32, 0x17, 0x2e, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61,
|
||||
0x74, 0x61, 0x2e, 0x53, 0x79, 0x73, 0x6c, 0x6f, 0x67, 0x48, 0x00, 0x52, 0x06, 0x73, 0x79, 0x73,
|
||||
0x6c, 0x6f, 0x67, 0x12, 0x5e, 0x0a, 0x15, 0x70, 0x75, 0x62, 0x6c, 0x69, 0x63, 0x45, 0x76, 0x65,
|
||||
0x6e, 0x74, 0x4d, 0x6f, 0x6e, 0x69, 0x74, 0x6f, 0x72, 0x69, 0x6e, 0x67, 0x18, 0x18, 0x20, 0x01,
|
||||
0x28, 0x0b, 0x32, 0x26, 0x2e, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f, 0x6d, 0x65, 0x74, 0x61,
|
||||
0x64, 0x61, 0x74, 0x61, 0x2e, 0x50, 0x75, 0x62, 0x6c, 0x69, 0x63, 0x45, 0x76, 0x65, 0x6e, 0x74,
|
||||
0x4d, 0x6f, 0x6e, 0x69, 0x74, 0x6f, 0x72, 0x69, 0x6e, 0x67, 0x48, 0x00, 0x52, 0x15, 0x70, 0x75,
|
||||
0x62, 0x6c, 0x69, 0x63, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x4d, 0x6f, 0x6e, 0x69, 0x74, 0x6f, 0x72,
|
||||
0x69, 0x6e, 0x67, 0x12, 0x3d, 0x0a, 0x0a, 0x73, 0x68, 0x61, 0x72, 0x65, 0x70, 0x6f, 0x69, 0x6e,
|
||||
0x74, 0x18, 0x19, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1b, 0x2e, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65,
|
||||
0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x53, 0x68, 0x61, 0x72, 0x65, 0x50,
|
||||
0x6f, 0x69, 0x6e, 0x74, 0x48, 0x00, 0x52, 0x0a, 0x73, 0x68, 0x61, 0x72, 0x65, 0x70, 0x6f, 0x69,
|
||||
0x6e, 0x74, 0x12, 0x40, 0x0a, 0x0b, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x44, 0x72, 0x69, 0x76,
|
||||
0x65, 0x18, 0x1a, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65,
|
||||
0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x47, 0x6f, 0x6f, 0x67, 0x6c, 0x65,
|
||||
0x44, 0x72, 0x69, 0x76, 0x65, 0x48, 0x00, 0x52, 0x0b, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x44,
|
||||
0x72, 0x69, 0x76, 0x65, 0x12, 0x3d, 0x0a, 0x0a, 0x61, 0x7a, 0x75, 0x72, 0x65, 0x52, 0x65, 0x70,
|
||||
0x6f, 0x73, 0x18, 0x1b, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1b, 0x2e, 0x73, 0x6f, 0x75, 0x72, 0x63,
|
||||
0x65, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x41, 0x7a, 0x75, 0x72, 0x65,
|
||||
0x52, 0x65, 0x70, 0x6f, 0x73, 0x48, 0x00, 0x52, 0x0a, 0x61, 0x7a, 0x75, 0x72, 0x65, 0x52, 0x65,
|
||||
0x70, 0x6f, 0x73, 0x42, 0x06, 0x0a, 0x04, 0x64, 0x61, 0x74, 0x61, 0x2a, 0x3e, 0x0a, 0x0a, 0x56,
|
||||
0x69, 0x73, 0x69, 0x62, 0x69, 0x6c, 0x69, 0x74, 0x79, 0x12, 0x0a, 0x0a, 0x06, 0x70, 0x75, 0x62,
|
||||
0x6c, 0x69, 0x63, 0x10, 0x00, 0x12, 0x0b, 0x0a, 0x07, 0x70, 0x72, 0x69, 0x76, 0x61, 0x74, 0x65,
|
||||
0x10, 0x01, 0x12, 0x0a, 0x0a, 0x06, 0x73, 0x68, 0x61, 0x72, 0x65, 0x64, 0x10, 0x02, 0x12, 0x0b,
|
||||
0x0a, 0x07, 0x75, 0x6e, 0x6b, 0x6e, 0x6f, 0x77, 0x6e, 0x10, 0x03, 0x42, 0x43, 0x5a, 0x41, 0x67,
|
||||
0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x74, 0x72, 0x75, 0x66, 0x66, 0x6c,
|
||||
0x65, 0x73, 0x65, 0x63, 0x75, 0x72, 0x69, 0x74, 0x79, 0x2f, 0x74, 0x72, 0x75, 0x66, 0x66, 0x6c,
|
||||
0x65, 0x68, 0x6f, 0x67, 0x2f, 0x76, 0x33, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x70, 0x62, 0x2f, 0x73,
|
||||
0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x70, 0x62,
|
||||
0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
|
||||
0x74, 0x61, 0x2e, 0x47, 0x69, 0x74, 0x68, 0x75, 0x62, 0x48, 0x00, 0x52, 0x06, 0x67, 0x69, 0x74,
|
||||
0x68, 0x75, 0x62, 0x12, 0x31, 0x0a, 0x06, 0x67, 0x69, 0x74, 0x6c, 0x61, 0x62, 0x18, 0x09, 0x20,
|
||||
0x01, 0x28, 0x0b, 0x32, 0x17, 0x2e, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f, 0x6d, 0x65, 0x74,
|
||||
0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x47, 0x69, 0x74, 0x6c, 0x61, 0x62, 0x48, 0x00, 0x52, 0x06,
|
||||
0x67, 0x69, 0x74, 0x6c, 0x61, 0x62, 0x12, 0x2b, 0x0a, 0x04, 0x6a, 0x69, 0x72, 0x61, 0x18, 0x0a,
|
||||
0x20, 0x01, 0x28, 0x0b, 0x32, 0x15, 0x2e, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f, 0x6d, 0x65,
|
||||
0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x4a, 0x69, 0x72, 0x61, 0x48, 0x00, 0x52, 0x04, 0x6a,
|
||||
0x69, 0x72, 0x61, 0x12, 0x28, 0x0a, 0x03, 0x6e, 0x70, 0x6d, 0x18, 0x0b, 0x20, 0x01, 0x28, 0x0b,
|
||||
0x32, 0x14, 0x2e, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61,
|
||||
0x74, 0x61, 0x2e, 0x4e, 0x50, 0x4d, 0x48, 0x00, 0x52, 0x03, 0x6e, 0x70, 0x6d, 0x12, 0x2b, 0x0a,
|
||||
0x04, 0x70, 0x79, 0x70, 0x69, 0x18, 0x0c, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x15, 0x2e, 0x73, 0x6f,
|
||||
0x75, 0x72, 0x63, 0x65, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x50, 0x79,
|
||||
0x50, 0x69, 0x48, 0x00, 0x52, 0x04, 0x70, 0x79, 0x70, 0x69, 0x12, 0x25, 0x0a, 0x02, 0x73, 0x33,
|
||||
0x18, 0x0d, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x13, 0x2e, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f,
|
||||
0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x53, 0x33, 0x48, 0x00, 0x52, 0x02, 0x73,
|
||||
0x33, 0x12, 0x2e, 0x0a, 0x05, 0x73, 0x6c, 0x61, 0x63, 0x6b, 0x18, 0x0e, 0x20, 0x01, 0x28, 0x0b,
|
||||
0x32, 0x16, 0x2e, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61,
|
||||
0x74, 0x61, 0x2e, 0x53, 0x6c, 0x61, 0x63, 0x6b, 0x48, 0x00, 0x52, 0x05, 0x73, 0x6c, 0x61, 0x63,
|
||||
0x6b, 0x12, 0x3d, 0x0a, 0x0a, 0x66, 0x69, 0x6c, 0x65, 0x73, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x18,
|
||||
0x0f, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1b, 0x2e, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f, 0x6d,
|
||||
0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x46, 0x69, 0x6c, 0x65, 0x73, 0x79, 0x73, 0x74,
|
||||
0x65, 0x6d, 0x48, 0x00, 0x52, 0x0a, 0x66, 0x69, 0x6c, 0x65, 0x73, 0x79, 0x73, 0x74, 0x65, 0x6d,
|
||||
0x12, 0x28, 0x0a, 0x03, 0x67, 0x69, 0x74, 0x18, 0x10, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x14, 0x2e,
|
||||
0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e,
|
||||
0x47, 0x69, 0x74, 0x48, 0x00, 0x52, 0x03, 0x67, 0x69, 0x74, 0x12, 0x2b, 0x0a, 0x04, 0x74, 0x65,
|
||||
0x73, 0x74, 0x18, 0x11, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x15, 0x2e, 0x73, 0x6f, 0x75, 0x72, 0x63,
|
||||
0x65, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x54, 0x65, 0x73, 0x74, 0x48,
|
||||
0x00, 0x52, 0x04, 0x74, 0x65, 0x73, 0x74, 0x12, 0x3a, 0x0a, 0x09, 0x62, 0x75, 0x69, 0x6c, 0x64,
|
||||
0x6b, 0x69, 0x74, 0x65, 0x18, 0x12, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x73, 0x6f, 0x75,
|
||||
0x72, 0x63, 0x65, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x42, 0x75, 0x69,
|
||||
0x6c, 0x64, 0x6b, 0x69, 0x74, 0x65, 0x48, 0x00, 0x52, 0x09, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x6b,
|
||||
0x69, 0x74, 0x65, 0x12, 0x31, 0x0a, 0x06, 0x67, 0x65, 0x72, 0x72, 0x69, 0x74, 0x18, 0x13, 0x20,
|
||||
0x01, 0x28, 0x0b, 0x32, 0x17, 0x2e, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f, 0x6d, 0x65, 0x74,
|
||||
0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x47, 0x65, 0x72, 0x72, 0x69, 0x74, 0x48, 0x00, 0x52, 0x06,
|
||||
0x67, 0x65, 0x72, 0x72, 0x69, 0x74, 0x12, 0x34, 0x0a, 0x07, 0x6a, 0x65, 0x6e, 0x6b, 0x69, 0x6e,
|
||||
0x73, 0x18, 0x14, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x18, 0x2e, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65,
|
||||
0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x4a, 0x65, 0x6e, 0x6b, 0x69, 0x6e,
|
||||
0x73, 0x48, 0x00, 0x52, 0x07, 0x6a, 0x65, 0x6e, 0x6b, 0x69, 0x6e, 0x73, 0x12, 0x2e, 0x0a, 0x05,
|
||||
0x74, 0x65, 0x61, 0x6d, 0x73, 0x18, 0x15, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x16, 0x2e, 0x73, 0x6f,
|
||||
0x75, 0x72, 0x63, 0x65, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x54, 0x65,
|
||||
0x61, 0x6d, 0x73, 0x48, 0x00, 0x52, 0x05, 0x74, 0x65, 0x61, 0x6d, 0x73, 0x12, 0x40, 0x0a, 0x0b,
|
||||
0x61, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x6f, 0x72, 0x79, 0x18, 0x16, 0x20, 0x01, 0x28,
|
||||
0x0b, 0x32, 0x1c, 0x2e, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64,
|
||||
0x61, 0x74, 0x61, 0x2e, 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x6f, 0x72, 0x79, 0x48,
|
||||
0x00, 0x52, 0x0b, 0x61, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x6f, 0x72, 0x79, 0x12, 0x31,
|
||||
0x0a, 0x06, 0x73, 0x79, 0x73, 0x6c, 0x6f, 0x67, 0x18, 0x17, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x17,
|
||||
0x2e, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61,
|
||||
0x2e, 0x53, 0x79, 0x73, 0x6c, 0x6f, 0x67, 0x48, 0x00, 0x52, 0x06, 0x73, 0x79, 0x73, 0x6c, 0x6f,
|
||||
0x67, 0x12, 0x5e, 0x0a, 0x15, 0x70, 0x75, 0x62, 0x6c, 0x69, 0x63, 0x45, 0x76, 0x65, 0x6e, 0x74,
|
||||
0x4d, 0x6f, 0x6e, 0x69, 0x74, 0x6f, 0x72, 0x69, 0x6e, 0x67, 0x18, 0x18, 0x20, 0x01, 0x28, 0x0b,
|
||||
0x32, 0x26, 0x2e, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61,
|
||||
0x74, 0x61, 0x2e, 0x50, 0x75, 0x62, 0x6c, 0x69, 0x63, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x4d, 0x6f,
|
||||
0x6e, 0x69, 0x74, 0x6f, 0x72, 0x69, 0x6e, 0x67, 0x48, 0x00, 0x52, 0x15, 0x70, 0x75, 0x62, 0x6c,
|
||||
0x69, 0x63, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x4d, 0x6f, 0x6e, 0x69, 0x74, 0x6f, 0x72, 0x69, 0x6e,
|
||||
0x67, 0x12, 0x3d, 0x0a, 0x0a, 0x73, 0x68, 0x61, 0x72, 0x65, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x18,
|
||||
0x19, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1b, 0x2e, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f, 0x6d,
|
||||
0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x53, 0x68, 0x61, 0x72, 0x65, 0x50, 0x6f, 0x69,
|
||||
0x6e, 0x74, 0x48, 0x00, 0x52, 0x0a, 0x73, 0x68, 0x61, 0x72, 0x65, 0x70, 0x6f, 0x69, 0x6e, 0x74,
|
||||
0x12, 0x40, 0x0a, 0x0b, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x44, 0x72, 0x69, 0x76, 0x65, 0x18,
|
||||
0x1a, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f, 0x6d,
|
||||
0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x47, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x44, 0x72,
|
||||
0x69, 0x76, 0x65, 0x48, 0x00, 0x52, 0x0b, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x44, 0x72, 0x69,
|
||||
0x76, 0x65, 0x42, 0x06, 0x0a, 0x04, 0x64, 0x61, 0x74, 0x61, 0x2a, 0x3e, 0x0a, 0x0a, 0x56, 0x69,
|
||||
0x73, 0x69, 0x62, 0x69, 0x6c, 0x69, 0x74, 0x79, 0x12, 0x0a, 0x0a, 0x06, 0x70, 0x75, 0x62, 0x6c,
|
||||
0x69, 0x63, 0x10, 0x00, 0x12, 0x0b, 0x0a, 0x07, 0x70, 0x72, 0x69, 0x76, 0x61, 0x74, 0x65, 0x10,
|
||||
0x01, 0x12, 0x0a, 0x0a, 0x06, 0x73, 0x68, 0x61, 0x72, 0x65, 0x64, 0x10, 0x02, 0x12, 0x0b, 0x0a,
|
||||
0x07, 0x75, 0x6e, 0x6b, 0x6e, 0x6f, 0x77, 0x6e, 0x10, 0x03, 0x42, 0x43, 0x5a, 0x41, 0x67, 0x69,
|
||||
0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x74, 0x72, 0x75, 0x66, 0x66, 0x6c, 0x65,
|
||||
0x73, 0x65, 0x63, 0x75, 0x72, 0x69, 0x74, 0x79, 0x2f, 0x74, 0x72, 0x75, 0x66, 0x66, 0x6c, 0x65,
|
||||
0x68, 0x6f, 0x67, 0x2f, 0x76, 0x33, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x70, 0x62, 0x2f, 0x73, 0x6f,
|
||||
0x75, 0x72, 0x63, 0x65, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x70, 0x62, 0x62,
|
||||
0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
|
||||
}
|
||||
|
||||
var (
|
||||
@@ -3348,7 +3173,7 @@ func file_source_metadata_proto_rawDescGZIP() []byte {
|
||||
}
|
||||
|
||||
var file_source_metadata_proto_enumTypes = make([]protoimpl.EnumInfo, 1)
|
||||
var file_source_metadata_proto_msgTypes = make([]protoimpl.MessageInfo, 28)
|
||||
var file_source_metadata_proto_msgTypes = make([]protoimpl.MessageInfo, 27)
|
||||
var file_source_metadata_proto_goTypes = []interface{}{
|
||||
(Visibility)(0), // 0: source_metadata.Visibility
|
||||
(*Azure)(nil), // 1: source_metadata.Azure
|
||||
@@ -3377,46 +3202,43 @@ var file_source_metadata_proto_goTypes = []interface{}{
|
||||
(*PublicEventMonitoring)(nil), // 24: source_metadata.PublicEventMonitoring
|
||||
(*SharePoint)(nil), // 25: source_metadata.SharePoint
|
||||
(*GoogleDrive)(nil), // 26: source_metadata.GoogleDrive
|
||||
(*AzureRepos)(nil), // 27: source_metadata.AzureRepos
|
||||
(*MetaData)(nil), // 28: source_metadata.MetaData
|
||||
(*MetaData)(nil), // 27: source_metadata.MetaData
|
||||
}
|
||||
var file_source_metadata_proto_depIdxs = []int32{
|
||||
0, // 0: source_metadata.Github.visibility:type_name -> source_metadata.Visibility
|
||||
0, // 1: source_metadata.Slack.visibility:type_name -> source_metadata.Visibility
|
||||
10, // 2: source_metadata.PublicEventMonitoring.github:type_name -> source_metadata.Github
|
||||
0, // 3: source_metadata.AzureRepos.visibility:type_name -> source_metadata.Visibility
|
||||
1, // 4: source_metadata.MetaData.azure:type_name -> source_metadata.Azure
|
||||
2, // 5: source_metadata.MetaData.bitbucket:type_name -> source_metadata.Bitbucket
|
||||
4, // 6: source_metadata.MetaData.circleci:type_name -> source_metadata.CircleCI
|
||||
5, // 7: source_metadata.MetaData.confluence:type_name -> source_metadata.Confluence
|
||||
6, // 8: source_metadata.MetaData.docker:type_name -> source_metadata.Docker
|
||||
7, // 9: source_metadata.MetaData.ecr:type_name -> source_metadata.ECR
|
||||
12, // 10: source_metadata.MetaData.gcs:type_name -> source_metadata.GCS
|
||||
10, // 11: source_metadata.MetaData.github:type_name -> source_metadata.Github
|
||||
11, // 12: source_metadata.MetaData.gitlab:type_name -> source_metadata.Gitlab
|
||||
13, // 13: source_metadata.MetaData.jira:type_name -> source_metadata.Jira
|
||||
14, // 14: source_metadata.MetaData.npm:type_name -> source_metadata.NPM
|
||||
15, // 15: source_metadata.MetaData.pypi:type_name -> source_metadata.PyPi
|
||||
16, // 16: source_metadata.MetaData.s3:type_name -> source_metadata.S3
|
||||
17, // 17: source_metadata.MetaData.slack:type_name -> source_metadata.Slack
|
||||
8, // 18: source_metadata.MetaData.filesystem:type_name -> source_metadata.Filesystem
|
||||
9, // 19: source_metadata.MetaData.git:type_name -> source_metadata.Git
|
||||
19, // 20: source_metadata.MetaData.test:type_name -> source_metadata.Test
|
||||
3, // 21: source_metadata.MetaData.buildkite:type_name -> source_metadata.Buildkite
|
||||
18, // 22: source_metadata.MetaData.gerrit:type_name -> source_metadata.Gerrit
|
||||
20, // 23: source_metadata.MetaData.jenkins:type_name -> source_metadata.Jenkins
|
||||
21, // 24: source_metadata.MetaData.teams:type_name -> source_metadata.Teams
|
||||
22, // 25: source_metadata.MetaData.artifactory:type_name -> source_metadata.Artifactory
|
||||
23, // 26: source_metadata.MetaData.syslog:type_name -> source_metadata.Syslog
|
||||
24, // 27: source_metadata.MetaData.publicEventMonitoring:type_name -> source_metadata.PublicEventMonitoring
|
||||
25, // 28: source_metadata.MetaData.sharepoint:type_name -> source_metadata.SharePoint
|
||||
26, // 29: source_metadata.MetaData.googleDrive:type_name -> source_metadata.GoogleDrive
|
||||
27, // 30: source_metadata.MetaData.azureRepos:type_name -> source_metadata.AzureRepos
|
||||
31, // [31:31] is the sub-list for method output_type
|
||||
31, // [31:31] is the sub-list for method input_type
|
||||
31, // [31:31] is the sub-list for extension type_name
|
||||
31, // [31:31] is the sub-list for extension extendee
|
||||
0, // [0:31] is the sub-list for field type_name
|
||||
1, // 3: source_metadata.MetaData.azure:type_name -> source_metadata.Azure
|
||||
2, // 4: source_metadata.MetaData.bitbucket:type_name -> source_metadata.Bitbucket
|
||||
4, // 5: source_metadata.MetaData.circleci:type_name -> source_metadata.CircleCI
|
||||
5, // 6: source_metadata.MetaData.confluence:type_name -> source_metadata.Confluence
|
||||
6, // 7: source_metadata.MetaData.docker:type_name -> source_metadata.Docker
|
||||
7, // 8: source_metadata.MetaData.ecr:type_name -> source_metadata.ECR
|
||||
12, // 9: source_metadata.MetaData.gcs:type_name -> source_metadata.GCS
|
||||
10, // 10: source_metadata.MetaData.github:type_name -> source_metadata.Github
|
||||
11, // 11: source_metadata.MetaData.gitlab:type_name -> source_metadata.Gitlab
|
||||
13, // 12: source_metadata.MetaData.jira:type_name -> source_metadata.Jira
|
||||
14, // 13: source_metadata.MetaData.npm:type_name -> source_metadata.NPM
|
||||
15, // 14: source_metadata.MetaData.pypi:type_name -> source_metadata.PyPi
|
||||
16, // 15: source_metadata.MetaData.s3:type_name -> source_metadata.S3
|
||||
17, // 16: source_metadata.MetaData.slack:type_name -> source_metadata.Slack
|
||||
8, // 17: source_metadata.MetaData.filesystem:type_name -> source_metadata.Filesystem
|
||||
9, // 18: source_metadata.MetaData.git:type_name -> source_metadata.Git
|
||||
19, // 19: source_metadata.MetaData.test:type_name -> source_metadata.Test
|
||||
3, // 20: source_metadata.MetaData.buildkite:type_name -> source_metadata.Buildkite
|
||||
18, // 21: source_metadata.MetaData.gerrit:type_name -> source_metadata.Gerrit
|
||||
20, // 22: source_metadata.MetaData.jenkins:type_name -> source_metadata.Jenkins
|
||||
21, // 23: source_metadata.MetaData.teams:type_name -> source_metadata.Teams
|
||||
22, // 24: source_metadata.MetaData.artifactory:type_name -> source_metadata.Artifactory
|
||||
23, // 25: source_metadata.MetaData.syslog:type_name -> source_metadata.Syslog
|
||||
24, // 26: source_metadata.MetaData.publicEventMonitoring:type_name -> source_metadata.PublicEventMonitoring
|
||||
25, // 27: source_metadata.MetaData.sharepoint:type_name -> source_metadata.SharePoint
|
||||
26, // 28: source_metadata.MetaData.googleDrive:type_name -> source_metadata.GoogleDrive
|
||||
29, // [29:29] is the sub-list for method output_type
|
||||
29, // [29:29] is the sub-list for method input_type
|
||||
29, // [29:29] is the sub-list for extension type_name
|
||||
29, // [29:29] is the sub-list for extension extendee
|
||||
0, // [0:29] is the sub-list for field type_name
|
||||
}
|
||||
|
||||
func init() { file_source_metadata_proto_init() }
|
||||
@@ -3738,18 +3560,6 @@ func file_source_metadata_proto_init() {
|
||||
}
|
||||
}
|
||||
file_source_metadata_proto_msgTypes[26].Exporter = func(v interface{}, i int) interface{} {
|
||||
switch v := v.(*AzureRepos); i {
|
||||
case 0:
|
||||
return &v.state
|
||||
case 1:
|
||||
return &v.sizeCache
|
||||
case 2:
|
||||
return &v.unknownFields
|
||||
default:
|
||||
return nil
|
||||
}
|
||||
}
|
||||
file_source_metadata_proto_msgTypes[27].Exporter = func(v interface{}, i int) interface{} {
|
||||
switch v := v.(*MetaData); i {
|
||||
case 0:
|
||||
return &v.state
|
||||
@@ -3765,7 +3575,7 @@ func file_source_metadata_proto_init() {
|
||||
file_source_metadata_proto_msgTypes[23].OneofWrappers = []interface{}{
|
||||
(*PublicEventMonitoring_Github)(nil),
|
||||
}
|
||||
file_source_metadata_proto_msgTypes[27].OneofWrappers = []interface{}{
|
||||
file_source_metadata_proto_msgTypes[26].OneofWrappers = []interface{}{
|
||||
(*MetaData_Azure)(nil),
|
||||
(*MetaData_Bitbucket)(nil),
|
||||
(*MetaData_Circleci)(nil),
|
||||
@@ -3792,7 +3602,6 @@ func file_source_metadata_proto_init() {
|
||||
(*MetaData_PublicEventMonitoring)(nil),
|
||||
(*MetaData_Sharepoint)(nil),
|
||||
(*MetaData_GoogleDrive)(nil),
|
||||
(*MetaData_AzureRepos)(nil),
|
||||
}
|
||||
type x struct{}
|
||||
out := protoimpl.TypeBuilder{
|
||||
@@ -3800,7 +3609,7 @@ func file_source_metadata_proto_init() {
|
||||
GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
|
||||
RawDescriptor: file_source_metadata_proto_rawDesc,
|
||||
NumEnums: 1,
|
||||
NumMessages: 28,
|
||||
NumMessages: 27,
|
||||
NumExtensions: 0,
|
||||
NumServices: 0,
|
||||
},
|
||||
|
||||
@@ -2865,8 +2865,6 @@ func (m *GoogleDrive) validate(all bool) error {
|
||||
|
||||
// no validation rules for LastModifiedBy
|
||||
|
||||
// no validation rules for Path
|
||||
|
||||
if len(errors) > 0 {
|
||||
return GoogleDriveMultiError(errors)
|
||||
}
|
||||
@@ -2944,127 +2942,6 @@ var _ interface {
|
||||
ErrorName() string
|
||||
} = GoogleDriveValidationError{}
|
||||
|
||||
// Validate checks the field values on AzureRepos with the rules defined in the
|
||||
// proto definition for this message. If any rules are violated, the first
|
||||
// error encountered is returned, or nil if there are no violations.
|
||||
func (m *AzureRepos) Validate() error {
|
||||
return m.validate(false)
|
||||
}
|
||||
|
||||
// ValidateAll checks the field values on AzureRepos with the rules defined in
|
||||
// the proto definition for this message. If any rules are violated, the
|
||||
// result is a list of violation errors wrapped in AzureReposMultiError, or
|
||||
// nil if none found.
|
||||
func (m *AzureRepos) ValidateAll() error {
|
||||
return m.validate(true)
|
||||
}
|
||||
|
||||
func (m *AzureRepos) validate(all bool) error {
|
||||
if m == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
var errors []error
|
||||
|
||||
// no validation rules for Link
|
||||
|
||||
// no validation rules for Username
|
||||
|
||||
// no validation rules for Repository
|
||||
|
||||
// no validation rules for Commit
|
||||
|
||||
// no validation rules for Email
|
||||
|
||||
// no validation rules for File
|
||||
|
||||
// no validation rules for Timestamp
|
||||
|
||||
// no validation rules for Line
|
||||
|
||||
// no validation rules for Visibility
|
||||
|
||||
// no validation rules for Project
|
||||
|
||||
// no validation rules for Organization
|
||||
|
||||
if len(errors) > 0 {
|
||||
return AzureReposMultiError(errors)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// AzureReposMultiError is an error wrapping multiple validation errors
|
||||
// returned by AzureRepos.ValidateAll() if the designated constraints aren't met.
|
||||
type AzureReposMultiError []error
|
||||
|
||||
// Error returns a concatenation of all the error messages it wraps.
|
||||
func (m AzureReposMultiError) Error() string {
|
||||
var msgs []string
|
||||
for _, err := range m {
|
||||
msgs = append(msgs, err.Error())
|
||||
}
|
||||
return strings.Join(msgs, "; ")
|
||||
}
|
||||
|
||||
// AllErrors returns a list of validation violation errors.
|
||||
func (m AzureReposMultiError) AllErrors() []error { return m }
|
||||
|
||||
// AzureReposValidationError is the validation error returned by
|
||||
// AzureRepos.Validate if the designated constraints aren't met.
|
||||
type AzureReposValidationError struct {
|
||||
field string
|
||||
reason string
|
||||
cause error
|
||||
key bool
|
||||
}
|
||||
|
||||
// Field function returns field value.
|
||||
func (e AzureReposValidationError) Field() string { return e.field }
|
||||
|
||||
// Reason function returns reason value.
|
||||
func (e AzureReposValidationError) Reason() string { return e.reason }
|
||||
|
||||
// Cause function returns cause value.
|
||||
func (e AzureReposValidationError) Cause() error { return e.cause }
|
||||
|
||||
// Key function returns key value.
|
||||
func (e AzureReposValidationError) Key() bool { return e.key }
|
||||
|
||||
// ErrorName returns error name.
|
||||
func (e AzureReposValidationError) ErrorName() string { return "AzureReposValidationError" }
|
||||
|
||||
// Error satisfies the builtin error interface
|
||||
func (e AzureReposValidationError) Error() string {
|
||||
cause := ""
|
||||
if e.cause != nil {
|
||||
cause = fmt.Sprintf(" | caused by: %v", e.cause)
|
||||
}
|
||||
|
||||
key := ""
|
||||
if e.key {
|
||||
key = "key for "
|
||||
}
|
||||
|
||||
return fmt.Sprintf(
|
||||
"invalid %sAzureRepos.%s: %s%s",
|
||||
key,
|
||||
e.field,
|
||||
e.reason,
|
||||
cause)
|
||||
}
|
||||
|
||||
var _ error = AzureReposValidationError{}
|
||||
|
||||
var _ interface {
|
||||
Field() string
|
||||
Reason() string
|
||||
Key() bool
|
||||
Cause() error
|
||||
ErrorName() string
|
||||
} = AzureReposValidationError{}
|
||||
|
||||
// Validate checks the field values on MetaData with the rules defined in the
|
||||
// proto definition for this message. If any rules are violated, the first
|
||||
// error encountered is returned, or nil if there are no violations.
|
||||
@@ -3895,37 +3772,6 @@ func (m *MetaData) validate(all bool) error {
|
||||
}
|
||||
}
|
||||
|
||||
case *MetaData_AzureRepos:
|
||||
|
||||
if all {
|
||||
switch v := interface{}(m.GetAzureRepos()).(type) {
|
||||
case interface{ ValidateAll() error }:
|
||||
if err := v.ValidateAll(); err != nil {
|
||||
errors = append(errors, MetaDataValidationError{
|
||||
field: "AzureRepos",
|
||||
reason: "embedded message failed validation",
|
||||
cause: err,
|
||||
})
|
||||
}
|
||||
case interface{ Validate() error }:
|
||||
if err := v.Validate(); err != nil {
|
||||
errors = append(errors, MetaDataValidationError{
|
||||
field: "AzureRepos",
|
||||
reason: "embedded message failed validation",
|
||||
cause: err,
|
||||
})
|
||||
}
|
||||
}
|
||||
} else if v, ok := interface{}(m.GetAzureRepos()).(interface{ Validate() error }); ok {
|
||||
if err := v.Validate(); err != nil {
|
||||
return MetaDataValidationError{
|
||||
field: "AzureRepos",
|
||||
reason: "embedded message failed validation",
|
||||
cause: err,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
if len(errors) > 0 {
|
||||
|
||||
+281
-494
@@ -59,7 +59,6 @@ const (
|
||||
SourceType_SOURCE_TYPE_GOOGLE_DRIVE SourceType = 28
|
||||
SourceType_SOURCE_TYPE_SHAREPOINT SourceType = 29
|
||||
SourceType_SOURCE_TYPE_GCS_UNAUTHED SourceType = 30
|
||||
SourceType_SOURCE_TYPE_AZURE_REPOS SourceType = 31
|
||||
)
|
||||
|
||||
// Enum value maps for SourceType.
|
||||
@@ -96,7 +95,6 @@ var (
|
||||
28: "SOURCE_TYPE_GOOGLE_DRIVE",
|
||||
29: "SOURCE_TYPE_SHAREPOINT",
|
||||
30: "SOURCE_TYPE_GCS_UNAUTHED",
|
||||
31: "SOURCE_TYPE_AZURE_REPOS",
|
||||
}
|
||||
SourceType_value = map[string]int32{
|
||||
"SOURCE_TYPE_AZURE_STORAGE": 0,
|
||||
@@ -130,7 +128,6 @@ var (
|
||||
"SOURCE_TYPE_GOOGLE_DRIVE": 28,
|
||||
"SOURCE_TYPE_SHAREPOINT": 29,
|
||||
"SOURCE_TYPE_GCS_UNAUTHED": 30,
|
||||
"SOURCE_TYPE_AZURE_REPOS": 31,
|
||||
}
|
||||
)
|
||||
|
||||
@@ -1428,7 +1425,6 @@ type GitHub struct {
|
||||
IncludeRepos []string `protobuf:"bytes,12,rep,name=includeRepos,proto3" json:"includeRepos,omitempty"`
|
||||
IncludePullRequestComments bool `protobuf:"varint,14,opt,name=includePullRequestComments,proto3" json:"includePullRequestComments,omitempty"`
|
||||
IncludeIssueComments bool `protobuf:"varint,15,opt,name=includeIssueComments,proto3" json:"includeIssueComments,omitempty"`
|
||||
IncludeGistComments bool `protobuf:"varint,16,opt,name=includeGistComments,proto3" json:"includeGistComments,omitempty"`
|
||||
}
|
||||
|
||||
func (x *GitHub) Reset() {
|
||||
@@ -1575,13 +1571,6 @@ func (x *GitHub) GetIncludeIssueComments() bool {
|
||||
return false
|
||||
}
|
||||
|
||||
func (x *GitHub) GetIncludeGistComments() bool {
|
||||
if x != nil {
|
||||
return x.IncludeGistComments
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
type isGitHub_Credential interface {
|
||||
isGitHub_Credential()
|
||||
}
|
||||
@@ -2988,158 +2977,6 @@ type Sharepoint_Oauth struct {
|
||||
|
||||
func (*Sharepoint_Oauth) isSharepoint_Credential() {}
|
||||
|
||||
type AzureRepos struct {
|
||||
state protoimpl.MessageState
|
||||
sizeCache protoimpl.SizeCache
|
||||
unknownFields protoimpl.UnknownFields
|
||||
|
||||
Endpoint string `protobuf:"bytes,1,opt,name=endpoint,proto3" json:"endpoint,omitempty"`
|
||||
// Types that are assignable to Credential:
|
||||
// *AzureRepos_Token
|
||||
// *AzureRepos_Oauth
|
||||
Credential isAzureRepos_Credential `protobuf_oneof:"credential"`
|
||||
Repositories []string `protobuf:"bytes,4,rep,name=repositories,proto3" json:"repositories,omitempty"`
|
||||
Organizations []string `protobuf:"bytes,5,rep,name=organizations,proto3" json:"organizations,omitempty"`
|
||||
Projects []string `protobuf:"bytes,6,rep,name=projects,proto3" json:"projects,omitempty"`
|
||||
IncludeForks bool `protobuf:"varint,7,opt,name=includeForks,proto3" json:"includeForks,omitempty"`
|
||||
IgnoreRepos []string `protobuf:"bytes,8,rep,name=ignoreRepos,proto3" json:"ignoreRepos,omitempty"`
|
||||
IncludeRepos []string `protobuf:"bytes,9,rep,name=includeRepos,proto3" json:"includeRepos,omitempty"`
|
||||
IncludeProjects []string `protobuf:"bytes,10,rep,name=includeProjects,proto3" json:"includeProjects,omitempty"`
|
||||
IgnoreProjects []string `protobuf:"bytes,11,rep,name=ignoreProjects,proto3" json:"ignoreProjects,omitempty"`
|
||||
}
|
||||
|
||||
func (x *AzureRepos) Reset() {
|
||||
*x = AzureRepos{}
|
||||
if protoimpl.UnsafeEnabled {
|
||||
mi := &file_sources_proto_msgTypes[28]
|
||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
||||
ms.StoreMessageInfo(mi)
|
||||
}
|
||||
}
|
||||
|
||||
func (x *AzureRepos) String() string {
|
||||
return protoimpl.X.MessageStringOf(x)
|
||||
}
|
||||
|
||||
func (*AzureRepos) ProtoMessage() {}
|
||||
|
||||
func (x *AzureRepos) ProtoReflect() protoreflect.Message {
|
||||
mi := &file_sources_proto_msgTypes[28]
|
||||
if protoimpl.UnsafeEnabled && x != nil {
|
||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
||||
if ms.LoadMessageInfo() == nil {
|
||||
ms.StoreMessageInfo(mi)
|
||||
}
|
||||
return ms
|
||||
}
|
||||
return mi.MessageOf(x)
|
||||
}
|
||||
|
||||
// Deprecated: Use AzureRepos.ProtoReflect.Descriptor instead.
|
||||
func (*AzureRepos) Descriptor() ([]byte, []int) {
|
||||
return file_sources_proto_rawDescGZIP(), []int{28}
|
||||
}
|
||||
|
||||
func (x *AzureRepos) GetEndpoint() string {
|
||||
if x != nil {
|
||||
return x.Endpoint
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func (m *AzureRepos) GetCredential() isAzureRepos_Credential {
|
||||
if m != nil {
|
||||
return m.Credential
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (x *AzureRepos) GetToken() string {
|
||||
if x, ok := x.GetCredential().(*AzureRepos_Token); ok {
|
||||
return x.Token
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func (x *AzureRepos) GetOauth() *credentialspb.Oauth2 {
|
||||
if x, ok := x.GetCredential().(*AzureRepos_Oauth); ok {
|
||||
return x.Oauth
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (x *AzureRepos) GetRepositories() []string {
|
||||
if x != nil {
|
||||
return x.Repositories
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (x *AzureRepos) GetOrganizations() []string {
|
||||
if x != nil {
|
||||
return x.Organizations
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (x *AzureRepos) GetProjects() []string {
|
||||
if x != nil {
|
||||
return x.Projects
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (x *AzureRepos) GetIncludeForks() bool {
|
||||
if x != nil {
|
||||
return x.IncludeForks
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func (x *AzureRepos) GetIgnoreRepos() []string {
|
||||
if x != nil {
|
||||
return x.IgnoreRepos
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (x *AzureRepos) GetIncludeRepos() []string {
|
||||
if x != nil {
|
||||
return x.IncludeRepos
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (x *AzureRepos) GetIncludeProjects() []string {
|
||||
if x != nil {
|
||||
return x.IncludeProjects
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (x *AzureRepos) GetIgnoreProjects() []string {
|
||||
if x != nil {
|
||||
return x.IgnoreProjects
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
type isAzureRepos_Credential interface {
|
||||
isAzureRepos_Credential()
|
||||
}
|
||||
|
||||
type AzureRepos_Token struct {
|
||||
Token string `protobuf:"bytes,2,opt,name=token,proto3,oneof"`
|
||||
}
|
||||
|
||||
type AzureRepos_Oauth struct {
|
||||
Oauth *credentialspb.Oauth2 `protobuf:"bytes,3,opt,name=oauth,proto3,oneof"`
|
||||
}
|
||||
|
||||
func (*AzureRepos_Token) isAzureRepos_Credential() {}
|
||||
|
||||
func (*AzureRepos_Oauth) isAzureRepos_Credential() {}
|
||||
|
||||
var File_sources_proto protoreflect.FileDescriptor
|
||||
|
||||
var file_sources_proto_rawDesc = []byte{
|
||||
@@ -3336,7 +3173,7 @@ var file_sources_proto_rawDesc = []byte{
|
||||
0x65, 0x70, 0x6f, 0x73, 0x69, 0x74, 0x6f, 0x72, 0x69, 0x65, 0x73, 0x12, 0x21, 0x0a, 0x0c, 0x69,
|
||||
0x67, 0x6e, 0x6f, 0x72, 0x65, 0x5f, 0x72, 0x65, 0x70, 0x6f, 0x73, 0x18, 0x06, 0x20, 0x03, 0x28,
|
||||
0x09, 0x52, 0x0b, 0x69, 0x67, 0x6e, 0x6f, 0x72, 0x65, 0x52, 0x65, 0x70, 0x6f, 0x73, 0x42, 0x0c,
|
||||
0x0a, 0x0a, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x22, 0xb0, 0x05, 0x0a,
|
||||
0x0a, 0x0a, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x22, 0xfe, 0x04, 0x0a,
|
||||
0x06, 0x47, 0x69, 0x74, 0x48, 0x75, 0x62, 0x12, 0x24, 0x0a, 0x08, 0x65, 0x6e, 0x64, 0x70, 0x6f,
|
||||
0x69, 0x6e, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x42, 0x08, 0xfa, 0x42, 0x05, 0x72, 0x03,
|
||||
0x90, 0x01, 0x01, 0x52, 0x08, 0x65, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x12, 0x37, 0x0a,
|
||||
@@ -3375,267 +3212,235 @@ var file_sources_proto_rawDesc = []byte{
|
||||
0x65, 0x73, 0x74, 0x43, 0x6f, 0x6d, 0x6d, 0x65, 0x6e, 0x74, 0x73, 0x12, 0x32, 0x0a, 0x14, 0x69,
|
||||
0x6e, 0x63, 0x6c, 0x75, 0x64, 0x65, 0x49, 0x73, 0x73, 0x75, 0x65, 0x43, 0x6f, 0x6d, 0x6d, 0x65,
|
||||
0x6e, 0x74, 0x73, 0x18, 0x0f, 0x20, 0x01, 0x28, 0x08, 0x52, 0x14, 0x69, 0x6e, 0x63, 0x6c, 0x75,
|
||||
0x64, 0x65, 0x49, 0x73, 0x73, 0x75, 0x65, 0x43, 0x6f, 0x6d, 0x6d, 0x65, 0x6e, 0x74, 0x73, 0x12,
|
||||
0x30, 0x0a, 0x13, 0x69, 0x6e, 0x63, 0x6c, 0x75, 0x64, 0x65, 0x47, 0x69, 0x73, 0x74, 0x43, 0x6f,
|
||||
0x6d, 0x6d, 0x65, 0x6e, 0x74, 0x73, 0x18, 0x10, 0x20, 0x01, 0x28, 0x08, 0x52, 0x13, 0x69, 0x6e,
|
||||
0x63, 0x6c, 0x75, 0x64, 0x65, 0x47, 0x69, 0x73, 0x74, 0x43, 0x6f, 0x6d, 0x6d, 0x65, 0x6e, 0x74,
|
||||
0x73, 0x42, 0x0c, 0x0a, 0x0a, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x22,
|
||||
0x42, 0x0a, 0x0b, 0x47, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x44, 0x72, 0x69, 0x76, 0x65, 0x12, 0x25,
|
||||
0x0a, 0x0d, 0x72, 0x65, 0x66, 0x72, 0x65, 0x73, 0x68, 0x5f, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x18,
|
||||
0x01, 0x20, 0x01, 0x28, 0x09, 0x48, 0x00, 0x52, 0x0c, 0x72, 0x65, 0x66, 0x72, 0x65, 0x73, 0x68,
|
||||
0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x42, 0x0c, 0x0a, 0x0a, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74,
|
||||
0x69, 0x61, 0x6c, 0x22, 0xc7, 0x02, 0x0a, 0x04, 0x4a, 0x49, 0x52, 0x41, 0x12, 0x24, 0x0a, 0x08,
|
||||
0x65, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x42, 0x08,
|
||||
0xfa, 0x42, 0x05, 0x72, 0x03, 0x90, 0x01, 0x01, 0x52, 0x08, 0x65, 0x6e, 0x64, 0x70, 0x6f, 0x69,
|
||||
0x6e, 0x74, 0x12, 0x37, 0x0a, 0x0a, 0x62, 0x61, 0x73, 0x69, 0x63, 0x5f, 0x61, 0x75, 0x74, 0x68,
|
||||
0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x16, 0x2e, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74,
|
||||
0x69, 0x61, 0x6c, 0x73, 0x2e, 0x42, 0x61, 0x73, 0x69, 0x63, 0x41, 0x75, 0x74, 0x68, 0x48, 0x00,
|
||||
0x52, 0x09, 0x62, 0x61, 0x73, 0x69, 0x63, 0x41, 0x75, 0x74, 0x68, 0x12, 0x48, 0x0a, 0x0f, 0x75,
|
||||
0x6e, 0x61, 0x75, 0x74, 0x68, 0x65, 0x6e, 0x74, 0x69, 0x63, 0x61, 0x74, 0x65, 0x64, 0x18, 0x03,
|
||||
0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61,
|
||||
0x6c, 0x73, 0x2e, 0x55, 0x6e, 0x61, 0x75, 0x74, 0x68, 0x65, 0x6e, 0x74, 0x69, 0x63, 0x61, 0x74,
|
||||
0x65, 0x64, 0x48, 0x00, 0x52, 0x0f, 0x75, 0x6e, 0x61, 0x75, 0x74, 0x68, 0x65, 0x6e, 0x74, 0x69,
|
||||
0x63, 0x61, 0x74, 0x65, 0x64, 0x12, 0x2b, 0x0a, 0x05, 0x6f, 0x61, 0x75, 0x74, 0x68, 0x18, 0x04,
|
||||
0x20, 0x01, 0x28, 0x0b, 0x32, 0x13, 0x2e, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61,
|
||||
0x6c, 0x73, 0x2e, 0x4f, 0x61, 0x75, 0x74, 0x68, 0x32, 0x48, 0x00, 0x52, 0x05, 0x6f, 0x61, 0x75,
|
||||
0x74, 0x68, 0x12, 0x16, 0x0a, 0x05, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x18, 0x06, 0x20, 0x01, 0x28,
|
||||
0x09, 0x48, 0x00, 0x52, 0x05, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x12, 0x1a, 0x0a, 0x08, 0x70, 0x72,
|
||||
0x6f, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x18, 0x05, 0x20, 0x03, 0x28, 0x09, 0x52, 0x08, 0x70, 0x72,
|
||||
0x6f, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x12, 0x27, 0x0a, 0x0f, 0x69, 0x67, 0x6e, 0x6f, 0x72, 0x65,
|
||||
0x5f, 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x18, 0x07, 0x20, 0x03, 0x28, 0x09, 0x52,
|
||||
0x0e, 0x69, 0x67, 0x6e, 0x6f, 0x72, 0x65, 0x50, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x42,
|
||||
0x0c, 0x0a, 0x0a, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x22, 0x73, 0x0a,
|
||||
0x19, 0x4e, 0x50, 0x4d, 0x55, 0x6e, 0x61, 0x75, 0x74, 0x68, 0x65, 0x6e, 0x74, 0x69, 0x63, 0x61,
|
||||
0x74, 0x65, 0x64, 0x50, 0x61, 0x63, 0x6b, 0x61, 0x67, 0x65, 0x12, 0x48, 0x0a, 0x0f, 0x75, 0x6e,
|
||||
0x61, 0x75, 0x74, 0x68, 0x65, 0x6e, 0x74, 0x69, 0x63, 0x61, 0x74, 0x65, 0x64, 0x18, 0x01, 0x20,
|
||||
0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c,
|
||||
0x73, 0x2e, 0x55, 0x6e, 0x61, 0x75, 0x74, 0x68, 0x65, 0x6e, 0x74, 0x69, 0x63, 0x61, 0x74, 0x65,
|
||||
0x64, 0x48, 0x00, 0x52, 0x0f, 0x75, 0x6e, 0x61, 0x75, 0x74, 0x68, 0x65, 0x6e, 0x74, 0x69, 0x63,
|
||||
0x61, 0x74, 0x65, 0x64, 0x42, 0x0c, 0x0a, 0x0a, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69,
|
||||
0x61, 0x6c, 0x22, 0x74, 0x0a, 0x1a, 0x50, 0x79, 0x50, 0x49, 0x55, 0x6e, 0x61, 0x75, 0x74, 0x68,
|
||||
0x65, 0x6e, 0x74, 0x69, 0x63, 0x61, 0x74, 0x65, 0x64, 0x50, 0x61, 0x63, 0x6b, 0x61, 0x67, 0x65,
|
||||
0x12, 0x48, 0x0a, 0x0f, 0x75, 0x6e, 0x61, 0x75, 0x74, 0x68, 0x65, 0x6e, 0x74, 0x69, 0x63, 0x61,
|
||||
0x74, 0x65, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x63, 0x72, 0x65, 0x64,
|
||||
0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x73, 0x2e, 0x55, 0x6e, 0x61, 0x75, 0x74, 0x68, 0x65, 0x6e,
|
||||
0x74, 0x69, 0x63, 0x61, 0x74, 0x65, 0x64, 0x48, 0x00, 0x52, 0x0f, 0x75, 0x6e, 0x61, 0x75, 0x74,
|
||||
0x68, 0x65, 0x6e, 0x74, 0x69, 0x63, 0x61, 0x74, 0x65, 0x64, 0x42, 0x0c, 0x0a, 0x0a, 0x63, 0x72,
|
||||
0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x22, 0xf0, 0x02, 0x0a, 0x02, 0x53, 0x33, 0x12,
|
||||
0x37, 0x0a, 0x0a, 0x61, 0x63, 0x63, 0x65, 0x73, 0x73, 0x5f, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20,
|
||||
0x01, 0x28, 0x0b, 0x32, 0x16, 0x2e, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c,
|
||||
0x73, 0x2e, 0x4b, 0x65, 0x79, 0x53, 0x65, 0x63, 0x72, 0x65, 0x74, 0x48, 0x00, 0x52, 0x09, 0x61,
|
||||
0x63, 0x63, 0x65, 0x73, 0x73, 0x4b, 0x65, 0x79, 0x12, 0x48, 0x0a, 0x0f, 0x75, 0x6e, 0x61, 0x75,
|
||||
0x74, 0x68, 0x65, 0x6e, 0x74, 0x69, 0x63, 0x61, 0x74, 0x65, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28,
|
||||
0x64, 0x65, 0x49, 0x73, 0x73, 0x75, 0x65, 0x43, 0x6f, 0x6d, 0x6d, 0x65, 0x6e, 0x74, 0x73, 0x42,
|
||||
0x0c, 0x0a, 0x0a, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x22, 0x42, 0x0a,
|
||||
0x0b, 0x47, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x44, 0x72, 0x69, 0x76, 0x65, 0x12, 0x25, 0x0a, 0x0d,
|
||||
0x72, 0x65, 0x66, 0x72, 0x65, 0x73, 0x68, 0x5f, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x18, 0x01, 0x20,
|
||||
0x01, 0x28, 0x09, 0x48, 0x00, 0x52, 0x0c, 0x72, 0x65, 0x66, 0x72, 0x65, 0x73, 0x68, 0x54, 0x6f,
|
||||
0x6b, 0x65, 0x6e, 0x42, 0x0c, 0x0a, 0x0a, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61,
|
||||
0x6c, 0x22, 0xc7, 0x02, 0x0a, 0x04, 0x4a, 0x49, 0x52, 0x41, 0x12, 0x24, 0x0a, 0x08, 0x65, 0x6e,
|
||||
0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x42, 0x08, 0xfa, 0x42,
|
||||
0x05, 0x72, 0x03, 0x90, 0x01, 0x01, 0x52, 0x08, 0x65, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74,
|
||||
0x12, 0x37, 0x0a, 0x0a, 0x62, 0x61, 0x73, 0x69, 0x63, 0x5f, 0x61, 0x75, 0x74, 0x68, 0x18, 0x02,
|
||||
0x20, 0x01, 0x28, 0x0b, 0x32, 0x16, 0x2e, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61,
|
||||
0x6c, 0x73, 0x2e, 0x42, 0x61, 0x73, 0x69, 0x63, 0x41, 0x75, 0x74, 0x68, 0x48, 0x00, 0x52, 0x09,
|
||||
0x62, 0x61, 0x73, 0x69, 0x63, 0x41, 0x75, 0x74, 0x68, 0x12, 0x48, 0x0a, 0x0f, 0x75, 0x6e, 0x61,
|
||||
0x75, 0x74, 0x68, 0x65, 0x6e, 0x74, 0x69, 0x63, 0x61, 0x74, 0x65, 0x64, 0x18, 0x03, 0x20, 0x01,
|
||||
0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x73,
|
||||
0x2e, 0x55, 0x6e, 0x61, 0x75, 0x74, 0x68, 0x65, 0x6e, 0x74, 0x69, 0x63, 0x61, 0x74, 0x65, 0x64,
|
||||
0x48, 0x00, 0x52, 0x0f, 0x75, 0x6e, 0x61, 0x75, 0x74, 0x68, 0x65, 0x6e, 0x74, 0x69, 0x63, 0x61,
|
||||
0x74, 0x65, 0x64, 0x12, 0x2b, 0x0a, 0x05, 0x6f, 0x61, 0x75, 0x74, 0x68, 0x18, 0x04, 0x20, 0x01,
|
||||
0x28, 0x0b, 0x32, 0x13, 0x2e, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x73,
|
||||
0x2e, 0x4f, 0x61, 0x75, 0x74, 0x68, 0x32, 0x48, 0x00, 0x52, 0x05, 0x6f, 0x61, 0x75, 0x74, 0x68,
|
||||
0x12, 0x16, 0x0a, 0x05, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x48,
|
||||
0x00, 0x52, 0x05, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x12, 0x1a, 0x0a, 0x08, 0x70, 0x72, 0x6f, 0x6a,
|
||||
0x65, 0x63, 0x74, 0x73, 0x18, 0x05, 0x20, 0x03, 0x28, 0x09, 0x52, 0x08, 0x70, 0x72, 0x6f, 0x6a,
|
||||
0x65, 0x63, 0x74, 0x73, 0x12, 0x27, 0x0a, 0x0f, 0x69, 0x67, 0x6e, 0x6f, 0x72, 0x65, 0x5f, 0x70,
|
||||
0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x18, 0x07, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0e, 0x69,
|
||||
0x67, 0x6e, 0x6f, 0x72, 0x65, 0x50, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x42, 0x0c, 0x0a,
|
||||
0x0a, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x22, 0x73, 0x0a, 0x19, 0x4e,
|
||||
0x50, 0x4d, 0x55, 0x6e, 0x61, 0x75, 0x74, 0x68, 0x65, 0x6e, 0x74, 0x69, 0x63, 0x61, 0x74, 0x65,
|
||||
0x64, 0x50, 0x61, 0x63, 0x6b, 0x61, 0x67, 0x65, 0x12, 0x48, 0x0a, 0x0f, 0x75, 0x6e, 0x61, 0x75,
|
||||
0x74, 0x68, 0x65, 0x6e, 0x74, 0x69, 0x63, 0x61, 0x74, 0x65, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28,
|
||||
0x0b, 0x32, 0x1c, 0x2e, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x73, 0x2e,
|
||||
0x55, 0x6e, 0x61, 0x75, 0x74, 0x68, 0x65, 0x6e, 0x74, 0x69, 0x63, 0x61, 0x74, 0x65, 0x64, 0x48,
|
||||
0x00, 0x52, 0x0f, 0x75, 0x6e, 0x61, 0x75, 0x74, 0x68, 0x65, 0x6e, 0x74, 0x69, 0x63, 0x61, 0x74,
|
||||
0x65, 0x64, 0x12, 0x4c, 0x0a, 0x11, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x5f, 0x65, 0x6e, 0x76, 0x69,
|
||||
0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1d, 0x2e,
|
||||
0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x73, 0x2e, 0x43, 0x6c, 0x6f, 0x75,
|
||||
0x64, 0x45, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x48, 0x00, 0x52, 0x10,
|
||||
0x63, 0x6c, 0x6f, 0x75, 0x64, 0x45, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74,
|
||||
0x12, 0x49, 0x0a, 0x0d, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x5f, 0x74, 0x6f, 0x6b, 0x65,
|
||||
0x6e, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x22, 0x2e, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e,
|
||||
0x74, 0x69, 0x61, 0x6c, 0x73, 0x2e, 0x41, 0x57, 0x53, 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e,
|
||||
0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x53, 0x65, 0x63, 0x72, 0x65, 0x74, 0x48, 0x00, 0x52, 0x0c, 0x73,
|
||||
0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x12, 0x18, 0x0a, 0x07, 0x62,
|
||||
0x75, 0x63, 0x6b, 0x65, 0x74, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x09, 0x52, 0x07, 0x62, 0x75,
|
||||
0x63, 0x6b, 0x65, 0x74, 0x73, 0x12, 0x26, 0x0a, 0x0f, 0x6d, 0x61, 0x78, 0x5f, 0x6f, 0x62, 0x6a,
|
||||
0x65, 0x63, 0x74, 0x5f, 0x73, 0x69, 0x7a, 0x65, 0x18, 0x06, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0d,
|
||||
0x6d, 0x61, 0x78, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x53, 0x69, 0x7a, 0x65, 0x42, 0x0c, 0x0a,
|
||||
0x0a, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x22, 0xc3, 0x01, 0x0a, 0x05,
|
||||
0x53, 0x6c, 0x61, 0x63, 0x6b, 0x12, 0x24, 0x0a, 0x08, 0x65, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e,
|
||||
0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x42, 0x08, 0xfa, 0x42, 0x05, 0x72, 0x03, 0x90, 0x01,
|
||||
0x01, 0x52, 0x08, 0x65, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x12, 0x16, 0x0a, 0x05, 0x74,
|
||||
0x6f, 0x6b, 0x65, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x48, 0x00, 0x52, 0x05, 0x74, 0x6f,
|
||||
0x6b, 0x65, 0x6e, 0x12, 0x32, 0x0a, 0x06, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x73, 0x18, 0x05, 0x20,
|
||||
0x01, 0x28, 0x0b, 0x32, 0x18, 0x2e, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c,
|
||||
0x73, 0x2e, 0x53, 0x6c, 0x61, 0x63, 0x6b, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x73, 0x48, 0x00, 0x52,
|
||||
0x06, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x73, 0x12, 0x1a, 0x0a, 0x08, 0x63, 0x68, 0x61, 0x6e, 0x6e,
|
||||
0x65, 0x6c, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x09, 0x52, 0x08, 0x63, 0x68, 0x61, 0x6e, 0x6e,
|
||||
0x65, 0x6c, 0x73, 0x12, 0x1e, 0x0a, 0x0a, 0x69, 0x67, 0x6e, 0x6f, 0x72, 0x65, 0x4c, 0x69, 0x73,
|
||||
0x74, 0x18, 0x04, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0a, 0x69, 0x67, 0x6e, 0x6f, 0x72, 0x65, 0x4c,
|
||||
0x69, 0x73, 0x74, 0x42, 0x0c, 0x0a, 0x0a, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61,
|
||||
0x6c, 0x22, 0x06, 0x0a, 0x04, 0x54, 0x65, 0x73, 0x74, 0x22, 0x31, 0x0a, 0x09, 0x42, 0x75, 0x69,
|
||||
0x6c, 0x64, 0x6b, 0x69, 0x74, 0x65, 0x12, 0x16, 0x0a, 0x05, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x18,
|
||||
0x01, 0x20, 0x01, 0x28, 0x09, 0x48, 0x00, 0x52, 0x05, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x42, 0x0c,
|
||||
0x0a, 0x0a, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x22, 0xdb, 0x01, 0x0a,
|
||||
0x06, 0x47, 0x65, 0x72, 0x72, 0x69, 0x74, 0x12, 0x24, 0x0a, 0x08, 0x65, 0x6e, 0x64, 0x70, 0x6f,
|
||||
0x69, 0x6e, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x42, 0x08, 0xfa, 0x42, 0x05, 0x72, 0x03,
|
||||
0x90, 0x01, 0x01, 0x52, 0x08, 0x65, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x12, 0x37, 0x0a,
|
||||
0x0a, 0x62, 0x61, 0x73, 0x69, 0x63, 0x5f, 0x61, 0x75, 0x74, 0x68, 0x18, 0x02, 0x20, 0x01, 0x28,
|
||||
0x65, 0x64, 0x42, 0x0c, 0x0a, 0x0a, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c,
|
||||
0x22, 0x74, 0x0a, 0x1a, 0x50, 0x79, 0x50, 0x49, 0x55, 0x6e, 0x61, 0x75, 0x74, 0x68, 0x65, 0x6e,
|
||||
0x74, 0x69, 0x63, 0x61, 0x74, 0x65, 0x64, 0x50, 0x61, 0x63, 0x6b, 0x61, 0x67, 0x65, 0x12, 0x48,
|
||||
0x0a, 0x0f, 0x75, 0x6e, 0x61, 0x75, 0x74, 0x68, 0x65, 0x6e, 0x74, 0x69, 0x63, 0x61, 0x74, 0x65,
|
||||
0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e,
|
||||
0x74, 0x69, 0x61, 0x6c, 0x73, 0x2e, 0x55, 0x6e, 0x61, 0x75, 0x74, 0x68, 0x65, 0x6e, 0x74, 0x69,
|
||||
0x63, 0x61, 0x74, 0x65, 0x64, 0x48, 0x00, 0x52, 0x0f, 0x75, 0x6e, 0x61, 0x75, 0x74, 0x68, 0x65,
|
||||
0x6e, 0x74, 0x69, 0x63, 0x61, 0x74, 0x65, 0x64, 0x42, 0x0c, 0x0a, 0x0a, 0x63, 0x72, 0x65, 0x64,
|
||||
0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x22, 0xf0, 0x02, 0x0a, 0x02, 0x53, 0x33, 0x12, 0x37, 0x0a,
|
||||
0x0a, 0x61, 0x63, 0x63, 0x65, 0x73, 0x73, 0x5f, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28,
|
||||
0x0b, 0x32, 0x16, 0x2e, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x73, 0x2e,
|
||||
0x42, 0x61, 0x73, 0x69, 0x63, 0x41, 0x75, 0x74, 0x68, 0x48, 0x00, 0x52, 0x09, 0x62, 0x61, 0x73,
|
||||
0x69, 0x63, 0x41, 0x75, 0x74, 0x68, 0x12, 0x48, 0x0a, 0x0f, 0x75, 0x6e, 0x61, 0x75, 0x74, 0x68,
|
||||
0x65, 0x6e, 0x74, 0x69, 0x63, 0x61, 0x74, 0x65, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32,
|
||||
0x4b, 0x65, 0x79, 0x53, 0x65, 0x63, 0x72, 0x65, 0x74, 0x48, 0x00, 0x52, 0x09, 0x61, 0x63, 0x63,
|
||||
0x65, 0x73, 0x73, 0x4b, 0x65, 0x79, 0x12, 0x48, 0x0a, 0x0f, 0x75, 0x6e, 0x61, 0x75, 0x74, 0x68,
|
||||
0x65, 0x6e, 0x74, 0x69, 0x63, 0x61, 0x74, 0x65, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32,
|
||||
0x1c, 0x2e, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x73, 0x2e, 0x55, 0x6e,
|
||||
0x61, 0x75, 0x74, 0x68, 0x65, 0x6e, 0x74, 0x69, 0x63, 0x61, 0x74, 0x65, 0x64, 0x48, 0x00, 0x52,
|
||||
0x0f, 0x75, 0x6e, 0x61, 0x75, 0x74, 0x68, 0x65, 0x6e, 0x74, 0x69, 0x63, 0x61, 0x74, 0x65, 0x64,
|
||||
0x12, 0x1a, 0x0a, 0x08, 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x18, 0x04, 0x20, 0x03,
|
||||
0x28, 0x09, 0x52, 0x08, 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x42, 0x0c, 0x0a, 0x0a,
|
||||
0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x22, 0xa5, 0x01, 0x0a, 0x07, 0x4a,
|
||||
0x65, 0x6e, 0x6b, 0x69, 0x6e, 0x73, 0x12, 0x24, 0x0a, 0x08, 0x65, 0x6e, 0x64, 0x70, 0x6f, 0x69,
|
||||
0x6e, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x42, 0x08, 0xfa, 0x42, 0x05, 0x72, 0x03, 0x90,
|
||||
0x01, 0x01, 0x52, 0x08, 0x65, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x12, 0x37, 0x0a, 0x0a,
|
||||
0x62, 0x61, 0x73, 0x69, 0x63, 0x5f, 0x61, 0x75, 0x74, 0x68, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b,
|
||||
0x32, 0x16, 0x2e, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x73, 0x2e, 0x42,
|
||||
0x61, 0x73, 0x69, 0x63, 0x41, 0x75, 0x74, 0x68, 0x48, 0x00, 0x52, 0x09, 0x62, 0x61, 0x73, 0x69,
|
||||
0x63, 0x41, 0x75, 0x74, 0x68, 0x12, 0x2d, 0x0a, 0x06, 0x68, 0x65, 0x61, 0x64, 0x65, 0x72, 0x18,
|
||||
0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x13, 0x2e, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69,
|
||||
0x61, 0x6c, 0x73, 0x2e, 0x48, 0x65, 0x61, 0x64, 0x65, 0x72, 0x48, 0x00, 0x52, 0x06, 0x68, 0x65,
|
||||
0x61, 0x64, 0x65, 0x72, 0x42, 0x0c, 0x0a, 0x0a, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69,
|
||||
0x61, 0x6c, 0x22, 0x9f, 0x02, 0x0a, 0x05, 0x54, 0x65, 0x61, 0x6d, 0x73, 0x12, 0x24, 0x0a, 0x08,
|
||||
0x65, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x42, 0x08,
|
||||
0xfa, 0x42, 0x05, 0x72, 0x03, 0x90, 0x01, 0x01, 0x52, 0x08, 0x65, 0x6e, 0x64, 0x70, 0x6f, 0x69,
|
||||
0x6e, 0x74, 0x12, 0x16, 0x0a, 0x05, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28,
|
||||
0x09, 0x48, 0x00, 0x52, 0x05, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x12, 0x46, 0x0a, 0x0d, 0x61, 0x75,
|
||||
0x74, 0x68, 0x65, 0x6e, 0x74, 0x69, 0x63, 0x61, 0x74, 0x65, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28,
|
||||
0x0b, 0x32, 0x1e, 0x2e, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x73, 0x2e,
|
||||
0x43, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x43, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c,
|
||||
0x73, 0x48, 0x00, 0x52, 0x0d, 0x61, 0x75, 0x74, 0x68, 0x65, 0x6e, 0x74, 0x69, 0x63, 0x61, 0x74,
|
||||
0x65, 0x64, 0x12, 0x2b, 0x0a, 0x05, 0x6f, 0x61, 0x75, 0x74, 0x68, 0x18, 0x07, 0x20, 0x01, 0x28,
|
||||
0x0b, 0x32, 0x13, 0x2e, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x73, 0x2e,
|
||||
0x4f, 0x61, 0x75, 0x74, 0x68, 0x32, 0x48, 0x00, 0x52, 0x05, 0x6f, 0x61, 0x75, 0x74, 0x68, 0x12,
|
||||
0x1a, 0x0a, 0x08, 0x63, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28,
|
||||
0x09, 0x52, 0x08, 0x63, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x73, 0x12, 0x1e, 0x0a, 0x0a, 0x69,
|
||||
0x67, 0x6e, 0x6f, 0x72, 0x65, 0x4c, 0x69, 0x73, 0x74, 0x18, 0x05, 0x20, 0x03, 0x28, 0x09, 0x52,
|
||||
0x0a, 0x69, 0x67, 0x6e, 0x6f, 0x72, 0x65, 0x4c, 0x69, 0x73, 0x74, 0x12, 0x19, 0x0a, 0x08, 0x74,
|
||||
0x65, 0x61, 0x6d, 0x5f, 0x69, 0x64, 0x73, 0x18, 0x06, 0x20, 0x03, 0x28, 0x09, 0x52, 0x07, 0x74,
|
||||
0x65, 0x61, 0x6d, 0x49, 0x64, 0x73, 0x42, 0x0c, 0x0a, 0x0a, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e,
|
||||
0x74, 0x69, 0x61, 0x6c, 0x22, 0xc3, 0x01, 0x0a, 0x0b, 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63,
|
||||
0x74, 0x6f, 0x72, 0x79, 0x12, 0x24, 0x0a, 0x08, 0x65, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74,
|
||||
0x12, 0x4c, 0x0a, 0x11, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x5f, 0x65, 0x6e, 0x76, 0x69, 0x72, 0x6f,
|
||||
0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1d, 0x2e, 0x63, 0x72,
|
||||
0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x73, 0x2e, 0x43, 0x6c, 0x6f, 0x75, 0x64, 0x45,
|
||||
0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x48, 0x00, 0x52, 0x10, 0x63, 0x6c,
|
||||
0x6f, 0x75, 0x64, 0x45, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x12, 0x49,
|
||||
0x0a, 0x0d, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x5f, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x18,
|
||||
0x05, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x22, 0x2e, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69,
|
||||
0x61, 0x6c, 0x73, 0x2e, 0x41, 0x57, 0x53, 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x54, 0x6f,
|
||||
0x6b, 0x65, 0x6e, 0x53, 0x65, 0x63, 0x72, 0x65, 0x74, 0x48, 0x00, 0x52, 0x0c, 0x73, 0x65, 0x73,
|
||||
0x73, 0x69, 0x6f, 0x6e, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x12, 0x18, 0x0a, 0x07, 0x62, 0x75, 0x63,
|
||||
0x6b, 0x65, 0x74, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x09, 0x52, 0x07, 0x62, 0x75, 0x63, 0x6b,
|
||||
0x65, 0x74, 0x73, 0x12, 0x26, 0x0a, 0x0f, 0x6d, 0x61, 0x78, 0x5f, 0x6f, 0x62, 0x6a, 0x65, 0x63,
|
||||
0x74, 0x5f, 0x73, 0x69, 0x7a, 0x65, 0x18, 0x06, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0d, 0x6d, 0x61,
|
||||
0x78, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x53, 0x69, 0x7a, 0x65, 0x42, 0x0c, 0x0a, 0x0a, 0x63,
|
||||
0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x22, 0xc3, 0x01, 0x0a, 0x05, 0x53, 0x6c,
|
||||
0x61, 0x63, 0x6b, 0x12, 0x24, 0x0a, 0x08, 0x65, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x18,
|
||||
0x01, 0x20, 0x01, 0x28, 0x09, 0x42, 0x08, 0xfa, 0x42, 0x05, 0x72, 0x03, 0x90, 0x01, 0x01, 0x52,
|
||||
0x08, 0x65, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x12, 0x16, 0x0a, 0x05, 0x74, 0x6f, 0x6b,
|
||||
0x65, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x48, 0x00, 0x52, 0x05, 0x74, 0x6f, 0x6b, 0x65,
|
||||
0x6e, 0x12, 0x32, 0x0a, 0x06, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x73, 0x18, 0x05, 0x20, 0x01, 0x28,
|
||||
0x0b, 0x32, 0x18, 0x2e, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x73, 0x2e,
|
||||
0x53, 0x6c, 0x61, 0x63, 0x6b, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x73, 0x48, 0x00, 0x52, 0x06, 0x74,
|
||||
0x6f, 0x6b, 0x65, 0x6e, 0x73, 0x12, 0x1a, 0x0a, 0x08, 0x63, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c,
|
||||
0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x09, 0x52, 0x08, 0x63, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c,
|
||||
0x73, 0x12, 0x1e, 0x0a, 0x0a, 0x69, 0x67, 0x6e, 0x6f, 0x72, 0x65, 0x4c, 0x69, 0x73, 0x74, 0x18,
|
||||
0x04, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0a, 0x69, 0x67, 0x6e, 0x6f, 0x72, 0x65, 0x4c, 0x69, 0x73,
|
||||
0x74, 0x42, 0x0c, 0x0a, 0x0a, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x22,
|
||||
0x06, 0x0a, 0x04, 0x54, 0x65, 0x73, 0x74, 0x22, 0x31, 0x0a, 0x09, 0x42, 0x75, 0x69, 0x6c, 0x64,
|
||||
0x6b, 0x69, 0x74, 0x65, 0x12, 0x16, 0x0a, 0x05, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x18, 0x01, 0x20,
|
||||
0x01, 0x28, 0x09, 0x48, 0x00, 0x52, 0x05, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x42, 0x0c, 0x0a, 0x0a,
|
||||
0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x22, 0xdb, 0x01, 0x0a, 0x06, 0x47,
|
||||
0x65, 0x72, 0x72, 0x69, 0x74, 0x12, 0x24, 0x0a, 0x08, 0x65, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e,
|
||||
0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x42, 0x08, 0xfa, 0x42, 0x05, 0x72, 0x03, 0x90, 0x01,
|
||||
0x01, 0x52, 0x08, 0x65, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x12, 0x37, 0x0a, 0x0a, 0x62,
|
||||
0x61, 0x73, 0x69, 0x63, 0x5f, 0x61, 0x75, 0x74, 0x68, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32,
|
||||
0x16, 0x2e, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x73, 0x2e, 0x42, 0x61,
|
||||
0x73, 0x69, 0x63, 0x41, 0x75, 0x74, 0x68, 0x48, 0x00, 0x52, 0x09, 0x62, 0x61, 0x73, 0x69, 0x63,
|
||||
0x41, 0x75, 0x74, 0x68, 0x12, 0x48, 0x0a, 0x0f, 0x75, 0x6e, 0x61, 0x75, 0x74, 0x68, 0x65, 0x6e,
|
||||
0x74, 0x69, 0x63, 0x61, 0x74, 0x65, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e,
|
||||
0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x73, 0x2e, 0x55, 0x6e, 0x61, 0x75,
|
||||
0x74, 0x68, 0x65, 0x6e, 0x74, 0x69, 0x63, 0x61, 0x74, 0x65, 0x64, 0x48, 0x00, 0x52, 0x0f, 0x75,
|
||||
0x6e, 0x61, 0x75, 0x74, 0x68, 0x65, 0x6e, 0x74, 0x69, 0x63, 0x61, 0x74, 0x65, 0x64, 0x12, 0x1a,
|
||||
0x0a, 0x08, 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x09,
|
||||
0x52, 0x08, 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x42, 0x0c, 0x0a, 0x0a, 0x63, 0x72,
|
||||
0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x22, 0xa5, 0x01, 0x0a, 0x07, 0x4a, 0x65, 0x6e,
|
||||
0x6b, 0x69, 0x6e, 0x73, 0x12, 0x24, 0x0a, 0x08, 0x65, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74,
|
||||
0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x42, 0x08, 0xfa, 0x42, 0x05, 0x72, 0x03, 0x90, 0x01, 0x01,
|
||||
0x52, 0x08, 0x65, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x12, 0x37, 0x0a, 0x0a, 0x62, 0x61,
|
||||
0x73, 0x69, 0x63, 0x5f, 0x61, 0x75, 0x74, 0x68, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x16,
|
||||
0x2e, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x73, 0x2e, 0x42, 0x61, 0x73,
|
||||
0x69, 0x63, 0x41, 0x75, 0x74, 0x68, 0x48, 0x00, 0x52, 0x09, 0x62, 0x61, 0x73, 0x69, 0x63, 0x41,
|
||||
0x75, 0x74, 0x68, 0x12, 0x23, 0x0a, 0x0c, 0x61, 0x63, 0x63, 0x65, 0x73, 0x73, 0x5f, 0x74, 0x6f,
|
||||
0x6b, 0x65, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x48, 0x00, 0x52, 0x0b, 0x61, 0x63, 0x63,
|
||||
0x65, 0x73, 0x73, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x12, 0x22, 0x0a, 0x0c, 0x72, 0x65, 0x70, 0x6f,
|
||||
0x73, 0x69, 0x74, 0x6f, 0x72, 0x69, 0x65, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0c,
|
||||
0x72, 0x65, 0x70, 0x6f, 0x73, 0x69, 0x74, 0x6f, 0x72, 0x69, 0x65, 0x73, 0x42, 0x0c, 0x0a, 0x0a,
|
||||
0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x22, 0x94, 0x01, 0x0a, 0x06, 0x53,
|
||||
0x79, 0x73, 0x6c, 0x6f, 0x67, 0x12, 0x1a, 0x0a, 0x08, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f,
|
||||
0x6c, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f,
|
||||
0x6c, 0x12, 0x24, 0x0a, 0x0d, 0x6c, 0x69, 0x73, 0x74, 0x65, 0x6e, 0x41, 0x64, 0x64, 0x72, 0x65,
|
||||
0x73, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0d, 0x6c, 0x69, 0x73, 0x74, 0x65, 0x6e,
|
||||
0x41, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x12, 0x18, 0x0a, 0x07, 0x74, 0x6c, 0x73, 0x43, 0x65,
|
||||
0x72, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x74, 0x6c, 0x73, 0x43, 0x65, 0x72,
|
||||
0x74, 0x12, 0x16, 0x0a, 0x06, 0x74, 0x6c, 0x73, 0x4b, 0x65, 0x79, 0x18, 0x04, 0x20, 0x01, 0x28,
|
||||
0x09, 0x52, 0x06, 0x74, 0x6c, 0x73, 0x4b, 0x65, 0x79, 0x12, 0x16, 0x0a, 0x06, 0x66, 0x6f, 0x72,
|
||||
0x6d, 0x61, 0x74, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x66, 0x6f, 0x72, 0x6d, 0x61,
|
||||
0x74, 0x22, 0xd6, 0x01, 0x0a, 0x15, 0x50, 0x75, 0x62, 0x6c, 0x69, 0x63, 0x45, 0x76, 0x65, 0x6e,
|
||||
0x74, 0x4d, 0x6f, 0x6e, 0x69, 0x74, 0x6f, 0x72, 0x69, 0x6e, 0x67, 0x12, 0x48, 0x0a, 0x0f, 0x75,
|
||||
0x6e, 0x61, 0x75, 0x74, 0x68, 0x65, 0x6e, 0x74, 0x69, 0x63, 0x61, 0x74, 0x65, 0x64, 0x18, 0x01,
|
||||
0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61,
|
||||
0x6c, 0x73, 0x2e, 0x55, 0x6e, 0x61, 0x75, 0x74, 0x68, 0x65, 0x6e, 0x74, 0x69, 0x63, 0x61, 0x74,
|
||||
0x65, 0x64, 0x48, 0x00, 0x52, 0x0f, 0x75, 0x6e, 0x61, 0x75, 0x74, 0x68, 0x65, 0x6e, 0x74, 0x69,
|
||||
0x63, 0x61, 0x74, 0x65, 0x64, 0x12, 0x16, 0x0a, 0x06, 0x64, 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x18,
|
||||
0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x64, 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x12, 0x1b, 0x0a,
|
||||
0x09, 0x6d, 0x61, 0x78, 0x5f, 0x64, 0x65, 0x70, 0x74, 0x68, 0x18, 0x03, 0x20, 0x01, 0x28, 0x03,
|
||||
0x52, 0x08, 0x6d, 0x61, 0x78, 0x44, 0x65, 0x70, 0x74, 0x68, 0x12, 0x30, 0x0a, 0x05, 0x73, 0x69,
|
||||
0x6e, 0x63, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67,
|
||||
0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65,
|
||||
0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x05, 0x73, 0x69, 0x6e, 0x63, 0x65, 0x42, 0x0c, 0x0a, 0x0a,
|
||||
0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x22, 0x51, 0x0a, 0x0d, 0x53, 0x6c,
|
||||
0x61, 0x63, 0x6b, 0x52, 0x65, 0x61, 0x6c, 0x74, 0x69, 0x6d, 0x65, 0x12, 0x32, 0x0a, 0x06, 0x74,
|
||||
0x6f, 0x6b, 0x65, 0x6e, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x18, 0x2e, 0x63, 0x72,
|
||||
0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x73, 0x2e, 0x53, 0x6c, 0x61, 0x63, 0x6b, 0x54,
|
||||
0x6f, 0x6b, 0x65, 0x6e, 0x73, 0x48, 0x00, 0x52, 0x06, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x73, 0x42,
|
||||
0x0c, 0x0a, 0x0a, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x22, 0x62, 0x0a,
|
||||
0x0a, 0x53, 0x68, 0x61, 0x72, 0x65, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x12, 0x2b, 0x0a, 0x05, 0x6f,
|
||||
0x61, 0x75, 0x74, 0x68, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x13, 0x2e, 0x63, 0x72, 0x65,
|
||||
0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x73, 0x2e, 0x4f, 0x61, 0x75, 0x74, 0x68, 0x32, 0x48,
|
||||
0x00, 0x52, 0x05, 0x6f, 0x61, 0x75, 0x74, 0x68, 0x12, 0x19, 0x0a, 0x08, 0x73, 0x69, 0x74, 0x65,
|
||||
0x5f, 0x75, 0x72, 0x6c, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x73, 0x69, 0x74, 0x65,
|
||||
0x55, 0x72, 0x6c, 0x42, 0x0c, 0x0a, 0x0a, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61,
|
||||
0x6c, 0x22, 0xa7, 0x03, 0x0a, 0x0a, 0x41, 0x7a, 0x75, 0x72, 0x65, 0x52, 0x65, 0x70, 0x6f, 0x73,
|
||||
0x12, 0x24, 0x0a, 0x08, 0x65, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x18, 0x01, 0x20, 0x01,
|
||||
0x28, 0x09, 0x42, 0x08, 0xfa, 0x42, 0x05, 0x72, 0x03, 0x90, 0x01, 0x01, 0x52, 0x08, 0x65, 0x6e,
|
||||
0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x12, 0x16, 0x0a, 0x05, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x18,
|
||||
0x02, 0x20, 0x01, 0x28, 0x09, 0x48, 0x00, 0x52, 0x05, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x12, 0x2b,
|
||||
0x0a, 0x05, 0x6f, 0x61, 0x75, 0x74, 0x68, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x13, 0x2e,
|
||||
0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x73, 0x2e, 0x4f, 0x61, 0x75, 0x74,
|
||||
0x68, 0x32, 0x48, 0x00, 0x52, 0x05, 0x6f, 0x61, 0x75, 0x74, 0x68, 0x12, 0x22, 0x0a, 0x0c, 0x72,
|
||||
0x65, 0x70, 0x6f, 0x73, 0x69, 0x74, 0x6f, 0x72, 0x69, 0x65, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28,
|
||||
0x09, 0x52, 0x0c, 0x72, 0x65, 0x70, 0x6f, 0x73, 0x69, 0x74, 0x6f, 0x72, 0x69, 0x65, 0x73, 0x12,
|
||||
0x24, 0x0a, 0x0d, 0x6f, 0x72, 0x67, 0x61, 0x6e, 0x69, 0x7a, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73,
|
||||
0x18, 0x05, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0d, 0x6f, 0x72, 0x67, 0x61, 0x6e, 0x69, 0x7a, 0x61,
|
||||
0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x1a, 0x0a, 0x08, 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74,
|
||||
0x73, 0x18, 0x06, 0x20, 0x03, 0x28, 0x09, 0x52, 0x08, 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74,
|
||||
0x73, 0x12, 0x22, 0x0a, 0x0c, 0x69, 0x6e, 0x63, 0x6c, 0x75, 0x64, 0x65, 0x46, 0x6f, 0x72, 0x6b,
|
||||
0x73, 0x18, 0x07, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0c, 0x69, 0x6e, 0x63, 0x6c, 0x75, 0x64, 0x65,
|
||||
0x46, 0x6f, 0x72, 0x6b, 0x73, 0x12, 0x20, 0x0a, 0x0b, 0x69, 0x67, 0x6e, 0x6f, 0x72, 0x65, 0x52,
|
||||
0x65, 0x70, 0x6f, 0x73, 0x18, 0x08, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0b, 0x69, 0x67, 0x6e, 0x6f,
|
||||
0x72, 0x65, 0x52, 0x65, 0x70, 0x6f, 0x73, 0x12, 0x22, 0x0a, 0x0c, 0x69, 0x6e, 0x63, 0x6c, 0x75,
|
||||
0x64, 0x65, 0x52, 0x65, 0x70, 0x6f, 0x73, 0x18, 0x09, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0c, 0x69,
|
||||
0x6e, 0x63, 0x6c, 0x75, 0x64, 0x65, 0x52, 0x65, 0x70, 0x6f, 0x73, 0x12, 0x28, 0x0a, 0x0f, 0x69,
|
||||
0x6e, 0x63, 0x6c, 0x75, 0x64, 0x65, 0x50, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x18, 0x0a,
|
||||
0x20, 0x03, 0x28, 0x09, 0x52, 0x0f, 0x69, 0x6e, 0x63, 0x6c, 0x75, 0x64, 0x65, 0x50, 0x72, 0x6f,
|
||||
0x6a, 0x65, 0x63, 0x74, 0x73, 0x12, 0x26, 0x0a, 0x0e, 0x69, 0x67, 0x6e, 0x6f, 0x72, 0x65, 0x50,
|
||||
0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x18, 0x0b, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0e, 0x69,
|
||||
0x67, 0x6e, 0x6f, 0x72, 0x65, 0x50, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x42, 0x0c, 0x0a,
|
||||
0x0a, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x2a, 0x84, 0x07, 0x0a, 0x0a,
|
||||
0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x54, 0x79, 0x70, 0x65, 0x12, 0x1d, 0x0a, 0x19, 0x53, 0x4f,
|
||||
0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x41, 0x5a, 0x55, 0x52, 0x45, 0x5f,
|
||||
0x53, 0x54, 0x4f, 0x52, 0x41, 0x47, 0x45, 0x10, 0x00, 0x12, 0x19, 0x0a, 0x15, 0x53, 0x4f, 0x55,
|
||||
0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x42, 0x49, 0x54, 0x42, 0x55, 0x43, 0x4b,
|
||||
0x45, 0x54, 0x10, 0x01, 0x12, 0x18, 0x0a, 0x14, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54,
|
||||
0x59, 0x50, 0x45, 0x5f, 0x43, 0x49, 0x52, 0x43, 0x4c, 0x45, 0x43, 0x49, 0x10, 0x02, 0x12, 0x1a,
|
||||
0x0a, 0x16, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x43, 0x4f,
|
||||
0x4e, 0x46, 0x4c, 0x55, 0x45, 0x4e, 0x43, 0x45, 0x10, 0x03, 0x12, 0x16, 0x0a, 0x12, 0x53, 0x4f,
|
||||
0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x44, 0x4f, 0x43, 0x4b, 0x45, 0x52,
|
||||
0x10, 0x04, 0x12, 0x13, 0x0a, 0x0f, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50,
|
||||
0x45, 0x5f, 0x45, 0x43, 0x52, 0x10, 0x05, 0x12, 0x13, 0x0a, 0x0f, 0x53, 0x4f, 0x55, 0x52, 0x43,
|
||||
0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x47, 0x43, 0x53, 0x10, 0x06, 0x12, 0x16, 0x0a, 0x12,
|
||||
0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x47, 0x49, 0x54, 0x48,
|
||||
0x55, 0x42, 0x10, 0x07, 0x12, 0x1a, 0x0a, 0x16, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54,
|
||||
0x59, 0x50, 0x45, 0x5f, 0x50, 0x55, 0x42, 0x4c, 0x49, 0x43, 0x5f, 0x47, 0x49, 0x54, 0x10, 0x08,
|
||||
0x75, 0x74, 0x68, 0x12, 0x2d, 0x0a, 0x06, 0x68, 0x65, 0x61, 0x64, 0x65, 0x72, 0x18, 0x03, 0x20,
|
||||
0x01, 0x28, 0x0b, 0x32, 0x13, 0x2e, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c,
|
||||
0x73, 0x2e, 0x48, 0x65, 0x61, 0x64, 0x65, 0x72, 0x48, 0x00, 0x52, 0x06, 0x68, 0x65, 0x61, 0x64,
|
||||
0x65, 0x72, 0x42, 0x0c, 0x0a, 0x0a, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c,
|
||||
0x22, 0x9f, 0x02, 0x0a, 0x05, 0x54, 0x65, 0x61, 0x6d, 0x73, 0x12, 0x24, 0x0a, 0x08, 0x65, 0x6e,
|
||||
0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x42, 0x08, 0xfa, 0x42,
|
||||
0x05, 0x72, 0x03, 0x90, 0x01, 0x01, 0x52, 0x08, 0x65, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74,
|
||||
0x12, 0x16, 0x0a, 0x05, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x48,
|
||||
0x00, 0x52, 0x05, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x12, 0x46, 0x0a, 0x0d, 0x61, 0x75, 0x74, 0x68,
|
||||
0x65, 0x6e, 0x74, 0x69, 0x63, 0x61, 0x74, 0x65, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32,
|
||||
0x1e, 0x2e, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x73, 0x2e, 0x43, 0x6c,
|
||||
0x69, 0x65, 0x6e, 0x74, 0x43, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x73, 0x48,
|
||||
0x00, 0x52, 0x0d, 0x61, 0x75, 0x74, 0x68, 0x65, 0x6e, 0x74, 0x69, 0x63, 0x61, 0x74, 0x65, 0x64,
|
||||
0x12, 0x2b, 0x0a, 0x05, 0x6f, 0x61, 0x75, 0x74, 0x68, 0x18, 0x07, 0x20, 0x01, 0x28, 0x0b, 0x32,
|
||||
0x13, 0x2e, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x73, 0x2e, 0x4f, 0x61,
|
||||
0x75, 0x74, 0x68, 0x32, 0x48, 0x00, 0x52, 0x05, 0x6f, 0x61, 0x75, 0x74, 0x68, 0x12, 0x1a, 0x0a,
|
||||
0x08, 0x63, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x09, 0x52,
|
||||
0x08, 0x63, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x73, 0x12, 0x1e, 0x0a, 0x0a, 0x69, 0x67, 0x6e,
|
||||
0x6f, 0x72, 0x65, 0x4c, 0x69, 0x73, 0x74, 0x18, 0x05, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0a, 0x69,
|
||||
0x67, 0x6e, 0x6f, 0x72, 0x65, 0x4c, 0x69, 0x73, 0x74, 0x12, 0x19, 0x0a, 0x08, 0x74, 0x65, 0x61,
|
||||
0x6d, 0x5f, 0x69, 0x64, 0x73, 0x18, 0x06, 0x20, 0x03, 0x28, 0x09, 0x52, 0x07, 0x74, 0x65, 0x61,
|
||||
0x6d, 0x49, 0x64, 0x73, 0x42, 0x0c, 0x0a, 0x0a, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69,
|
||||
0x61, 0x6c, 0x22, 0xc3, 0x01, 0x0a, 0x0b, 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x6f,
|
||||
0x72, 0x79, 0x12, 0x24, 0x0a, 0x08, 0x65, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x18, 0x01,
|
||||
0x20, 0x01, 0x28, 0x09, 0x42, 0x08, 0xfa, 0x42, 0x05, 0x72, 0x03, 0x90, 0x01, 0x01, 0x52, 0x08,
|
||||
0x65, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x12, 0x37, 0x0a, 0x0a, 0x62, 0x61, 0x73, 0x69,
|
||||
0x63, 0x5f, 0x61, 0x75, 0x74, 0x68, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x16, 0x2e, 0x63,
|
||||
0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x73, 0x2e, 0x42, 0x61, 0x73, 0x69, 0x63,
|
||||
0x41, 0x75, 0x74, 0x68, 0x48, 0x00, 0x52, 0x09, 0x62, 0x61, 0x73, 0x69, 0x63, 0x41, 0x75, 0x74,
|
||||
0x68, 0x12, 0x23, 0x0a, 0x0c, 0x61, 0x63, 0x63, 0x65, 0x73, 0x73, 0x5f, 0x74, 0x6f, 0x6b, 0x65,
|
||||
0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x48, 0x00, 0x52, 0x0b, 0x61, 0x63, 0x63, 0x65, 0x73,
|
||||
0x73, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x12, 0x22, 0x0a, 0x0c, 0x72, 0x65, 0x70, 0x6f, 0x73, 0x69,
|
||||
0x74, 0x6f, 0x72, 0x69, 0x65, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0c, 0x72, 0x65,
|
||||
0x70, 0x6f, 0x73, 0x69, 0x74, 0x6f, 0x72, 0x69, 0x65, 0x73, 0x42, 0x0c, 0x0a, 0x0a, 0x63, 0x72,
|
||||
0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x22, 0x94, 0x01, 0x0a, 0x06, 0x53, 0x79, 0x73,
|
||||
0x6c, 0x6f, 0x67, 0x12, 0x1a, 0x0a, 0x08, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x18,
|
||||
0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x12,
|
||||
0x24, 0x0a, 0x0d, 0x6c, 0x69, 0x73, 0x74, 0x65, 0x6e, 0x41, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73,
|
||||
0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0d, 0x6c, 0x69, 0x73, 0x74, 0x65, 0x6e, 0x41, 0x64,
|
||||
0x64, 0x72, 0x65, 0x73, 0x73, 0x12, 0x18, 0x0a, 0x07, 0x74, 0x6c, 0x73, 0x43, 0x65, 0x72, 0x74,
|
||||
0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x74, 0x6c, 0x73, 0x43, 0x65, 0x72, 0x74, 0x12,
|
||||
0x16, 0x0a, 0x06, 0x74, 0x6c, 0x73, 0x4b, 0x65, 0x79, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52,
|
||||
0x06, 0x74, 0x6c, 0x73, 0x4b, 0x65, 0x79, 0x12, 0x16, 0x0a, 0x06, 0x66, 0x6f, 0x72, 0x6d, 0x61,
|
||||
0x74, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x66, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x22,
|
||||
0xd6, 0x01, 0x0a, 0x15, 0x50, 0x75, 0x62, 0x6c, 0x69, 0x63, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x4d,
|
||||
0x6f, 0x6e, 0x69, 0x74, 0x6f, 0x72, 0x69, 0x6e, 0x67, 0x12, 0x48, 0x0a, 0x0f, 0x75, 0x6e, 0x61,
|
||||
0x75, 0x74, 0x68, 0x65, 0x6e, 0x74, 0x69, 0x63, 0x61, 0x74, 0x65, 0x64, 0x18, 0x01, 0x20, 0x01,
|
||||
0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x73,
|
||||
0x2e, 0x55, 0x6e, 0x61, 0x75, 0x74, 0x68, 0x65, 0x6e, 0x74, 0x69, 0x63, 0x61, 0x74, 0x65, 0x64,
|
||||
0x48, 0x00, 0x52, 0x0f, 0x75, 0x6e, 0x61, 0x75, 0x74, 0x68, 0x65, 0x6e, 0x74, 0x69, 0x63, 0x61,
|
||||
0x74, 0x65, 0x64, 0x12, 0x16, 0x0a, 0x06, 0x64, 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x18, 0x02, 0x20,
|
||||
0x01, 0x28, 0x09, 0x52, 0x06, 0x64, 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x12, 0x1b, 0x0a, 0x09, 0x6d,
|
||||
0x61, 0x78, 0x5f, 0x64, 0x65, 0x70, 0x74, 0x68, 0x18, 0x03, 0x20, 0x01, 0x28, 0x03, 0x52, 0x08,
|
||||
0x6d, 0x61, 0x78, 0x44, 0x65, 0x70, 0x74, 0x68, 0x12, 0x30, 0x0a, 0x05, 0x73, 0x69, 0x6e, 0x63,
|
||||
0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65,
|
||||
0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74,
|
||||
0x61, 0x6d, 0x70, 0x52, 0x05, 0x73, 0x69, 0x6e, 0x63, 0x65, 0x42, 0x0c, 0x0a, 0x0a, 0x63, 0x72,
|
||||
0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x22, 0x51, 0x0a, 0x0d, 0x53, 0x6c, 0x61, 0x63,
|
||||
0x6b, 0x52, 0x65, 0x61, 0x6c, 0x74, 0x69, 0x6d, 0x65, 0x12, 0x32, 0x0a, 0x06, 0x74, 0x6f, 0x6b,
|
||||
0x65, 0x6e, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x18, 0x2e, 0x63, 0x72, 0x65, 0x64,
|
||||
0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x73, 0x2e, 0x53, 0x6c, 0x61, 0x63, 0x6b, 0x54, 0x6f, 0x6b,
|
||||
0x65, 0x6e, 0x73, 0x48, 0x00, 0x52, 0x06, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x73, 0x42, 0x0c, 0x0a,
|
||||
0x0a, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x22, 0x62, 0x0a, 0x0a, 0x53,
|
||||
0x68, 0x61, 0x72, 0x65, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x12, 0x2b, 0x0a, 0x05, 0x6f, 0x61, 0x75,
|
||||
0x74, 0x68, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x13, 0x2e, 0x63, 0x72, 0x65, 0x64, 0x65,
|
||||
0x6e, 0x74, 0x69, 0x61, 0x6c, 0x73, 0x2e, 0x4f, 0x61, 0x75, 0x74, 0x68, 0x32, 0x48, 0x00, 0x52,
|
||||
0x05, 0x6f, 0x61, 0x75, 0x74, 0x68, 0x12, 0x19, 0x0a, 0x08, 0x73, 0x69, 0x74, 0x65, 0x5f, 0x75,
|
||||
0x72, 0x6c, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x73, 0x69, 0x74, 0x65, 0x55, 0x72,
|
||||
0x6c, 0x42, 0x0c, 0x0a, 0x0a, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x2a,
|
||||
0xe7, 0x06, 0x0a, 0x0a, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x54, 0x79, 0x70, 0x65, 0x12, 0x1d,
|
||||
0x0a, 0x19, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x41, 0x5a,
|
||||
0x55, 0x52, 0x45, 0x5f, 0x53, 0x54, 0x4f, 0x52, 0x41, 0x47, 0x45, 0x10, 0x00, 0x12, 0x19, 0x0a,
|
||||
0x15, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x42, 0x49, 0x54,
|
||||
0x42, 0x55, 0x43, 0x4b, 0x45, 0x54, 0x10, 0x01, 0x12, 0x18, 0x0a, 0x14, 0x53, 0x4f, 0x55, 0x52,
|
||||
0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x43, 0x49, 0x52, 0x43, 0x4c, 0x45, 0x43, 0x49,
|
||||
0x10, 0x02, 0x12, 0x1a, 0x0a, 0x16, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50,
|
||||
0x45, 0x5f, 0x43, 0x4f, 0x4e, 0x46, 0x4c, 0x55, 0x45, 0x4e, 0x43, 0x45, 0x10, 0x03, 0x12, 0x16,
|
||||
0x0a, 0x12, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x44, 0x4f,
|
||||
0x43, 0x4b, 0x45, 0x52, 0x10, 0x04, 0x12, 0x13, 0x0a, 0x0f, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45,
|
||||
0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x45, 0x43, 0x52, 0x10, 0x05, 0x12, 0x13, 0x0a, 0x0f, 0x53,
|
||||
0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x47, 0x43, 0x53, 0x10, 0x06,
|
||||
0x12, 0x16, 0x0a, 0x12, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f,
|
||||
0x47, 0x49, 0x54, 0x4c, 0x41, 0x42, 0x10, 0x09, 0x12, 0x14, 0x0a, 0x10, 0x53, 0x4f, 0x55, 0x52,
|
||||
0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x4a, 0x49, 0x52, 0x41, 0x10, 0x0a, 0x12, 0x24,
|
||||
0x0a, 0x20, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x4e, 0x50,
|
||||
0x4d, 0x5f, 0x55, 0x4e, 0x41, 0x55, 0x54, 0x48, 0x44, 0x5f, 0x50, 0x41, 0x43, 0x4b, 0x41, 0x47,
|
||||
0x45, 0x53, 0x10, 0x0b, 0x12, 0x25, 0x0a, 0x21, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54,
|
||||
0x59, 0x50, 0x45, 0x5f, 0x50, 0x59, 0x50, 0x49, 0x5f, 0x55, 0x4e, 0x41, 0x55, 0x54, 0x48, 0x44,
|
||||
0x5f, 0x50, 0x41, 0x43, 0x4b, 0x41, 0x47, 0x45, 0x53, 0x10, 0x0c, 0x12, 0x12, 0x0a, 0x0e, 0x53,
|
||||
0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x53, 0x33, 0x10, 0x0d, 0x12,
|
||||
0x15, 0x0a, 0x11, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x53,
|
||||
0x4c, 0x41, 0x43, 0x4b, 0x10, 0x0e, 0x12, 0x1a, 0x0a, 0x16, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45,
|
||||
0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x46, 0x49, 0x4c, 0x45, 0x53, 0x59, 0x53, 0x54, 0x45, 0x4d,
|
||||
0x10, 0x0f, 0x12, 0x13, 0x0a, 0x0f, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50,
|
||||
0x45, 0x5f, 0x47, 0x49, 0x54, 0x10, 0x10, 0x12, 0x14, 0x0a, 0x10, 0x53, 0x4f, 0x55, 0x52, 0x43,
|
||||
0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x54, 0x45, 0x53, 0x54, 0x10, 0x11, 0x12, 0x1b, 0x0a,
|
||||
0x17, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x53, 0x33, 0x5f,
|
||||
0x55, 0x4e, 0x41, 0x55, 0x54, 0x48, 0x45, 0x44, 0x10, 0x12, 0x12, 0x2a, 0x0a, 0x26, 0x53, 0x4f,
|
||||
0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x47, 0x49, 0x54, 0x48, 0x55, 0x42,
|
||||
0x5f, 0x55, 0x4e, 0x41, 0x55, 0x54, 0x48, 0x45, 0x4e, 0x54, 0x49, 0x43, 0x41, 0x54, 0x45, 0x44,
|
||||
0x5f, 0x4f, 0x52, 0x47, 0x10, 0x13, 0x12, 0x19, 0x0a, 0x15, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45,
|
||||
0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x42, 0x55, 0x49, 0x4c, 0x44, 0x4b, 0x49, 0x54, 0x45, 0x10,
|
||||
0x14, 0x12, 0x16, 0x0a, 0x12, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45,
|
||||
0x5f, 0x47, 0x45, 0x52, 0x52, 0x49, 0x54, 0x10, 0x15, 0x12, 0x17, 0x0a, 0x13, 0x53, 0x4f, 0x55,
|
||||
0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x4a, 0x45, 0x4e, 0x4b, 0x49, 0x4e, 0x53,
|
||||
0x10, 0x16, 0x12, 0x15, 0x0a, 0x11, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50,
|
||||
0x45, 0x5f, 0x54, 0x45, 0x41, 0x4d, 0x53, 0x10, 0x17, 0x12, 0x21, 0x0a, 0x1d, 0x53, 0x4f, 0x55,
|
||||
0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x4a, 0x46, 0x52, 0x4f, 0x47, 0x5f, 0x41,
|
||||
0x52, 0x54, 0x49, 0x46, 0x41, 0x43, 0x54, 0x4f, 0x52, 0x59, 0x10, 0x18, 0x12, 0x16, 0x0a, 0x12,
|
||||
0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x53, 0x59, 0x53, 0x4c,
|
||||
0x4f, 0x47, 0x10, 0x19, 0x12, 0x27, 0x0a, 0x23, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54,
|
||||
0x59, 0x50, 0x45, 0x5f, 0x50, 0x55, 0x42, 0x4c, 0x49, 0x43, 0x5f, 0x45, 0x56, 0x45, 0x4e, 0x54,
|
||||
0x5f, 0x4d, 0x4f, 0x4e, 0x49, 0x54, 0x4f, 0x52, 0x49, 0x4e, 0x47, 0x10, 0x1a, 0x12, 0x1e, 0x0a,
|
||||
0x1a, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x53, 0x4c, 0x41,
|
||||
0x43, 0x4b, 0x5f, 0x52, 0x45, 0x41, 0x4c, 0x54, 0x49, 0x4d, 0x45, 0x10, 0x1b, 0x12, 0x1c, 0x0a,
|
||||
0x18, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x47, 0x4f, 0x4f,
|
||||
0x47, 0x4c, 0x45, 0x5f, 0x44, 0x52, 0x49, 0x56, 0x45, 0x10, 0x1c, 0x12, 0x1a, 0x0a, 0x16, 0x53,
|
||||
0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x53, 0x48, 0x41, 0x52, 0x45,
|
||||
0x50, 0x4f, 0x49, 0x4e, 0x54, 0x10, 0x1d, 0x12, 0x1c, 0x0a, 0x18, 0x53, 0x4f, 0x55, 0x52, 0x43,
|
||||
0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x47, 0x43, 0x53, 0x5f, 0x55, 0x4e, 0x41, 0x55, 0x54,
|
||||
0x48, 0x45, 0x44, 0x10, 0x1e, 0x12, 0x1b, 0x0a, 0x17, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f,
|
||||
0x54, 0x59, 0x50, 0x45, 0x5f, 0x41, 0x5a, 0x55, 0x52, 0x45, 0x5f, 0x52, 0x45, 0x50, 0x4f, 0x53,
|
||||
0x10, 0x1f, 0x42, 0x3b, 0x5a, 0x39, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d,
|
||||
0x2f, 0x74, 0x72, 0x75, 0x66, 0x66, 0x6c, 0x65, 0x73, 0x65, 0x63, 0x75, 0x72, 0x69, 0x74, 0x79,
|
||||
0x2f, 0x74, 0x72, 0x75, 0x66, 0x66, 0x6c, 0x65, 0x68, 0x6f, 0x67, 0x2f, 0x76, 0x33, 0x2f, 0x70,
|
||||
0x6b, 0x67, 0x2f, 0x70, 0x62, 0x2f, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x73, 0x70, 0x62, 0x62,
|
||||
0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
|
||||
0x47, 0x49, 0x54, 0x48, 0x55, 0x42, 0x10, 0x07, 0x12, 0x1a, 0x0a, 0x16, 0x53, 0x4f, 0x55, 0x52,
|
||||
0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x50, 0x55, 0x42, 0x4c, 0x49, 0x43, 0x5f, 0x47,
|
||||
0x49, 0x54, 0x10, 0x08, 0x12, 0x16, 0x0a, 0x12, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54,
|
||||
0x59, 0x50, 0x45, 0x5f, 0x47, 0x49, 0x54, 0x4c, 0x41, 0x42, 0x10, 0x09, 0x12, 0x14, 0x0a, 0x10,
|
||||
0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x4a, 0x49, 0x52, 0x41,
|
||||
0x10, 0x0a, 0x12, 0x24, 0x0a, 0x20, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50,
|
||||
0x45, 0x5f, 0x4e, 0x50, 0x4d, 0x5f, 0x55, 0x4e, 0x41, 0x55, 0x54, 0x48, 0x44, 0x5f, 0x50, 0x41,
|
||||
0x43, 0x4b, 0x41, 0x47, 0x45, 0x53, 0x10, 0x0b, 0x12, 0x25, 0x0a, 0x21, 0x53, 0x4f, 0x55, 0x52,
|
||||
0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x50, 0x59, 0x50, 0x49, 0x5f, 0x55, 0x4e, 0x41,
|
||||
0x55, 0x54, 0x48, 0x44, 0x5f, 0x50, 0x41, 0x43, 0x4b, 0x41, 0x47, 0x45, 0x53, 0x10, 0x0c, 0x12,
|
||||
0x12, 0x0a, 0x0e, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x53,
|
||||
0x33, 0x10, 0x0d, 0x12, 0x15, 0x0a, 0x11, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59,
|
||||
0x50, 0x45, 0x5f, 0x53, 0x4c, 0x41, 0x43, 0x4b, 0x10, 0x0e, 0x12, 0x1a, 0x0a, 0x16, 0x53, 0x4f,
|
||||
0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x46, 0x49, 0x4c, 0x45, 0x53, 0x59,
|
||||
0x53, 0x54, 0x45, 0x4d, 0x10, 0x0f, 0x12, 0x13, 0x0a, 0x0f, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45,
|
||||
0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x47, 0x49, 0x54, 0x10, 0x10, 0x12, 0x14, 0x0a, 0x10, 0x53,
|
||||
0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x54, 0x45, 0x53, 0x54, 0x10,
|
||||
0x11, 0x12, 0x1b, 0x0a, 0x17, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45,
|
||||
0x5f, 0x53, 0x33, 0x5f, 0x55, 0x4e, 0x41, 0x55, 0x54, 0x48, 0x45, 0x44, 0x10, 0x12, 0x12, 0x2a,
|
||||
0x0a, 0x26, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x47, 0x49,
|
||||
0x54, 0x48, 0x55, 0x42, 0x5f, 0x55, 0x4e, 0x41, 0x55, 0x54, 0x48, 0x45, 0x4e, 0x54, 0x49, 0x43,
|
||||
0x41, 0x54, 0x45, 0x44, 0x5f, 0x4f, 0x52, 0x47, 0x10, 0x13, 0x12, 0x19, 0x0a, 0x15, 0x53, 0x4f,
|
||||
0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x42, 0x55, 0x49, 0x4c, 0x44, 0x4b,
|
||||
0x49, 0x54, 0x45, 0x10, 0x14, 0x12, 0x16, 0x0a, 0x12, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f,
|
||||
0x54, 0x59, 0x50, 0x45, 0x5f, 0x47, 0x45, 0x52, 0x52, 0x49, 0x54, 0x10, 0x15, 0x12, 0x17, 0x0a,
|
||||
0x13, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x4a, 0x45, 0x4e,
|
||||
0x4b, 0x49, 0x4e, 0x53, 0x10, 0x16, 0x12, 0x15, 0x0a, 0x11, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45,
|
||||
0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x54, 0x45, 0x41, 0x4d, 0x53, 0x10, 0x17, 0x12, 0x21, 0x0a,
|
||||
0x1d, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x4a, 0x46, 0x52,
|
||||
0x4f, 0x47, 0x5f, 0x41, 0x52, 0x54, 0x49, 0x46, 0x41, 0x43, 0x54, 0x4f, 0x52, 0x59, 0x10, 0x18,
|
||||
0x12, 0x16, 0x0a, 0x12, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f,
|
||||
0x53, 0x59, 0x53, 0x4c, 0x4f, 0x47, 0x10, 0x19, 0x12, 0x27, 0x0a, 0x23, 0x53, 0x4f, 0x55, 0x52,
|
||||
0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x50, 0x55, 0x42, 0x4c, 0x49, 0x43, 0x5f, 0x45,
|
||||
0x56, 0x45, 0x4e, 0x54, 0x5f, 0x4d, 0x4f, 0x4e, 0x49, 0x54, 0x4f, 0x52, 0x49, 0x4e, 0x47, 0x10,
|
||||
0x1a, 0x12, 0x1e, 0x0a, 0x1a, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45,
|
||||
0x5f, 0x53, 0x4c, 0x41, 0x43, 0x4b, 0x5f, 0x52, 0x45, 0x41, 0x4c, 0x54, 0x49, 0x4d, 0x45, 0x10,
|
||||
0x1b, 0x12, 0x1c, 0x0a, 0x18, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45,
|
||||
0x5f, 0x47, 0x4f, 0x4f, 0x47, 0x4c, 0x45, 0x5f, 0x44, 0x52, 0x49, 0x56, 0x45, 0x10, 0x1c, 0x12,
|
||||
0x1a, 0x0a, 0x16, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x53,
|
||||
0x48, 0x41, 0x52, 0x45, 0x50, 0x4f, 0x49, 0x4e, 0x54, 0x10, 0x1d, 0x12, 0x1c, 0x0a, 0x18, 0x53,
|
||||
0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x47, 0x43, 0x53, 0x5f, 0x55,
|
||||
0x4e, 0x41, 0x55, 0x54, 0x48, 0x45, 0x44, 0x10, 0x1e, 0x42, 0x3b, 0x5a, 0x39, 0x67, 0x69, 0x74,
|
||||
0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x74, 0x72, 0x75, 0x66, 0x66, 0x6c, 0x65, 0x73,
|
||||
0x65, 0x63, 0x75, 0x72, 0x69, 0x74, 0x79, 0x2f, 0x74, 0x72, 0x75, 0x66, 0x66, 0x6c, 0x65, 0x68,
|
||||
0x6f, 0x67, 0x2f, 0x76, 0x33, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x70, 0x62, 0x2f, 0x73, 0x6f, 0x75,
|
||||
0x72, 0x63, 0x65, 0x73, 0x70, 0x62, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
|
||||
}
|
||||
|
||||
var (
|
||||
@@ -3651,7 +3456,7 @@ func file_sources_proto_rawDescGZIP() []byte {
|
||||
}
|
||||
|
||||
var file_sources_proto_enumTypes = make([]protoimpl.EnumInfo, 2)
|
||||
var file_sources_proto_msgTypes = make([]protoimpl.MessageInfo, 29)
|
||||
var file_sources_proto_msgTypes = make([]protoimpl.MessageInfo, 28)
|
||||
var file_sources_proto_goTypes = []interface{}{
|
||||
(SourceType)(0), // 0: sources.SourceType
|
||||
(Confluence_GetAllSpacesScope)(0), // 1: sources.Confluence.GetAllSpacesScope
|
||||
@@ -3683,73 +3488,71 @@ var file_sources_proto_goTypes = []interface{}{
|
||||
(*PublicEventMonitoring)(nil), // 27: sources.PublicEventMonitoring
|
||||
(*SlackRealtime)(nil), // 28: sources.SlackRealtime
|
||||
(*Sharepoint)(nil), // 29: sources.Sharepoint
|
||||
(*AzureRepos)(nil), // 30: sources.AzureRepos
|
||||
(*durationpb.Duration)(nil), // 31: google.protobuf.Duration
|
||||
(*anypb.Any)(nil), // 32: google.protobuf.Any
|
||||
(*credentialspb.BasicAuth)(nil), // 33: credentials.BasicAuth
|
||||
(*credentialspb.Unauthenticated)(nil), // 34: credentials.Unauthenticated
|
||||
(*credentialspb.Oauth2)(nil), // 35: credentials.Oauth2
|
||||
(*credentialspb.KeySecret)(nil), // 36: credentials.KeySecret
|
||||
(*credentialspb.CloudEnvironment)(nil), // 37: credentials.CloudEnvironment
|
||||
(*credentialspb.SSHAuth)(nil), // 38: credentials.SSHAuth
|
||||
(*credentialspb.GitHubApp)(nil), // 39: credentials.GitHubApp
|
||||
(*credentialspb.AWSSessionTokenSecret)(nil), // 40: credentials.AWSSessionTokenSecret
|
||||
(*credentialspb.SlackTokens)(nil), // 41: credentials.SlackTokens
|
||||
(*credentialspb.Header)(nil), // 42: credentials.Header
|
||||
(*credentialspb.ClientCredentials)(nil), // 43: credentials.ClientCredentials
|
||||
(*timestamppb.Timestamp)(nil), // 44: google.protobuf.Timestamp
|
||||
(*durationpb.Duration)(nil), // 30: google.protobuf.Duration
|
||||
(*anypb.Any)(nil), // 31: google.protobuf.Any
|
||||
(*credentialspb.BasicAuth)(nil), // 32: credentials.BasicAuth
|
||||
(*credentialspb.Unauthenticated)(nil), // 33: credentials.Unauthenticated
|
||||
(*credentialspb.Oauth2)(nil), // 34: credentials.Oauth2
|
||||
(*credentialspb.KeySecret)(nil), // 35: credentials.KeySecret
|
||||
(*credentialspb.CloudEnvironment)(nil), // 36: credentials.CloudEnvironment
|
||||
(*credentialspb.SSHAuth)(nil), // 37: credentials.SSHAuth
|
||||
(*credentialspb.GitHubApp)(nil), // 38: credentials.GitHubApp
|
||||
(*credentialspb.AWSSessionTokenSecret)(nil), // 39: credentials.AWSSessionTokenSecret
|
||||
(*credentialspb.SlackTokens)(nil), // 40: credentials.SlackTokens
|
||||
(*credentialspb.Header)(nil), // 41: credentials.Header
|
||||
(*credentialspb.ClientCredentials)(nil), // 42: credentials.ClientCredentials
|
||||
(*timestamppb.Timestamp)(nil), // 43: google.protobuf.Timestamp
|
||||
}
|
||||
var file_sources_proto_depIdxs = []int32{
|
||||
31, // 0: sources.LocalSource.scan_interval:type_name -> google.protobuf.Duration
|
||||
32, // 1: sources.LocalSource.connection:type_name -> google.protobuf.Any
|
||||
33, // 2: sources.AzureStorage.basic_auth:type_name -> credentials.BasicAuth
|
||||
34, // 3: sources.AzureStorage.unauthenticated:type_name -> credentials.Unauthenticated
|
||||
35, // 4: sources.Bitbucket.oauth:type_name -> credentials.Oauth2
|
||||
33, // 5: sources.Bitbucket.basic_auth:type_name -> credentials.BasicAuth
|
||||
34, // 6: sources.Confluence.unauthenticated:type_name -> credentials.Unauthenticated
|
||||
33, // 7: sources.Confluence.basic_auth:type_name -> credentials.BasicAuth
|
||||
30, // 0: sources.LocalSource.scan_interval:type_name -> google.protobuf.Duration
|
||||
31, // 1: sources.LocalSource.connection:type_name -> google.protobuf.Any
|
||||
32, // 2: sources.AzureStorage.basic_auth:type_name -> credentials.BasicAuth
|
||||
33, // 3: sources.AzureStorage.unauthenticated:type_name -> credentials.Unauthenticated
|
||||
34, // 4: sources.Bitbucket.oauth:type_name -> credentials.Oauth2
|
||||
32, // 5: sources.Bitbucket.basic_auth:type_name -> credentials.BasicAuth
|
||||
33, // 6: sources.Confluence.unauthenticated:type_name -> credentials.Unauthenticated
|
||||
32, // 7: sources.Confluence.basic_auth:type_name -> credentials.BasicAuth
|
||||
1, // 8: sources.Confluence.spaces_scope:type_name -> sources.Confluence.GetAllSpacesScope
|
||||
34, // 9: sources.Docker.unauthenticated:type_name -> credentials.Unauthenticated
|
||||
33, // 10: sources.Docker.basic_auth:type_name -> credentials.BasicAuth
|
||||
36, // 11: sources.ECR.access_key:type_name -> credentials.KeySecret
|
||||
34, // 12: sources.GCS.unauthenticated:type_name -> credentials.Unauthenticated
|
||||
37, // 13: sources.GCS.adc:type_name -> credentials.CloudEnvironment
|
||||
35, // 14: sources.GCS.oauth:type_name -> credentials.Oauth2
|
||||
33, // 15: sources.Git.basic_auth:type_name -> credentials.BasicAuth
|
||||
34, // 16: sources.Git.unauthenticated:type_name -> credentials.Unauthenticated
|
||||
38, // 17: sources.Git.ssh_auth:type_name -> credentials.SSHAuth
|
||||
35, // 18: sources.GitLab.oauth:type_name -> credentials.Oauth2
|
||||
33, // 19: sources.GitLab.basic_auth:type_name -> credentials.BasicAuth
|
||||
39, // 20: sources.GitHub.github_app:type_name -> credentials.GitHubApp
|
||||
34, // 21: sources.GitHub.unauthenticated:type_name -> credentials.Unauthenticated
|
||||
33, // 22: sources.GitHub.basic_auth:type_name -> credentials.BasicAuth
|
||||
33, // 23: sources.JIRA.basic_auth:type_name -> credentials.BasicAuth
|
||||
34, // 24: sources.JIRA.unauthenticated:type_name -> credentials.Unauthenticated
|
||||
35, // 25: sources.JIRA.oauth:type_name -> credentials.Oauth2
|
||||
34, // 26: sources.NPMUnauthenticatedPackage.unauthenticated:type_name -> credentials.Unauthenticated
|
||||
34, // 27: sources.PyPIUnauthenticatedPackage.unauthenticated:type_name -> credentials.Unauthenticated
|
||||
36, // 28: sources.S3.access_key:type_name -> credentials.KeySecret
|
||||
34, // 29: sources.S3.unauthenticated:type_name -> credentials.Unauthenticated
|
||||
37, // 30: sources.S3.cloud_environment:type_name -> credentials.CloudEnvironment
|
||||
40, // 31: sources.S3.session_token:type_name -> credentials.AWSSessionTokenSecret
|
||||
41, // 32: sources.Slack.tokens:type_name -> credentials.SlackTokens
|
||||
33, // 33: sources.Gerrit.basic_auth:type_name -> credentials.BasicAuth
|
||||
34, // 34: sources.Gerrit.unauthenticated:type_name -> credentials.Unauthenticated
|
||||
33, // 35: sources.Jenkins.basic_auth:type_name -> credentials.BasicAuth
|
||||
42, // 36: sources.Jenkins.header:type_name -> credentials.Header
|
||||
43, // 37: sources.Teams.authenticated:type_name -> credentials.ClientCredentials
|
||||
35, // 38: sources.Teams.oauth:type_name -> credentials.Oauth2
|
||||
33, // 39: sources.Artifactory.basic_auth:type_name -> credentials.BasicAuth
|
||||
34, // 40: sources.PublicEventMonitoring.unauthenticated:type_name -> credentials.Unauthenticated
|
||||
44, // 41: sources.PublicEventMonitoring.since:type_name -> google.protobuf.Timestamp
|
||||
41, // 42: sources.SlackRealtime.tokens:type_name -> credentials.SlackTokens
|
||||
35, // 43: sources.Sharepoint.oauth:type_name -> credentials.Oauth2
|
||||
35, // 44: sources.AzureRepos.oauth:type_name -> credentials.Oauth2
|
||||
45, // [45:45] is the sub-list for method output_type
|
||||
45, // [45:45] is the sub-list for method input_type
|
||||
45, // [45:45] is the sub-list for extension type_name
|
||||
45, // [45:45] is the sub-list for extension extendee
|
||||
0, // [0:45] is the sub-list for field type_name
|
||||
33, // 9: sources.Docker.unauthenticated:type_name -> credentials.Unauthenticated
|
||||
32, // 10: sources.Docker.basic_auth:type_name -> credentials.BasicAuth
|
||||
35, // 11: sources.ECR.access_key:type_name -> credentials.KeySecret
|
||||
33, // 12: sources.GCS.unauthenticated:type_name -> credentials.Unauthenticated
|
||||
36, // 13: sources.GCS.adc:type_name -> credentials.CloudEnvironment
|
||||
34, // 14: sources.GCS.oauth:type_name -> credentials.Oauth2
|
||||
32, // 15: sources.Git.basic_auth:type_name -> credentials.BasicAuth
|
||||
33, // 16: sources.Git.unauthenticated:type_name -> credentials.Unauthenticated
|
||||
37, // 17: sources.Git.ssh_auth:type_name -> credentials.SSHAuth
|
||||
34, // 18: sources.GitLab.oauth:type_name -> credentials.Oauth2
|
||||
32, // 19: sources.GitLab.basic_auth:type_name -> credentials.BasicAuth
|
||||
38, // 20: sources.GitHub.github_app:type_name -> credentials.GitHubApp
|
||||
33, // 21: sources.GitHub.unauthenticated:type_name -> credentials.Unauthenticated
|
||||
32, // 22: sources.GitHub.basic_auth:type_name -> credentials.BasicAuth
|
||||
32, // 23: sources.JIRA.basic_auth:type_name -> credentials.BasicAuth
|
||||
33, // 24: sources.JIRA.unauthenticated:type_name -> credentials.Unauthenticated
|
||||
34, // 25: sources.JIRA.oauth:type_name -> credentials.Oauth2
|
||||
33, // 26: sources.NPMUnauthenticatedPackage.unauthenticated:type_name -> credentials.Unauthenticated
|
||||
33, // 27: sources.PyPIUnauthenticatedPackage.unauthenticated:type_name -> credentials.Unauthenticated
|
||||
35, // 28: sources.S3.access_key:type_name -> credentials.KeySecret
|
||||
33, // 29: sources.S3.unauthenticated:type_name -> credentials.Unauthenticated
|
||||
36, // 30: sources.S3.cloud_environment:type_name -> credentials.CloudEnvironment
|
||||
39, // 31: sources.S3.session_token:type_name -> credentials.AWSSessionTokenSecret
|
||||
40, // 32: sources.Slack.tokens:type_name -> credentials.SlackTokens
|
||||
32, // 33: sources.Gerrit.basic_auth:type_name -> credentials.BasicAuth
|
||||
33, // 34: sources.Gerrit.unauthenticated:type_name -> credentials.Unauthenticated
|
||||
32, // 35: sources.Jenkins.basic_auth:type_name -> credentials.BasicAuth
|
||||
41, // 36: sources.Jenkins.header:type_name -> credentials.Header
|
||||
42, // 37: sources.Teams.authenticated:type_name -> credentials.ClientCredentials
|
||||
34, // 38: sources.Teams.oauth:type_name -> credentials.Oauth2
|
||||
32, // 39: sources.Artifactory.basic_auth:type_name -> credentials.BasicAuth
|
||||
33, // 40: sources.PublicEventMonitoring.unauthenticated:type_name -> credentials.Unauthenticated
|
||||
43, // 41: sources.PublicEventMonitoring.since:type_name -> google.protobuf.Timestamp
|
||||
40, // 42: sources.SlackRealtime.tokens:type_name -> credentials.SlackTokens
|
||||
34, // 43: sources.Sharepoint.oauth:type_name -> credentials.Oauth2
|
||||
44, // [44:44] is the sub-list for method output_type
|
||||
44, // [44:44] is the sub-list for method input_type
|
||||
44, // [44:44] is the sub-list for extension type_name
|
||||
44, // [44:44] is the sub-list for extension extendee
|
||||
0, // [0:44] is the sub-list for field type_name
|
||||
}
|
||||
|
||||
func init() { file_sources_proto_init() }
|
||||
@@ -4094,18 +3897,6 @@ func file_sources_proto_init() {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
file_sources_proto_msgTypes[28].Exporter = func(v interface{}, i int) interface{} {
|
||||
switch v := v.(*AzureRepos); i {
|
||||
case 0:
|
||||
return &v.state
|
||||
case 1:
|
||||
return &v.sizeCache
|
||||
case 2:
|
||||
return &v.unknownFields
|
||||
default:
|
||||
return nil
|
||||
}
|
||||
}
|
||||
}
|
||||
file_sources_proto_msgTypes[1].OneofWrappers = []interface{}{
|
||||
(*AzureStorage_ConnectionString)(nil),
|
||||
@@ -4213,17 +4004,13 @@ func file_sources_proto_init() {
|
||||
file_sources_proto_msgTypes[27].OneofWrappers = []interface{}{
|
||||
(*Sharepoint_Oauth)(nil),
|
||||
}
|
||||
file_sources_proto_msgTypes[28].OneofWrappers = []interface{}{
|
||||
(*AzureRepos_Token)(nil),
|
||||
(*AzureRepos_Oauth)(nil),
|
||||
}
|
||||
type x struct{}
|
||||
out := protoimpl.TypeBuilder{
|
||||
File: protoimpl.DescBuilder{
|
||||
GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
|
||||
RawDescriptor: file_sources_proto_rawDesc,
|
||||
NumEnums: 2,
|
||||
NumMessages: 29,
|
||||
NumMessages: 28,
|
||||
NumExtensions: 0,
|
||||
NumServices: 0,
|
||||
},
|
||||
|
||||
@@ -1884,8 +1884,6 @@ func (m *GitHub) validate(all bool) error {
|
||||
|
||||
// no validation rules for IncludeIssueComments
|
||||
|
||||
// no validation rules for IncludeGistComments
|
||||
|
||||
switch m.Credential.(type) {
|
||||
|
||||
case *GitHub_GithubApp:
|
||||
@@ -4461,154 +4459,3 @@ var _ interface {
|
||||
Cause() error
|
||||
ErrorName() string
|
||||
} = SharepointValidationError{}
|
||||
|
||||
// Validate checks the field values on AzureRepos with the rules defined in the
|
||||
// proto definition for this message. If any rules are violated, the first
|
||||
// error encountered is returned, or nil if there are no violations.
|
||||
func (m *AzureRepos) Validate() error {
|
||||
return m.validate(false)
|
||||
}
|
||||
|
||||
// ValidateAll checks the field values on AzureRepos with the rules defined in
|
||||
// the proto definition for this message. If any rules are violated, the
|
||||
// result is a list of violation errors wrapped in AzureReposMultiError, or
|
||||
// nil if none found.
|
||||
func (m *AzureRepos) ValidateAll() error {
|
||||
return m.validate(true)
|
||||
}
|
||||
|
||||
func (m *AzureRepos) validate(all bool) error {
|
||||
if m == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
var errors []error
|
||||
|
||||
if _, err := url.Parse(m.GetEndpoint()); err != nil {
|
||||
err = AzureReposValidationError{
|
||||
field: "Endpoint",
|
||||
reason: "value must be a valid URI",
|
||||
cause: err,
|
||||
}
|
||||
if !all {
|
||||
return err
|
||||
}
|
||||
errors = append(errors, err)
|
||||
}
|
||||
|
||||
// no validation rules for IncludeForks
|
||||
|
||||
switch m.Credential.(type) {
|
||||
|
||||
case *AzureRepos_Token:
|
||||
// no validation rules for Token
|
||||
|
||||
case *AzureRepos_Oauth:
|
||||
|
||||
if all {
|
||||
switch v := interface{}(m.GetOauth()).(type) {
|
||||
case interface{ ValidateAll() error }:
|
||||
if err := v.ValidateAll(); err != nil {
|
||||
errors = append(errors, AzureReposValidationError{
|
||||
field: "Oauth",
|
||||
reason: "embedded message failed validation",
|
||||
cause: err,
|
||||
})
|
||||
}
|
||||
case interface{ Validate() error }:
|
||||
if err := v.Validate(); err != nil {
|
||||
errors = append(errors, AzureReposValidationError{
|
||||
field: "Oauth",
|
||||
reason: "embedded message failed validation",
|
||||
cause: err,
|
||||
})
|
||||
}
|
||||
}
|
||||
} else if v, ok := interface{}(m.GetOauth()).(interface{ Validate() error }); ok {
|
||||
if err := v.Validate(); err != nil {
|
||||
return AzureReposValidationError{
|
||||
field: "Oauth",
|
||||
reason: "embedded message failed validation",
|
||||
cause: err,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
if len(errors) > 0 {
|
||||
return AzureReposMultiError(errors)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// AzureReposMultiError is an error wrapping multiple validation errors
|
||||
// returned by AzureRepos.ValidateAll() if the designated constraints aren't met.
|
||||
type AzureReposMultiError []error
|
||||
|
||||
// Error returns a concatenation of all the error messages it wraps.
|
||||
func (m AzureReposMultiError) Error() string {
|
||||
var msgs []string
|
||||
for _, err := range m {
|
||||
msgs = append(msgs, err.Error())
|
||||
}
|
||||
return strings.Join(msgs, "; ")
|
||||
}
|
||||
|
||||
// AllErrors returns a list of validation violation errors.
|
||||
func (m AzureReposMultiError) AllErrors() []error { return m }
|
||||
|
||||
// AzureReposValidationError is the validation error returned by
|
||||
// AzureRepos.Validate if the designated constraints aren't met.
|
||||
type AzureReposValidationError struct {
|
||||
field string
|
||||
reason string
|
||||
cause error
|
||||
key bool
|
||||
}
|
||||
|
||||
// Field function returns field value.
|
||||
func (e AzureReposValidationError) Field() string { return e.field }
|
||||
|
||||
// Reason function returns reason value.
|
||||
func (e AzureReposValidationError) Reason() string { return e.reason }
|
||||
|
||||
// Cause function returns cause value.
|
||||
func (e AzureReposValidationError) Cause() error { return e.cause }
|
||||
|
||||
// Key function returns key value.
|
||||
func (e AzureReposValidationError) Key() bool { return e.key }
|
||||
|
||||
// ErrorName returns error name.
|
||||
func (e AzureReposValidationError) ErrorName() string { return "AzureReposValidationError" }
|
||||
|
||||
// Error satisfies the builtin error interface
|
||||
func (e AzureReposValidationError) Error() string {
|
||||
cause := ""
|
||||
if e.cause != nil {
|
||||
cause = fmt.Sprintf(" | caused by: %v", e.cause)
|
||||
}
|
||||
|
||||
key := ""
|
||||
if e.key {
|
||||
key = "key for "
|
||||
}
|
||||
|
||||
return fmt.Sprintf(
|
||||
"invalid %sAzureRepos.%s: %s%s",
|
||||
key,
|
||||
e.field,
|
||||
e.reason,
|
||||
cause)
|
||||
}
|
||||
|
||||
var _ error = AzureReposValidationError{}
|
||||
|
||||
var _ interface {
|
||||
Field() string
|
||||
Reason() string
|
||||
Key() bool
|
||||
Cause() error
|
||||
ErrorName() string
|
||||
} = AzureReposValidationError{}
|
||||
|
||||
+12
-120
@@ -5,8 +5,6 @@ import (
|
||||
"bytes"
|
||||
"errors"
|
||||
"io"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/context"
|
||||
)
|
||||
|
||||
const (
|
||||
@@ -27,7 +25,6 @@ func Chunker(originalChunk *Chunk) chan *Chunk {
|
||||
chunkChan <- originalChunk
|
||||
return
|
||||
}
|
||||
|
||||
r := bytes.NewReader(originalChunk.Data)
|
||||
reader := bufio.NewReaderSize(bufio.NewReader(r), ChunkSize)
|
||||
for {
|
||||
@@ -35,126 +32,21 @@ func Chunker(originalChunk *Chunk) chan *Chunk {
|
||||
chunk := *originalChunk
|
||||
chunkBytes = chunkBytes[:ChunkSize]
|
||||
n, err := reader.Read(chunkBytes)
|
||||
if n > 0 {
|
||||
peekData, _ := reader.Peek(TotalChunkSize - n)
|
||||
chunkBytes = append(chunkBytes[:n], peekData...)
|
||||
chunk.Data = chunkBytes
|
||||
chunkChan <- &chunk
|
||||
}
|
||||
if err != nil {
|
||||
if err != nil && !errors.Is(err, io.EOF) {
|
||||
break
|
||||
}
|
||||
if n == 0 {
|
||||
if errors.Is(err, io.EOF) {
|
||||
break
|
||||
}
|
||||
continue
|
||||
}
|
||||
peekData, _ := reader.Peek(PeekSize)
|
||||
copy(chunkBytes[n:], peekData)
|
||||
chunk.Data = chunkBytes[:n+len(peekData)]
|
||||
|
||||
chunkChan <- &chunk
|
||||
}
|
||||
}()
|
||||
return chunkChan
|
||||
}
|
||||
|
||||
type chunkReaderConfig struct {
|
||||
chunkSize int
|
||||
totalSize int
|
||||
peekSize int
|
||||
}
|
||||
|
||||
// ConfigOption is a function that configures a chunker.
|
||||
type ConfigOption func(*chunkReaderConfig)
|
||||
|
||||
// WithChunkSize sets the chunk size.
|
||||
func WithChunkSize(size int) ConfigOption {
|
||||
return func(c *chunkReaderConfig) {
|
||||
c.chunkSize = size
|
||||
}
|
||||
}
|
||||
|
||||
// WithPeekSize sets the peek size.
|
||||
func WithPeekSize(size int) ConfigOption {
|
||||
return func(c *chunkReaderConfig) {
|
||||
c.peekSize = size
|
||||
}
|
||||
}
|
||||
|
||||
// ChunkResult is the output unit of a ChunkReader,
|
||||
// it contains the data and error of a chunk.
|
||||
type ChunkResult struct {
|
||||
data []byte
|
||||
err error
|
||||
}
|
||||
|
||||
// Bytes for a ChunkResult.
|
||||
func (cr ChunkResult) Bytes() []byte {
|
||||
return cr.data
|
||||
}
|
||||
|
||||
// Error for a ChunkResult.
|
||||
func (cr ChunkResult) Error() error {
|
||||
return cr.err
|
||||
}
|
||||
|
||||
// ChunkReader reads chunks from a reader and returns a channel of chunks and a channel of errors.
|
||||
// The channel of chunks is closed when the reader is closed.
|
||||
// This should be used whenever a large amount of data is read from a reader.
|
||||
// Ex: reading attachments, archives, etc.
|
||||
type ChunkReader func(ctx context.Context, reader io.Reader) <-chan ChunkResult
|
||||
|
||||
// NewChunkReader returns a ChunkReader with the given options.
|
||||
func NewChunkReader(opts ...ConfigOption) ChunkReader {
|
||||
config := applyOptions(opts)
|
||||
return createReaderFn(config)
|
||||
}
|
||||
|
||||
func applyOptions(opts []ConfigOption) *chunkReaderConfig {
|
||||
// Set defaults.
|
||||
config := &chunkReaderConfig{
|
||||
chunkSize: ChunkSize, // default
|
||||
peekSize: PeekSize, // default
|
||||
}
|
||||
|
||||
for _, opt := range opts {
|
||||
opt(config)
|
||||
}
|
||||
|
||||
config.totalSize = config.chunkSize + config.peekSize
|
||||
|
||||
return config
|
||||
}
|
||||
|
||||
func createReaderFn(config *chunkReaderConfig) ChunkReader {
|
||||
return func(ctx context.Context, reader io.Reader) <-chan ChunkResult {
|
||||
return readInChunks(ctx, reader, config)
|
||||
}
|
||||
}
|
||||
|
||||
func readInChunks(ctx context.Context, reader io.Reader, config *chunkReaderConfig) <-chan ChunkResult {
|
||||
const channelSize = 1
|
||||
chunkReader := bufio.NewReaderSize(reader, config.chunkSize)
|
||||
chunkResultChan := make(chan ChunkResult, channelSize)
|
||||
|
||||
go func() {
|
||||
defer close(chunkResultChan)
|
||||
|
||||
for {
|
||||
chunkRes := ChunkResult{}
|
||||
chunkBytes := make([]byte, config.totalSize)
|
||||
chunkBytes = chunkBytes[:config.chunkSize]
|
||||
n, err := chunkReader.Read(chunkBytes)
|
||||
if n > 0 {
|
||||
peekData, _ := chunkReader.Peek(config.totalSize - n)
|
||||
chunkBytes = append(chunkBytes[:n], peekData...)
|
||||
chunkRes.data = chunkBytes
|
||||
}
|
||||
|
||||
// If there is an error other than EOF, or if we have read some bytes, send the chunk.
|
||||
if err != nil && !errors.Is(err, io.EOF) || n > 0 {
|
||||
if err != nil && !errors.Is(err, io.EOF) {
|
||||
ctx.Logger().Error(err, "error reading chunk")
|
||||
chunkRes.err = err
|
||||
}
|
||||
chunkResultChan <- chunkRes
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
}
|
||||
}()
|
||||
return chunkResultChan
|
||||
}
|
||||
|
||||
+47
-159
@@ -1,15 +1,13 @@
|
||||
package sources
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
"errors"
|
||||
"io"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
diskbufferreader "github.com/bill-rich/disk-buffer-reader"
|
||||
"github.com/stretchr/testify/assert"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/context"
|
||||
)
|
||||
|
||||
func TestChunker(t *testing.T) {
|
||||
@@ -20,39 +18,59 @@ func TestChunker(t *testing.T) {
|
||||
}
|
||||
defer reReader.Close()
|
||||
|
||||
baseChunks := make([]*Chunk, 0, 9)
|
||||
baseChunkCount := 0
|
||||
|
||||
// Count chunks from looping using chunk size.
|
||||
for {
|
||||
tmpChunk := make([]byte, ChunkSize)
|
||||
_, err := reReader.Read(tmpChunk)
|
||||
if err != nil {
|
||||
if errors.Is(err, io.EOF) {
|
||||
break
|
||||
}
|
||||
t.Fatal(err)
|
||||
}
|
||||
baseChunkCount++
|
||||
}
|
||||
_ = reReader.Reset()
|
||||
|
||||
// Get the first two chunks for comparing later.
|
||||
baseChunkOne := make([]byte, ChunkSize)
|
||||
baseChunkTwo := make([]byte, ChunkSize)
|
||||
|
||||
baseReader := bufio.NewReaderSize(reReader, ChunkSize)
|
||||
_, _ = baseReader.Read(baseChunkOne)
|
||||
peek, _ := baseReader.Peek(PeekSize)
|
||||
baseChunkOne = append(baseChunkOne, peek...)
|
||||
_, _ = baseReader.Read(baseChunkTwo)
|
||||
peek, _ = baseReader.Peek(PeekSize)
|
||||
baseChunkTwo = append(baseChunkTwo, peek...)
|
||||
|
||||
// Reset the reader to the beginning and use ChunkReader.
|
||||
_ = reReader.Reset()
|
||||
|
||||
testChunkCount := 0
|
||||
chunkData, _ := io.ReadAll(reReader)
|
||||
originalChunk := &Chunk{
|
||||
Data: chunkData,
|
||||
}
|
||||
|
||||
for chunk := range Chunker(originalChunk) {
|
||||
baseChunks = append(baseChunks, chunk)
|
||||
}
|
||||
|
||||
_ = reReader.Reset()
|
||||
|
||||
testChunks := make([]*Chunk, 0, 9)
|
||||
|
||||
testData, _ := io.ReadAll(reReader)
|
||||
testOriginalChunk := &Chunk{
|
||||
Data: testData,
|
||||
}
|
||||
|
||||
for chunk := range Chunker(testOriginalChunk) {
|
||||
testChunks = append(testChunks, chunk)
|
||||
}
|
||||
|
||||
if len(testChunks) != len(baseChunks) {
|
||||
t.Errorf("Wrong number of chunks received. Got %d, expected: %d.", len(testChunks), len(baseChunks))
|
||||
}
|
||||
|
||||
for i, baseChunk := range baseChunks {
|
||||
if !bytes.Equal(baseChunk.Data, testChunks[i].Data) {
|
||||
t.Errorf("Chunk %d did not match expected. Got: %d bytes, expected: %d bytes", i+1, len(testChunks[i].Data), len(baseChunk.Data))
|
||||
testChunkCount++
|
||||
switch testChunkCount {
|
||||
case 1:
|
||||
if !bytes.Equal(baseChunkOne, chunk.Data) {
|
||||
t.Errorf("First chunk did not match expected. Got: %d bytes, expected: %d bytes", len(chunk.Data), len(baseChunkOne))
|
||||
}
|
||||
case 2:
|
||||
if !bytes.Equal(baseChunkTwo, chunk.Data) {
|
||||
t.Errorf("Second chunk did not match expected. Got: %d bytes, expected: %d bytes", len(chunk.Data), len(baseChunkTwo))
|
||||
}
|
||||
}
|
||||
}
|
||||
if testChunkCount != baseChunkCount {
|
||||
t.Errorf("Wrong number of chunks received. Got %d, expected: %d.", testChunkCount, baseChunkCount)
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
func BenchmarkChunker(b *testing.B) {
|
||||
@@ -66,133 +84,3 @@ func BenchmarkChunker(b *testing.B) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewChunkedReader(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
input string
|
||||
chunkSize int
|
||||
peekSize int
|
||||
wantChunks []string
|
||||
wantErr bool
|
||||
}{
|
||||
{
|
||||
name: "Smaller data than default chunkSize and peekSize",
|
||||
input: "example input",
|
||||
chunkSize: ChunkSize,
|
||||
peekSize: PeekSize,
|
||||
wantChunks: []string{"example input"},
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "Reader with no data",
|
||||
input: "",
|
||||
chunkSize: 1024,
|
||||
peekSize: 512,
|
||||
wantChunks: []string{},
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "Smaller data than chunkSize and peekSize",
|
||||
input: "small data",
|
||||
chunkSize: 1024,
|
||||
peekSize: 512,
|
||||
wantChunks: []string{"small data"},
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "Equal to chunkSize",
|
||||
input: strings.Repeat("a", 1024),
|
||||
chunkSize: 1024,
|
||||
peekSize: 512,
|
||||
wantChunks: []string{strings.Repeat("a", 1024)},
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "Equal to chunkSize + peekSize",
|
||||
input: strings.Repeat("a", 1536),
|
||||
chunkSize: 1024,
|
||||
peekSize: 512,
|
||||
wantChunks: []string{strings.Repeat("a", 1536), strings.Repeat("a", 512)},
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "EOF during peeking",
|
||||
input: strings.Repeat("a", 1300),
|
||||
chunkSize: 1024,
|
||||
peekSize: 512,
|
||||
wantChunks: []string{strings.Repeat("a", 1300), strings.Repeat("a", 276)},
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "EOF during reading",
|
||||
input: strings.Repeat("a", 512),
|
||||
chunkSize: 1024,
|
||||
peekSize: 512,
|
||||
wantChunks: []string{strings.Repeat("a", 512)},
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "Equal to totalSize",
|
||||
input: strings.Repeat("a", 2048),
|
||||
chunkSize: 1024,
|
||||
peekSize: 1024,
|
||||
wantChunks: []string{strings.Repeat("a", 2048), strings.Repeat("a", 1024)},
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "Larger than totalSize",
|
||||
input: strings.Repeat("a", 4096),
|
||||
chunkSize: 1024,
|
||||
peekSize: 1024,
|
||||
wantChunks: []string{strings.Repeat("a", 2048), strings.Repeat("a", 2048), strings.Repeat("a", 2048), strings.Repeat("a", 1024)},
|
||||
wantErr: false,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
readerFunc := NewChunkReader(WithChunkSize(tt.chunkSize), WithPeekSize(tt.peekSize))
|
||||
reader := strings.NewReader(tt.input)
|
||||
ctx := context.Background()
|
||||
chunkResChan := readerFunc(ctx, reader)
|
||||
|
||||
var err error
|
||||
chunks := make([]string, 0)
|
||||
for data := range chunkResChan {
|
||||
chunks = append(chunks, string(data.Bytes()))
|
||||
err = data.Error()
|
||||
}
|
||||
|
||||
assert.Equal(t, tt.wantChunks, chunks, "Chunks do not match")
|
||||
if tt.wantErr {
|
||||
assert.Error(t, err, "Expected an error")
|
||||
} else {
|
||||
assert.NoError(t, err, "Unexpected error")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkChunkReader(b *testing.B) {
|
||||
var bigChunk = make([]byte, 1<<24) // 16MB
|
||||
|
||||
reader := bytes.NewReader(bigChunk)
|
||||
chunkReader := NewChunkReader(WithChunkSize(ChunkSize), WithPeekSize(PeekSize))
|
||||
|
||||
b.ReportAllocs()
|
||||
b.ResetTimer()
|
||||
|
||||
for i := 0; i < b.N; i++ {
|
||||
b.StartTimer()
|
||||
chunkResChan := chunkReader(context.Background(), reader)
|
||||
|
||||
// Drain the channel.
|
||||
for range chunkResChan {
|
||||
}
|
||||
|
||||
b.StopTimer()
|
||||
_, err := reader.Seek(0, 0)
|
||||
assert.Nil(b, err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -4,6 +4,7 @@ import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"sync/atomic"
|
||||
|
||||
@@ -218,7 +219,7 @@ func (s *Source) stepsForBuild(_ context.Context, proj project, bld build) ([]bu
|
||||
return bldRes.Steps, nil
|
||||
}
|
||||
|
||||
func (s *Source) chunkAction(ctx context.Context, proj project, bld build, act action, stepName string, chunksChan chan *sources.Chunk) error {
|
||||
func (s *Source) chunkAction(_ context.Context, proj project, bld build, act action, stepName string, chunksChan chan *sources.Chunk) error {
|
||||
req, err := http.NewRequest("GET", act.OutputURL, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -228,40 +229,35 @@ func (s *Source) chunkAction(ctx context.Context, proj project, bld build, act a
|
||||
return err
|
||||
}
|
||||
defer res.Body.Close()
|
||||
logOutput, err := io.ReadAll(res.Body)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
linkURL := fmt.Sprintf("https://app.circleci.com/pipelines/%s/%s/%s/%d", proj.VCS, proj.Username, proj.RepoName, bld.BuildNum)
|
||||
|
||||
chunkReader := sources.NewChunkReader()
|
||||
chunkResChan := chunkReader(ctx, res.Body)
|
||||
for data := range chunkResChan {
|
||||
chunk := &sources.Chunk{
|
||||
SourceType: s.Type(),
|
||||
SourceName: s.name,
|
||||
SourceID: s.SourceID(),
|
||||
Data: removeCircleSha1Line(data.Bytes()),
|
||||
SourceMetadata: &source_metadatapb.MetaData{
|
||||
Data: &source_metadatapb.MetaData_Circleci{
|
||||
Circleci: &source_metadatapb.CircleCI{
|
||||
VcsType: proj.VCS,
|
||||
Username: proj.Username,
|
||||
Repository: proj.RepoName,
|
||||
BuildNumber: int64(bld.BuildNum),
|
||||
BuildStep: stepName,
|
||||
Link: linkURL,
|
||||
},
|
||||
chunk := &sources.Chunk{
|
||||
SourceType: s.Type(),
|
||||
SourceName: s.name,
|
||||
SourceID: s.SourceID(),
|
||||
Data: removeCircleSha1Line(logOutput),
|
||||
SourceMetadata: &source_metadatapb.MetaData{
|
||||
Data: &source_metadatapb.MetaData_Circleci{
|
||||
Circleci: &source_metadatapb.CircleCI{
|
||||
VcsType: proj.VCS,
|
||||
Username: proj.Username,
|
||||
Repository: proj.RepoName,
|
||||
BuildNumber: int64(bld.BuildNum),
|
||||
BuildStep: stepName,
|
||||
Link: linkURL,
|
||||
},
|
||||
},
|
||||
Verify: s.verify,
|
||||
}
|
||||
chunk.Data = data.Bytes()
|
||||
if err := data.Error(); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := common.CancellableWrite(ctx, chunksChan, chunk); err != nil {
|
||||
return err
|
||||
}
|
||||
},
|
||||
Verify: s.verify,
|
||||
}
|
||||
|
||||
chunksChan <- chunk
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
|
||||
@@ -84,8 +84,6 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) err
|
||||
var img v1.Image
|
||||
var err error
|
||||
var base, tag string
|
||||
var hasDigest bool
|
||||
var imageName name.Reference
|
||||
|
||||
if strings.HasPrefix(image, "file://") {
|
||||
image = strings.TrimPrefix(image, "file://")
|
||||
@@ -95,18 +93,10 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) err
|
||||
return err
|
||||
}
|
||||
} else {
|
||||
base, tag, hasDigest = baseAndTagFromImage(image)
|
||||
|
||||
if hasDigest {
|
||||
imageName, err = name.NewDigest(image)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
} else {
|
||||
imageName, err = name.NewTag(image)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
base, tag = baseAndTagFromImage(image)
|
||||
imageName, err := name.NewTag(image)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
img, err = remote.Image(imageName, remoteOpts...)
|
||||
@@ -204,20 +194,10 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) err
|
||||
return nil
|
||||
}
|
||||
|
||||
func baseAndTagFromImage(image string) (base, tag string, hasDigest bool) {
|
||||
func baseAndTagFromImage(image string) (base, tag string) {
|
||||
regRepoDelimiter := "/"
|
||||
tagDelim := ":"
|
||||
digestDelim := "@"
|
||||
|
||||
// Split on digest first, if present.
|
||||
parts := strings.SplitN(image, digestDelim, 2)
|
||||
if len(parts) > 1 {
|
||||
base = parts[0]
|
||||
tag = parts[1]
|
||||
hasDigest = true
|
||||
return
|
||||
}
|
||||
parts = strings.Split(image, tagDelim)
|
||||
parts := strings.Split(image, tagDelim)
|
||||
// Verify that we aren't confusing a tag for a hostname w/ port for the purposes of weak validation.
|
||||
if len(parts) > 1 && !strings.Contains(parts[len(parts)-1], regRepoDelimiter) {
|
||||
base = strings.Join(parts[:len(parts)-1], tagDelim)
|
||||
|
||||
@@ -1,3 +1,5 @@
|
||||
//go:build integration
|
||||
|
||||
package docker
|
||||
|
||||
import (
|
||||
@@ -49,64 +51,3 @@ func TestDockerImageScan(t *testing.T) {
|
||||
|
||||
assert.Equal(t, 1, chunkCounter)
|
||||
}
|
||||
|
||||
func TestDockerImageScanWithDigest(t *testing.T) {
|
||||
dockerConn := &sourcespb.Docker{
|
||||
Credential: &sourcespb.Docker_Unauthenticated{
|
||||
Unauthenticated: &credentialspb.Unauthenticated{},
|
||||
},
|
||||
Images: []string{"trufflesecurity/secrets@sha256:864f6d41209462d8e37fc302ba1532656e265f7c361f11e29fed6ca1f4208e11"},
|
||||
}
|
||||
|
||||
conn := &anypb.Any{}
|
||||
err := conn.MarshalFrom(dockerConn)
|
||||
assert.NoError(t, err)
|
||||
|
||||
s := &Source{}
|
||||
err = s.Init(context.TODO(), "test source", 0, 0, false, conn, 1)
|
||||
assert.NoError(t, err)
|
||||
|
||||
var wg sync.WaitGroup
|
||||
chunksChan := make(chan *sources.Chunk, 1)
|
||||
chunkCounter := 0
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
for chunk := range chunksChan {
|
||||
assert.NotEmpty(t, chunk)
|
||||
chunkCounter++
|
||||
}
|
||||
}()
|
||||
|
||||
err = s.Chunks(context.TODO(), chunksChan)
|
||||
assert.NoError(t, err)
|
||||
|
||||
close(chunksChan)
|
||||
wg.Wait()
|
||||
|
||||
assert.Equal(t, 1, chunkCounter)
|
||||
}
|
||||
|
||||
func TestBaseAndTagFromImage(t *testing.T) {
|
||||
tests := []struct {
|
||||
image string
|
||||
wantBase string
|
||||
wantTag string
|
||||
wantDigest bool
|
||||
}{
|
||||
{"golang:1.16", "golang", "1.16", false},
|
||||
{"golang@sha256:abcdef", "golang", "sha256:abcdef", true},
|
||||
{"ghcr.io/golang:1.16", "ghcr.io/golang", "1.16", false},
|
||||
{"ghcr.io/golang:nightly", "ghcr.io/golang", "nightly", false},
|
||||
{"ghcr.io/golang", "ghcr.io/golang", "latest", false},
|
||||
{"ghcr.io/trufflesecurity/secrets", "ghcr.io/trufflesecurity/secrets", "latest", false},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
gotBase, gotTag, gotDigest := baseAndTagFromImage(tt.image)
|
||||
if gotBase != tt.wantBase || gotTag != tt.wantTag || gotDigest != tt.wantDigest {
|
||||
t.Errorf("baseAndTagFromImage(%q) = (%q, %q, %v), want (%q, %q, %v)",
|
||||
tt.image, gotBase, gotTag, gotDigest, tt.wantBase, tt.wantTag, tt.wantDigest)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
package sources
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"sync"
|
||||
)
|
||||
@@ -33,13 +32,5 @@ func (s *ScanErrors) Count() uint64 {
|
||||
}
|
||||
|
||||
func (s *ScanErrors) String() string {
|
||||
s.mu.RLock()
|
||||
defer s.mu.RUnlock()
|
||||
return fmt.Sprintf("%v", s.errors)
|
||||
}
|
||||
|
||||
func (s *ScanErrors) Errors() error {
|
||||
s.mu.RLock()
|
||||
defer s.mu.RUnlock()
|
||||
return errors.Join(s.errors...)
|
||||
}
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
package filesystem
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/fs"
|
||||
@@ -22,6 +23,13 @@ import (
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
|
||||
)
|
||||
|
||||
const (
|
||||
// These buffer sizes are mainly driven by our largest credential size, which is GCP @ ~2.25KB.
|
||||
// Having a peek size larger than that ensures that we have complete credential coverage in our chunks.
|
||||
BufferSize = 10 * 1024 // 10KB
|
||||
PeekSize = 3 * 1024 // 3KB
|
||||
)
|
||||
|
||||
type Source struct {
|
||||
name string
|
||||
sourceId int64
|
||||
@@ -179,33 +187,34 @@ func (s *Source) scanFile(ctx context.Context, path string, chunksChan chan *sou
|
||||
}
|
||||
reReader.Stop()
|
||||
|
||||
chunkReader := sources.NewChunkReader()
|
||||
chunkResChan := chunkReader(ctx, reReader)
|
||||
for data := range chunkResChan {
|
||||
if err := data.Error(); err != nil {
|
||||
s.log.Error(err, "error reading chunk.")
|
||||
continue
|
||||
for {
|
||||
chunkBytes := make([]byte, BufferSize)
|
||||
reader := bufio.NewReaderSize(reReader, BufferSize)
|
||||
n, err := reader.Read(chunkBytes)
|
||||
if err != nil && !errors.Is(err, io.EOF) {
|
||||
break
|
||||
}
|
||||
|
||||
chunk := &sources.Chunk{
|
||||
SourceType: s.Type(),
|
||||
SourceName: s.name,
|
||||
SourceID: s.SourceID(),
|
||||
Data: data.Bytes(),
|
||||
SourceMetadata: &source_metadatapb.MetaData{
|
||||
Data: &source_metadatapb.MetaData_Filesystem{
|
||||
Filesystem: &source_metadatapb.Filesystem{
|
||||
File: sanitizer.UTF8(path),
|
||||
peekData, _ := reader.Peek(PeekSize)
|
||||
if n > 0 {
|
||||
chunksChan <- &sources.Chunk{
|
||||
SourceType: s.Type(),
|
||||
SourceName: s.name,
|
||||
SourceID: s.SourceID(),
|
||||
Data: append(chunkBytes[:n], peekData...),
|
||||
SourceMetadata: &source_metadatapb.MetaData{
|
||||
Data: &source_metadatapb.MetaData_Filesystem{
|
||||
Filesystem: &source_metadatapb.Filesystem{
|
||||
File: sanitizer.UTF8(path),
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
Verify: s.verify,
|
||||
Verify: s.verify,
|
||||
}
|
||||
}
|
||||
if err := common.CancellableWrite(ctx, chunksChan, chunk); err != nil {
|
||||
return err
|
||||
if errors.Is(err, io.EOF) {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
|
||||
@@ -1,14 +1,10 @@
|
||||
package filesystem
|
||||
|
||||
import (
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/go-logr/logr"
|
||||
"github.com/kylelemons/godebug/pretty"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"google.golang.org/protobuf/types/known/anypb"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/context"
|
||||
@@ -82,41 +78,3 @@ func TestSource_Scan(t *testing.T) {
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestScanFile(t *testing.T) {
|
||||
tmpfile, err := os.CreateTemp("", "example.txt")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer os.Remove(tmpfile.Name())
|
||||
|
||||
chunkSize := sources.ChunkSize
|
||||
secretPart1 := "SECRET"
|
||||
secretPart2 := "SPLIT"
|
||||
// Split the secret into two parts and pad the rest of the chunk with A's.
|
||||
data := strings.Repeat("A", chunkSize-len(secretPart1)) + secretPart1 + secretPart2 + strings.Repeat("A", chunkSize-len(secretPart2))
|
||||
|
||||
_, err = tmpfile.Write([]byte(data))
|
||||
assert.Nil(t, err)
|
||||
|
||||
err = tmpfile.Close()
|
||||
assert.Nil(t, err)
|
||||
|
||||
source := &Source{}
|
||||
chunksChan := make(chan *sources.Chunk, 2)
|
||||
|
||||
ctx := context.WithLogger(context.Background(), logr.Discard())
|
||||
go func() {
|
||||
defer close(chunksChan)
|
||||
err = source.scanFile(ctx, tmpfile.Name(), chunksChan)
|
||||
assert.Nil(t, err)
|
||||
}()
|
||||
|
||||
// Read from the channel and validate the secrets.
|
||||
foundSecret := ""
|
||||
for chunkCh := range chunksChan {
|
||||
foundSecret += string(chunkCh.Data)
|
||||
}
|
||||
|
||||
assert.Contains(t, foundSecret, secretPart1+secretPart2)
|
||||
}
|
||||
|
||||
+48
-98
@@ -4,6 +4,7 @@ import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/url"
|
||||
"os"
|
||||
"os/exec"
|
||||
@@ -11,7 +12,6 @@ import (
|
||||
"runtime"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
diskbufferreader "github.com/bill-rich/disk-buffer-reader"
|
||||
@@ -25,7 +25,6 @@ import (
|
||||
"google.golang.org/protobuf/proto"
|
||||
"google.golang.org/protobuf/types/known/anypb"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/context"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/gitparse"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/handlers"
|
||||
@@ -42,11 +41,7 @@ type Source struct {
|
||||
verify bool
|
||||
git *Git
|
||||
sources.Progress
|
||||
conn *sourcespb.Git
|
||||
scanOptions *ScanOptions
|
||||
// Kludge to preserve engine.ScanGit functionality which doesn't expect
|
||||
// the scanning to clean up the directory.
|
||||
preserveTempDirs bool
|
||||
conn *sourcespb.Git
|
||||
}
|
||||
|
||||
type Git struct {
|
||||
@@ -61,7 +56,7 @@ type Git struct {
|
||||
}
|
||||
|
||||
type metrics struct {
|
||||
commitsScanned uint64
|
||||
commitsScanned int
|
||||
}
|
||||
|
||||
func NewGit(sourceType sourcespb.SourceType, jobID, sourceID int64, sourceName string, verify bool, concurrency int,
|
||||
@@ -96,28 +91,12 @@ func (s *Source) JobID() int64 {
|
||||
return s.jobId
|
||||
}
|
||||
|
||||
// WithScanOptions sets the scan options.
|
||||
func (s *Source) WithScanOptions(scanOptions *ScanOptions) {
|
||||
s.scanOptions = scanOptions
|
||||
}
|
||||
|
||||
// WithPreserveTempDirs sets whether to preserve temp directories when scanning
|
||||
// the provided list of s.conn.Directories. NOTE: This is *only* for
|
||||
// s.conn.Directories, not all temp directories created. This is also a kludge
|
||||
// and should be refactored away.
|
||||
func (s *Source) WithPreserveTempDirs(preserve bool) {
|
||||
s.preserveTempDirs = preserve
|
||||
}
|
||||
|
||||
// Init returns an initialized GitHub source.
|
||||
func (s *Source) Init(aCtx context.Context, name string, jobId, sourceId int64, verify bool, connection *anypb.Any, concurrency int) error {
|
||||
s.name = name
|
||||
s.sourceId = sourceId
|
||||
s.jobId = jobId
|
||||
s.verify = verify
|
||||
if s.scanOptions == nil {
|
||||
s.scanOptions = &ScanOptions{}
|
||||
}
|
||||
|
||||
var conn sourcespb.Git
|
||||
if err := anypb.UnmarshalTo(connection, &conn, proto.UnmarshalOptions{}); err != nil {
|
||||
@@ -155,29 +134,8 @@ func (s *Source) Init(aCtx context.Context, name string, jobId, sourceId int64,
|
||||
|
||||
// Chunks emits chunks of bytes over a channel.
|
||||
func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) error {
|
||||
if err := s.scanRepos(ctx, chunksChan); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := s.scanDirs(ctx, chunksChan); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
totalRepos := len(s.conn.Repositories) + len(s.conn.Directories)
|
||||
ctx.Logger().V(1).Info("Git source finished scanning", "repo_count", totalRepos)
|
||||
s.SetProgressComplete(
|
||||
totalRepos, totalRepos,
|
||||
fmt.Sprintf("Completed scanning source %s", s.name), "",
|
||||
)
|
||||
return nil
|
||||
}
|
||||
|
||||
// scanRepos scans the configured repositories in s.conn.Repositories.
|
||||
func (s *Source) scanRepos(ctx context.Context, chunksChan chan *sources.Chunk) error {
|
||||
if len(s.conn.Repositories) == 0 {
|
||||
return nil
|
||||
}
|
||||
totalRepos := len(s.conn.Repositories) + len(s.conn.Directories)
|
||||
// TODO: refactor to remove duplicate code
|
||||
totalRepos := len(s.conn.Repositories) + len(s.conn.Directories)
|
||||
switch cred := s.conn.GetCredential().(type) {
|
||||
case *sourcespb.Git_BasicAuth:
|
||||
user := cred.BasicAuth.Username
|
||||
@@ -194,7 +152,7 @@ func (s *Source) scanRepos(ctx context.Context, chunksChan chan *sources.Chunk)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return s.git.ScanRepo(ctx, repo, path, s.scanOptions, chunksChan)
|
||||
return s.git.ScanRepo(ctx, repo, path, NewScanOptions(), chunksChan)
|
||||
}(repoURI)
|
||||
if err != nil {
|
||||
ctx.Logger().Info("error scanning repository", "repo", repoURI, "error", err)
|
||||
@@ -213,7 +171,7 @@ func (s *Source) scanRepos(ctx context.Context, chunksChan chan *sources.Chunk)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return s.git.ScanRepo(ctx, repo, path, s.scanOptions, chunksChan)
|
||||
return s.git.ScanRepo(ctx, repo, path, NewScanOptions(), chunksChan)
|
||||
}(repoURI)
|
||||
if err != nil {
|
||||
ctx.Logger().Info("error scanning repository", "repo", repoURI, "error", err)
|
||||
@@ -232,7 +190,7 @@ func (s *Source) scanRepos(ctx context.Context, chunksChan chan *sources.Chunk)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return s.git.ScanRepo(ctx, repo, path, s.scanOptions, chunksChan)
|
||||
return s.git.ScanRepo(ctx, repo, path, NewScanOptions(), chunksChan)
|
||||
}(repoURI)
|
||||
if err != nil {
|
||||
ctx.Logger().Info("error scanning repository", "repo", repoURI, "error", err)
|
||||
@@ -242,50 +200,48 @@ func (s *Source) scanRepos(ctx context.Context, chunksChan chan *sources.Chunk)
|
||||
default:
|
||||
return errors.New("invalid connection type for git source")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// scanDirs scans the configured directories in s.conn.Directories.
|
||||
func (s *Source) scanDirs(ctx context.Context, chunksChan chan *sources.Chunk) error {
|
||||
totalRepos := len(s.conn.Repositories) + len(s.conn.Directories)
|
||||
for i, gitDir := range s.conn.Directories {
|
||||
s.SetProgressComplete(len(s.conn.Repositories)+i, totalRepos, fmt.Sprintf("Repo: %s", gitDir), "")
|
||||
|
||||
if len(gitDir) == 0 {
|
||||
continue
|
||||
}
|
||||
if !s.scanOptions.Bare && strings.HasSuffix(gitDir, "git") {
|
||||
// TODO: Figure out why we skip directories ending in "git".
|
||||
continue
|
||||
}
|
||||
// try paths instead of url
|
||||
repo, err := RepoFromPath(gitDir, s.scanOptions.Bare)
|
||||
if err != nil {
|
||||
ctx.Logger().Info("error scanning repository", "repo", gitDir, "error", err)
|
||||
continue
|
||||
}
|
||||
|
||||
err = func(repoPath string) error {
|
||||
if !s.preserveTempDirs && strings.HasPrefix(repoPath, filepath.Join(os.TempDir(), "trufflehog")) {
|
||||
defer os.RemoveAll(repoPath)
|
||||
if !strings.HasSuffix(gitDir, "git") {
|
||||
// try paths instead of url
|
||||
repo, err := RepoFromPath(gitDir)
|
||||
if err != nil {
|
||||
ctx.Logger().Info("error scanning repository", "repo", gitDir, "error", err)
|
||||
continue
|
||||
}
|
||||
|
||||
return s.git.ScanRepo(ctx, repo, repoPath, s.scanOptions, chunksChan)
|
||||
}(gitDir)
|
||||
if err != nil {
|
||||
ctx.Logger().Info("error scanning repository", "repo", gitDir, "error", err)
|
||||
continue
|
||||
err = func(repoPath string) error {
|
||||
if strings.HasPrefix(repoPath, filepath.Join(os.TempDir(), "trufflehog")) {
|
||||
defer os.RemoveAll(repoPath)
|
||||
}
|
||||
|
||||
return s.git.ScanRepo(ctx, repo, repoPath, NewScanOptions(), chunksChan)
|
||||
}(gitDir)
|
||||
if err != nil {
|
||||
ctx.Logger().Info("error scanning repository", "repo", gitDir, "error", err)
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
ctx.Logger().V(1).Info("Git source finished scanning", "repo-count", totalRepos)
|
||||
s.SetProgressComplete(
|
||||
totalRepos, totalRepos,
|
||||
fmt.Sprintf("Completed scanning source %s", s.name), "",
|
||||
)
|
||||
return nil
|
||||
}
|
||||
|
||||
func RepoFromPath(path string, isBare bool) (*git.Repository, error) {
|
||||
options := &git.PlainOpenOptions{}
|
||||
if !isBare {
|
||||
options.DetectDotGit = true
|
||||
options.EnableDotGitCommonDir = true
|
||||
func RepoFromPath(path string) (*git.Repository, error) {
|
||||
options := &git.PlainOpenOptions{
|
||||
DetectDotGit: true,
|
||||
EnableDotGitCommonDir: true,
|
||||
}
|
||||
return git.PlainOpenWithOptions(path, options)
|
||||
}
|
||||
@@ -388,8 +344,8 @@ func CloneRepoUsingSSH(ctx context.Context, gitUrl string, args ...string) (stri
|
||||
return CloneRepo(ctx, userInfo, gitUrl, args...)
|
||||
}
|
||||
|
||||
func (s *Git) CommitsScanned() uint64 {
|
||||
return atomic.LoadUint64(&s.metrics.commitsScanned)
|
||||
func (s *Git) CommitsScanned() int {
|
||||
return s.metrics.commitsScanned
|
||||
}
|
||||
|
||||
func (s *Git) ScanCommits(ctx context.Context, repo *git.Repository, path string, scanOptions *ScanOptions, chunksChan chan *sources.Chunk) error {
|
||||
@@ -397,7 +353,7 @@ func (s *Git) ScanCommits(ctx context.Context, repo *git.Repository, path string
|
||||
return err
|
||||
}
|
||||
|
||||
commitChan, err := gitparse.NewParser().RepoPath(ctx, path, scanOptions.HeadHash, scanOptions.BaseHash == "", scanOptions.ExcludeGlobs, scanOptions.Bare)
|
||||
commitChan, err := gitparse.NewParser().RepoPath(ctx, path, scanOptions.HeadHash, scanOptions.BaseHash == "", scanOptions.ExcludeGlobs)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -424,7 +380,7 @@ func (s *Git) ScanCommits(ctx context.Context, repo *git.Repository, path string
|
||||
break
|
||||
}
|
||||
depth++
|
||||
atomic.AddUint64(&s.metrics.commitsScanned, 1)
|
||||
s.metrics.commitsScanned++
|
||||
logger.V(5).Info("scanning commit", "commit", commit.Hash)
|
||||
for _, diff := range commit.Diffs {
|
||||
if !scanOptions.Filter.Pass(diff.PathB) {
|
||||
@@ -625,10 +581,8 @@ func (s *Git) ScanRepo(ctx context.Context, repo *git.Repository, repoPath strin
|
||||
if err := s.ScanCommits(ctx, repo, repoPath, scanOptions, chunksChan); err != nil {
|
||||
return err
|
||||
}
|
||||
if !scanOptions.Bare {
|
||||
if err := s.ScanStaged(ctx, repo, repoPath, scanOptions, chunksChan); err != nil {
|
||||
ctx.Logger().V(1).Info("error scanning unstaged changes", "error", err)
|
||||
}
|
||||
if err := s.ScanStaged(ctx, repo, repoPath, scanOptions, chunksChan); err != nil {
|
||||
ctx.Logger().V(1).Info("error scanning unstaged changes", "error", err)
|
||||
}
|
||||
|
||||
// We're logging time, but the repoPath is usually a dynamically generated folder in /tmp
|
||||
@@ -640,7 +594,7 @@ func (s *Git) ScanRepo(ctx context.Context, repo *git.Repository, repoPath strin
|
||||
}
|
||||
|
||||
scanTime := time.Now().Unix() - start
|
||||
ctx.Logger().V(1).Info("scanning git repo complete", "repo", repoUrl, "path", repoPath, "time_seconds", scanTime, "commits_scanned", atomic.LoadUint64(&s.metrics.commitsScanned))
|
||||
ctx.Logger().V(1).Info("scanning git repo complete", "repo", repoUrl, "path", repoPath, "time_seconds", scanTime)
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -930,19 +884,15 @@ func handleBinary(ctx context.Context, repo *git.Repository, chunksChan chan *so
|
||||
}
|
||||
reader.Stop()
|
||||
|
||||
chunkReader := sources.NewChunkReader()
|
||||
chunkResChan := chunkReader(ctx, reader)
|
||||
for data := range chunkResChan {
|
||||
chunk := *chunkSkel
|
||||
chunk.Data = data.Bytes()
|
||||
if err := data.Error(); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := common.CancellableWrite(ctx, chunksChan, &chunk); err != nil {
|
||||
return err
|
||||
}
|
||||
chunkData, err := io.ReadAll(reader)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
chunk := *chunkSkel
|
||||
chunk.Data = chunkData
|
||||
chunksChan <- &chunk
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
|
||||
@@ -10,7 +10,6 @@ type ScanOptions struct {
|
||||
BaseHash string // When scanning a git.Log, this is the oldest/first commit.
|
||||
HeadHash string
|
||||
MaxDepth int64
|
||||
Bare bool
|
||||
ExcludeGlobs []string
|
||||
LogOptions *git.LogOptions
|
||||
}
|
||||
@@ -53,12 +52,6 @@ func ScanOptionLogOptions(logOptions *git.LogOptions) ScanOption {
|
||||
}
|
||||
}
|
||||
|
||||
func ScanOptionBare(bare bool) ScanOption {
|
||||
return func(scanOptions *ScanOptions) {
|
||||
scanOptions.Bare = bare
|
||||
}
|
||||
}
|
||||
|
||||
func NewScanOptions(options ...ScanOption) *ScanOptions {
|
||||
scanOptions := &ScanOptions{
|
||||
Filter: common.FilterEmpty(),
|
||||
|
||||
+111
-230
@@ -44,41 +44,32 @@ const (
|
||||
)
|
||||
|
||||
type Source struct {
|
||||
name string
|
||||
// Protects the user and token.
|
||||
userMu sync.Mutex
|
||||
githubUser string
|
||||
githubToken string
|
||||
|
||||
sourceID int64
|
||||
jobID int64
|
||||
verify bool
|
||||
repos []string
|
||||
members []string
|
||||
orgsCache cache.Cache
|
||||
filteredRepoCache *filteredRepoCache
|
||||
memberCache map[string]struct{}
|
||||
repoSizes repoSize
|
||||
totalRepoSize int // total size in bytes of all repos
|
||||
git *git.Git
|
||||
|
||||
scanOptMu sync.Mutex // protects the scanOptions
|
||||
scanOptions *git.ScanOptions
|
||||
|
||||
httpClient *http.Client
|
||||
log logr.Logger
|
||||
conn *sourcespb.GitHub
|
||||
jobPool *errgroup.Group
|
||||
resumeInfoMutex sync.Mutex
|
||||
resumeInfoSlice []string
|
||||
apiClient *github.Client
|
||||
|
||||
mu sync.Mutex // protects the visibility maps
|
||||
publicMap map[string]source_metadatapb.Visibility
|
||||
|
||||
name string
|
||||
githubUser string
|
||||
githubToken string
|
||||
sourceID int64
|
||||
jobID int64
|
||||
verify bool
|
||||
repos []string
|
||||
members []string
|
||||
orgsCache cache.Cache
|
||||
filteredRepoCache *filteredRepoCache
|
||||
memberCache map[string]struct{}
|
||||
repoSizes repoSize
|
||||
totalRepoSize int // total size in bytes of all repos
|
||||
git *git.Git
|
||||
scanOptions *git.ScanOptions
|
||||
httpClient *http.Client
|
||||
log logr.Logger
|
||||
conn *sourcespb.GitHub
|
||||
jobPool *errgroup.Group
|
||||
resumeInfoMutex sync.Mutex
|
||||
resumeInfoSlice []string
|
||||
apiClient *github.Client
|
||||
mu sync.Mutex
|
||||
publicMap map[string]source_metadatapb.Visibility
|
||||
includePRComments bool
|
||||
includeIssueComments bool
|
||||
includeGistComments bool
|
||||
sources.Progress
|
||||
sources.CommonSourceUnitUnmarshaller
|
||||
}
|
||||
@@ -87,13 +78,6 @@ func (s *Source) WithScanOptions(scanOptions *git.ScanOptions) {
|
||||
s.scanOptions = scanOptions
|
||||
}
|
||||
|
||||
func (s *Source) setScanOptions(base, head string) {
|
||||
s.scanOptMu.Lock()
|
||||
defer s.scanOptMu.Unlock()
|
||||
s.scanOptions.BaseHash = base
|
||||
s.scanOptions.HeadHash = head
|
||||
}
|
||||
|
||||
// Ensure the Source satisfies the interfaces at compile time
|
||||
var _ sources.Source = (*Source)(nil)
|
||||
var _ sources.SourceUnitUnmarshaller = (*Source)(nil)
|
||||
@@ -233,7 +217,6 @@ func (s *Source) Init(aCtx context.Context, name string, jobID, sourceID int64,
|
||||
|
||||
s.includeIssueComments = s.conn.IncludeIssueComments
|
||||
s.includePRComments = s.conn.IncludePullRequestComments
|
||||
s.includeGistComments = s.conn.IncludeGistComments
|
||||
|
||||
s.orgsCache = memory.New()
|
||||
for _, org := range s.conn.Organizations {
|
||||
@@ -373,11 +356,11 @@ func (s *Source) enumerate(ctx context.Context, apiEndpoint string) (*github.Cli
|
||||
|
||||
switch cred := s.conn.GetCredential().(type) {
|
||||
case *sourcespb.GitHub_BasicAuth:
|
||||
if err = s.enumerateBasicAuth(ctx, apiEndpoint, cred.BasicAuth); err != nil {
|
||||
if err = s.enumerateBasicAuth(ctx, cred.BasicAuth); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
case *sourcespb.GitHub_Unauthenticated:
|
||||
s.enumerateUnauthenticated(ctx, apiEndpoint)
|
||||
s.enumerateUnauthenticated(ctx)
|
||||
case *sourcespb.GitHub_Token:
|
||||
if err = s.enumerateWithToken(ctx, apiEndpoint, cred.Token); err != nil {
|
||||
return nil, err
|
||||
@@ -399,16 +382,11 @@ func (s *Source) enumerate(ctx context.Context, apiEndpoint string) (*github.Cli
|
||||
return installationClient, nil
|
||||
}
|
||||
|
||||
func (s *Source) enumerateBasicAuth(ctx context.Context, apiEndpoint string, basicAuth *credentialspb.BasicAuth) error {
|
||||
s.httpClient.Transport = &github.BasicAuthTransport{
|
||||
func (s *Source) enumerateBasicAuth(ctx context.Context, basicAuth *credentialspb.BasicAuth) error {
|
||||
s.apiClient = github.NewClient(&http.Client{Transport: &github.BasicAuthTransport{
|
||||
Username: basicAuth.Username,
|
||||
Password: basicAuth.Password,
|
||||
}
|
||||
ghClient, err := createGitHubClient(s.httpClient, apiEndpoint)
|
||||
if err != nil {
|
||||
s.log.Error(err, "error creating GitHub client")
|
||||
}
|
||||
s.apiClient = ghClient
|
||||
}})
|
||||
|
||||
for _, org := range s.orgsCache.Keys() {
|
||||
if err := s.getReposByOrg(ctx, org); err != nil {
|
||||
@@ -419,12 +397,8 @@ func (s *Source) enumerateBasicAuth(ctx context.Context, apiEndpoint string, bas
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Source) enumerateUnauthenticated(ctx context.Context, apiEndpoint string) {
|
||||
ghClient, err := createGitHubClient(s.httpClient, apiEndpoint)
|
||||
if err != nil {
|
||||
s.log.Error(err, "error creating GitHub client")
|
||||
}
|
||||
s.apiClient = ghClient
|
||||
func (s *Source) enumerateUnauthenticated(ctx context.Context) {
|
||||
s.apiClient = github.NewClient(s.httpClient)
|
||||
if s.orgsCache.Count() > unauthGithubOrgRateLimt {
|
||||
s.log.Info("You may experience rate limiting when using the unauthenticated GitHub api. Consider using an authenticated scan instead.")
|
||||
}
|
||||
@@ -458,14 +432,19 @@ func (s *Source) enumerateWithToken(ctx context.Context, apiEndpoint, token stri
|
||||
Source: oauth2.ReuseTokenSource(nil, ts),
|
||||
}
|
||||
|
||||
// If we're using public GitHub, make a regular client.
|
||||
var err error
|
||||
// If we're using public Github, make a regular client.
|
||||
// Otherwise, make an enterprise client.
|
||||
var isGHE bool = apiEndpoint != "https://api.github.com"
|
||||
ghClient, err := createGitHubClient(s.httpClient, apiEndpoint)
|
||||
if err != nil {
|
||||
s.log.Error(err, "error creating GitHub client")
|
||||
var isGHE bool
|
||||
if apiEndpoint == "https://api.github.com" {
|
||||
s.apiClient = github.NewClient(s.httpClient)
|
||||
} else {
|
||||
isGHE = true
|
||||
s.apiClient, err = github.NewEnterpriseClient(apiEndpoint, apiEndpoint, s.httpClient)
|
||||
if err != nil {
|
||||
return errors.New(err)
|
||||
}
|
||||
}
|
||||
s.apiClient = ghClient
|
||||
|
||||
// TODO: this should support scanning users too
|
||||
|
||||
@@ -571,28 +550,6 @@ func (s *Source) enumerateWithApp(ctx context.Context, apiEndpoint string, app *
|
||||
return nil, errors.New(err)
|
||||
}
|
||||
|
||||
// This client is required to create installation tokens for cloning.
|
||||
// Otherwise, the required JWT is not in the request for the token :/
|
||||
// This client uses the source's original HTTP transport, so make sure
|
||||
// to build it before modifying that transport (such as is done during
|
||||
// the creation of the other API client below).
|
||||
appItr, err := ghinstallation.NewAppsTransport(
|
||||
s.httpClient.Transport,
|
||||
appID,
|
||||
[]byte(app.PrivateKey))
|
||||
if err != nil {
|
||||
return nil, errors.New(err)
|
||||
}
|
||||
appItr.BaseURL = apiEndpoint
|
||||
|
||||
// Does this need to be separate from |s.httpClient|?
|
||||
instHttpClient := common.RetryableHttpClientTimeout(60)
|
||||
instHttpClient.Transport = appItr
|
||||
installationClient, err = github.NewEnterpriseClient(apiEndpoint, apiEndpoint, instHttpClient)
|
||||
if err != nil {
|
||||
return nil, errors.New(err)
|
||||
}
|
||||
|
||||
// This client is used for most APIs.
|
||||
itr, err := ghinstallation.New(
|
||||
s.httpClient.Transport,
|
||||
@@ -603,9 +560,22 @@ func (s *Source) enumerateWithApp(ctx context.Context, apiEndpoint string, app *
|
||||
return nil, errors.New(err)
|
||||
}
|
||||
itr.BaseURL = apiEndpoint
|
||||
s.apiClient, err = github.NewEnterpriseClient(apiEndpoint, apiEndpoint, &http.Client{Transport: itr})
|
||||
if err != nil {
|
||||
return nil, errors.New(err)
|
||||
}
|
||||
|
||||
s.httpClient.Transport = itr
|
||||
s.apiClient, err = github.NewEnterpriseClient(apiEndpoint, apiEndpoint, s.httpClient)
|
||||
// This client is required to create installation tokens for cloning.
|
||||
// Otherwise, the required JWT is not in the request for the token :/
|
||||
appItr, err := ghinstallation.NewAppsTransport(
|
||||
s.httpClient.Transport,
|
||||
appID,
|
||||
[]byte(app.PrivateKey))
|
||||
if err != nil {
|
||||
return nil, errors.New(err)
|
||||
}
|
||||
appItr.BaseURL = apiEndpoint
|
||||
installationClient, err = github.NewEnterpriseClient(apiEndpoint, apiEndpoint, &http.Client{Transport: appItr})
|
||||
if err != nil {
|
||||
return nil, errors.New(err)
|
||||
}
|
||||
@@ -638,21 +608,6 @@ func (s *Source) enumerateWithApp(ctx context.Context, apiEndpoint string, app *
|
||||
return installationClient, nil
|
||||
}
|
||||
|
||||
func createGitHubClient(httpClient *http.Client, apiEndpoint string) (ghClient *github.Client, err error) {
|
||||
// If we're using public GitHub, make a regular client.
|
||||
// Otherwise, make an enterprise client.
|
||||
if apiEndpoint == "https://api.github.com" {
|
||||
ghClient = github.NewClient(httpClient)
|
||||
} else {
|
||||
ghClient, err = github.NewEnterpriseClient(apiEndpoint, apiEndpoint, httpClient)
|
||||
if err != nil {
|
||||
return nil, errors.New(err)
|
||||
}
|
||||
}
|
||||
|
||||
return ghClient, err
|
||||
}
|
||||
|
||||
func (s *Source) scan(ctx context.Context, installationClient *github.Client, chunksChan chan *sources.Chunk) error {
|
||||
var scanned uint64
|
||||
|
||||
@@ -698,7 +653,6 @@ func (s *Source) scan(ctx context.Context, installationClient *github.Client, ch
|
||||
path, repo, err = s.cloneRepo(ctx, repoURL, installationClient)
|
||||
if err != nil {
|
||||
scanErrs.Add(err)
|
||||
return nil
|
||||
}
|
||||
|
||||
defer os.RemoveAll(path)
|
||||
@@ -707,7 +661,8 @@ func (s *Source) scan(ctx context.Context, installationClient *github.Client, ch
|
||||
return nil
|
||||
}
|
||||
|
||||
s.setScanOptions(s.conn.Base, s.conn.Head)
|
||||
s.scanOptions.BaseHash = s.conn.Base
|
||||
s.scanOptions.HeadHash = s.conn.Head
|
||||
|
||||
repoSize := s.repoSizes.getRepo(repoURL)
|
||||
logger.V(2).Info(fmt.Sprintf("scanning repo %d/%d", i, len(s.repos)), "repo_size", repoSize)
|
||||
@@ -717,16 +672,15 @@ func (s *Source) scan(ctx context.Context, installationClient *github.Client, ch
|
||||
logger.V(2).Info(fmt.Sprintf("scanned %d/%d repos", scanned, len(s.repos)), "repo_size", repoSize, "duration_seconds", time.Since(start).Seconds())
|
||||
}(now)
|
||||
|
||||
if err = s.git.ScanRepo(ctx, repo, path, s.scanOptions, chunksChan); err != nil {
|
||||
scanErrs.Add(fmt.Errorf("error scanning repo %s: %w", repoURL, err))
|
||||
return nil
|
||||
}
|
||||
|
||||
if err = s.scanComments(ctx, repoURL, chunksChan); err != nil {
|
||||
scanErrs.Add(fmt.Errorf("error scanning comments in repo %s: %w", repoURL, err))
|
||||
return nil
|
||||
}
|
||||
|
||||
if err = s.git.ScanRepo(ctx, repo, path, s.scanOptions, chunksChan); err != nil {
|
||||
scanErrs.Add(fmt.Errorf("error scanning repo %s: %w", repoURL, err))
|
||||
return nil
|
||||
}
|
||||
atomic.AddUint64(&scanned, 1)
|
||||
|
||||
return nil
|
||||
@@ -978,7 +932,7 @@ func (s *Source) setProgressCompleteWithRepo(index int, offset int, repoURL stri
|
||||
}
|
||||
|
||||
func (s *Source) scanComments(ctx context.Context, repoPath string, chunksChan chan *sources.Chunk) error {
|
||||
s.log.Info("scanning comments", "repository", repoPath)
|
||||
s.log.Info("scanning comments")
|
||||
|
||||
// Support ssh and https URLs
|
||||
repoURL, err := git.GitURLParse(repoPath)
|
||||
@@ -987,118 +941,79 @@ func (s *Source) scanComments(ctx context.Context, repoPath string, chunksChan c
|
||||
}
|
||||
|
||||
trimmedURL := removeURLAndSplit(repoURL.String())
|
||||
if repoURL.Host == "gist.github.com" && s.includeGistComments {
|
||||
// GitHub Gist URL.
|
||||
var gistId string
|
||||
if len(trimmedURL) == 2 {
|
||||
// https://gist.github.com/<id>
|
||||
gistId = trimmedURL[1]
|
||||
} else if len(trimmedURL) == 3 {
|
||||
// https://gist.github.com/<owner>/<id>
|
||||
gistId = trimmedURL[2]
|
||||
} else {
|
||||
return fmt.Errorf("failed to parse Gist URL: '%s'", repoURL.String())
|
||||
owner := trimmedURL[1]
|
||||
repo := trimmedURL[2]
|
||||
|
||||
var (
|
||||
sortType = "created"
|
||||
directionType = "desc"
|
||||
allComments = 0
|
||||
)
|
||||
|
||||
if s.includeIssueComments {
|
||||
|
||||
issueOpts := &github.IssueListCommentsOptions{
|
||||
Sort: &sortType,
|
||||
Direction: &directionType,
|
||||
ListOptions: github.ListOptions{
|
||||
PerPage: defaultPagination,
|
||||
Page: 1,
|
||||
},
|
||||
}
|
||||
|
||||
options := &github.ListOptions{
|
||||
PerPage: defaultPagination,
|
||||
Page: 1,
|
||||
}
|
||||
for {
|
||||
comments, resp, err := s.apiClient.Gists.ListComments(ctx, gistId, options)
|
||||
issueComments, resp, err := s.apiClient.Issues.ListComments(ctx, owner, repo, allComments, issueOpts)
|
||||
if s.handleRateLimit(err, resp) {
|
||||
break
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
err = s.chunkGistComments(ctx, repoURL.String(), comments, chunksChan)
|
||||
err = s.chunkIssueComments(ctx, repo, issueComments, chunksChan, repoPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
options.Page++
|
||||
if len(comments) < options.PerPage {
|
||||
issueOpts.ListOptions.Page++
|
||||
|
||||
if len(issueComments) < defaultPagination {
|
||||
break
|
||||
}
|
||||
}
|
||||
} else {
|
||||
// Normal repository URL (https://github.com/<owner>/<repo>).
|
||||
owner := trimmedURL[1]
|
||||
repo := trimmedURL[2]
|
||||
|
||||
var (
|
||||
sortType = "created"
|
||||
directionType = "desc"
|
||||
allComments = 0
|
||||
)
|
||||
|
||||
if s.includeIssueComments {
|
||||
|
||||
issueOpts := &github.IssueListCommentsOptions{
|
||||
Sort: &sortType,
|
||||
Direction: &directionType,
|
||||
ListOptions: github.ListOptions{
|
||||
PerPage: defaultPagination,
|
||||
Page: 1,
|
||||
},
|
||||
}
|
||||
|
||||
for {
|
||||
issueComments, resp, err := s.apiClient.Issues.ListComments(ctx, owner, repo, allComments, issueOpts)
|
||||
if s.handleRateLimit(err, resp) {
|
||||
break
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
err = s.chunkIssueComments(ctx, repo, issueComments, chunksChan, repoPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
issueOpts.ListOptions.Page++
|
||||
|
||||
if len(issueComments) < defaultPagination {
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if s.includePRComments {
|
||||
prOpts := &github.PullRequestListCommentsOptions{
|
||||
Sort: sortType,
|
||||
Direction: directionType,
|
||||
ListOptions: github.ListOptions{
|
||||
PerPage: defaultPagination,
|
||||
Page: 1,
|
||||
},
|
||||
}
|
||||
|
||||
if s.includePRComments {
|
||||
prOpts := &github.PullRequestListCommentsOptions{
|
||||
Sort: sortType,
|
||||
Direction: directionType,
|
||||
ListOptions: github.ListOptions{
|
||||
PerPage: defaultPagination,
|
||||
Page: 1,
|
||||
},
|
||||
for {
|
||||
prComments, resp, err := s.apiClient.PullRequests.ListComments(ctx, owner, repo, allComments, prOpts)
|
||||
if s.handleRateLimit(err, resp) {
|
||||
break
|
||||
}
|
||||
|
||||
for {
|
||||
prComments, resp, err := s.apiClient.PullRequests.ListComments(ctx, owner, repo, allComments, prOpts)
|
||||
if s.handleRateLimit(err, resp) {
|
||||
break
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
err = s.chunkPullRequestComments(ctx, repo, prComments, chunksChan, repoPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
err = s.chunkPullRequestComments(ctx, repo, prComments, chunksChan, repoPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
prOpts.ListOptions.Page++
|
||||
|
||||
prOpts.ListOptions.Page++
|
||||
|
||||
if len(prComments) < defaultPagination {
|
||||
break
|
||||
}
|
||||
if len(prComments) < defaultPagination {
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1169,40 +1084,6 @@ func (s *Source) chunkPullRequestComments(ctx context.Context, repo string, comm
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Source) chunkGistComments(ctx context.Context, gistUrl string, comments []*github.GistComment, chunksChan chan *sources.Chunk) error {
|
||||
for _, comment := range comments {
|
||||
// Create chunk and send it to the channel.
|
||||
chunk := &sources.Chunk{
|
||||
SourceName: s.name,
|
||||
SourceID: s.SourceID(),
|
||||
SourceType: s.Type(),
|
||||
SourceMetadata: &source_metadatapb.MetaData{
|
||||
Data: &source_metadatapb.MetaData_Github{
|
||||
Github: &source_metadatapb.Github{
|
||||
Link: sanitizer.UTF8(comment.GetURL()),
|
||||
Username: sanitizer.UTF8(comment.GetUser().GetLogin()),
|
||||
Email: sanitizer.UTF8(comment.GetUser().GetEmail()),
|
||||
Repository: sanitizer.UTF8(gistUrl),
|
||||
Timestamp: sanitizer.UTF8(comment.GetCreatedAt().String()),
|
||||
// Fetching this information requires making an additional API call.
|
||||
// We may want to include this in the future.
|
||||
//Visibility: s.visibilityOf(ctx, repoPath),
|
||||
},
|
||||
},
|
||||
},
|
||||
Data: []byte(sanitizer.UTF8(comment.GetBody())),
|
||||
Verify: s.verify,
|
||||
}
|
||||
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
case chunksChan <- chunk:
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func removeURLAndSplit(url string) []string {
|
||||
trimmedURL := strings.TrimPrefix(url, "https://")
|
||||
trimmedURL = strings.TrimSuffix(trimmedURL, ".git")
|
||||
|
||||
@@ -223,77 +223,6 @@ func TestSource_ScanComments(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestSource_ScanChunks(t *testing.T) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), time.Second*30)
|
||||
defer cancel()
|
||||
|
||||
secret, err := common.GetTestSecret(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(fmt.Errorf("failed to access secret: %v", err))
|
||||
}
|
||||
|
||||
// For the personal access token test.
|
||||
githubToken := secret.MustGetField("GITHUB_TOKEN")
|
||||
|
||||
type init struct {
|
||||
name string
|
||||
verify bool
|
||||
connection *sourcespb.GitHub
|
||||
}
|
||||
tests := []struct {
|
||||
name string
|
||||
init init
|
||||
wantChunks int
|
||||
}{
|
||||
{
|
||||
name: "token authenticated, 4 repos",
|
||||
init: init{
|
||||
name: "test source",
|
||||
connection: &sourcespb.GitHub{
|
||||
Repositories: []string{
|
||||
"https://github.com/truffle-test-integration-org/another-test-repo.git",
|
||||
"https://github.com/trufflesecurity/trufflehog.git",
|
||||
"https://github.com/Akash-goyal-github/Inventory-Management-System.git",
|
||||
"https://github.com/R1ck404/Crypto-Exchange-Example.git",
|
||||
"https://github.com/Stability-AI/generative-models.git",
|
||||
"https://github.com/bloomberg/blazingmq.git",
|
||||
"https://github.com/Kong/kong.git",
|
||||
},
|
||||
Credential: &sourcespb.GitHub_Token{Token: githubToken},
|
||||
},
|
||||
},
|
||||
wantChunks: 20000,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
s := Source{}
|
||||
|
||||
conn, err := anypb.New(tt.init.connection)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
err = s.Init(ctx, tt.init.name, 0, 0, tt.init.verify, conn, 8)
|
||||
assert.Nil(t, err)
|
||||
|
||||
chunksCh := make(chan *sources.Chunk, 1)
|
||||
go func() {
|
||||
defer close(chunksCh)
|
||||
err = s.Chunks(ctx, chunksCh)
|
||||
assert.Nil(t, err)
|
||||
}()
|
||||
|
||||
i := 0
|
||||
for range chunksCh {
|
||||
i++
|
||||
}
|
||||
assert.GreaterOrEqual(t, i, tt.wantChunks)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestSource_Scan(t *testing.T) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), time.Second*300)
|
||||
defer cancel()
|
||||
|
||||
@@ -339,8 +339,7 @@ func TestHandleRateLimit(t *testing.T) {
|
||||
func TestEnumerateUnauthenticated(t *testing.T) {
|
||||
defer gock.Off()
|
||||
|
||||
apiEndpoint := "https://api.github.com"
|
||||
gock.New(apiEndpoint).
|
||||
gock.New("https://api.github.com").
|
||||
Get("/orgs/super-secret-org/repos").
|
||||
Reply(200).
|
||||
JSON([]map[string]string{{"clone_url": "https://github.com/super-secret-repo.git", "full_name": "super-secret-repo"}})
|
||||
@@ -348,7 +347,7 @@ func TestEnumerateUnauthenticated(t *testing.T) {
|
||||
s := initTestSource(nil)
|
||||
s.orgsCache = memory.New()
|
||||
s.orgsCache.Set("super-secret-org", "super-secret-org")
|
||||
s.enumerateUnauthenticated(context.Background(), apiEndpoint)
|
||||
s.enumerateUnauthenticated(context.Background())
|
||||
assert.Equal(t, 1, s.filteredRepoCache.Count())
|
||||
ok := s.filteredRepoCache.Exists("super-secret-repo")
|
||||
assert.True(t, ok)
|
||||
|
||||
@@ -49,8 +49,12 @@ func (s *Source) cloneRepo(
|
||||
}
|
||||
|
||||
case *sourcespb.GitHub_Token:
|
||||
if err := s.getUserAndToken(ctx, repoURL, installationClient); err != nil {
|
||||
return "", nil, fmt.Errorf("error getting token for repo %s: %w", repoURL, err)
|
||||
// We never refresh user provided tokens, so if we already have them, we never need to try and fetch them again.
|
||||
if s.githubUser == "" || s.githubToken == "" {
|
||||
s.githubUser, s.githubToken, err = s.userAndToken(ctx, installationClient)
|
||||
if err != nil {
|
||||
return "", nil, fmt.Errorf("error getting token for repo %s: %w", repoURL, err)
|
||||
}
|
||||
}
|
||||
path, repo, err = git.CloneRepoUsingToken(ctx, s.githubToken, repoURL, s.githubUser)
|
||||
if err != nil {
|
||||
@@ -62,20 +66,6 @@ func (s *Source) cloneRepo(
|
||||
return path, repo, nil
|
||||
}
|
||||
|
||||
func (s *Source) getUserAndToken(ctx context.Context, repoURL string, installationClient *github.Client) error {
|
||||
// We never refresh user provided tokens, so if we already have them, we never need to try and fetch them again.
|
||||
s.userMu.Lock()
|
||||
defer s.userMu.Unlock()
|
||||
if s.githubUser == "" || s.githubToken == "" {
|
||||
var err error
|
||||
s.githubUser, s.githubToken, err = s.userAndToken(ctx, installationClient)
|
||||
if err != nil {
|
||||
return fmt.Errorf("error getting token for repo %s: %w", repoURL, err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Source) userAndToken(ctx context.Context, installationClient *github.Client) (string, string, error) {
|
||||
switch cred := s.conn.GetCredential().(type) {
|
||||
case *sourcespb.GitHub_BasicAuth:
|
||||
|
||||
@@ -1,286 +0,0 @@
|
||||
package sources
|
||||
|
||||
//go:generate mockgen --source=./job_progress.go --destination=mock_job_progress_test.go --package=sources
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
type JobProgressHook interface {
|
||||
// Start and End marks the overall start and end time for this job.
|
||||
Start(JobProgressRef, time.Time)
|
||||
End(JobProgressRef, time.Time)
|
||||
// StartEnumerating and EndEnumerating marks the start and end time for
|
||||
// calling the source's Enumerate method. If the source does not
|
||||
// support enumeration these methods will never be called.
|
||||
StartEnumerating(JobProgressRef, time.Time)
|
||||
EndEnumerating(JobProgressRef, time.Time)
|
||||
// StartUnitChunking and EndUnitChunking marks the start and end time
|
||||
// for calling the source's ChunkUnit method for a given unit. If the
|
||||
// source does not support enumeration these methods will never be
|
||||
// called.
|
||||
StartUnitChunking(JobProgressRef, SourceUnit, time.Time)
|
||||
EndUnitChunking(JobProgressRef, SourceUnit, time.Time)
|
||||
// ReportError is called when any general error is encountered, usually
|
||||
// from enumeration.
|
||||
ReportError(JobProgressRef, error)
|
||||
// ReportUnit is called when a unit has been enumerated. If the source
|
||||
// does not support enumeration this method will never be called.
|
||||
ReportUnit(JobProgressRef, SourceUnit)
|
||||
// ReportChunk is called when a chunk has been produced for the given
|
||||
// unit. The unit will be nil if the source does not support
|
||||
// enumeration.
|
||||
ReportChunk(JobProgressRef, SourceUnit, *Chunk)
|
||||
// Finish marks the job as done.
|
||||
Finish(JobProgressRef)
|
||||
}
|
||||
|
||||
// JobProgressRef is a wrapper of a JobProgress for read-only access to its state.
|
||||
type JobProgressRef struct {
|
||||
SourceID int64
|
||||
JobID int64
|
||||
jobProgress *JobProgress
|
||||
}
|
||||
|
||||
// Snapshot returns a snapshot of the job's current metrics.
|
||||
func (r *JobProgressRef) Snapshot() JobProgressMetrics {
|
||||
if r.jobProgress == nil {
|
||||
return JobProgressMetrics{}
|
||||
}
|
||||
return r.jobProgress.Snapshot()
|
||||
}
|
||||
|
||||
// Done returns a channel that will block until the job has completed.
|
||||
func (r *JobProgressRef) Done() <-chan struct{} {
|
||||
if r.jobProgress == nil {
|
||||
// Return a closed channel so it does not block.
|
||||
ch := make(chan struct{})
|
||||
close(ch)
|
||||
return ch
|
||||
}
|
||||
return r.jobProgress.Done()
|
||||
}
|
||||
|
||||
// Fatal is a wrapper around error to differentiate non-fatal errors from fatal
|
||||
// ones. A fatal error is typically from a finished context or any error
|
||||
// returned from a source's Init, Chunks, Enumerate, or ChunkUnit methods.
|
||||
type Fatal struct{ error }
|
||||
|
||||
func (f Fatal) Error() string { return fmt.Sprintf("fatal: %s", f.error.Error()) }
|
||||
func (f Fatal) Unwrap() error { return f.error }
|
||||
|
||||
// ChunkError is a custom error type for errors encountered during chunking of
|
||||
// a specific unit.
|
||||
type ChunkError struct {
|
||||
unit SourceUnit
|
||||
err error
|
||||
}
|
||||
|
||||
func (f ChunkError) Error() string {
|
||||
return fmt.Sprintf("error chunking unit %q: %s", f.unit.SourceUnitID(), f.err.Error())
|
||||
}
|
||||
func (f ChunkError) Unwrap() error { return f.err }
|
||||
|
||||
// JobProgress aggregates information about a run of a Source.
|
||||
type JobProgress struct {
|
||||
// Unique identifiers for this job.
|
||||
SourceID int64
|
||||
JobID int64
|
||||
// Tracks whether the job is finished or not.
|
||||
ctx context.Context
|
||||
cancel context.CancelFunc
|
||||
// Metrics.
|
||||
metrics JobProgressMetrics
|
||||
metricsLock sync.Mutex
|
||||
// Coarse grained hooks for adding extra functionality when events trigger.
|
||||
hooks []JobProgressHook
|
||||
}
|
||||
|
||||
// JobProgressMetrics tracks the metrics of a job.
|
||||
type JobProgressMetrics struct {
|
||||
StartTime time.Time
|
||||
EndTime time.Time
|
||||
// Total number of units found by the Source.
|
||||
TotalUnits uint64
|
||||
// Total number of units that have finished chunking.
|
||||
FinishedUnits uint64
|
||||
// Total number of chunks produced. This metric updates before the
|
||||
// chunk is sent on the output channel.
|
||||
TotalChunks uint64
|
||||
Errors []error
|
||||
DoneEnumerating bool
|
||||
}
|
||||
|
||||
// WithHooks adds hooks to be called when an event triggers.
|
||||
func WithHooks(hooks ...JobProgressHook) func(*JobProgress) {
|
||||
return func(jp *JobProgress) { jp.hooks = append(jp.hooks, hooks...) }
|
||||
}
|
||||
|
||||
// NewJobProgress creates a new job report for the given source and job ID.
|
||||
func NewJobProgress(sourceID, jobID int64, opts ...func(*JobProgress)) *JobProgress {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
jp := &JobProgress{
|
||||
SourceID: sourceID,
|
||||
JobID: jobID,
|
||||
ctx: ctx,
|
||||
cancel: cancel,
|
||||
}
|
||||
for _, opt := range opts {
|
||||
opt(jp)
|
||||
}
|
||||
return jp
|
||||
}
|
||||
|
||||
// executeHooks is a helper method to execute all the hooks for the given
|
||||
// closure.
|
||||
func (jp *JobProgress) executeHooks(todo func(hook JobProgressHook)) {
|
||||
for _, hook := range jp.hooks {
|
||||
// TODO: Non-blocking?
|
||||
todo(hook)
|
||||
}
|
||||
}
|
||||
|
||||
// TODO: Comment all this mess. They are mostly implementing JobProgressHook but
|
||||
// without the JobProgressRef parameter.
|
||||
func (jp *JobProgress) Start(start time.Time) {
|
||||
jp.metricsLock.Lock()
|
||||
jp.metrics.StartTime = start
|
||||
jp.metricsLock.Unlock()
|
||||
|
||||
jp.executeHooks(func(hook JobProgressHook) { hook.Start(jp.Ref(), start) })
|
||||
}
|
||||
func (jp *JobProgress) End(end time.Time) {
|
||||
jp.metricsLock.Lock()
|
||||
jp.metrics.EndTime = end
|
||||
jp.metricsLock.Unlock()
|
||||
|
||||
jp.executeHooks(func(hook JobProgressHook) { hook.End(jp.Ref(), end) })
|
||||
}
|
||||
func (jp *JobProgress) Finish() {
|
||||
jp.cancel()
|
||||
jp.executeHooks(func(hook JobProgressHook) { hook.Finish(jp.Ref()) })
|
||||
}
|
||||
func (jp *JobProgress) Done() <-chan struct{} { return jp.ctx.Done() }
|
||||
func (jp *JobProgress) ReportUnit(unit SourceUnit) {
|
||||
jp.metricsLock.Lock()
|
||||
jp.metrics.TotalUnits++
|
||||
jp.metricsLock.Unlock()
|
||||
jp.executeHooks(func(hook JobProgressHook) { hook.ReportUnit(jp.Ref(), unit) })
|
||||
}
|
||||
func (jp *JobProgress) ReportChunk(unit SourceUnit, chunk *Chunk) {
|
||||
jp.metricsLock.Lock()
|
||||
jp.metrics.TotalChunks++
|
||||
jp.metricsLock.Unlock()
|
||||
jp.executeHooks(func(hook JobProgressHook) { hook.ReportChunk(jp.Ref(), unit, chunk) })
|
||||
}
|
||||
func (jp *JobProgress) StartUnitChunking(unit SourceUnit, start time.Time) {
|
||||
// TODO: Record time.
|
||||
jp.executeHooks(func(hook JobProgressHook) { hook.StartUnitChunking(jp.Ref(), unit, start) })
|
||||
}
|
||||
func (jp *JobProgress) EndUnitChunking(unit SourceUnit, end time.Time) {
|
||||
// TODO: Record time.
|
||||
jp.metricsLock.Lock()
|
||||
jp.metrics.FinishedUnits++
|
||||
jp.metricsLock.Unlock()
|
||||
jp.executeHooks(func(hook JobProgressHook) { hook.EndUnitChunking(jp.Ref(), unit, end) })
|
||||
}
|
||||
func (jp *JobProgress) StartEnumerating(start time.Time) {
|
||||
// TODO: Record time.
|
||||
jp.executeHooks(func(hook JobProgressHook) { hook.StartEnumerating(jp.Ref(), start) })
|
||||
}
|
||||
func (jp *JobProgress) EndEnumerating(end time.Time) {
|
||||
jp.metricsLock.Lock()
|
||||
// TODO: Record time.
|
||||
jp.metrics.DoneEnumerating = true
|
||||
jp.metricsLock.Unlock()
|
||||
|
||||
jp.executeHooks(func(hook JobProgressHook) { hook.EndEnumerating(jp.Ref(), end) })
|
||||
}
|
||||
|
||||
// Snapshot safely gets the job's current metrics.
|
||||
func (jp *JobProgress) Snapshot() JobProgressMetrics {
|
||||
jp.metricsLock.Lock()
|
||||
defer jp.metricsLock.Unlock()
|
||||
|
||||
metrics := jp.metrics
|
||||
metrics.Errors = make([]error, len(metrics.Errors))
|
||||
copy(metrics.Errors, jp.metrics.Errors)
|
||||
|
||||
return metrics
|
||||
}
|
||||
|
||||
// ReportError adds a non-nil error to the aggregate of errors
|
||||
// encountered during scanning.
|
||||
func (jp *JobProgress) ReportError(err error) {
|
||||
if err == nil {
|
||||
return
|
||||
}
|
||||
jp.metricsLock.Lock()
|
||||
jp.metrics.Errors = append(jp.metrics.Errors, err)
|
||||
jp.metricsLock.Unlock()
|
||||
|
||||
jp.executeHooks(func(hook JobProgressHook) { hook.ReportError(jp.Ref(), err) })
|
||||
}
|
||||
|
||||
// Ref provides a read-only reference to the JobProgress.
|
||||
func (jp *JobProgress) Ref() JobProgressRef {
|
||||
return JobProgressRef{
|
||||
SourceID: jp.SourceID,
|
||||
JobID: jp.JobID,
|
||||
jobProgress: jp,
|
||||
}
|
||||
}
|
||||
|
||||
// EnumerationErrors joins all errors encountered during initialization or
|
||||
// enumeration.
|
||||
func (m JobProgressMetrics) EnumerationError() error {
|
||||
return errors.Join(m.Errors...)
|
||||
}
|
||||
|
||||
// ChunkErrors joins all errors encountered during chunking.
|
||||
func (m JobProgressMetrics) ChunkError() error {
|
||||
var aggregate []error
|
||||
for _, err := range m.Errors {
|
||||
var chunkErr ChunkError
|
||||
if ok := errors.As(err, &chunkErr); ok {
|
||||
aggregate = append(aggregate, err)
|
||||
}
|
||||
}
|
||||
return errors.Join(aggregate...)
|
||||
}
|
||||
|
||||
// FatalError returns the first Fatal error, if any, encountered in the scan.
|
||||
func (m JobProgressMetrics) FatalError() error {
|
||||
for _, err := range m.Errors {
|
||||
var fatalErr Fatal
|
||||
if found := errors.As(err, &fatalErr); found {
|
||||
return fatalErr
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// FatalErrors returns all of the encountered fatal errors joined together.
|
||||
func (m JobProgressMetrics) FatalErrors() error {
|
||||
var aggregate []error
|
||||
for _, err := range m.Errors {
|
||||
var fatalErr Fatal
|
||||
if found := errors.As(err, &fatalErr); found {
|
||||
aggregate = append(aggregate, fatalErr)
|
||||
}
|
||||
}
|
||||
return errors.Join(aggregate...)
|
||||
}
|
||||
|
||||
func (m JobProgressMetrics) PercentComplete() int {
|
||||
num := m.FinishedUnits
|
||||
den := m.TotalUnits
|
||||
if num == 0 && den == 0 {
|
||||
return 0
|
||||
}
|
||||
return int(num * 100 / den)
|
||||
}
|
||||
@@ -1,117 +0,0 @@
|
||||
package sources
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"go.uber.org/mock/gomock"
|
||||
)
|
||||
|
||||
func TestJobProgressFatalErrors(t *testing.T) {
|
||||
var jp JobProgress
|
||||
|
||||
// Add a non-fatal error.
|
||||
jp.ReportError(fmt.Errorf("oh no"))
|
||||
assert.Greater(t, len(jp.Snapshot().Errors), 0)
|
||||
assert.NoError(t, jp.Snapshot().FatalError())
|
||||
assert.NoError(t, jp.Snapshot().ChunkError())
|
||||
|
||||
// Add a fatal error and make sure we can test comparison.
|
||||
err := fmt.Errorf("fatal error")
|
||||
jp.ReportError(Fatal{err})
|
||||
assert.Greater(t, len(jp.Snapshot().Errors), 0)
|
||||
assert.Error(t, jp.Snapshot().FatalError())
|
||||
assert.NoError(t, jp.Snapshot().ChunkError())
|
||||
assert.True(t, errors.Is(jp.Snapshot().FatalError(), err))
|
||||
|
||||
// Add another fatal error and test we still return the first.
|
||||
jp.ReportError(Fatal{fmt.Errorf("second fatal error")})
|
||||
assert.Greater(t, len(jp.Snapshot().Errors), 0)
|
||||
assert.Error(t, jp.Snapshot().FatalError())
|
||||
assert.NoError(t, jp.Snapshot().ChunkError())
|
||||
assert.True(t, errors.Is(jp.Snapshot().FatalError(), err))
|
||||
}
|
||||
|
||||
func TestJobProgressRef(t *testing.T) {
|
||||
jp := NewJobProgress(123, 456)
|
||||
ref := jp.Ref()
|
||||
assert.Equal(t, int64(123), ref.SourceID)
|
||||
assert.Equal(t, int64(456), ref.JobID)
|
||||
|
||||
// Test Done() blocks until Finish() is called.
|
||||
select {
|
||||
case <-jp.Done():
|
||||
assert.FailNow(t, "job should not be finished")
|
||||
default:
|
||||
}
|
||||
|
||||
jp.Finish()
|
||||
select {
|
||||
case <-jp.Done():
|
||||
default:
|
||||
assert.FailNow(t, "job should be finished")
|
||||
}
|
||||
}
|
||||
|
||||
func TestJobProgressHook(t *testing.T) {
|
||||
ctrl := gomock.NewController(t)
|
||||
defer ctrl.Finish()
|
||||
|
||||
hook := NewMockJobProgressHook(ctrl)
|
||||
jp := NewJobProgress(123, 456, WithHooks(hook))
|
||||
|
||||
// Start(JobProgressRef, time.Time)
|
||||
// End(JobProgressRef, time.Time)
|
||||
// StartEnumerating(JobProgressRef, time.Time)
|
||||
// EndEnumerating(JobProgressRef, time.Time)
|
||||
// StartUnitChunking(JobProgressRef, SourceUnit, time.Time)
|
||||
// EndUnitChunking(JobProgressRef, SourceUnit, time.Time)
|
||||
// ReportError(JobProgressRef, error)
|
||||
// ReportUnit(JobProgressRef, SourceUnit)
|
||||
// ReportChunk(JobProgressRef, SourceUnit, *Chunk)
|
||||
// Finish(JobProgressRef)
|
||||
|
||||
startTime := time.Now()
|
||||
endTime := time.Now().Add(10 * time.Second)
|
||||
startEnum := time.Now().Add(20 * time.Second)
|
||||
endEnum := time.Now().Add(30 * time.Second)
|
||||
startChunk := time.Now().Add(40 * time.Second)
|
||||
endChunk := time.Now().Add(50 * time.Second)
|
||||
reportErr := fmt.Errorf("reporting error")
|
||||
reportUnit := CommonSourceUnit{"reporting unit"}
|
||||
reportChunk := &Chunk{Data: []byte("reporting chunk")}
|
||||
|
||||
hook.EXPECT().Start(gomock.Any(), startTime)
|
||||
hook.EXPECT().End(gomock.Any(), endTime)
|
||||
hook.EXPECT().StartEnumerating(gomock.Any(), startEnum)
|
||||
hook.EXPECT().EndEnumerating(gomock.Any(), endEnum)
|
||||
hook.EXPECT().StartUnitChunking(gomock.Any(), reportUnit, startChunk)
|
||||
hook.EXPECT().EndUnitChunking(gomock.Any(), reportUnit, endChunk)
|
||||
hook.EXPECT().ReportError(gomock.Any(), reportErr)
|
||||
hook.EXPECT().ReportUnit(gomock.Any(), reportUnit)
|
||||
hook.EXPECT().ReportChunk(gomock.Any(), reportUnit, reportChunk)
|
||||
hook.EXPECT().Finish(gomock.Any())
|
||||
|
||||
jp.Start(startTime)
|
||||
jp.End(endTime)
|
||||
jp.StartEnumerating(startEnum)
|
||||
jp.EndEnumerating(endEnum)
|
||||
jp.StartUnitChunking(reportUnit, startChunk)
|
||||
jp.EndUnitChunking(reportUnit, endChunk)
|
||||
jp.ReportError(reportErr)
|
||||
jp.ReportUnit(reportUnit)
|
||||
jp.ReportChunk(reportUnit, reportChunk)
|
||||
jp.Finish()
|
||||
}
|
||||
|
||||
func TestJobProgressDone(t *testing.T) {
|
||||
ref := JobProgressRef{}
|
||||
select {
|
||||
case <-ref.Done():
|
||||
default:
|
||||
assert.FailNow(t, "done should not block for a nil job")
|
||||
}
|
||||
}
|
||||
@@ -1,155 +0,0 @@
|
||||
// Code generated by MockGen. DO NOT EDIT.
|
||||
// Source: ./job_progress.go
|
||||
|
||||
// Package sources is a generated GoMock package.
|
||||
package sources
|
||||
|
||||
import (
|
||||
reflect "reflect"
|
||||
time "time"
|
||||
|
||||
gomock "go.uber.org/mock/gomock"
|
||||
)
|
||||
|
||||
// MockJobProgressHook is a mock of JobProgressHook interface.
|
||||
type MockJobProgressHook struct {
|
||||
ctrl *gomock.Controller
|
||||
recorder *MockJobProgressHookMockRecorder
|
||||
}
|
||||
|
||||
// MockJobProgressHookMockRecorder is the mock recorder for MockJobProgressHook.
|
||||
type MockJobProgressHookMockRecorder struct {
|
||||
mock *MockJobProgressHook
|
||||
}
|
||||
|
||||
// NewMockJobProgressHook creates a new mock instance.
|
||||
func NewMockJobProgressHook(ctrl *gomock.Controller) *MockJobProgressHook {
|
||||
mock := &MockJobProgressHook{ctrl: ctrl}
|
||||
mock.recorder = &MockJobProgressHookMockRecorder{mock}
|
||||
return mock
|
||||
}
|
||||
|
||||
// EXPECT returns an object that allows the caller to indicate expected use.
|
||||
func (m *MockJobProgressHook) EXPECT() *MockJobProgressHookMockRecorder {
|
||||
return m.recorder
|
||||
}
|
||||
|
||||
// End mocks base method.
|
||||
func (m *MockJobProgressHook) End(arg0 JobProgressRef, arg1 time.Time) {
|
||||
m.ctrl.T.Helper()
|
||||
m.ctrl.Call(m, "End", arg0, arg1)
|
||||
}
|
||||
|
||||
// End indicates an expected call of End.
|
||||
func (mr *MockJobProgressHookMockRecorder) End(arg0, arg1 interface{}) *gomock.Call {
|
||||
mr.mock.ctrl.T.Helper()
|
||||
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "End", reflect.TypeOf((*MockJobProgressHook)(nil).End), arg0, arg1)
|
||||
}
|
||||
|
||||
// EndEnumerating mocks base method.
|
||||
func (m *MockJobProgressHook) EndEnumerating(arg0 JobProgressRef, arg1 time.Time) {
|
||||
m.ctrl.T.Helper()
|
||||
m.ctrl.Call(m, "EndEnumerating", arg0, arg1)
|
||||
}
|
||||
|
||||
// EndEnumerating indicates an expected call of EndEnumerating.
|
||||
func (mr *MockJobProgressHookMockRecorder) EndEnumerating(arg0, arg1 interface{}) *gomock.Call {
|
||||
mr.mock.ctrl.T.Helper()
|
||||
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "EndEnumerating", reflect.TypeOf((*MockJobProgressHook)(nil).EndEnumerating), arg0, arg1)
|
||||
}
|
||||
|
||||
// EndUnitChunking mocks base method.
|
||||
func (m *MockJobProgressHook) EndUnitChunking(arg0 JobProgressRef, arg1 SourceUnit, arg2 time.Time) {
|
||||
m.ctrl.T.Helper()
|
||||
m.ctrl.Call(m, "EndUnitChunking", arg0, arg1, arg2)
|
||||
}
|
||||
|
||||
// EndUnitChunking indicates an expected call of EndUnitChunking.
|
||||
func (mr *MockJobProgressHookMockRecorder) EndUnitChunking(arg0, arg1, arg2 interface{}) *gomock.Call {
|
||||
mr.mock.ctrl.T.Helper()
|
||||
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "EndUnitChunking", reflect.TypeOf((*MockJobProgressHook)(nil).EndUnitChunking), arg0, arg1, arg2)
|
||||
}
|
||||
|
||||
// Finish mocks base method.
|
||||
func (m *MockJobProgressHook) Finish(arg0 JobProgressRef) {
|
||||
m.ctrl.T.Helper()
|
||||
m.ctrl.Call(m, "Finish", arg0)
|
||||
}
|
||||
|
||||
// Finish indicates an expected call of Finish.
|
||||
func (mr *MockJobProgressHookMockRecorder) Finish(arg0 interface{}) *gomock.Call {
|
||||
mr.mock.ctrl.T.Helper()
|
||||
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Finish", reflect.TypeOf((*MockJobProgressHook)(nil).Finish), arg0)
|
||||
}
|
||||
|
||||
// ReportChunk mocks base method.
|
||||
func (m *MockJobProgressHook) ReportChunk(arg0 JobProgressRef, arg1 SourceUnit, arg2 *Chunk) {
|
||||
m.ctrl.T.Helper()
|
||||
m.ctrl.Call(m, "ReportChunk", arg0, arg1, arg2)
|
||||
}
|
||||
|
||||
// ReportChunk indicates an expected call of ReportChunk.
|
||||
func (mr *MockJobProgressHookMockRecorder) ReportChunk(arg0, arg1, arg2 interface{}) *gomock.Call {
|
||||
mr.mock.ctrl.T.Helper()
|
||||
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ReportChunk", reflect.TypeOf((*MockJobProgressHook)(nil).ReportChunk), arg0, arg1, arg2)
|
||||
}
|
||||
|
||||
// ReportError mocks base method.
|
||||
func (m *MockJobProgressHook) ReportError(arg0 JobProgressRef, arg1 error) {
|
||||
m.ctrl.T.Helper()
|
||||
m.ctrl.Call(m, "ReportError", arg0, arg1)
|
||||
}
|
||||
|
||||
// ReportError indicates an expected call of ReportError.
|
||||
func (mr *MockJobProgressHookMockRecorder) ReportError(arg0, arg1 interface{}) *gomock.Call {
|
||||
mr.mock.ctrl.T.Helper()
|
||||
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ReportError", reflect.TypeOf((*MockJobProgressHook)(nil).ReportError), arg0, arg1)
|
||||
}
|
||||
|
||||
// ReportUnit mocks base method.
|
||||
func (m *MockJobProgressHook) ReportUnit(arg0 JobProgressRef, arg1 SourceUnit) {
|
||||
m.ctrl.T.Helper()
|
||||
m.ctrl.Call(m, "ReportUnit", arg0, arg1)
|
||||
}
|
||||
|
||||
// ReportUnit indicates an expected call of ReportUnit.
|
||||
func (mr *MockJobProgressHookMockRecorder) ReportUnit(arg0, arg1 interface{}) *gomock.Call {
|
||||
mr.mock.ctrl.T.Helper()
|
||||
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ReportUnit", reflect.TypeOf((*MockJobProgressHook)(nil).ReportUnit), arg0, arg1)
|
||||
}
|
||||
|
||||
// Start mocks base method.
|
||||
func (m *MockJobProgressHook) Start(arg0 JobProgressRef, arg1 time.Time) {
|
||||
m.ctrl.T.Helper()
|
||||
m.ctrl.Call(m, "Start", arg0, arg1)
|
||||
}
|
||||
|
||||
// Start indicates an expected call of Start.
|
||||
func (mr *MockJobProgressHookMockRecorder) Start(arg0, arg1 interface{}) *gomock.Call {
|
||||
mr.mock.ctrl.T.Helper()
|
||||
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Start", reflect.TypeOf((*MockJobProgressHook)(nil).Start), arg0, arg1)
|
||||
}
|
||||
|
||||
// StartEnumerating mocks base method.
|
||||
func (m *MockJobProgressHook) StartEnumerating(arg0 JobProgressRef, arg1 time.Time) {
|
||||
m.ctrl.T.Helper()
|
||||
m.ctrl.Call(m, "StartEnumerating", arg0, arg1)
|
||||
}
|
||||
|
||||
// StartEnumerating indicates an expected call of StartEnumerating.
|
||||
func (mr *MockJobProgressHookMockRecorder) StartEnumerating(arg0, arg1 interface{}) *gomock.Call {
|
||||
mr.mock.ctrl.T.Helper()
|
||||
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "StartEnumerating", reflect.TypeOf((*MockJobProgressHook)(nil).StartEnumerating), arg0, arg1)
|
||||
}
|
||||
|
||||
// StartUnitChunking mocks base method.
|
||||
func (m *MockJobProgressHook) StartUnitChunking(arg0 JobProgressRef, arg1 SourceUnit, arg2 time.Time) {
|
||||
m.ctrl.T.Helper()
|
||||
m.ctrl.Call(m, "StartUnitChunking", arg0, arg1, arg2)
|
||||
}
|
||||
|
||||
// StartUnitChunking indicates an expected call of StartUnitChunking.
|
||||
func (mr *MockJobProgressHookMockRecorder) StartUnitChunking(arg0, arg1, arg2 interface{}) *gomock.Call {
|
||||
mr.mock.ctrl.T.Helper()
|
||||
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "StartUnitChunking", reflect.TypeOf((*MockJobProgressHook)(nil).StartUnitChunking), arg0, arg1, arg2)
|
||||
}
|
||||
+11
-15
@@ -2,6 +2,7 @@ package s3
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
@@ -228,19 +229,19 @@ func (s *Source) pageChunker(ctx context.Context, client *s3.S3, chunksChan chan
|
||||
// ignore large files
|
||||
if *obj.Size > s.maxObjectSize {
|
||||
s.log.V(3).Info("Skipping %d byte file (over maxObjectSize limit)", "object", *obj.Key)
|
||||
continue
|
||||
return
|
||||
}
|
||||
|
||||
// file empty file
|
||||
if *obj.Size == 0 {
|
||||
s.log.V(5).Info("Skipping 0 byte file", "object", *obj.Key)
|
||||
continue
|
||||
return
|
||||
}
|
||||
|
||||
// skip incompatible extensions
|
||||
if common.SkipFile(*obj.Key) {
|
||||
s.log.V(5).Info("Skipping file with incompatible extension", "object", *obj.Key)
|
||||
continue
|
||||
return
|
||||
}
|
||||
|
||||
s.jobPool.Go(func() error {
|
||||
@@ -335,21 +336,16 @@ func (s *Source) pageChunker(ctx context.Context, client *s3.S3, chunksChan chan
|
||||
reader.Stop()
|
||||
|
||||
chunk := *chunkSkel
|
||||
chunkReader := sources.NewChunkReader()
|
||||
chunkResChan := chunkReader(ctx, reader)
|
||||
for data := range chunkResChan {
|
||||
chunk.Data = data.Bytes()
|
||||
if err := data.Error(); err != nil {
|
||||
s.log.Error(err, "error reading chunk.")
|
||||
continue
|
||||
}
|
||||
if err := common.CancellableWrite(ctx, chunksChan, &chunk); err != nil {
|
||||
return err
|
||||
}
|
||||
chunkData, err := io.ReadAll(reader)
|
||||
if err != nil {
|
||||
s.log.Error(err, "Could not read file data.")
|
||||
return nil
|
||||
}
|
||||
|
||||
atomic.AddUint64(objectCount, 1)
|
||||
s.log.V(5).Info("S3 object scanned.", "object_count", objectCount, "page_number", pageNumber)
|
||||
chunk.Data = chunkData
|
||||
chunksChan <- &chunk
|
||||
|
||||
nErr, ok = errorCount.Load(prefix)
|
||||
if !ok {
|
||||
nErr = 0
|
||||
|
||||
+64
-298
@@ -4,7 +4,6 @@ import (
|
||||
"fmt"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/context"
|
||||
@@ -19,30 +18,17 @@ type handle int64
|
||||
|
||||
// SourceInitFunc is a function that takes a source and job ID and returns an
|
||||
// initialized Source.
|
||||
type SourceInitFunc func(ctx context.Context, sourceID int64, jobID int64) (Source, error)
|
||||
|
||||
// sourceInfo is an aggregate struct to store source information provided on
|
||||
// initialization.
|
||||
type sourceInfo struct {
|
||||
initFunc SourceInitFunc
|
||||
name string
|
||||
}
|
||||
type SourceInitFunc func(sourceID int64, jobID int64) (Source, error)
|
||||
|
||||
type SourceManager struct {
|
||||
api apiClient
|
||||
hooks []JobProgressHook
|
||||
api apiClient
|
||||
// Map of handle to source initializer.
|
||||
handles map[handle]sourceInfo
|
||||
handles map[handle]SourceInitFunc
|
||||
handlesLock sync.Mutex
|
||||
// Pool limiting the amount of concurrent sources running.
|
||||
pool errgroup.Group
|
||||
concurrentUnits int
|
||||
// Run the sources using source unit enumeration / chunking if available.
|
||||
useSourceUnits bool
|
||||
pool errgroup.Group
|
||||
// Downstream chunks channel to be scanned.
|
||||
outputChunks chan *Chunk
|
||||
// Set when Wait() returns.
|
||||
done bool
|
||||
}
|
||||
|
||||
// apiClient is an interface for optionally communicating with an external API.
|
||||
@@ -53,59 +39,32 @@ type apiClient interface {
|
||||
GetJobID(ctx context.Context, id int64) (int64, error)
|
||||
}
|
||||
|
||||
// WithAPI adds an API client to the manager for tracking jobs and progress. If
|
||||
// the API is also a JobProgressHook, it will be added to the list of event hooks.
|
||||
// WithAPI adds an API client to the manager for tracking jobs and progress.
|
||||
func WithAPI(api apiClient) func(*SourceManager) {
|
||||
return func(mgr *SourceManager) { mgr.api = api }
|
||||
return func(man *SourceManager) { man.api = api }
|
||||
}
|
||||
|
||||
func WithReportHook(hook JobProgressHook) func(*SourceManager) {
|
||||
return func(mgr *SourceManager) {
|
||||
mgr.hooks = append(mgr.hooks, hook)
|
||||
}
|
||||
}
|
||||
|
||||
// WithConcurrentSources limits the concurrent number of sources a manager can run.
|
||||
func WithConcurrentSources(concurrency int) func(*SourceManager) {
|
||||
return func(mgr *SourceManager) { mgr.pool.SetLimit(concurrency) }
|
||||
}
|
||||
|
||||
// WithBufferedOutput sets the size of the buffer used for the Chunks() channel.
|
||||
func WithBufferedOutput(size int) func(*SourceManager) {
|
||||
return func(mgr *SourceManager) { mgr.outputChunks = make(chan *Chunk, size) }
|
||||
}
|
||||
|
||||
// WithSourceUnits enables using source unit enumeration and chunking if the
|
||||
// source supports it.
|
||||
func WithSourceUnits() func(*SourceManager) {
|
||||
return func(mgr *SourceManager) { mgr.useSourceUnits = true }
|
||||
}
|
||||
|
||||
// WithConcurrentUnits limits the number of units to be scanned concurrently.
|
||||
// The default is unlimited.
|
||||
func WithConcurrentUnits(n int) func(*SourceManager) {
|
||||
return func(mgr *SourceManager) { mgr.concurrentUnits = n }
|
||||
// WithConcurrency limits the concurrent number of sources a manager can run.
|
||||
func WithConcurrency(concurrency int) func(*SourceManager) {
|
||||
return func(man *SourceManager) { man.pool.SetLimit(concurrency) }
|
||||
}
|
||||
|
||||
// NewManager creates a new manager with the provided options.
|
||||
func NewManager(opts ...func(*SourceManager)) *SourceManager {
|
||||
mgr := SourceManager{
|
||||
func NewManager(outputChunks chan *Chunk, opts ...func(*SourceManager)) *SourceManager {
|
||||
man := SourceManager{
|
||||
// Default to the headless API. Can be overwritten by the WithAPI option.
|
||||
api: &headlessAPI{},
|
||||
handles: make(map[handle]sourceInfo),
|
||||
outputChunks: make(chan *Chunk),
|
||||
handles: make(map[handle]SourceInitFunc),
|
||||
outputChunks: outputChunks,
|
||||
}
|
||||
for _, opt := range opts {
|
||||
opt(&mgr)
|
||||
opt(&man)
|
||||
}
|
||||
return &mgr
|
||||
return &man
|
||||
}
|
||||
|
||||
// Enroll informs the SourceManager to track and manage a Source.
|
||||
func (s *SourceManager) Enroll(ctx context.Context, name string, kind sourcespb.SourceType, f SourceInitFunc) (handle, error) {
|
||||
if s.done {
|
||||
return 0, fmt.Errorf("manager is done")
|
||||
}
|
||||
id, err := s.api.RegisterSource(ctx, name, kind)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
@@ -117,232 +76,72 @@ func (s *SourceManager) Enroll(ctx context.Context, name string, kind sourcespb.
|
||||
// TODO: smartly handle this?
|
||||
return 0, fmt.Errorf("handle ID '%d' already in use", handleID)
|
||||
}
|
||||
s.handles[handleID] = sourceInfo{
|
||||
initFunc: f,
|
||||
name: name,
|
||||
}
|
||||
return handleID, nil
|
||||
s.handles[handleID] = f
|
||||
return 0, nil
|
||||
}
|
||||
|
||||
// Run blocks until a resource is available to run the source, then
|
||||
// synchronously runs it. The first fatal error, if any, will be returned.
|
||||
func (s *SourceManager) Run(ctx context.Context, handle handle) (JobProgressRef, error) {
|
||||
progress, err := s.asyncRun(ctx, handle)
|
||||
if err != nil {
|
||||
return progress, err
|
||||
// Run blocks until a resource is available to run the source, then synchronously runs it.
|
||||
func (s *SourceManager) Run(ctx context.Context, handle handle) error {
|
||||
// Check the handle is valid before waiting on the pool.
|
||||
if _, ok := s.getInitFunc(handle); !ok {
|
||||
return fmt.Errorf("unrecognized handle")
|
||||
}
|
||||
<-progress.Done()
|
||||
return progress, progress.Snapshot().FatalError()
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
ch := make(chan error)
|
||||
s.pool.Go(func() error {
|
||||
defer common.Recover(ctx)
|
||||
// TODO: The manager should record these errors.
|
||||
ch <- s.run(ctx, handle)
|
||||
return nil
|
||||
})
|
||||
return <-ch
|
||||
}
|
||||
|
||||
// ScheduleRun blocks until a resource is available to run the source, then
|
||||
// asynchronously runs it. Error information is stored and accessible via the
|
||||
// JobProgressRef as it becomes available.
|
||||
func (s *SourceManager) ScheduleRun(ctx context.Context, handle handle) (JobProgressRef, error) {
|
||||
return s.asyncRun(ctx, handle)
|
||||
}
|
||||
|
||||
// asyncRun is a helper method to asynchronously run the Source. It calls out
|
||||
// to the API to get a job ID for this run, creates a JobProgress object, then
|
||||
// waits for an available goroutine to asynchronously run it.
|
||||
func (s *SourceManager) asyncRun(ctx context.Context, handle handle) (JobProgressRef, error) {
|
||||
// Do preflight checks before waiting on the pool.
|
||||
if err := s.preflightChecks(ctx, handle); err != nil {
|
||||
return JobProgressRef{}, err
|
||||
}
|
||||
// Get a Job ID.
|
||||
ctx = context.WithValue(ctx, "source_id", int64(handle))
|
||||
jobID, err := s.api.GetJobID(ctx, int64(handle))
|
||||
if err != nil {
|
||||
return JobProgressRef{SourceID: int64(handle)}, err
|
||||
}
|
||||
// Create a JobProgress object for tracking progress.
|
||||
progress := NewJobProgress(int64(handle), jobID, WithHooks(s.hooks...))
|
||||
s.pool.Go(func() error {
|
||||
ctx := context.WithValues(ctx,
|
||||
"job_id", jobID,
|
||||
"source_manager_worker_id", common.RandomID(5),
|
||||
)
|
||||
defer common.Recover(ctx)
|
||||
return s.run(ctx, handle, jobID, progress)
|
||||
})
|
||||
return progress.Ref(), nil
|
||||
}
|
||||
|
||||
// Chunks returns the read only channel of all the chunks produced by all of
|
||||
// the sources managed by this manager.
|
||||
func (s *SourceManager) Chunks() <-chan *Chunk {
|
||||
return s.outputChunks
|
||||
}
|
||||
|
||||
// Wait blocks until all running sources are completed and closes the channel
|
||||
// returned by Chunks(). The manager should not be reused after calling this
|
||||
// method. This current implementation is not thread safe and should only be
|
||||
// called by one thread.
|
||||
func (s *SourceManager) Wait() error {
|
||||
// Check if the manager has been Waited.
|
||||
if s.done {
|
||||
return s.pool.Wait()
|
||||
}
|
||||
defer close(s.outputChunks)
|
||||
defer func() { s.done = true }()
|
||||
|
||||
// Return the first error returned by run.
|
||||
return s.pool.Wait()
|
||||
}
|
||||
|
||||
// preflightChecks is a helper method to check the Manager or the context isn't
|
||||
// done and that the handle is valid.
|
||||
func (s *SourceManager) preflightChecks(ctx context.Context, handle handle) error {
|
||||
// Check if the manager has been Waited.
|
||||
if s.done {
|
||||
return fmt.Errorf("manager is done")
|
||||
}
|
||||
// Check the handle is valid.
|
||||
if _, ok := s.getSourceInfo(handle); !ok {
|
||||
// asynchronously runs it. Error information is lost in this case.
|
||||
func (s *SourceManager) ScheduleRun(ctx context.Context, handle handle) error {
|
||||
// Check the handle is valid before waiting on the pool.
|
||||
if _, ok := s.getInitFunc(handle); !ok {
|
||||
return fmt.Errorf("unrecognized handle")
|
||||
}
|
||||
return ctx.Err()
|
||||
}
|
||||
|
||||
// run is a helper method to sychronously run the source. It does not check for
|
||||
// acquired resources. An error is returned if there was a fatal error during
|
||||
// the run. This information is also recorded in the JobProgress.
|
||||
func (s *SourceManager) run(ctx context.Context, handle handle, jobID int64, report *JobProgress) error {
|
||||
defer report.Finish()
|
||||
report.Start(time.Now())
|
||||
defer func() { report.End(time.Now()) }()
|
||||
|
||||
// Initialize the source.
|
||||
sourceInfo, ok := s.getSourceInfo(handle)
|
||||
if !ok {
|
||||
// Shouldn't happen due to preflight checks.
|
||||
err := fmt.Errorf("unrecognized handle")
|
||||
report.ReportError(Fatal{err})
|
||||
return Fatal{err}
|
||||
}
|
||||
source, err := sourceInfo.initFunc(ctx, int64(handle), jobID)
|
||||
if err != nil {
|
||||
report.ReportError(Fatal{err})
|
||||
return Fatal{err}
|
||||
}
|
||||
ctx = context.WithValues(ctx,
|
||||
"source_type", source.Type().String(),
|
||||
"source_name", sourceInfo.name,
|
||||
)
|
||||
// Check for the preferred method of tracking source units.
|
||||
if enumChunker, ok := source.(SourceUnitEnumChunker); ok && s.useSourceUnits {
|
||||
return s.runWithUnits(ctx, handle, enumChunker, report)
|
||||
}
|
||||
return s.runWithoutUnits(ctx, handle, source, report)
|
||||
}
|
||||
|
||||
// runWithoutUnits is a helper method to run a Source. It has coarse-grained
|
||||
// job reporting.
|
||||
func (s *SourceManager) runWithoutUnits(ctx context.Context, handle handle, source Source, report *JobProgress) error {
|
||||
// Introspect on the chunks we get from the Chunks method.
|
||||
ch := make(chan *Chunk, 1)
|
||||
var wg sync.WaitGroup
|
||||
// Consume chunks and export chunks.
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
for chunk := range ch {
|
||||
report.ReportChunk(nil, chunk)
|
||||
s.outputChunks <- chunk
|
||||
}
|
||||
}()
|
||||
// Don't return from this function until the goroutine has finished
|
||||
// outputting chunks to the downstream channel. Closing the channel
|
||||
// will stop the goroutine, so that needs to happen first in the defer
|
||||
// stack.
|
||||
defer wg.Wait()
|
||||
defer close(ch)
|
||||
if err := source.Chunks(ctx, ch); err != nil {
|
||||
report.ReportError(Fatal{err})
|
||||
return Fatal{err}
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
s.pool.Go(func() error {
|
||||
defer common.Recover(ctx)
|
||||
// TODO: The manager should record these errors.
|
||||
_ = s.run(ctx, handle)
|
||||
return nil
|
||||
})
|
||||
// TODO: Maybe wait for a signal here that initialization was successful?
|
||||
return nil
|
||||
}
|
||||
|
||||
// runWithUnits is a helper method to run a Source that is also a
|
||||
// SourceUnitEnumChunker. This allows better introspection of what is getting
|
||||
// scanned and any errors encountered.
|
||||
func (s *SourceManager) runWithUnits(ctx context.Context, handle handle, source SourceUnitEnumChunker, report *JobProgress) error {
|
||||
unitReporter := &mgrUnitReporter{
|
||||
unitCh: make(chan SourceUnit, 1),
|
||||
report: report,
|
||||
}
|
||||
// Create a function that will save the first error encountered (if
|
||||
// any) and discard the rest.
|
||||
fatalErr := make(chan error, 1)
|
||||
catchFirstFatal := func(err error) {
|
||||
select {
|
||||
case fatalErr <- err:
|
||||
default:
|
||||
}
|
||||
}
|
||||
// Produce units.
|
||||
go func() {
|
||||
// TODO: Catch panics and add to report.
|
||||
report.StartEnumerating(time.Now())
|
||||
defer func() { report.EndEnumerating(time.Now()) }()
|
||||
defer close(unitReporter.unitCh)
|
||||
ctx.Logger().V(2).Info("enumerating source")
|
||||
if err := source.Enumerate(ctx, unitReporter); err != nil {
|
||||
report.ReportError(Fatal{err})
|
||||
catchFirstFatal(Fatal{err})
|
||||
}
|
||||
}()
|
||||
var wg sync.WaitGroup
|
||||
// TODO: Maybe switch to using a semaphore.Weighted.
|
||||
var unitPool errgroup.Group
|
||||
if s.concurrentUnits != 0 {
|
||||
// Negative values indicated no limit.
|
||||
unitPool.SetLimit(s.concurrentUnits)
|
||||
}
|
||||
for unit := range unitReporter.unitCh {
|
||||
unit := unit
|
||||
chunkReporter := &mgrChunkReporter{
|
||||
unit: unit,
|
||||
chunkCh: make(chan *Chunk, 1),
|
||||
report: report,
|
||||
}
|
||||
// Consume units and produce chunks.
|
||||
unitPool.Go(func() error {
|
||||
report.StartUnitChunking(unit, time.Now())
|
||||
// TODO: Catch panics and add to report.
|
||||
defer close(chunkReporter.chunkCh)
|
||||
ctx := context.WithValue(ctx, "unit", unit.SourceUnitID())
|
||||
ctx.Logger().V(3).Info("chunking unit")
|
||||
if err := source.ChunkUnit(ctx, unit, chunkReporter); err != nil {
|
||||
report.ReportError(Fatal{err})
|
||||
catchFirstFatal(Fatal{err})
|
||||
}
|
||||
return nil
|
||||
})
|
||||
// Consume chunks and export chunks.
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
defer func() { report.EndUnitChunking(unit, time.Now()) }()
|
||||
for chunk := range chunkReporter.chunkCh {
|
||||
s.outputChunks <- chunk
|
||||
}
|
||||
}()
|
||||
}
|
||||
wg.Wait()
|
||||
select {
|
||||
case err := <-fatalErr:
|
||||
// run is a helper method to sychronously run the source. It does not check for
|
||||
// acquired resources.
|
||||
func (s *SourceManager) run(ctx context.Context, handle handle) error {
|
||||
jobID, err := s.api.GetJobID(ctx, int64(handle))
|
||||
if err != nil {
|
||||
return err
|
||||
default:
|
||||
return nil
|
||||
}
|
||||
initFunc, ok := s.getInitFunc(handle)
|
||||
if !ok {
|
||||
return fmt.Errorf("unrecognized handle")
|
||||
}
|
||||
source, err := initFunc(jobID, int64(handle))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// TODO: Support UnitChunker and SourceUnitEnumerator.
|
||||
// TODO: This is where we can introspect on the chunks collected.
|
||||
return source.Chunks(ctx, s.outputChunks)
|
||||
}
|
||||
|
||||
// getSourceInfo is a helper method for safe concurrent access to the
|
||||
// getInitFunc is a helper method for safe concurrent access to the
|
||||
// map[handle]SourceInitFunc map.
|
||||
func (s *SourceManager) getSourceInfo(handle handle) (sourceInfo, bool) {
|
||||
func (s *SourceManager) getInitFunc(handle handle) (SourceInitFunc, bool) {
|
||||
s.handlesLock.Lock()
|
||||
defer s.handlesLock.Unlock()
|
||||
f, ok := s.handles[handle]
|
||||
@@ -363,36 +162,3 @@ func (api *headlessAPI) RegisterSource(ctx context.Context, name string, kind so
|
||||
func (api *headlessAPI) GetJobID(ctx context.Context, id int64) (int64, error) {
|
||||
return atomic.AddInt64(&api.jobIDCounter, 1), nil
|
||||
}
|
||||
|
||||
// mgrUnitReporter implements the UnitReporter interface.
|
||||
type mgrUnitReporter struct {
|
||||
unitCh chan SourceUnit
|
||||
report *JobProgress
|
||||
}
|
||||
|
||||
func (s *mgrUnitReporter) UnitOk(ctx context.Context, unit SourceUnit) error {
|
||||
s.report.ReportUnit(unit)
|
||||
return common.CancellableWrite(ctx, s.unitCh, unit)
|
||||
}
|
||||
|
||||
func (s *mgrUnitReporter) UnitErr(ctx context.Context, err error) error {
|
||||
s.report.ReportError(err)
|
||||
return nil
|
||||
}
|
||||
|
||||
// mgrChunkReporter implements the ChunkReporter interface.
|
||||
type mgrChunkReporter struct {
|
||||
unit SourceUnit
|
||||
chunkCh chan *Chunk
|
||||
report *JobProgress
|
||||
}
|
||||
|
||||
func (s *mgrChunkReporter) ChunkOk(ctx context.Context, chunk Chunk) error {
|
||||
s.report.ReportChunk(s.unit, &chunk)
|
||||
return common.CancellableWrite(ctx, s.chunkCh, &chunk)
|
||||
}
|
||||
|
||||
func (s *mgrChunkReporter) ChunkErr(ctx context.Context, err error) error {
|
||||
s.report.ReportError(ChunkError{s.unit, err})
|
||||
return nil
|
||||
}
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user