Compare commits

..

2 Commits

Author SHA1 Message Date
Ahrav Dutta 52bb5d44f8 Set total count during enumeration. 2023-03-17 14:15:15 -07:00
Ahrav Dutta 825c751775 Remove progress object populating. 2023-03-17 14:04:23 -07:00
103 changed files with 3280 additions and 6567 deletions
+9 -4
View File
@@ -4,6 +4,8 @@ on:
push:
tags:
- v*
branches:
- main
pull_request:
permissions:
@@ -15,9 +17,9 @@ jobs:
name: lint
runs-on: ubuntu-latest
steps:
- uses: actions/setup-go@v4
- uses: actions/setup-go@v3
with:
go-version: '1.20'
go-version: '1.18'
- uses: actions/checkout@v3
- name: golangci-lint
uses: golangci/golangci-lint-action@v3
@@ -28,10 +30,13 @@ jobs:
# working-directory: somedir
# Optional: golangci-lint command line arguments.
args: --enable bodyclose --enable exportloopref --timeout 10m
args: --enable bodyclose --timeout 10m
# Optional: show only new issues if it's a pull request. The default value is `false`.
only-new-issues: true
# Optional: if set to true then the action don't cache or restore ~/go/pkg.
# skip-pkg-cache: true
# Optional: if set to true then the action don't cache or restore ~/.cache/go-build.
# skip-build-cache: true
# skip-build-cache: true
-88
View File
@@ -1,88 +0,0 @@
name: Performance Test
on: [pull_request]
jobs:
speed:
runs-on: ubuntu-latest
steps:
- name: Install Go
uses: actions/setup-go@v4
with:
go-version: '1.18'
- name: Checkout code
uses: actions/checkout@v3
with:
fetch-depth: 0
ref: ${{ github.head_ref }}
- name: Run Head
run: |
go build -o current .
repo_tmp=$(mktemp -d)
git clone https://github.com/trufflesecurity/trufflehog.git $repo_tmp
user_time_sum=0
for i in {1..5}
do
tmpfile=$(mktemp)
/usr/bin/time -o $tmpfile $GITHUB_WORKSPACE/current git "file://$repo_tmp" --no-verification --no-update > out.txt
cat $tmpfile
time_output=$(cat $tmpfile)
rm $tmpfile
user_time=$(echo $time_output | awk '{print $1}' | sed 's/user//')
# Add the user time to the sum
user_time_sum=$(echo "$user_time_sum + $user_time" | bc)
done
average_user_time=$(echo "scale=3; $user_time_sum / 5" | bc)
echo HEAD_TIME=$average_user_time >> $GITHUB_ENV
- name: Figure out previous tag
run: |
git fetch --tags
git tag -l --sort=-v:refname | head -n 1 > previous_tag.txt
echo PREVIOUS_TAG=$(cat previous_tag.txt) >> $GITHUB_ENV
- name: Checkout code
uses: actions/checkout@v3
with:
fetch-depth: 0
ref: ${{ env.PREVIOUS_TAG }}
- name: Run Previous
run: |
go build -o previous .
repo_tmp=$(mktemp -d)
git clone https://github.com/trufflesecurity/trufflehog.git $repo_tmp
user_time_sum=0
for i in {1..5}
do
tmpfile=$(mktemp)
/usr/bin/time -o $tmpfile $GITHUB_WORKSPACE/previous git "file://$repo_tmp" --no-verification --no-update > out.txt
cat $tmpfile
time_output=$(cat $tmpfile)
rm $tmpfile
user_time=$(echo $time_output | awk '{print $1}' | sed 's/user//')
# Add the user time to the sum
user_time_sum=$(echo "$user_time_sum + $user_time" | bc)
done
average_user_time=$(echo "scale=3; $user_time_sum / 5" | bc)
echo PREVIOUS_TIME=$average_user_time >> $GITHUB_ENV
- name: Compare Results
run: |
echo "head ($GITHUB_SHA) avg time (n=5): $HEAD_TIME"
echo "$PREVIOUS_TAG avg time (n=5): $PREVIOUS_TIME"
if [ $(echo "$HEAD_TIME > $PREVIOUS_TIME * 1.1" | bc) -eq 1 ]
then
echo "HEAD run time is at least 10% slower than PREVIOUS run time"
exit 1
fi
+2 -2
View File
@@ -33,9 +33,9 @@ jobs:
username: ${{ github.repository_owner }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Set up Go
uses: actions/setup-go@v4
uses: actions/setup-go@v3
with:
go-version: '1.20'
go-version: '1.18'
- name: Run GoReleaser
uses: goreleaser/goreleaser-action@v4
with:
+4 -5
View File
@@ -7,16 +7,16 @@ on:
- v*
branches:
- main
pull_request:
pull_request:
jobs:
test:
runs-on: ubuntu-latest
steps:
- name: Install Go
uses: actions/setup-go@v4
uses: actions/setup-go@v3
with:
go-version: '1.20'
go-version: '1.18'
- name: Checkout code
uses: actions/checkout@v3
with:
@@ -28,5 +28,4 @@ jobs:
with:
path: ./
base: ${{ github.event.repository.default_branch }}
head: HEAD
extra_args: --only-verified
head: HEAD
+1 -1
View File
@@ -9,7 +9,7 @@ jobs:
runs-on: ubuntu-latest
steps:
- name: Install Go
uses: actions/setup-go@v4
uses: actions/setup-go@v3
with:
go-version: '1.18'
- name: Checkout code
+2 -2
View File
@@ -16,9 +16,9 @@ jobs:
name: Run Snifftest
runs-on: ubuntu-latest
steps:
- uses: actions/setup-go@v4
- uses: actions/setup-go@v3
with:
go-version: '1.20'
go-version: '1.18'
- uses: actions/checkout@v3
- name: Run Snifftest
run: make snifftest
+4 -4
View File
@@ -17,9 +17,9 @@ jobs:
id-token: 'write'
steps:
- name: Install Go
uses: actions/setup-go@v4
uses: actions/setup-go@v3
with:
go-version: '1.20'
go-version: '1.18'
- name: Checkout code
uses: actions/checkout@v3
- id: 'auth'
@@ -36,9 +36,9 @@ jobs:
id-token: 'write'
steps:
- name: Install Go
uses: actions/setup-go@v4
uses: actions/setup-go@v3
with:
go-version: '1.20'
go-version: '1.18'
- name: Checkout code
uses: actions/checkout@v3
- id: 'auth'
+2 -2
View File
@@ -22,7 +22,7 @@ check:
go vet $(shell go list ./... | grep -v /vendor/)
lint:
golangci-lint run --enable bodyclose --enable exportloopref --out-format=colored-line-number --timeout 10m
golangci-lint run --enable bodyclose --out-format=colored-line-number --timeout 10m
test-failing:
CGO_ENABLED=0 go test -timeout=5m $(shell go list ./... | grep -v /vendor/) | grep FAIL
@@ -62,4 +62,4 @@ snifftest:
./hack/snifftest/snifftest.sh
test-release:
goreleaser release --rm-dist --skip-publish --snapshot
goreleaser release --rm-dist --skip-publish --snapshot
+86 -65
View File
@@ -17,23 +17,13 @@
---
# :mag_right: _Now Scanning_
<div align="center">
<img src="assets/scanning_logos.svg">
**...and more**
</div>
# :loudspeaker: Join Our Community
## Join The Slack
Have questions? Feedback? Jump in slack and hang out with us
https://join.slack.com/t/trufflehog-community/shared_invite/zt-pw2qbi43-Aa86hkiimstfdKH9UCpPzQ
# :tv: Demo
## Demo
![GitHub scanning demo](https://storage.googleapis.com/truffle-demos/non-interactive.svg)
@@ -41,30 +31,9 @@ https://join.slack.com/t/trufflehog-community/shared_invite/zt-pw2qbi43-Aa86hkii
docker run --rm -it -v "$PWD:/pwd" trufflesecurity/trufflehog:latest github --org=trufflesecurity
```
# :floppy_disk: Installation
## Examples
Several options available for you:
```bash
# MacOS users
brew install trufflesecurity/trufflehog/trufflehog
# Docker
docker run --rm -it -v "$PWD:/pwd" trufflesecurity/trufflehog:latest github --repo https://github.com/trufflesecurity/test_keys
# Docker for M1 and M2 Mac
docker run --platform linux/arm64 --rm -it -v "$PWD:/pwd" trufflesecurity/trufflehog:latest github --repo https://github.com/trufflesecurity/test_keys
# Binary releases
Download and unpack from https://github.com/trufflesecurity/trufflehog/releases
# Compile from source
git clone https://github.com/trufflesecurity/trufflehog.git
cd trufflehog; go install
```
# :rocket: Quick Start
## 1: Scan a repo for only verified secrets
### Example 1: Scan a repo for only verified secrets
Command:
@@ -89,13 +58,13 @@ Timestamp: 2022-06-16 10:17:40 -0700 PDT
...
```
## 2: Scan a GitHub Org for only verified secrets
### Example 2: Scan a GitHub Org for only verified secrets
```bash
trufflehog github --org=trufflesecurity --only-verified
```
## 3: Scan a GitHub Repo for only verified keys and get JSON output
### Example 3: Scan a GitHub Repo for only verified keys and get JSON output
Command:
@@ -109,31 +78,31 @@ Expected output:
...
```
## 4: Scan an S3 bucket for verified keys
### Example 4: Scan an S3 bucket for verified keys
```bash
trufflehog s3 --bucket=<bucket name> --only-verified
```
## 5: Scan a Github Repo using SSH authentication in docker
### Example 5: Scan a Github Repo using SSH authentication in docker
```bash
docker run --rm -v "$HOME/.ssh:/root/.ssh:ro" trufflesecurity/trufflehog:latest git ssh://github.com/trufflesecurity/test_keys
```
## 6: Scan individual files or directories
### Example 6: Scan individual files or directories
```bash
trufflehog filesystem path/to/file1.txt path/to/file2.txt path/to/dir
```
## 7: Scan GCS buckets for verified secrets.
### Example 7: Scan GCS buckets for verified secrets.
```bash
trufflehog gcs --project-id=<project-ID> --cloud-environment --only-verified
```
# :question: FAQ
### FAQ
+ All I see is `🐷🔑🐷 TruffleHog. Unearth your secrets. 🐷🔑🐷` and the program exits, what gives?
+ That means no secrets were detected
@@ -143,21 +112,66 @@ trufflehog gcs --project-id=<project-ID> --cloud-environment --only-verified
+ Check out our Driftwood blog post to learn how to do this, in short we've confirmed the key can be used live for SSH or SSL [Blog post](https://trufflesecurity.com/blog/driftwood-know-if-private-keys-are-sensitive/)
# :newspaper: What's new in v3?
# What's new in v3?
TruffleHog v3 is a complete rewrite in Go with many new powerful features.
- We've **added over 700 credential detectors that support active verification against their respective APIs**.
- We've also added native **support for scanning GitHub, GitLab, filesystems, S3, GCS and Circle CI**.
- We've also added native **support for scanning GitHub, GitLab, filesystems, S3, and Circle CI**.
- **Instantly verify private keys** against millions of github users and **billions** of TLS certificates using our [Driftwood](https://trufflesecurity.com/blog/driftwood) technology.
- Scan binaries and other file formats
- Available as a GitHub Action and a pre-commit hook
## What is credential verification?
For every potential credential that is detected, we've painstakingly implemented programmatic verification against the API that we think it belongs to. Verification eliminates false positives. For example, the [AWS credential detector](pkg/detectors/aws/aws.go) performs a `GetCallerIdentity` API call against the AWS API to verify if an AWS credential is active.
# :memo: Usage
## Installation
Several options:
### 1. Go
```bash
git clone https://github.com/trufflesecurity/trufflehog.git
cd trufflehog; go install
```
### 2. [Release binaries](https://github.com/trufflesecurity/trufflehog/releases)
### 3. Docker
> Note: Apple M1 hardware users should run with `docker run --rm --platform linux/arm64` for better performance.
#### **Most users**
```bash
docker run --rm -it -v "$PWD:/pwd" trufflesecurity/trufflehog:latest github --repo https://github.com/trufflesecurity/test_keys
```
#### **Apple M1 users**
The `linux/arm64` image is better to run on the M1 than the amd64 image.
Even better is running the native darwin binary available, but there is no container image for that.
```bash
docker run --rm --platform linux/arm64 -it -v "$PWD:/pwd" trufflesecurity/trufflehog:latest github --repo https://github.com/trufflesecurity/test_keys
```
### 4. Pip (help wanted)
It's possible to distribute binaries in pip wheels.
Here is an example of a [project that does it](https://github.com/Yelp/dumb-init).
Help with setting up this packaging would be appreciated!
### 5. Brew
```bash
brew install trufflesecurity/trufflehog/trufflehog
```
## Usage
TruffleHog has a sub-command for each source of data that you may want to scan:
@@ -211,10 +225,18 @@ Exit Codes:
- 1: An error was encountered. Sources may not have completed scans.
- 183: No errors were encountered, but results were found. Will only be returned if `--fail` flag is used.
# :octocat: TruffleHog Github Action
#### Scanning an organization
Try scanning an entire GitHub organization with the following:
```bash
docker run --rm -it -v "$PWD:/pwd" trufflesecurity/trufflehog:latest github --org=trufflesecurity
```
### TruffleHog OSS Github Action
```yaml
- name: TruffleHog
- name: TruffleHog OSS
uses: trufflesecurity/trufflehog@main
with:
# Repository path
@@ -232,7 +254,7 @@ any results are found.
For example, to scan the contents of pull requests you could use the following workflow:
```yaml
name: TruffleHog Secrets Scan
name: Leaked Secrets Scan
on: [pull_request]
jobs:
TruffleHog:
@@ -251,7 +273,7 @@ jobs:
extra_args: --debug --only-verified
```
# Precommit Hook
### Precommit Hook
Trufflehog can be used in a precommit hook to prevent credentials from leaking before they ever leave your computer.
An example `.pre-commit-config.yaml` is provided (see [pre-commit.com](https://pre-commit.com/) for installation).
@@ -270,7 +292,7 @@ repos:
stages: ["commit", "push"]
```
# Regex Detector (alpha)
## Regex Detector (alpha)
Trufflehog supports detection and verification of custom regular expressions.
For detection, at least one **regular expression** and **keyword** is required.
@@ -284,7 +306,7 @@ status code, the secret is considered verified.
**NB:** This feature is alpha and subject to change.
## Regex Detector Example
### Regex Detector Example
```yaml
# config.yaml
@@ -313,7 +335,7 @@ Raw result: hogs are cool
File: /tmp/hog-facts.txt
```
## Verification Server Example (Python)
#### Verification Server Example (Python)
Unless you run a verification server, secrets found by the custom regex
detector will be unverified. Here is an example Python implementation of a
@@ -363,7 +385,12 @@ with HTTPServer(('', 8000), Verifier) as server:
pass
```
# :heart: Contributors
## Use as a library
Currently, trufflehog is in heavy development and no guarantees can be made on
the stability of the public APIs at this time.
## Contributors
This project exists thanks to all the people who contribute. [[Contribute](CONTRIBUTING.md)].
@@ -373,28 +400,22 @@ This project exists thanks to all the people who contribute. [[Contribute](CONTR
</a>
# :computer: Contributing
## Contributing
Contributions are very welcome! Please see our [contribution guidelines first](CONTRIBUTING.md).
We no longer accept contributions to TruffleHog v2, but that code is available in the `v2` branch.
## Adding new secret detectors
### Adding new secret detectors
We have published some [documentation and tooling to get started on adding new secret detectors](hack/docs/Adding_Detectors_external.md). Let's improve detection together!
# Use as a library
Currently, trufflehog is in heavy development and no guarantees can be made on
the stability of the public APIs at this time.
# License Change
## License Change
Since v3.0, TruffleHog is released under a AGPL 3 license, included in [`LICENSE`](LICENSE). TruffleHog v3.0 uses none of the previous codebase, but care was taken to preserve backwards compatibility on the command line interface. The work previous to this release is still available licensed under GPL 2.0 in the history of this repository and the previous package releases and tags. A completed CLA is required for us to accept contributions going forward.
# :money_with_wings: Enterprise product
## Enterprise product
Are you interested in continously monitoring your Git, Jira, Slack, Confluence, etc.. for credentials? We have an enterprise product that can help. Reach out here to learn more https://trufflesecurity.com/contact/
-1
View File
@@ -32,5 +32,4 @@ runs:
- ${{ inputs.head }}
- --fail
- --no-update
- --github-actions
- ${{ inputs.extra_args }}
-1
View File
@@ -1 +0,0 @@
<svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24"><path d="M12 0c-6.626 0-12 5.373-12 12 0 5.302 3.438 9.8 8.207 11.387.599.111.793-.261.793-.577v-2.234c-3.338.726-4.033-1.416-4.033-1.416-.546-1.387-1.333-1.756-1.333-1.756-1.089-.745.083-.729.083-.729 1.205.084 1.839 1.237 1.839 1.237 1.07 1.834 2.807 1.304 3.492.997.107-.775.418-1.305.762-1.604-2.665-.305-5.467-1.334-5.467-5.931 0-1.311.469-2.381 1.236-3.221-.124-.303-.535-1.524.117-3.176 0 0 1.008-.322 3.301 1.23.957-.266 1.983-.399 3.003-.404 1.02.005 2.047.138 3.006.404 2.291-1.552 3.297-1.23 3.297-1.23.653 1.653.242 2.874.118 3.176.77.84 1.235 1.911 1.235 3.221 0 4.609-2.807 5.624-5.479 5.921.43.372.823 1.102.823 2.222v3.293c0 .319.192.694.801.576 4.765-1.589 8.199-6.086 8.199-11.386 0-6.627-5.373-12-12-12z"/></svg>

Before

Width:  |  Height:  |  Size: 814 B

-167
View File
@@ -1,167 +0,0 @@
<?xml version="1.0" encoding="UTF-8" standalone="no"?>
<!-- Created with Inkscape (http://www.inkscape.org/) -->
<svg
width="173.78879mm"
height="26.198219mm"
viewBox="0 0 173.78879 26.198219"
version="1.1"
id="svg5"
xml:space="preserve"
inkscape:version="1.2.2 (b0a84865, 2022-12-01)"
sodipodi:docname="scanning_logos.svg"
xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape"
xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd"
xmlns:xlink="http://www.w3.org/1999/xlink"
xmlns="http://www.w3.org/2000/svg"
xmlns:svg="http://www.w3.org/2000/svg"><sodipodi:namedview
id="namedview7"
pagecolor="#ffffff"
bordercolor="#999999"
borderopacity="1"
inkscape:showpageshadow="0"
inkscape:pageopacity="0"
inkscape:pagecheckerboard="0"
inkscape:deskcolor="#d1d1d1"
inkscape:document-units="mm"
showgrid="false"
inkscape:zoom="0.67348485"
inkscape:cx="376.40045"
inkscape:cy="538.98763"
inkscape:window-width="1250"
inkscape:window-height="906"
inkscape:window-x="1226"
inkscape:window-y="265"
inkscape:window-maximized="0"
inkscape:current-layer="layer1" /><defs
id="defs2"><style
id="style344">.cls-1{fill:#e24329;}.cls-2{fill:#fc6d26;}.cls-3{fill:#fca326;}</style><style
id="style433">
.cls-1 {
fill: #e25444;
}
.cls-1, .cls-2, .cls-3 {
fill-rule: evenodd;
}
.cls-2 {
fill: #7b1d13;
}
.cls-3 {
fill: #58150d;
}
</style><path
d="M 27.791,115.217 1.541,69.749 a 11.499,11.499 0 0 1 0,-11.499 l 26.25,-45.467 a 11.499,11.499 0 0 1 9.959,-5.75 h 52.5 a 11.5,11.5 0 0 1 9.96,5.75 l 26.25,45.467 a 11.499,11.499 0 0 1 0,11.5 l -26.25,45.466 a 11.499,11.499 0 0 1 -9.96,5.75 h -52.5 a 11.499,11.499 0 0 1 -9.959,-5.75 z"
id="b" /></defs><g
inkscape:label="Layer 1"
inkscape:groupmode="layer"
id="layer1"
transform="translate(-5.467857,-5.738096)"><g
id="g706"><path
fill="#f05133"
d="M 88.304634,18.167061 77.689818,7.5527709 c -0.611187,-0.611452 -1.602581,-0.611452 -2.214298,0 l -2.203979,2.204508 2.795852,2.7958521 c 0.649817,-0.219604 1.394883,-0.07223 1.912673,0.445823 0.520965,0.521229 0.667015,1.272646 0.441854,1.924844 l 2.695045,2.694781 c 0.652198,-0.224895 1.404144,-0.07938 1.924844,0.442119 0.727604,0.727604 0.727604,1.906588 0,2.634721 -0.728134,0.727869 -1.907117,0.727869 -2.635515,0 -0.547157,-0.547688 -0.682624,-1.352021 -0.405077,-2.026179 l -2.513541,-2.513277 v 6.613789 c 0.177271,0.08784 0.344753,0.204788 0.49239,0.352425 0.727604,0.727604 0.727604,1.906588 0,2.634986 -0.727604,0.727339 -1.907381,0.727339 -2.634456,0 -0.727604,-0.728663 -0.727604,-1.907646 0,-2.634986 0.179916,-0.179652 0.388144,-0.315648 0.610394,-0.406664 v -6.675438 c -0.22225,-0.09102 -0.429948,-0.225689 -0.610394,-0.406664 -0.551127,-0.550863 -0.683684,-1.359959 -0.401109,-2.036763 l -2.756164,-2.756694 -7.278688,7.277894 c -0.611452,0.611981 -0.611452,1.603375 0,2.214827 l 10.615348,10.614554 c 0.611188,0.611452 1.602317,0.611452 2.214298,0 L 88.304634,20.381888 c 0.61145,-0.611452 0.61145,-1.603375 0,-2.214827 z"
id="path79"
style="stroke-width:0.264583" /><path
d="m 17.874996,7.9032889 c -6.850808,0 -12.407139,5.3892601 -12.407139,12.0363631 0,5.318065 3.554648,9.829669 8.485453,11.421504 0.619321,0.111453 0.819902,-0.261967 0.819902,-0.578771 v -2.240776 c -3.451252,0.728214 -4.169831,-1.420293 -4.169831,-1.420293 -0.564526,-1.391177 -1.378228,-1.76129 -1.378228,-1.76129 -1.125947,-0.747278 0.08582,-0.731237 0.08582,-0.731237 1.245884,0.08414 1.901392,1.240753 1.901392,1.240753 1.106307,1.83957 2.902239,1.307954 3.610476,1.000022 0.110631,-0.777343 0.432184,-1.308957 0.787857,-1.608858 -2.75542,-0.305935 -5.652486,-1.338031 -5.652486,-5.948955 0,-1.314982 0.484911,-2.38821 1.277936,-3.230747 -0.128214,-0.304036 -0.553152,-1.528637 0.120985,-3.185643 0,0 1.0422,-0.322993 3.412995,1.233734 0.989472,-0.266877 2.050281,-0.400211 3.104887,-0.405253 1.054607,0.0033 2.116454,0.138376 3.107992,0.405253 2.368727,-1.556727 3.408859,-1.233734 3.408859,-1.233734 0.675156,1.657988 0.250214,2.882698 0.122,3.185643 0.796124,0.842537 1.276903,1.916758 1.276903,3.230747 0,4.62295 -2.902238,5.641012 -5.664896,5.938948 0.444591,0.373028 0.850923,1.105332 0.850923,2.228716 v 3.30297 c 0,0.319958 0.198515,0.696095 0.828178,0.577744 4.926667,-1.593809 8.477177,-6.104433 8.477177,-11.420477 0,-6.647103 -5.555295,-12.0363631 -12.407141,-12.0363631 z"
id="path286"
style="stroke-width:1.01836" /><g
id="LOGO"
transform="matrix(0.12753077,0,0,0.12866528,23.501297,-4.4119714)"><path
class="cls-1"
d="m 282.83,170.73 -0.27,-0.69 -26.14,-68.22 a 6.81,6.81 0 0 0 -2.69,-3.24 7,7 0 0 0 -8,0.43 7,7 0 0 0 -2.32,3.52 l -17.65,54 h -71.47 l -17.65,-54 a 6.86,6.86 0 0 0 -2.32,-3.53 7,7 0 0 0 -8,-0.43 6.87,6.87 0 0 0 -2.69,3.24 L 97.44,170 l -0.26,0.69 a 48.54,48.54 0 0 0 16.1,56.1 l 0.09,0.07 0.24,0.17 39.82,29.82 19.7,14.91 12,9.06 a 8.07,8.07 0 0 0 9.76,0 l 12,-9.06 19.7,-14.91 40.06,-30 0.1,-0.08 a 48.56,48.56 0 0 0 16.08,-56.04 z"
id="path348" /><path
class="cls-2"
d="m 282.83,170.73 -0.27,-0.69 a 88.3,88.3 0 0 0 -35.15,15.8 L 190,229.25 c 19.55,14.79 36.57,27.64 36.57,27.64 l 40.06,-30 0.1,-0.08 a 48.56,48.56 0 0 0 16.1,-56.08 z"
id="path350" /><path
class="cls-3"
d="m 153.43,256.89 19.7,14.91 12,9.06 a 8.07,8.07 0 0 0 9.76,0 l 12,-9.06 19.7,-14.91 c 0,0 -17.04,-12.89 -36.59,-27.64 -19.55,14.75 -36.57,27.64 -36.57,27.64 z"
id="path352" /><path
class="cls-2"
d="M 132.58,185.84 A 88.19,88.19 0 0 0 97.44,170 l -0.26,0.69 a 48.54,48.54 0 0 0 16.1,56.1 l 0.09,0.07 0.24,0.17 39.82,29.82 c 0,0 17,-12.85 36.57,-27.64 z"
id="path354" /></g><g
id="g481"
transform="matrix(0.06220237,0,0,0.05342262,92.880356,5.1809529)"><path
class="cls-1"
d="m 378,99 -83,158 83,158 34,-19 V 118 Z"
id="path437" /><path
class="cls-2"
d="M 378,99 212,118 127.5,257 212,396 378,415 Z"
id="path439" /><path
class="cls-3"
d="M 43,99 16,111 V 403 L 43,415 212,257 Z"
id="path441" /><path
class="cls-1"
d="m 42.637,98.667 169.587,47.111 V 372.444 L 42.637,415.111 Z"
id="path443" /><path
class="cls-3"
d="m 212.313,170.667 -72.008,-11.556 72.008,-81.778 71.83,81.778 z"
id="path445" /><path
class="cls-3"
d="m 284.143,159.111 -71.919,11.733 -71.919,-11.733 V 77.333"
id="path447" /><path
class="cls-3"
d="m 212.313,342.222 -72.008,13.334 72.008,70.222 71.83,-70.222 z"
id="path449" /><path
class="cls-2"
d="m 212,16 -72,38 v 105 l 72.224,-20.333 z"
id="path451" /><path
class="cls-2"
d="m 212.224,196.444 -71.919,7.823 v 104.838 l 71.919,8.228 z"
id="path453" /><path
class="cls-2"
d="M 212.224,373.333 140.305,355.3 V 458.363 L 212.224,496 Z"
id="path455" /><path
class="cls-1"
d="m 284.143,355.3 -71.919,18.038 V 496 l 71.919,-37.637 z"
id="path457" /><path
class="cls-1"
d="m 212.224,196.444 71.919,7.823 v 104.838 l -71.919,8.228 z"
id="path459" /><path
class="cls-1"
d="m 212,16 72,38 v 105 l -72,-20 z"
id="path461" /></g><g
id="g583"
transform="matrix(0.20423993,0,0,0.21790562,123.68704,6.5803506)"><g
transform="translate(0,-7.034)"
id="g549"><linearGradient
y2="120.789"
x2="64"
y1="7.0339999"
x1="64"
gradientUnits="userSpaceOnUse"
id="a"><stop
offset="0"
stop-color="#4387fd"
id="stop542" /><stop
offset="1"
stop-color="#4683ea"
id="stop544" /></linearGradient><path
d="M 27.79,115.217 1.54,69.749 a 11.499,11.499 0 0 1 0,-11.499 L 27.79,12.783 a 11.5,11.5 0 0 1 9.96,-5.75 h 52.5 a 11.5,11.5 0 0 1 9.959,5.75 l 26.25,45.467 a 11.499,11.499 0 0 1 0,11.5 l -26.25,45.466 a 11.5,11.5 0 0 1 -9.959,5.75 h -52.5 a 11.499,11.499 0 0 1 -9.96,-5.75 z"
fill="url(#a)"
id="path547"
style="fill:url(#a)" /></g><g
transform="translate(0,-7.034)"
id="g559"><defs
id="defs552"><path
d="M 27.791,115.217 1.541,69.749 a 11.499,11.499 0 0 1 0,-11.499 l 26.25,-45.467 a 11.499,11.499 0 0 1 9.959,-5.75 h 52.5 a 11.5,11.5 0 0 1 9.96,5.75 l 26.25,45.467 a 11.499,11.499 0 0 1 0,11.5 l -26.25,45.466 a 11.499,11.499 0 0 1 -9.96,5.75 h -52.5 a 11.499,11.499 0 0 1 -9.959,-5.75 z"
id="path575" /></defs><clipPath
id="c"><use
height="100%"
width="100%"
xlink:href="#b"
overflow="visible"
id="use554" /></clipPath><path
clip-path="url(#c)"
opacity="0.07"
d="m 49.313,53.875 -7.01,6.99 5.957,5.958 -5.898,10.476 44.635,44.636 10.816,0.002 L 118.936,84 85.489,50.55 Z"
id="path557" /></g><path
d="M 84.7,43.236 H 43.264 c -0.667,0 -1.212,0.546 -1.212,1.214 v 8.566 c 0,0.666 0.546,1.212 1.212,1.212 H 84.7 c 0.667,0 1.213,-0.546 1.213,-1.212 v -8.568 c 0,-0.666 -0.545,-1.213 -1.212,-1.213 m -6.416,7.976 a 2.484,2.484 0 0 1 -2.477,-2.48 2.475,2.475 0 0 1 2.477,-2.477 c 1.37,0 2.48,1.103 2.48,2.477 a 2.48,2.48 0 0 1 -2.48,2.48 m 6.415,8.491 -41.436,0.002 c -0.667,0 -1.212,0.546 -1.212,1.214 v 8.565 c 0,0.666 0.546,1.213 1.212,1.213 H 84.7 c 0.667,0 1.213,-0.547 1.213,-1.213 v -8.567 c 0,-0.666 -0.545,-1.214 -1.212,-1.214 m -6.416,7.976 a 2.483,2.483 0 0 1 -2.477,-2.48 2.475,2.475 0 0 1 2.477,-2.477 2.48,2.48 0 1 1 0,4.956"
fill="#ffffff"
id="path561" /></g><g
style="fill:#000000"
id="g668"
transform="matrix(1.0414114,0,0,1.0768849,154.45231,5.738096)"><path
d="m 8.963,12 c 0,-1.584 1.284,-2.855 2.855,-2.855 1.572,0 2.856,1.284 2.856,2.855 0,1.572 -1.284,2.856 -2.856,2.856 -1.57,0 -2.855,-1.284 -2.855,-2.856 z M 11.818,0 C 6.215,0 1.522,3.84 0.19,9.025 0.18,9.061 0.18,9.095 0.18,9.145 c 0,0.313 0.252,0.576 0.575,0.576 H 5.59 c 0.23,0 0.433,-0.13 0.517,-0.333 0.997,-2.16 3.18,-3.672 5.712,-3.672 3.466,0 6.286,2.82 6.286,6.287 0,3.47 -2.82,6.29 -6.29,6.29 -2.53,0 -4.714,-1.5 -5.71,-3.673 C 6.008,14.43 5.815,14.284 5.588,14.284 H 0.755 c -0.312,0 -0.575,0.253 -0.575,0.576 0,0.037 0.014,0.072 0.014,0.12 1.32,5.18 6.02,9.02 11.624,9.02 6.624,0 12,-5.375 12,-12 0,-6.623 -5.376,-12 -12,-12 z"
id="path659" /></g></g></g></svg>

Before

Width:  |  Height:  |  Size: 11 KiB

+1 -1
View File
@@ -3,4 +3,4 @@
# Parse the last argument into an array of extra_args.
mapfile -t extra_args < <(bash -c "for arg in ${*: -1}; do echo \$arg; done")
/usr/bin/trufflehog "${@: 1: $#-1}" "${extra_args[@]}"
/usr/bin/trufflehog "${@: 1: $#-1}" "${extra_args[@]}"
+34 -36
View File
@@ -1,29 +1,33 @@
module github.com/trufflesecurity/trufflehog/v3
go 1.20
go 1.18
replace github.com/jpillora/overseer => github.com/trufflesecurity/overseer v1.1.7-custom5
replace github.com/zricethezav/gitleaks/v8 => github.com/trufflesecurity/gitleaks/v8 v8.6.1-custom10
replace github.com/gitleaks/go-gitdiff => github.com/trufflesecurity/go-gitdiff v0.7.6-zombies2
require (
cloud.google.com/go/secretmanager v1.10.0
cloud.google.com/go/storage v1.30.1
cloud.google.com/go/storage v1.29.0
github.com/Azure/go-autorest/autorest/azure/auth v0.5.11
github.com/TheZeroSlave/zapsentry v1.15.0
github.com/TheZeroSlave/zapsentry v1.14.0
github.com/aws/aws-sdk-go v1.44.83
github.com/bill-rich/disk-buffer-reader v0.1.7
github.com/bill-rich/go-syslog v0.0.0-20220413021637-49edb52a574c
github.com/bitfinexcom/bitfinex-api-go v0.0.0-20210608095005-9e0b26f200fb
github.com/bradleyfalzon/ghinstallation/v2 v2.4.0
github.com/bradleyfalzon/ghinstallation/v2 v2.2.0
github.com/crewjam/rfc5424 v0.1.0
github.com/denisenkom/go-mssqldb v0.12.3
github.com/envoyproxy/protoc-gen-validate v1.0.0
github.com/envoyproxy/protoc-gen-validate v0.9.1
github.com/fatih/color v1.15.0
github.com/felixge/fgprof v0.9.3
github.com/getsentry/sentry-go v0.20.0
github.com/getsentry/sentry-go v0.19.0
github.com/go-errors/errors v1.4.2
github.com/go-git/go-git/v5 v5.6.1
github.com/go-git/go-git/v5 v5.6.0
github.com/go-ldap/ldap/v3 v3.4.4
github.com/go-logr/logr v1.2.4
github.com/go-logr/logr v1.2.3
github.com/go-logr/zapr v1.2.3
github.com/go-redis/redis v6.15.9+incompatible
github.com/go-sql-driver/mysql v1.7.0
@@ -38,27 +42,26 @@ require (
github.com/joho/godotenv v1.5.1
github.com/jpillora/overseer v1.1.6
github.com/kylelemons/godebug v1.1.0
github.com/lib/pq v1.10.9
github.com/lib/pq v1.10.7
github.com/mattn/go-sqlite3 v1.14.16
github.com/mholt/archiver/v4 v4.0.0-alpha.8
github.com/patrickmn/go-cache v2.1.0+incompatible
github.com/mholt/archiver/v4 v4.0.0-alpha.7
github.com/paulbellamy/ratecounter v0.2.0
github.com/petar-dambovaliev/aho-corasick v0.0.0-20211021192214-5ab2d9280aa9
github.com/pkg/errors v0.9.1
github.com/rabbitmq/amqp091-go v1.8.0
github.com/rabbitmq/amqp091-go v1.7.0
github.com/sergi/go-diff v1.3.1
github.com/stretchr/testify v1.8.2
github.com/tailscale/depaware v0.0.0-20210622194025-720c4b409502
github.com/xanzy/go-gitlab v0.83.0
go.mongodb.org/mongo-driver v1.11.4
github.com/xanzy/go-gitlab v0.81.0
go.mongodb.org/mongo-driver v1.11.2
go.uber.org/zap v1.24.0
golang.org/x/crypto v0.8.0
golang.org/x/crypto v0.7.0
golang.org/x/exp v0.0.0-20221018205818-5c77f4b2bbd7
golang.org/x/oauth2 v0.7.0
golang.org/x/oauth2 v0.6.0
golang.org/x/sync v0.1.0
golang.org/x/text v0.9.0
google.golang.org/api v0.114.0
google.golang.org/protobuf v1.30.0
golang.org/x/text v0.8.0
google.golang.org/api v0.111.0
google.golang.org/protobuf v1.29.1
gopkg.in/alecthomas/kingpin.v2 v2.2.6
gopkg.in/h2non/gock.v1 v1.1.2
sigs.k8s.io/yaml v1.3.0
@@ -78,24 +81,20 @@ require (
github.com/Azure/go-autorest/tracing v0.6.0 // indirect
github.com/Azure/go-ntlmssp v0.0.0-20220621081337-cb9428e4ac1e // indirect
github.com/Microsoft/go-winio v0.5.2 // indirect
github.com/ProtonMail/go-crypto v0.0.0-20230217124315-7d5c6f04bbb8 // indirect
github.com/acomagu/bufpipe v1.0.4 // indirect
github.com/ProtonMail/go-crypto v0.0.0-20221026131551-cf6655e29de4 // indirect
github.com/acomagu/bufpipe v1.0.3 // indirect
github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 // indirect
github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 // indirect
github.com/andybalholm/brotli v1.0.5 // indirect
github.com/andybalholm/brotli v1.0.4 // indirect
github.com/benbjohnson/clock v1.1.0 // indirect
github.com/bodgit/plumbing v1.2.0 // indirect
github.com/bodgit/sevenzip v1.3.0 // indirect
github.com/bodgit/windows v1.0.0 // indirect
github.com/cloudflare/circl v1.1.0 // indirect
github.com/connesc/cipherio v0.2.1 // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/dimchansky/utfbom v1.1.1 // indirect
github.com/dsnet/compress v0.0.1 // indirect
github.com/emirpasic/gods v1.18.1 // indirect
github.com/go-asn1-ber/asn1-ber v1.5.4 // indirect
github.com/go-git/gcfg v1.5.0 // indirect
github.com/go-git/go-billy/v5 v5.4.1 // indirect
github.com/go-git/go-billy/v5 v5.4.0 // indirect
github.com/go-ole/go-ole v1.2.6 // indirect
github.com/golang-jwt/jwt/v4 v4.5.0 // indirect
github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe // indirect
@@ -103,7 +102,7 @@ require (
github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e // indirect
github.com/golang/protobuf v1.5.3 // indirect
github.com/golang/snappy v0.0.4 // indirect
github.com/google/go-github/v52 v52.0.0 // indirect
github.com/google/go-github/v50 v50.1.0 // indirect
github.com/google/go-querystring v1.1.0 // indirect
github.com/google/pprof v0.0.0-20211214055906-6f57359322fd // indirect
github.com/google/uuid v1.3.0 // indirect
@@ -117,7 +116,7 @@ require (
github.com/jmespath/go-jmespath v0.4.0 // indirect
github.com/jpillora/s3 v1.1.4 // indirect
github.com/kevinburke/ssh_config v1.2.0 // indirect
github.com/klauspost/compress v1.16.0 // indirect
github.com/klauspost/compress v1.15.11 // indirect
github.com/klauspost/pgzip v1.2.5 // indirect
github.com/mattn/go-colorable v0.1.13 // indirect
github.com/mattn/go-isatty v0.0.17 // indirect
@@ -126,7 +125,7 @@ require (
github.com/nwaples/rardecode/v2 v2.0.0-beta.2 // indirect
github.com/onsi/ginkgo v1.16.5 // indirect
github.com/onsi/gomega v1.23.0 // indirect
github.com/pierrec/lz4/v4 v4.1.15 // indirect
github.com/pierrec/lz4/v4 v4.1.14 // indirect
github.com/pjbgf/sha1cd v0.3.0 // indirect
github.com/pkg/diff v0.0.0-20200914180035-5b29258ca4f7 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
@@ -142,15 +141,14 @@ require (
go.opencensus.io v0.24.0 // indirect
go.uber.org/atomic v1.7.0 // indirect
go.uber.org/multierr v1.6.0 // indirect
go4.org v0.0.0-20200411211856-f5505b9728dd // indirect
golang.org/x/mod v0.10.0 // indirect
golang.org/x/net v0.9.0 // indirect
golang.org/x/sys v0.7.0 // indirect
golang.org/x/mod v0.8.0 // indirect
golang.org/x/net v0.8.0 // indirect
golang.org/x/sys v0.6.0 // indirect
golang.org/x/time v0.3.0 // indirect
golang.org/x/tools v0.8.0 // indirect
golang.org/x/tools v0.6.0 // indirect
golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect
google.golang.org/appengine v1.6.7 // indirect
google.golang.org/genproto v0.0.0-20230320184635-7606e756e683 // indirect
google.golang.org/genproto v0.0.0-20230303212802-e74f57abe488 // indirect
google.golang.org/grpc v1.53.0 // indirect
gopkg.in/warnings.v0 v0.1.2 // indirect
gopkg.in/yaml.v2 v2.4.0 // indirect
+66 -235
View File
@@ -1,34 +1,17 @@
cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU=
cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU=
cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY=
cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc=
cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0=
cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To=
cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M=
cloud.google.com/go v0.110.0 h1:Zc8gqp3+a9/Eyph2KDmcGaPtbKRIoqq4YTlL4NMD0Ys=
cloud.google.com/go v0.110.0/go.mod h1:SJnCLqQ0FCFGSZMUNUf84MV3Aia54kn7pi8st7tMzaY=
cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o=
cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE=
cloud.google.com/go/compute v1.18.0 h1:FEigFqoDbys2cvFkZ9Fjq4gnHBP55anJ0yQyau2f9oY=
cloud.google.com/go/compute v1.18.0/go.mod h1:1X7yHxec2Ga+Ss6jPyjxRxpu2uu7PLgsOVXvgU0yacs=
cloud.google.com/go/compute/metadata v0.2.0/go.mod h1:zFmK7XCadkQkj6TtorcaGlCW1hT1fIilQDwofLpJ20k=
cloud.google.com/go/compute/metadata v0.2.3 h1:mg4jlk7mCAj6xXp9UJ4fjI9VUI5rubuGBW5aJ7UnBMY=
cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA=
cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE=
cloud.google.com/go/iam v0.12.0 h1:DRtTY29b75ciH6Ov1PHb4/iat2CLCvrOm40Q0a6DFpE=
cloud.google.com/go/iam v0.12.0/go.mod h1:knyHGviacl11zrtZUoDuYpDgLjvr28sLQaG0YB2GYAY=
cloud.google.com/go/longrunning v0.4.1 h1:v+yFJOfKC3yZdY6ZUI933pIYdhyhV8S3NpWrXWmg7jM=
cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I=
cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw=
cloud.google.com/go/secretmanager v1.10.0 h1:pu03bha7ukxF8otyPKTFdDz+rr9sE3YauS5PliDXK60=
cloud.google.com/go/secretmanager v1.10.0/go.mod h1:MfnrdvKMPNra9aZtQFvBcvRU54hbPD8/HayQdlUgJpU=
cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw=
cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos=
cloud.google.com/go/storage v1.30.1 h1:uOdMxAs8HExqBlnLtnQyP0YkvbiDpdGShGKtx6U/oNM=
cloud.google.com/go/storage v1.30.1/go.mod h1:NfxhC0UJE1aXSx7CIIbCf7y9HKT7BiccwkR7+P7gN8E=
dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
cloud.google.com/go/storage v1.29.0 h1:6weCgzRvMg7lzuUurI4697AqIRPU1SvzHhynwpW31jI=
cloud.google.com/go/storage v1.29.0/go.mod h1:4puEjyTKnku6gfKoTfNOU/W+a9JyuVNxjpS5GBrB8h4=
github.com/Azure/azure-sdk-for-go/sdk/azcore v0.19.0/go.mod h1:h6H6c8enJmmocHUbLiiGY6sx7f9i+X3m1CHdd5c6Rdw=
github.com/Azure/azure-sdk-for-go/sdk/azidentity v0.11.0/go.mod h1:HcM1YX14R7CJcghJGOYCgdezslRSVzqwLf/q+4Y2r/0=
github.com/Azure/azure-sdk-for-go/sdk/internal v0.7.0/go.mod h1:yqy467j36fJxcRV2TzfVZ1pCb5vxm4BtZPUdYWe/Xo8=
@@ -53,21 +36,20 @@ github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBp
github.com/Azure/go-ntlmssp v0.0.0-20220621081337-cb9428e4ac1e h1:NeAW1fUYUEWhft7pkxDf6WoUvEZJ/uOKsvtpjLnn8MU=
github.com/Azure/go-ntlmssp v0.0.0-20220621081337-cb9428e4ac1e/go.mod h1:chxPXzSsl7ZWRAuOIE23GDNzjWuZquvFlgA8xmpunjU=
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
github.com/Microsoft/go-winio v0.5.2 h1:a9IhgEQBCUEk6QCdml9CiJGhAws+YwffDHEMp1VMrpA=
github.com/Microsoft/go-winio v0.5.2/go.mod h1:WpS1mjBmmwHBEWmogvA2mj8546UReBk4v8QkMxJ6pZY=
github.com/ProtonMail/go-crypto v0.0.0-20230217124315-7d5c6f04bbb8 h1:wPbRQzjjwFc0ih8puEVAOFGELsn1zoIIYdxvML7mDxA=
github.com/ProtonMail/go-crypto v0.0.0-20230217124315-7d5c6f04bbb8/go.mod h1:I0gYDMZ6Z5GRU7l58bNFSkPTFN6Yl12dsUlAZ8xy98g=
github.com/TheZeroSlave/zapsentry v1.15.0 h1:w/YglG4Hc2L2VoEH6JKC+3YzhUL18/OZRjJRSvyXZp4=
github.com/TheZeroSlave/zapsentry v1.15.0/go.mod h1:D1YMfSuu6xnkhwFXxrronesmsiyDhIqo+86I3Ok+r64=
github.com/acomagu/bufpipe v1.0.4 h1:e3H4WUzM3npvo5uv95QuJM3cQspFNtFBzvJ2oNjKIDQ=
github.com/acomagu/bufpipe v1.0.4/go.mod h1:mxdxdup/WdsKVreO5GpW4+M/1CE2sMG4jeGJ2sYmHc4=
github.com/ProtonMail/go-crypto v0.0.0-20221026131551-cf6655e29de4 h1:ra2OtmuW0AE5csawV4YXMNGNQQXvLRps3z2Z59OPO+I=
github.com/ProtonMail/go-crypto v0.0.0-20221026131551-cf6655e29de4/go.mod h1:UBYPn8k0D56RtnR8RFQMjmh4KrZzWJ5o7Z9SYjossQ8=
github.com/TheZeroSlave/zapsentry v1.14.0 h1:z6H0lIyKVpdXsP5K8J11b4/+V+QCDLDslZmtXsnduMk=
github.com/TheZeroSlave/zapsentry v1.14.0/go.mod h1:0hUR1Et5tSroEYhWOZcxE2iBqcFkuG7jGGlUmOA7XAs=
github.com/acomagu/bufpipe v1.0.3 h1:fxAGrHZTgQ9w5QqVItgzwj235/uYZYgbXitB+dLupOk=
github.com/acomagu/bufpipe v1.0.3/go.mod h1:mxdxdup/WdsKVreO5GpW4+M/1CE2sMG4jeGJ2sYmHc4=
github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 h1:JYp7IbQjafoB+tBA3gMyHYHrpOtNuDiK/uB5uXxq5wM=
github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 h1:s6gZFSlWYmbqAuRjVTiNNhvNRfY2Wxp9nhfyel4rklc=
github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137/go.mod h1:OMCwj8VM1Kc9e19TLln2VL61YJF0x1XFtfdL4JdbSyE=
github.com/andybalholm/brotli v1.0.5 h1:8uQZIdzKmjc/iuPu7O2ioW48L81FgatrcpfFmiq/cCs=
github.com/andybalholm/brotli v1.0.5/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig=
github.com/andybalholm/brotli v1.0.4 h1:V7DdXeJtZscaqfNuAdSRuRFzuiKlHSC/Zh3zl9qY3JY=
github.com/andybalholm/brotli v1.0.4/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig=
github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be h1:9AeTilPcZAjCFIImctFaOjnTIavg87rW78vTPkQqLI8=
github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be/go.mod h1:ySMOLuWl6zY27l47sB3qLNK6tF2fkHG55UZxx8oIVo4=
github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio=
@@ -82,14 +64,8 @@ github.com/bill-rich/go-syslog v0.0.0-20220413021637-49edb52a574c h1:tSME5FDS02q
github.com/bill-rich/go-syslog v0.0.0-20220413021637-49edb52a574c/go.mod h1:+sCc6hztur+oZCLOsNk6wCCy+GLrnSNHSRmTnnL+8iQ=
github.com/bitfinexcom/bitfinex-api-go v0.0.0-20210608095005-9e0b26f200fb h1:9v7Bzlg+1EBYi2IYcUmOwHReBEfqBbYIj3ZCi9cIe1Q=
github.com/bitfinexcom/bitfinex-api-go v0.0.0-20210608095005-9e0b26f200fb/go.mod h1:EkOqCuelvo7DY8vCOoZ09p7pHvAK9B1PHI9GeM4Rdxc=
github.com/bodgit/plumbing v1.2.0 h1:gg4haxoKphLjml+tgnecR4yLBV5zo4HAZGCtAh3xCzM=
github.com/bodgit/plumbing v1.2.0/go.mod h1:b9TeRi7Hvc6Y05rjm8VML3+47n4XTZPtQ/5ghqic2n8=
github.com/bodgit/sevenzip v1.3.0 h1:1ljgELgtHqvgIp8W8kgeEGHIWP4ch3xGI8uOBZgLVKY=
github.com/bodgit/sevenzip v1.3.0/go.mod h1:omwNcgZTEooWM8gA/IJ2Nk/+ZQ94+GsytRzOJJ8FBlM=
github.com/bodgit/windows v1.0.0 h1:rLQ/XjsleZvx4fR1tB/UxQrK+SJ2OFHzfPjLWWOhDIA=
github.com/bodgit/windows v1.0.0/go.mod h1:a6JLwrB4KrTR5hBpp8FI9/9W9jJfeQ2h4XDXU74ZCdM=
github.com/bradleyfalzon/ghinstallation/v2 v2.4.0 h1:zYSzkoIwekCQAr6GT6KxISLt4YRS6kd4/ixfzMN+7yc=
github.com/bradleyfalzon/ghinstallation/v2 v2.4.0/go.mod h1:4MwZLSgBJJgg4i3nJwZJ95AMooSqN8fJDmegLVn9Q2U=
github.com/bradleyfalzon/ghinstallation/v2 v2.2.0 h1:AVvVU33rE8wdTS1aNnenwpigEBA9mvzI5OhjhZfH/LU=
github.com/bradleyfalzon/ghinstallation/v2 v2.2.0/go.mod h1:xo3iIfK0lDKECe0s19nbxT0KKvk7LsrGc4NxR5ckKMA=
github.com/bwesterb/go-ristretto v1.2.0/go.mod h1:fUIoIZaG73pV5biE2Blr2xEzDoMj7NFEuV9ekS419A0=
github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
@@ -99,8 +75,6 @@ github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDk
github.com/cloudflare/circl v1.1.0 h1:bZgT/A+cikZnKIwn7xL2OBj012Bmvho/o6RpRvv3GKY=
github.com/cloudflare/circl v1.1.0/go.mod h1:prBCrKB9DV4poKZY1l9zBXg2QJY7mvgRvtMxxK7fi4I=
github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
github.com/connesc/cipherio v0.2.1 h1:FGtpTPMbKNNWByNrr9aEBtaJtXjqOzkIXNYJp6OEycw=
github.com/connesc/cipherio v0.2.1/go.mod h1:ukY0MWJDFnJEbXMQtOcn2VmTpRfzcTz4OoVrWGGJZcA=
github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
github.com/crewjam/rfc5424 v0.1.0 h1:MSeXJm22oKovLzWj44AHwaItjIMUMugYGkEzfa831H8=
github.com/crewjam/rfc5424 v0.1.0/go.mod h1:RCi9M3xHVOeerf6ULZzqv2xOGRO/zYaVUeRyPnBW3gQ=
@@ -121,8 +95,8 @@ github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymF
github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
github.com/envoyproxy/protoc-gen-validate v1.0.0 h1:FPFO7LWZ2pfphahSUMX8L5p/6FqSzRYRxq6V74eG8ZI=
github.com/envoyproxy/protoc-gen-validate v1.0.0/go.mod h1:0vj8bNkYbSTNS2PIyH87KZaeN4x9zpL9Qt8fQC7d+vs=
github.com/envoyproxy/protoc-gen-validate v0.9.1 h1:PS7VIOgmSVhWUEeZwTe7z7zouA22Cr590PzXKbZHOVY=
github.com/envoyproxy/protoc-gen-validate v0.9.1/go.mod h1:OKNgG7TCp5pF4d6XftA0++PMirau2/yoOwVac3AbF2w=
github.com/fatih/color v1.15.0 h1:kOqh6YHBtK8aywxGerMG2Eq3H6Qgoqeo13Bk2Mv/nBs=
github.com/fatih/color v1.15.0/go.mod h1:0h5ZqXfHYED7Bhv2ZJamyIOUej9KtShiJESRwBDUSsw=
github.com/felixge/fgprof v0.9.3 h1:VvyZxILNuCiUCSXtPtYmmtGvb65nqXh2QFWc0Wpf2/g=
@@ -130,8 +104,8 @@ github.com/felixge/fgprof v0.9.3/go.mod h1:RdbpDgzqYVh/T9fPELJyV7EYJuHB55UTEULNu
github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4=
github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
github.com/getsentry/sentry-go v0.20.0 h1:bwXW98iMRIWxn+4FgPW7vMrjmbym6HblXALmhjHmQaQ=
github.com/getsentry/sentry-go v0.20.0/go.mod h1:lc76E2QywIyW8WuBnwl8Lc4bkmQH4+w1gwTf25trprY=
github.com/getsentry/sentry-go v0.19.0 h1:BcCH3CN5tXt5aML+gwmbFwVptLLQA+eT866fCO9wVOM=
github.com/getsentry/sentry-go v0.19.0/go.mod h1:y3+lGEFEFexZtpbG1GUE2WD/f9zGyKYwpEqryTOC/nE=
github.com/gliderlabs/ssh v0.3.5 h1:OcaySEmAQJgyYcArR+gGGTHCyE7nvhEMTlYY+Dp8CpY=
github.com/gliderlabs/ssh v0.3.5/go.mod h1:8XB4KraRrX39qHhT6yxPsHedjA08I/uBVwj4xC+/+z4=
github.com/go-asn1-ber/asn1-ber v1.5.4 h1:vXT6d/FNDiELJnLb6hGNa309LMsrCoYFvpwHDF0+Y1A=
@@ -141,19 +115,17 @@ github.com/go-errors/errors v1.4.2/go.mod h1:sIVyrIiJhuEF+Pj9Ebtd6P/rEYROXFi3Bop
github.com/go-git/gcfg v1.5.0 h1:Q5ViNfGF8zFgyJWPqYwA7qGFoMTEiBmdlkcfRmpIMa4=
github.com/go-git/gcfg v1.5.0/go.mod h1:5m20vg6GwYabIxaOonVkTdrILxQMpEShl1xiMF4ua+E=
github.com/go-git/go-billy/v5 v5.3.1/go.mod h1:pmpqyWchKfYfrkb/UVH4otLvyi/5gJlGI4Hb3ZqZ3W0=
github.com/go-git/go-billy/v5 v5.4.1 h1:Uwp5tDRkPr+l/TnbHOQzp+tmJfLceOlbVucgpTz8ix4=
github.com/go-git/go-billy/v5 v5.4.1/go.mod h1:vjbugF6Fz7JIflbVpl1hJsGjSHNltrSw45YK/ukIvQg=
github.com/go-git/go-billy/v5 v5.4.0 h1:Vaw7LaSTRJOUric7pe4vnzBSgyuf2KrLsu2Y4ZpQBDE=
github.com/go-git/go-billy/v5 v5.4.0/go.mod h1:vjbugF6Fz7JIflbVpl1hJsGjSHNltrSw45YK/ukIvQg=
github.com/go-git/go-git-fixtures/v4 v4.3.1 h1:y5z6dd3qi8Hl+stezc8p3JxDkoTRqMAlKnXHuzrfjTQ=
github.com/go-git/go-git-fixtures/v4 v4.3.1/go.mod h1:8LHG1a3SRW71ettAD/jW13h8c6AqjVSeL11RAdgaqpo=
github.com/go-git/go-git/v5 v5.6.1 h1:q4ZRqQl4pR/ZJHc1L5CFjGA1a10u76aV1iC+nh+bHsk=
github.com/go-git/go-git/v5 v5.6.1/go.mod h1:mvyoL6Unz0PiTQrGQfSfiLFhBH1c1e84ylC2MDs4ee8=
github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
github.com/go-git/go-git/v5 v5.6.0 h1:JvBdYfcttd+0kdpuWO7KTu0FYgCf5W0t5VwkWGobaa4=
github.com/go-git/go-git/v5 v5.6.0/go.mod h1:6nmJ0tJ3N4noMV1Omv7rC5FG3/o8Cm51TB4CJp7mRmE=
github.com/go-ldap/ldap/v3 v3.4.4 h1:qPjipEpt+qDa6SI/h1fzuGWoRUY+qqQ9sOZq67/PYUs=
github.com/go-ldap/ldap/v3 v3.4.4/go.mod h1:fe1MsuN5eJJ1FeLT/LEBVdWfNWKh459R7aXgXtJC+aI=
github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ=
github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
github.com/go-logr/logr v1.2.3 h1:2DntVwHkVopvECVRSlL5PSo9eG+cAkDCuckLubN+rq0=
github.com/go-logr/logr v1.2.3/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
github.com/go-logr/zapr v1.2.3 h1:a9vnzlIBPQBBkeaR9IuMUfmVOrQlkoC4YfPoFkX3T7A=
github.com/go-logr/zapr v1.2.3/go.mod h1:eIauM6P8qSvTw5o2ez6UEAfGjQKrxQTl5EoK+Qa2oG4=
github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY=
@@ -179,20 +151,12 @@ github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe/go.mod h1:8vg3r2V
github.com/golang-sql/sqlexp v0.1.0 h1:ZCD6MBpcuOVfGVqsEmY5/4FtYiKz6tSyUv9LPEDei6A=
github.com/golang-sql/sqlexp v0.1.0/go.mod h1:J4ad9Vo8ZCWQ2GMrC4UCQy1JpCbwU9m3EOqtpKwwwHI=
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e h1:1r7pUrabqp18hOBcwBwiTsbnFeTZHV9eER/QT5JVZxY=
github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y=
github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw=
github.com/golang/mock v1.4.3 h1:GV+pQPG/EUUbkh47niozDcADz6go/dUwhVzdUQHIVRw=
github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw=
github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw=
github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8=
github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA=
github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs=
@@ -202,14 +166,11 @@ github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QD
github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg=
github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM=
github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
@@ -218,31 +179,22 @@ github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/
github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
github.com/google/go-github/v42 v42.0.0 h1:YNT0FwjPrEysRkLIiKuEfSvBPCGKphW5aS5PxwaoLec=
github.com/google/go-github/v42 v42.0.0/go.mod h1:jgg/jvyI0YlDOM1/ps6XYh04HNQ3vKf0CVko62/EhRg=
github.com/google/go-github/v52 v52.0.0 h1:uyGWOY+jMQ8GVGSX8dkSwCzlehU3WfdxQ7GweO/JP7M=
github.com/google/go-github/v52 v52.0.0/go.mod h1:WJV6VEEUPuMo5pXqqa2ZCZEdbQqua4zAk2MZTIo+m+4=
github.com/google/go-github/v50 v50.1.0 h1:hMUpkZjklC5GJ+c3GquSqOP/T4BNsB7XohaPhtMOzRk=
github.com/google/go-github/v50 v50.1.0/go.mod h1:Ev4Tre8QoKiolvbpOSG3FIi4Mlon3S2Nt9W5JYqKiwA=
github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8=
github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU=
github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPgecwXBIDzw5no=
github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
github.com/google/martian/v3 v3.3.2 h1:IqNFLAmvJOgVlpdEBiQbDc2EwKW77amAycfTuWKdfvw=
github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
github.com/google/pprof v0.0.0-20211214055906-6f57359322fd h1:1FjCyPC+syAzJ5/2S8fqdZK1R22vvA0J7JZKcuOIQ7Y=
github.com/google/pprof v0.0.0-20211214055906-6f57359322fd/go.mod h1:KgnwoLYCZ8IQu3XUZ8Nc/bM9CCZFOyjUNOSygVozoDg=
github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I=
github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/googleapis/enterprise-certificate-proxy v0.2.3 h1:yk9/cqRKtT9wXZSsRH9aurXEpJX+U6FLtpYTdC3R06k=
github.com/googleapis/enterprise-certificate-proxy v0.2.3/go.mod h1:AwSRAtLfXpU5Nm3pW+v7rGDHp09LsPtGY9MduiEsR9k=
github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
github.com/googleapis/gax-go/v2 v2.7.1 h1:gF4c0zjUP2H/s/hEGyLA3I0fA2ZWjzYiONAD6cvPr8A=
github.com/googleapis/gax-go/v2 v2.7.1/go.mod h1:4orTrqY6hXxxaUL4LHIPl6lGo8vAE38/qKbhSAKP6QI=
github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI=
@@ -260,10 +212,7 @@ github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+l
github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM=
github.com/hashicorp/go-retryablehttp v0.7.2 h1:AcYqCvkpalPnPF2pn0KamgwamS42TqUDDYFRKq/RAd0=
github.com/hashicorp/go-retryablehttp v0.7.2/go.mod h1:Jy/gPYAdjqffZ/yFGCFV2doI5wjtH1ewM9u8iYVjtX8=
github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
github.com/ianlancetaylor/demangle v0.0.0-20210905161508-09a460cdf81d/go.mod h1:aYm2/VgdVmcIU8iMfdMvDMsRAQjcfZSKFby6HOFvi/w=
github.com/imdario/mergo v0.3.13 h1:lFzP57bqS/wsqKssCGmtLAb8A0wKjLGrve2q3PPVcBk=
github.com/imdario/mergo v0.3.13/go.mod h1:4lJ1jqUDcsbIECGy0RUJAXNIhg+6ocWgb1ALK2O4oXg=
@@ -280,15 +229,12 @@ github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0=
github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4=
github.com/jpillora/s3 v1.1.4 h1:YCCKDWzb/Ye9EBNd83ATRF/8wPEy0xd43Rezb6u6fzc=
github.com/jpillora/s3 v1.1.4/go.mod h1:yedE603V+crlFi1Kl/5vZJaBu9pUzE9wvKegU/lF2zs=
github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk=
github.com/kevinburke/ssh_config v1.2.0 h1:x584FjTGwHzMwvHx18PXxbBVzfnxogHaAReU4gf13a4=
github.com/kevinburke/ssh_config v1.2.0/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM=
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=
github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk=
github.com/klauspost/compress v1.16.0 h1:iULayQNOReoYUe+1qtKOqw9CwJv3aNQu8ivo7lw1HU4=
github.com/klauspost/compress v1.16.0/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE=
github.com/klauspost/compress v1.15.11 h1:Lcadnb3RKGin4FYM/orgq0qde+nc15E5Cbqg4B9Sx9c=
github.com/klauspost/compress v1.15.11/go.mod h1:QPwzmACJjUTFsnSHH934V6woptycfrDDJnH7hvFVbGM=
github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek=
github.com/klauspost/pgzip v1.2.5 h1:qnWYvvKqedOF2ulHpMG72XQol4ILEJ8k2wwRl/Km8oE=
github.com/klauspost/pgzip v1.2.5/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs=
@@ -301,8 +247,8 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw=
github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
github.com/lib/pq v1.10.7 h1:p7ZhMD+KsSRozJr34udlUrhboJwWAgCg34+/ZZNvZZw=
github.com/lib/pq v1.10.7/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
github.com/matryer/is v1.2.0 h1:92UTHpy8CDwaJ08GqLDzhhuixiBUUD1p3AU6PHddz4A=
github.com/matryer/is v1.2.0/go.mod h1:2fLPjFQM9rhQ15aVEtbuwhJinnOqrmgXPNdZsdwlWXA=
github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=
@@ -312,8 +258,8 @@ github.com/mattn/go-isatty v0.0.17 h1:BTarxUcIeDqL27Mc+vyvdWYSL28zpIhv3RoTdsLMPn
github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
github.com/mattn/go-sqlite3 v1.14.16 h1:yOQRA0RpS5PFz/oikGwBEqvAWhWg5ufRz4ETLjwpU1Y=
github.com/mattn/go-sqlite3 v1.14.16/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg=
github.com/mholt/archiver/v4 v4.0.0-alpha.8 h1:tRGQuDVPh66WCOelqe6LIGh0gwmfwxUrSSDunscGsRM=
github.com/mholt/archiver/v4 v4.0.0-alpha.8/go.mod h1:5f7FUYGXdJWUjESffJaYR4R60VhnHxb2X3T1teMyv5A=
github.com/mholt/archiver/v4 v4.0.0-alpha.7 h1:xzByj8G8tj0Oq7ZYYU4+ixL/CVb5ruWCm0EZQ1PjOkE=
github.com/mholt/archiver/v4 v4.0.0-alpha.7/go.mod h1:Fs8qUkO74HHaidabihzYephJH8qmGD/nCP6tE5xC9BM=
github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y=
github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
github.com/mmcloughlin/avo v0.5.0/go.mod h1:ChHFdoV7ql95Wi7vuq2YT1bwCJqiWdZrQ1im3VujLYM=
@@ -337,14 +283,12 @@ github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1y
github.com/onsi/gomega v1.23.0 h1:/oxKu9c2HVap+F3PfKort2Hw5DEU+HGlW8n+tguWsys=
github.com/onsi/gomega v1.23.0/go.mod h1:Z/NWtiqwBrwUt4/2loMmHL63EDLnYHmVbuBpDr2vQAg=
github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk=
github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc=
github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ=
github.com/paulbellamy/ratecounter v0.2.0 h1:2L/RhJq+HA8gBQImDXtLPrDXK5qAj6ozWVK/zFXVJGs=
github.com/paulbellamy/ratecounter v0.2.0/go.mod h1:Hfx1hDpSGoqxkVVpBi/IlYD7kChlfo5C6hzIHwPqfFE=
github.com/petar-dambovaliev/aho-corasick v0.0.0-20211021192214-5ab2d9280aa9 h1:lL+y4Xv20pVlCGyLzNHRC0I0rIHhIL1lTvHizoS/dU8=
github.com/petar-dambovaliev/aho-corasick v0.0.0-20211021192214-5ab2d9280aa9/go.mod h1:EHPiTAKtiFmrMldLUNswFwfZ2eJIYBHktdaUTZxYWRw=
github.com/pierrec/lz4/v4 v4.1.15 h1:MO0/ucJhngq7299dKLwIMtgTfbkoSPF6AoMYDd8Q4q0=
github.com/pierrec/lz4/v4 v4.1.15/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
github.com/pierrec/lz4/v4 v4.1.14 h1:+fL8AQEZtz/ijeNnpduH0bROTu0O3NZAlPjQxGn8LwE=
github.com/pierrec/lz4/v4 v4.1.14/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
github.com/pingcap/errors v0.11.4 h1:lFuQV/oaUMGcD2tqt+01ROSmJs75VG1ToEOkZIZ4nE4=
github.com/pjbgf/sha1cd v0.3.0 h1:4D5XXmUUBUl/xQ6IjCkEAbqXskkq/4O7LmGn0AqMDs4=
github.com/pjbgf/sha1cd v0.3.0/go.mod h1:nZ1rrWOcGJ5uZgEEVL1VUM9iRQiZvWdbZjkKyFzPPsI=
@@ -357,10 +301,8 @@ github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
github.com/rabbitmq/amqp091-go v1.8.0 h1:GBFy5PpLQ5jSVVSYv8ecHGqeX7UTLYR4ItQbDCss9MM=
github.com/rabbitmq/amqp091-go v1.8.0/go.mod h1:+jPrT9iY2eLjRaMSRHUhc3z14E/l85kv/f+6luSD3pc=
github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
github.com/rwcarlsen/goexif v0.0.0-20190401172101-9e8deecbddbd/go.mod h1:hPqNNc0+uJM6H+SuU8sEs5K5IQeKccPqeSjfgcKGgPk=
github.com/rabbitmq/amqp091-go v1.7.0 h1:V5CF5qPem5OGSnEo8BoSbsDGwejg6VUJsKEdneaoTUo=
github.com/rabbitmq/amqp091-go v1.7.0/go.mod h1:wfClAtY0C7bOHxd3GjmF26jEHn+rR/0B3+YV+Vn9/NI=
github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM=
github.com/sergi/go-diff v1.3.1 h1:xkr+Oxo4BOQKmkn/B9eMK0g5Kg/983T9DqqPHwYqD+8=
@@ -398,8 +340,8 @@ github.com/trufflesecurity/overseer v1.1.7-custom5/go.mod h1:nT9w37AiO1Nop2VhVhN
github.com/ulikunitz/xz v0.5.6/go.mod h1:2bypXElzHzzJZwzH67Y6wb67pO62Rzfn7BSiF4ABRW8=
github.com/ulikunitz/xz v0.5.10 h1:t92gobL9l3HE202wg3rlk19F6X+JOxl9BBrCCMYEYd8=
github.com/ulikunitz/xz v0.5.10/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14=
github.com/xanzy/go-gitlab v0.83.0 h1:37p0MpTPNbsTMKX/JnmJtY8Ch1sFiJzVF342+RvZEGw=
github.com/xanzy/go-gitlab v0.83.0/go.mod h1:5ryv+MnpZStBH8I/77HuQBsMbBGANtVpLWC15qOjWAw=
github.com/xanzy/go-gitlab v0.81.0 h1:ofbhZ5ZY9AjHATWQie4qd2JfncdUmvcSA/zfQB767Dk=
github.com/xanzy/go-gitlab v0.81.0/go.mod h1:VMbY3JIWdZ/ckvHbQqkyd3iYk2aViKrNIQ23IbFMQDo=
github.com/xanzy/ssh-agent v0.3.3 h1:+/15pJfg/RsTxqYcX6fHqOXZwwMP+2VyYWJeWM2qQFM=
github.com/xanzy/ssh-agent v0.3.3/go.mod h1:6dzNDKs0J9rVPHPhaGCukekBHKqfl+L3KghI1Bc68Uw=
github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c=
@@ -411,123 +353,81 @@ github.com/xdg-go/stringprep v1.0.3/go.mod h1:W3f5j4i+9rC0kuIEJL0ky1VpHXQU3ocBgk
github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d h1:splanxYIlg+5LfHAM6xpdFEAYOk8iySO56hMFq6uLyA=
github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA=
github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
github.com/yusufpapurcu/wmi v1.2.2 h1:KBNDSne4vP5mbSWnJbO+51IMOXJB67QiYCSBrubbPRg=
github.com/yusufpapurcu/wmi v1.2.2/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0=
go.mongodb.org/mongo-driver v1.11.4 h1:4ayjakA013OdpGyL2K3ZqylTac/rMjrJOMZ1EHizXas=
go.mongodb.org/mongo-driver v1.11.4/go.mod h1:PTSz5yu21bkT/wXpkS7WR5f0ddqw5quethTUn9WM+2g=
go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
go.mongodb.org/mongo-driver v1.11.2 h1:+1v2rDQUWNcGW7/7E0Jvdz51V38XXxJfhzbV17aNHCw=
go.mongodb.org/mongo-driver v1.11.2/go.mod h1:s7p5vEtfbeR1gYi6pnj3c3/urpbLv2T5Sfd6Rp2HBB8=
go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0=
go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo=
go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw=
go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A=
go.uber.org/goleak v1.2.1 h1:NBol2c7O1ZokfZ0LEU9K6Whx/KnwvepVetCUhtKja4A=
go.uber.org/goleak v1.2.1/go.mod h1:qlT2yGI9QafXHhZZLxlSuNsMw3FFLxBr+tBRlmO1xH4=
go.uber.org/goleak v1.2.0 h1:xqgm/S+aQvhWFTtR0XK3Jvg7z8kGV8P4X14IzwN3Eqk=
go.uber.org/goleak v1.2.0/go.mod h1:XJYK+MuIchqpmGmUSAzotztawfKvYLUIgg7guXrwVUo=
go.uber.org/multierr v1.6.0 h1:y6IPFStTAIT5Ytl7/XYmHvzXQ7S3g/IeZW9hyZ5thw4=
go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU=
go.uber.org/zap v1.19.0/go.mod h1:xg/QME4nWcxGxrpdeYfq7UvYrLh66cuVKdrbD1XF/NI=
go.uber.org/zap v1.24.0 h1:FiJd5l1UOLj0wCgbSE0rwwXHzEdAZS6hiiSnxJN/D60=
go.uber.org/zap v1.24.0/go.mod h1:2kMP+WWQ8aoFoedH3T2sq6iJ2yDWpHbP0f6MQbS9Gkg=
go4.org v0.0.0-20200411211856-f5505b9728dd h1:BNJlw5kRTzdmyfh5U8F93HA2OwkP7ZGwA51eJ/0wKOU=
go4.org v0.0.0-20200411211856-f5505b9728dd/go.mod h1:CIiUVy99QCPfoE13bO4EZaz5GZMZXMSBGhxRdsvzbkg=
golang.org/x/arch v0.1.0/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/crypto v0.0.0-20201016220609-9e8e0b390897/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/crypto v0.0.0-20210817164053-32db794688a5/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
golang.org/x/crypto v0.0.0-20211215153901-e495a2d5b3d3/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
golang.org/x/crypto v0.0.0-20220525230936-793ad666bf5e/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
golang.org/x/crypto v0.0.0-20220826181053-bd7e27e6170d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw=
golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58=
golang.org/x/crypto v0.3.0/go.mod h1:hebNnKkNXi2UzZN1eVRvBB7co0a+JxK6XbPiWVs/3J4=
golang.org/x/crypto v0.7.0 h1:AvwMYaRytfdeVt3u6mLaxYtErKYjxA2OXjJ1HHq6t3A=
golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU=
golang.org/x/crypto v0.8.0 h1:pd9TJtTueMTVQXzk8E2XESSMQDj/U7OUu0PqJqPXQjQ=
golang.org/x/crypto v0.8.0/go.mod h1:mRqEX+O9/h5TFCrQhkgjo2yKi0yYA+9ecGkdQoHrywE=
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek=
golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY=
golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM=
golang.org/x/exp v0.0.0-20221018205818-5c77f4b2bbd7 h1:lhh9BuEmjIBOdOhMIGSQzdW5LbTtFbUV1k9O/Rlq3SE=
golang.org/x/exp v0.0.0-20221018205818-5c77f4b2bbd7/go.mod h1:cyybsKvd6eL0RnXn6p/Grxp8F5bW7iYuBgsNCOHpMYE=
golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs=
golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE=
golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o=
golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc=
golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY=
golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
golang.org/x/mod v0.6.0/go.mod h1:4mET923SAdbXp2ki8ey+zGs1SLqsuM2Y0uvdZR/fUNI=
golang.org/x/mod v0.8.0 h1:LUYupSeNrTNCGzR/hVBk2NHZO4hXcVaW1k4Qx7rjPx8=
golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
golang.org/x/mod v0.10.0 h1:lFO9qtOdlre5W1jxS3r/4szv2/6iXxScdzjoBMXNhYk=
golang.org/x/mod v0.10.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
golang.org/x/net v0.0.0-20210610132358-84b48f89b13b/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
golang.org/x/net v0.0.0-20220826154423-83b083e8dc8b/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk=
golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco=
golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY=
golang.org/x/net v0.8.0 h1:Zrh2ngAOFYneWTAIAPethzeaQLuHwhuBkuV6ZiRnUaQ=
golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc=
golang.org/x/net v0.9.0 h1:aWJ/m6xSmxWBx+V0XRHTlrYrPG56jKsLdTFmsSsCzOM=
golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
golang.org/x/oauth2 v0.7.0 h1:qe6s0zUXlPX80/dITx3440hWZ7GwMwgDDyrSGTPJG/g=
golang.org/x/oauth2 v0.7.0/go.mod h1:hPLQkd9LyjfXTiRohC/41GhcFqxisoUQ99sCUOHO9x4=
golang.org/x/oauth2 v0.6.0 h1:Lh8GPgSKBfWSwFvtuWOfeI3aAAnbXTSutYxJiOJFgIw=
golang.org/x/oauth2 v0.6.0/go.mod h1:ycmewcwgD4Rpr3eZJLSB4Kyyljb3qDh40vJ8STE5HKw=
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@@ -536,22 +436,12 @@ golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -560,7 +450,9 @@ golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7w
golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
@@ -571,114 +463,61 @@ golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBc
golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220825204002-c680a09ffe64/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.6.0 h1:MVltZSvRTcU2ljQOhs94SXPftV6DCNnZViHeQps87pQ=
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.7.0 h1:3jlCCIQZPdOYu1h8BkNvLz8Kgwtae2cagcG/VamtZRU=
golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
golang.org/x/term v0.0.0-20220722155259-a9ba230a4035/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U=
golang.org/x/term v0.7.0 h1:BEvjmm5fURWqcfbSKTdpkDXYBrUS1c0m8agp14W48vQ=
golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY=
golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc=
golang.org/x/term v0.6.0 h1:clScbb1cHjoCkyRbWwBEUZ5H/tIFu5TAXIqaZD0Gcjw=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
golang.org/x/text v0.8.0 h1:57P1ETyNKtuIjB4SRd15iJxuhj8Gc416Y78H3qgMh68=
golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
golang.org/x/text v0.9.0 h1:2sjJmO8cDvYveuX97RDLsxlyUxLl+GHoLxBiRdHllBE=
golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4=
golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20191108193012-7d206e10da11/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
golang.org/x/tools v0.0.0-20201211185031-d93e913c1a58/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
golang.org/x/tools v0.2.0/go.mod h1:y4OqIKeOV/fWJetJ8bXPU1sEVniLMIyDAZWeHdV+NTA=
golang.org/x/tools v0.6.0 h1:BOw41kyTf3PuCW1pVQf8+Cyg8pMlkYB1oo9iJ6D/lKM=
golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
golang.org/x/tools v0.8.0 h1:vSDcovVPld282ceKgDimkRSC8kpaH1dgyc9UMzlt84Y=
golang.org/x/tools v0.8.0/go.mod h1:JxBZ99ISMI5ViVkT1tr6tdNmXeTrcpVSD3vZ1RsRdN4=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 h1:H2TDz8ibqkAF6YGhCdN3jS9O0/s90v0rJh3X/OLHEUk=
golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8=
google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE=
google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M=
google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg=
google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg=
google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI=
google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI=
google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI=
google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
google.golang.org/api v0.114.0 h1:1xQPji6cO2E2vLiI+C/XiFAnsn1WV3mjaEwGLhi3grE=
google.golang.org/api v0.114.0/go.mod h1:ifYI2ZsFK6/uGddGfAD5BMxlnkBqCmqHSDUVi45N5Yg=
google.golang.org/api v0.111.0 h1:bwKi+z2BsdwYFRKrqwutM+axAlYLz83gt5pDSXCJT+0=
google.golang.org/api v0.111.0/go.mod h1:qtFHvU9mhgTJegR31csQ+rwxyUTHOKFqCKWp1J0fdw0=
google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0=
google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c=
google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8=
google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=
google.golang.org/genproto v0.0.0-20230320184635-7606e756e683 h1:khxVcsk/FhnzxMKOyD+TDGwjbEOpcPuIpmafPGFmhMA=
google.golang.org/genproto v0.0.0-20230320184635-7606e756e683/go.mod h1:NWraEVixdDnqcqQ30jipen1STv2r/n24Wb7twVTGR4s=
google.golang.org/genproto v0.0.0-20230303212802-e74f57abe488 h1:QQF+HdiI4iocoxUjjpLgvTYDHKm99C/VtTBFnfiCJos=
google.golang.org/genproto v0.0.0-20230303212802-e74f57abe488/go.mod h1:TvhZT5f700eVlTNwND1xoEZQeWTB2RY/65kplwl/bFA=
google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY=
google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc=
google.golang.org/grpc v1.53.0 h1:LAv2ds7cmFV/XTS3XG1NneeENYrXGmorPxsBbptIjNc=
google.golang.org/grpc v1.53.0/go.mod h1:OnIrk0ipVdj4N5d9IUoFUx72/VlD7+jUsHwZgwSMQpw=
@@ -693,9 +532,8 @@ google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpAD
google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng=
google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
google.golang.org/protobuf v1.29.1 h1:7QBf+IK2gx70Ap/hDsOmam3GE0v9HicjfEdAxE62UoM=
google.golang.org/protobuf v1.29.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
gopkg.in/alecthomas/kingpin.v2 v2.2.6 h1:jMFz6MfLP0/4fUyZle81rXUoxOBFi19VUFKVDOQfozc=
gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
@@ -704,7 +542,6 @@ gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8
gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
gopkg.in/h2non/gock.v1 v1.1.2 h1:jBbHXgGBK/AoPVfJh5x4r/WxIrElvbLel8TCZkkZJoY=
gopkg.in/h2non/gock.v1 v1.1.2/go.mod h1:n7UGz/ckNChHiK05rDoiC4MYSunEC/lyaUm2WWaDva0=
@@ -724,13 +561,7 @@ gopkg.in/yaml.v3 v3.0.0/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg=
rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=
rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4=
rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0=
rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA=
sigs.k8s.io/yaml v1.3.0 h1:a2VclLzOGrwOHDiV8EfBGhvjHvP46CtW5j6POvhYGGo=
sigs.k8s.io/yaml v1.3.0/go.mod h1:GeOyir5tyXNByN85N/dRIT9es5UQNerPYEKK56eTBm8=
-10
View File
@@ -1,10 +0,0 @@
set terminal png size 800,600
set output "hack/bench/versions.png"
set title "User Time vs. Version"
set xlabel "Version"
set ylabel "Average User Time (s)"
set xtics rotate by -45
plot "hack/bench/plot.txt" using 2:xtic(1) with linespoints linestyle 1 notitle
-15
View File
@@ -1,15 +0,0 @@
#!/bin/bash
if [ $# -ne 2 ]; then
echo "Usage: $0 <repository to clone> <number_of_versions_back_to_test>"
exit 1
fi
# Get the number of versions back to test from command line argument
num_versions="$2"
test_repo="$1"
bash hack/bench/versions.sh $test_repo $num_versions | tee hack/bench/plot.txt
gnuplot hack/bench/plot.gp
-24
View File
@@ -1,24 +0,0 @@
v3.33.0: 1.402
v3.32.2: 1.298
v3.32.1: 1.332
v3.32.0: 1.348
v3.31.6: 2.470
v3.31.5: 2.462
v3.31.4: 2.460
v3.31.3: 2.418
v3.31.2: 1.384
v3.31.1: 1.344
v3.31.0: 1.354
v3.30.0: 1.392
v3.29.1: 1.382
v3.29.0: 1.340
v3.28.7: 1.380
v3.28.6: 1.308
v3.28.5: 2.596
v3.28.4: 2.554
v3.28.3: 2.582
v3.28.1: 2.578
v3.28.2: 2.566
v3.28.0: 2.552
v3.27.1: 2.574
v3.26.0: 2.538
Binary file not shown.

Before

Width:  |  Height:  |  Size: 9.3 KiB

-80
View File
@@ -1,80 +0,0 @@
#!/bin/bash
if [ $# -ne 2 ]; then
echo "Usage: $0 <repository to clone> <number_of_versions_back_to_test>"
exit 1
fi
# Get the number of versions back to test from command line argument
num_versions="$2"
test_repo="$1"
num_iterations=5
# Create a temporary folder to clone the repository
repo_tmp=$(mktemp -d)
# Set up a trap to remove the temporary folder on exit or failure
trap "rm -rf $repo_tmp" EXIT
# Clone the test repository to a temporary folder
git clone --quiet "$test_repo" $repo_tmp
# Get list of git tags, sorted from newest to oldest
tags=$(git tag --sort=-creatordate)
# Counter to keep track of number of tags checked out
count=0
# Loop over tags and checkout each one in turn, up to the specified number of versions
for tag in $tags
do
if [[ $count -eq $num_versions ]]; then
break
fi
# Skip RC tags
if [[ $tag == *"rc"* ]]; then
continue
fi
# Skip alpha tags
if [[ $tag == *"alpha"* ]]; then
continue
fi
# Use git checkout with the quiet flag to suppress output
git checkout $tag --quiet
# Run make install with suppressed output
make install > /dev/null
# Initialize the variable to store the sum of user times
user_time_sum=0
# Run each iteration 5 times and calculate the average user time
for i in {1..$num_iterations}
do
# Run trufflehog with suppressed output and capture user time with /usr/bin/time
tmpfile=$(mktemp)
/usr/bin/time -o $tmpfile trufflehog git "file://$repo_tmp" --no-verification --no-update >/dev/null 2>&1
time_output=$(cat $tmpfile)
rm $tmpfile
# Extract the user time from the output
user_time=$(echo $time_output | awk '{print $3}')
# Add the user time to the sum
user_time_sum=$(echo "$user_time_sum + $user_time" | bc)
done
# Calculate the average user time
average_user_time=$(echo "scale=3; $user_time_sum / $num_iterations" | bc)
# Print the average user time output for this iteration in the specified format
echo "$tag: $average_user_time"
# Increment the counter
count=$((count+1))
done
+59 -141
View File
@@ -26,6 +26,7 @@ import (
"github.com/trufflesecurity/trufflehog/v3/pkg/handlers"
"github.com/trufflesecurity/trufflehog/v3/pkg/log"
"github.com/trufflesecurity/trufflehog/v3/pkg/output"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
"github.com/trufflesecurity/trufflehog/v3/pkg/sources/git"
"github.com/trufflesecurity/trufflehog/v3/pkg/updater"
@@ -33,24 +34,22 @@ import (
)
var (
cli = kingpin.New("TruffleHog", "TruffleHog is a tool for finding credentials.")
cmd string
debug = cli.Flag("debug", "Run in debug mode.").Bool()
trace = cli.Flag("trace", "Run in trace mode.").Bool()
profile = cli.Flag("profile", "Enables profiling and sets a pprof and fgprof server on :18066.").Bool()
jsonOut = cli.Flag("json", "Output in JSON format.").Short('j').Bool()
jsonLegacy = cli.Flag("json-legacy", "Use the pre-v3.0 JSON format. Only works with git, gitlab, and github sources.").Bool()
gitHubActionsFormat = cli.Flag("github-actions", "Output in GitHub Actions format.").Bool()
concurrency = cli.Flag("concurrency", "Number of concurrent workers.").Default(strconv.Itoa(runtime.NumCPU())).Int()
noVerification = cli.Flag("no-verification", "Don't verify the results.").Bool()
onlyVerified = cli.Flag("only-verified", "Only output verified results.").Bool()
filterUnverified = cli.Flag("filter-unverified", "Only output first unverified result per chunk per detector if there are more than one results.").Bool()
configFilename = cli.Flag("config", "Path to configuration file.").ExistingFile()
cli = kingpin.New("TruffleHog", "TruffleHog is a tool for finding credentials.")
cmd string
debug = cli.Flag("debug", "Run in debug mode.").Bool()
trace = cli.Flag("trace", "Run in trace mode.").Bool()
profile = cli.Flag("profile", "Enables profiling and sets a pprof and fgprof server on :18066.").Bool()
jsonOut = cli.Flag("json", "Output in JSON format.").Short('j').Bool()
jsonLegacy = cli.Flag("json-legacy", "Use the pre-v3.0 JSON format. Only works with git, gitlab, and github sources.").Bool()
concurrency = cli.Flag("concurrency", "Number of concurrent workers.").Default(strconv.Itoa(runtime.NumCPU())).Int()
noVerification = cli.Flag("no-verification", "Don't verify the results.").Bool()
onlyVerified = cli.Flag("only-verified", "Only output verified results.").Bool()
filterUnverified = cli.Flag("filter-unverified", "Only output first unverified result per chunk per detector if there are more than one results.").Bool()
configFilename = cli.Flag("config", "Path to configuration file.").ExistingFile()
// rules = cli.Flag("rules", "Path to file with custom rules.").String()
printAvgDetectorTime = cli.Flag("print-avg-detector-time", "Print the average time spent on each detector.").Bool()
noUpdate = cli.Flag("no-update", "Don't check for updates.").Bool()
fail = cli.Flag("fail", "Exit with code 183 if results are found.").Bool()
verifiers = cli.Flag("verifier", "Set custom verification endpoints.").StringMap()
archiveMaxSize = cli.Flag("archive-max-size", "Maximum size of archive to scan. (Byte units eg. 512B, 2KB, 4MB)").Bytes()
archiveMaxDepth = cli.Flag("archive-max-depth", "Maximum depth of archive to scan.").Int()
archiveTimeout = cli.Flag("archive-timeout", "Maximum time to spend extracting an archive.").Duration()
@@ -61,7 +60,6 @@ var (
gitScanURI = gitScan.Arg("uri", "Git repository URL. https://, file://, or ssh:// schema expected.").Required().String()
gitScanIncludePaths = gitScan.Flag("include-paths", "Path to file with newline separated regexes for files to include in scan.").Short('i').String()
gitScanExcludePaths = gitScan.Flag("exclude-paths", "Path to file with newline separated regexes for files to exclude in scan.").Short('x').String()
gitScanExcludeGlobs = gitScan.Flag("exclude-globs", "Comma separated list of globs to exclude in scan. This option filters at the `git log` level, resulting in faster scans.").String()
gitScanSinceCommit = gitScan.Flag("since-commit", "Commit to start scan from.").String()
gitScanBranch = gitScan.Flag("branch", "Branch to scan.").String()
gitScanMaxDepth = gitScan.Flag("max-depth", "Maximum depth of commits to scan.").Int()
@@ -98,13 +96,11 @@ var (
filesystemScanIncludePaths = filesystemScan.Flag("include-paths", "Path to file with newline separated regexes for files to include in scan.").Short('i').String()
filesystemScanExcludePaths = filesystemScan.Flag("exclude-paths", "Path to file with newline separated regexes for files to exclude in scan.").Short('x').String()
s3Scan = cli.Command("s3", "Find credentials in S3 buckets.")
s3ScanKey = s3Scan.Flag("key", "S3 key used to authenticate. Can be provided with environment variable AWS_ACCESS_KEY_ID.").Envar("AWS_ACCESS_KEY_ID").String()
s3ScanSecret = s3Scan.Flag("secret", "S3 secret used to authenticate. Can be provided with environment variable AWS_SECRET_ACCESS_KEY.").Envar("AWS_SECRET_ACCESS_KEY").String()
s3ScanSessionToken = s3Scan.Flag("session-token", "S3 session token used to authenticate temporary credentials. Can be provided with environment variable AWS_SESSION_TOKEN.").Envar("AWS_SESSION_TOKEN").String()
s3ScanCloudEnv = s3Scan.Flag("cloud-environment", "Use IAM credentials in cloud environment.").Bool()
s3ScanBuckets = s3Scan.Flag("bucket", "Name of S3 bucket to scan. You can repeat this flag.").Strings()
s3ScanMaxObjectSize = s3Scan.Flag("max-object-size", "Maximum size of objects to scan. Objects larger than this will be skipped. (Byte units eg. 512B, 2KB, 4MB)").Default("250MB").Bytes()
s3Scan = cli.Command("s3", "Find credentials in S3 buckets.")
s3ScanKey = s3Scan.Flag("key", "S3 key used to authenticate. Can be provided with environment variable AWS_ACCESS_KEY_ID.").Envar("AWS_ACCESS_KEY_ID").String()
s3ScanSecret = s3Scan.Flag("secret", "S3 secret used to authenticate. Can be provided with environment variable AWS_SECRET_ACCESS_KEY.").Envar("AWS_SECRET_ACCESS_KEY").String()
s3ScanCloudEnv = s3Scan.Flag("cloud-environment", "Use IAM credentials in cloud environment.").Bool()
s3ScanBuckets = s3Scan.Flag("bucket", "Name of S3 bucket to scan. You can repeat this flag.").Strings()
gcsScan = cli.Command("gcs", "Find credentials in GCS buckets.")
gcsProjectID = gcsScan.Flag("project-id", "GCS project ID used to authenticate. Can NOT be used with unauth scan. Can be provided with environment variable GOOGLE_CLOUD_PROJECT.").Envar("GOOGLE_CLOUD_PROJECT").String()
@@ -231,85 +227,53 @@ func run(state overseer.State) {
handlers.SetArchiveMaxTimeout(*archiveTimeout)
}
// Build include and exclude detector sets for filtering on engine initialization.
// Exit if there was an error to inform the user of the misconfiguration.
var includeDetectorSet, excludeDetectorSet map[config.DetectorID]struct{}
var detectorsWithCustomVerifierEndpoints map[config.DetectorID][]string
// Build include and exclude detector filter sets.
var includeDetectorTypes, excludeDetectorTypes map[detectorspb.DetectorType]config.DetectorID
{
includeList, err := config.ParseDetectors(*includeDetectors)
if err != nil {
// Exit if there was an error to inform the user of the misconfiguration.
logFatal(err, "invalid include list detector configuration")
}
excludeList, err := config.ParseDetectors(*excludeDetectors)
if err != nil {
// Exit if there was an error to inform the user of the misconfiguration.
logFatal(err, "invalid exclude list detector configuration")
}
detectorsWithCustomVerifierEndpoints, err = config.ParseVerifierEndpoints(*verifiers)
if err != nil {
logFatal(err, "invalid verifier detector configuration")
}
includeDetectorSet = detectorTypeToSet(includeList)
excludeDetectorSet = detectorTypeToSet(excludeList)
includeDetectorTypes = detectorTypeToMap(includeList)
excludeDetectorTypes = detectorTypeToMap(excludeList)
}
// Verify that all the user-provided detectors support the optional
// detector features.
{
if err, id := verifyDetectorsAreVersioner(includeDetectorSet); err != nil {
logFatal(err, "invalid include list detector configuration", "detector", id)
}
if err, id := verifyDetectorsAreVersioner(excludeDetectorSet); err != nil {
logFatal(err, "invalid exclude list detector configuration", "detector", id)
}
if err, id := verifyDetectorsAreVersioner(detectorsWithCustomVerifierEndpoints); err != nil {
logFatal(err, "invalid verifier detector configuration", "detector", id)
}
// Extra check for endpoint customization.
isEndpointCustomizer := engine.DefaultDetectorTypesImplementing[detectors.EndpointCustomizer]()
for id := range detectorsWithCustomVerifierEndpoints {
if _, ok := isEndpointCustomizer[id.ID]; !ok {
logFatal(
fmt.Errorf("endpoint provided but detector does not support endpoint customization"),
"invalid custom verifier endpoint detector configuration",
"detector", id,
)
}
}
}
includeFilter := func(d detectors.Detector) bool {
_, ok := getWithDetectorID(d, includeDetectorSet)
return ok
}
excludeFilter := func(d detectors.Detector) bool {
_, ok := getWithDetectorID(d, excludeDetectorSet)
return !ok
}
// Abuse filter to cause a side-effect.
endpointCustomizer := func(d detectors.Detector) bool {
urls, ok := getWithDetectorID(d, detectorsWithCustomVerifierEndpoints)
if !ok {
return true
id, ok := includeDetectorTypes[d.Type()]
if id.Version == 0 {
return ok
}
id := config.GetDetectorID(d)
customizer, ok := d.(detectors.EndpointCustomizer)
versionD, ok := d.(detectors.Versioner)
if !ok {
// NOTE: We should never reach here due to validation above.
// Error: version provided but not a detectors.Versioner
logFatal(
fmt.Errorf("failed to configure a detector endpoint"),
"the provided detector does not support endpoint configuration",
fmt.Errorf("version provided but detector does not have a version"),
"invalid include list detector configuration",
"detector", id,
)
}
// TODO: Add flag to ignore the default endpoint.
urls = append(urls, customizer.DefaultEndpoint())
if err := customizer.SetEndpoints(urls...); err != nil {
logFatal(err, "failed configuring custom endpoint for detector", "detector", id)
return versionD.Version() == id.Version
}
excludeFilter := func(d detectors.Detector) bool {
id, ok := excludeDetectorTypes[d.Type()]
if id.Version == 0 {
return !ok
}
logger.Info("configured detector with verification urls",
"detector", id, "urls", urls,
)
return true
versionD, ok := d.(detectors.Versioner)
if !ok {
// Error: version provided but not a detectors.Versioner
logFatal(
fmt.Errorf("version provided but detector does not have a version"),
"invalid exclude list detector configuration",
"detector", id,
)
}
return versionD.Version() != id.Version
}
e := engine.Start(ctx,
@@ -319,7 +283,6 @@ func run(state overseer.State) {
engine.WithDetectors(!*noVerification, conf.Detectors...),
engine.WithFilterDetectors(includeFilter),
engine.WithFilterDetectors(excludeFilter),
engine.WithFilterDetectors(endpointCustomizer),
engine.WithFilterUnverified(*filterUnverified),
)
@@ -338,18 +301,13 @@ func run(state overseer.State) {
if remote {
defer os.RemoveAll(repoPath)
}
excludedGlobs := []string{}
if *gitScanExcludeGlobs != "" {
excludedGlobs = strings.Split(*gitScanExcludeGlobs, ",")
}
cfg := sources.GitConfig{
RepoPath: repoPath,
HeadRef: *gitScanBranch,
BaseRef: *gitScanSinceCommit,
MaxDepth: *gitScanMaxDepth,
Filter: filter,
ExcludeGlobs: excludedGlobs,
RepoPath: repoPath,
HeadRef: *gitScanBranch,
BaseRef: *gitScanSinceCommit,
MaxDepth: *gitScanMaxDepth,
Filter: filter,
}
if err = e.ScanGit(ctx, cfg); err != nil {
logFatal(err, "Failed to scan Git.")
@@ -413,12 +371,10 @@ func run(state overseer.State) {
}
case s3Scan.FullCommand():
cfg := sources.S3Config{
Key: *s3ScanKey,
Secret: *s3ScanSecret,
SessionToken: *s3ScanSessionToken,
Buckets: *s3ScanBuckets,
CloudCred: *s3ScanCloudEnv,
MaxObjectSize: int64(*s3ScanMaxObjectSize),
Key: *s3ScanKey,
Secret: *s3ScanSecret,
Buckets: *s3ScanBuckets,
CloudCred: *s3ScanCloudEnv,
}
if err := e.ScanS3(ctx, cfg); err != nil {
logFatal(err, "Failed to scan S3.")
@@ -479,8 +435,6 @@ func run(state overseer.State) {
err = output.PrintLegacyJSON(ctx, &r)
case *jsonOut:
err = output.PrintJSON(&r)
case *gitHubActionsFormat:
err = output.PrintGitHubActionsOutput(&r)
default:
err = output.PrintPlainOutput(&r)
}
@@ -542,46 +496,10 @@ func logFatalFunc(logger logr.Logger) func(error, string, ...any) {
}
}
// detectorTypeToSet is a helper function to convert a slice of detector IDs into a set.
func detectorTypeToSet(detectors []config.DetectorID) map[config.DetectorID]struct{} {
output := make(map[config.DetectorID]struct{}, len(detectors))
func detectorTypeToMap(detectors []config.DetectorID) map[detectorspb.DetectorType]config.DetectorID {
output := make(map[detectorspb.DetectorType]config.DetectorID, len(detectors))
for _, d := range detectors {
output[d] = struct{}{}
output[d.ID] = d
}
return output
}
// getWithDetectorID is a helper function to get a value from a map using a
// detector's ID. This function behaves like a normal map lookup, with an extra
// step of checking for the non-specific version of a detector.
func getWithDetectorID[T any](d detectors.Detector, data map[config.DetectorID]T) (T, bool) {
key := config.GetDetectorID(d)
// Check if the specific ID is provided.
if t, ok := data[key]; ok || key.Version == 0 {
return t, ok
}
// Check if the generic type is provided without a version.
// This means "all" versions of a type.
key.Version = 0
t, ok := data[key]
return t, ok
}
// verifyDetectorsAreVersioner checks all keys in a provided map to verify the
// provided type is actually a Versioner.
func verifyDetectorsAreVersioner[T any](data map[config.DetectorID]T) (error, config.DetectorID) {
isVersioner := engine.DefaultDetectorTypesImplementing[detectors.Versioner]()
for id := range data {
if id.Version == 0 {
// Version not provided.
continue
}
if _, ok := isVersioner[id.ID]; ok {
// Version provided for a Versioner detector.
continue
}
// Version provided on a non-Versioner detector.
return fmt.Errorf("version provided but detector does not have a version"), id
}
return nil, config.DetectorID{}
}
-24
View File
@@ -1,24 +0,0 @@
// Package cache provides an interface which can be implemented by different cache types.
package cache
// Cache is used to store key/value pairs.
type Cache interface {
// Set stores the given key/value pair.
Set(string, string)
// Get returns the value for the given key and a boolean indicating if the key was found.
Get(string) (string, bool)
// Exists returns true if the given key exists in the cache.
Exists(string) bool
// Delete the given key from the cache.
Delete(string)
// Clear all key/value pairs from the cache.
Clear()
// Count the number of key/value pairs in the cache.
Count() int
// Keys returns all keys in the cache.
Keys() []string
// Values returns all values in the cache.
Values() []string
// Contents returns all keys in the cache encoded as a string.
Contents() string
}
-105
View File
@@ -1,105 +0,0 @@
package memory
import (
"strings"
"time"
"github.com/patrickmn/go-cache"
"github.com/trufflesecurity/trufflehog/v3/pkg/context"
)
const (
expirationInterval = 12 * time.Hour
purgeInterval = 13 * time.Hour
defaultExpiration = cache.DefaultExpiration
)
// Cache is a wrapper around the go-cache library.
type Cache struct {
c *cache.Cache
}
// New constructs a new in-memory cache.
func New() *Cache {
c := cache.New(expirationInterval, purgeInterval)
return &Cache{c: c}
}
// NewWithData constructs a new in-memory cache with existing data.
func NewWithData(ctx context.Context, data []string) *Cache {
ctx.Logger().V(3).Info("Loading cache", "num-items", len(data))
items := make(map[string]cache.Item, len(data))
for _, d := range data {
items[d] = cache.Item{Object: d, Expiration: int64(defaultExpiration)}
}
c := cache.NewFrom(expirationInterval, purgeInterval, items)
return &Cache{c: c}
}
// Set adds a key-value pair to the cache.
func (c *Cache) Set(key, value string) {
c.c.Set(key, value, defaultExpiration)
}
// Get returns the value for the given key.
func (c *Cache) Get(key string) (string, bool) {
res, ok := c.c.Get(key)
if !ok {
return "", ok
}
return res.(string), ok
}
// Exists returns true if the given key exists in the cache.
func (c *Cache) Exists(key string) bool {
_, ok := c.c.Get(key)
return ok
}
// Delete removes the key-value pair from the cache.
func (c *Cache) Delete(key string) {
c.c.Delete(key)
}
// Clear removes all key-value pairs from the cache.
func (c *Cache) Clear() {
c.c.Flush()
}
// Count returns the number of key-value pairs in the cache.
func (c *Cache) Count() int {
return c.c.ItemCount()
}
// Keys returns all keys in the cache.
func (c *Cache) Keys() []string {
items := c.c.Items()
res := make([]string, 0, len(items))
for k := range items {
res = append(res, k)
}
return res
}
// Values returns all values in the cache.
func (c *Cache) Values() []string {
items := c.c.Items()
res := make([]string, 0, len(items))
for _, v := range items {
res = append(res, v.Object.(string))
}
return res
}
// Contents returns all key-value pairs in the cache encodes as a string.
func (c *Cache) Contents() string {
items := c.c.Items()
res := make([]string, 0, len(items))
for k := range items {
res = append(res, k)
}
return strings.Join(res, ",")
}
-169
View File
@@ -1,169 +0,0 @@
package memory
import (
"fmt"
"sort"
"strings"
"testing"
"github.com/google/go-cmp/cmp"
logContext "github.com/trufflesecurity/trufflehog/v3/pkg/context"
)
func TestCache(t *testing.T) {
c := New()
// Test set and get.
c.Set("key1", "key1")
v, ok := c.Get("key1")
if !ok || v != "key1" {
t.Fatalf("Unexpected value for key1: %v, %v", v, ok)
}
// Test exists.
if !c.Exists("key1") {
t.Fatalf("Expected key1 to exist")
}
// Test the count.
if c.Count() != 1 {
t.Fatalf("Unexpected count: %d", c.Count())
}
// Test delete.
c.Delete("key1")
v, ok = c.Get("key1")
if ok || v != "" {
t.Fatalf("Unexpected value for key1 after delete: %v, %v", v, ok)
}
// Test clear.
c.Set("key10", "key10")
c.Clear()
v, ok = c.Get("key10")
if ok || v != "" {
t.Fatalf("Unexpected value for key10 after clear: %v, %v", v, ok)
}
// Test getting only the keys.
keys := []string{"key1", "key2", "key3"}
values := []string{"value1", "value2", "value3"}
for i, k := range keys {
c.Set(k, values[i])
}
k := c.Keys()
sort.Strings(keys)
sort.Strings(k)
if !cmp.Equal(keys, k) {
t.Fatalf("Unexpected keys: %v", k)
}
// Test getting only the values.
vals := c.Values()
sort.Strings(vals)
sort.Strings(values)
if !cmp.Equal(values, vals) {
t.Fatalf("Unexpected values: %v", vals)
}
// Test contents.
items := c.Contents()
sort.Strings(keys)
res := strings.Split(items, ",")
sort.Strings(res)
if len(keys) != len(res) {
t.Fatalf("Unexpected length of items: %d", len(res))
}
if !cmp.Equal(keys, res) {
t.Fatalf("Unexpected items: %v", res)
}
}
func TestCache_NewWithData(t *testing.T) {
c := NewWithData(logContext.Background(), []string{"key1", "key2", "key3"})
// Test the count.
if c.Count() != 3 {
t.Fatalf("Unexpected count: %d", c.Count())
}
// Test contents.
keys := []string{"key1", "key2", "key3"}
items := c.Contents()
sort.Strings(keys)
res := strings.Split(items, ",")
sort.Strings(res)
if len(keys) != len(res) {
t.Fatalf("Unexpected length of items: %d", len(res))
}
if !cmp.Equal(keys, res) {
t.Fatalf("Unexpected items: %v", res)
}
}
func setupBenchmarks(b *testing.B) *Cache {
b.Helper()
c := New()
for i := 0; i < 500_000; i++ {
key := fmt.Sprintf("key%d", i)
c.Set(key, key)
}
return c
}
func BenchmarkSet(b *testing.B) {
c := New()
for i := 0; i < b.N; i++ {
key := fmt.Sprintf("key%d", i)
c.Set(key, key)
}
}
func BenchmarkGet(b *testing.B) {
c := setupBenchmarks(b)
b.ResetTimer()
for i := 0; i < b.N; i++ {
key := fmt.Sprintf("key%d", i)
c.Get(key)
}
}
func BenchmarkDelete(b *testing.B) {
c := setupBenchmarks(b)
b.ResetTimer()
for i := 0; i < b.N; i++ {
key := fmt.Sprintf("key%d", i)
c.Delete(key)
}
}
func BenchmarkCount(b *testing.B) {
c := setupBenchmarks(b)
b.ResetTimer()
for i := 0; i < b.N; i++ {
c.Count()
}
}
func BenchmarkContents(b *testing.B) {
c := setupBenchmarks(b)
b.ResetTimer()
var s string
for i := 0; i < b.N; i++ {
s = c.Contents()
}
_ = s
}
+2 -50
View File
@@ -2,12 +2,10 @@ package config
import (
"fmt"
"net/url"
"sort"
"strconv"
"strings"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
dpb "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
)
@@ -33,26 +31,11 @@ func init() {
}
}
// DetectorID identifies a detector type and version. This struct is used as a
// way for users to identify detectors, whether unique or not. A DetectorID
// with Version = 0 indicates all possible versions of a detector.
type DetectorID struct {
ID dpb.DetectorType
Version int
}
// GetDetectorID extracts the DetectorID from a Detector.
func GetDetectorID(d detectors.Detector) DetectorID {
var version int
if v, ok := d.(detectors.Versioner); ok {
version = v.Version()
}
return DetectorID{
ID: d.Type(),
Version: version,
}
}
// ParseDetectors parses user supplied string into a list of detectors types.
// "all" will return the list of all available detectors. The input is comma
// separated and may use the case-insensitive detector name defined in the
@@ -85,37 +68,6 @@ func ParseDetectors(input string) ([]DetectorID, error) {
return output, nil
}
// ParseDetector parses a user supplied string into a single DetectorID. Input
// is case-insensitive and either the detector name or ID may be used.
func ParseDetector(input string) (DetectorID, error) {
return asDetectorID(strings.TrimSpace(input))
}
// ParseVerifierEndpoints parses a map of user supplied verifier URLs. The
// input keys are detector IDs and the values are a comma separated list of
// URLs. The URLs are validated as HTTPS endpoints.
func ParseVerifierEndpoints(verifierURLs map[string]string) (map[DetectorID][]string, error) {
verifiers := make(map[DetectorID][]string, len(verifierURLs))
for detectorID, urls := range verifierURLs {
key, err := ParseDetector(detectorID)
if err != nil {
return nil, fmt.Errorf("invalid detector ID for verifier: %w", err)
}
verifierURLs := strings.Split(urls, ",")
for i, rawEndpoint := range verifierURLs {
rawEndpoint := strings.TrimSpace(rawEndpoint)
verifierURLs[i] = rawEndpoint
if endpoint, err := url.Parse(rawEndpoint); err != nil {
return nil, fmt.Errorf("invalid verifier url %q: %w", rawEndpoint, err)
} else if endpoint.Scheme != "https" {
return nil, fmt.Errorf("verifier url must be https: %q", rawEndpoint)
}
}
verifiers[key] = append(verifiers[key], verifierURLs...)
}
return verifiers, nil
}
func (id DetectorID) String() string {
name := dpb.DetectorType_name[int32(id.ID)]
if name == "" {
@@ -187,8 +139,8 @@ func asRange(input string) ([]DetectorID, error) {
return append(output, dtEnd), nil
}
// asDetectorID converts the case-insensitive input into a DetectorID. Name or
// ID may be used. Input is expected to be already trimmed of whitespace.
// asDetectorID converts the case-insensitive input into a DetectorID.
// Name or ID may be used.
func asDetectorID(input string) (DetectorID, error) {
if input == "" {
return DetectorID{}, fmt.Errorf("empty detector")
-1
View File
@@ -117,7 +117,6 @@ func (c *customRegexWebhook) createResults(ctx context.Context, match map[string
}
result := detectors.Result{
DetectorType: detectorspb.DetectorType_CustomRegex,
DetectorName: c.GetName(),
Raw: []byte(raw),
}
+43 -82
View File
@@ -3,6 +3,7 @@ package decoders
import (
"bytes"
"encoding/base64"
"strings"
"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
)
@@ -12,20 +13,52 @@ type Base64 struct{}
var (
b64Charset = []byte("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/=")
b64EndChars = "+/="
// Given characters are mostly ASCII, we can use a simple array to map.
b64CharsetMapping [128]bool
)
func init() {
// Build an array of all the characters in the base64 charset.
for _, char := range b64Charset {
b64CharsetMapping[char] = true
func getSubstringsOfCharacterSet(data []byte, charset []byte, threshold int) []string {
count := 0
substrings := []string{}
letters := strings.Builder{}
if len(data) == 0 {
return nil
}
for _, char := range string(data) {
if bytes.ContainsRune(charset, char) {
letters.WriteRune(char)
count++
} else {
if count > threshold {
substrings = appendB64Substring(letters, substrings)
}
letters.Reset()
count = 0
}
}
if count > threshold && len(letters.String()) > 0 {
substrings = appendB64Substring(letters, substrings)
}
return substrings
}
func appendB64Substring(letters strings.Builder, substrings []string) []string {
substring := strings.TrimLeft(letters.String(), b64EndChars)
// handle key=value
if strings.Contains(strings.TrimRight(substring, b64EndChars), "=") {
split := strings.SplitN(substring, "=", 2)
substrings = append(substrings, split[len(split)-1])
} else {
substrings = append(substrings, substring)
}
return substrings
}
func (d *Base64) FromChunk(chunk *sources.Chunk) *sources.Chunk {
encodedSubstrings := getSubstringsOfCharacterSet(chunk.Data, 20)
decodedSubstrings := make(map[string][]byte)
encodedSubstrings := getSubstringsOfCharacterSet(chunk.Data, b64Charset, 20)
decodedSubstrings := map[string][]byte{}
for _, str := range encodedSubstrings {
dec, err := base64.StdEncoding.DecodeString(str)
@@ -35,83 +68,11 @@ func (d *Base64) FromChunk(chunk *sources.Chunk) *sources.Chunk {
}
if len(decodedSubstrings) > 0 {
var result bytes.Buffer
result.Grow(len(chunk.Data))
start := 0
for _, encoded := range encodedSubstrings {
if decoded, ok := decodedSubstrings[encoded]; ok {
end := bytes.Index(chunk.Data[start:], []byte(encoded))
if end != -1 {
result.Write(chunk.Data[start : start+end])
result.Write(decoded)
start += end + len(encoded)
}
}
for substring, dec := range decodedSubstrings {
chunk.Data = bytes.Replace(chunk.Data, []byte(substring), dec, 1)
}
result.Write(chunk.Data[start:])
chunk.Data = result.Bytes()
return chunk
}
return nil
}
func getSubstringsOfCharacterSet(data []byte, threshold int) []string {
if len(data) == 0 {
return nil
}
count := 0
substringsCount := 0
// Determine the number of substrings that will be returned.
// Pre-allocate the slice to avoid reallocations.
for _, char := range data {
if char < 128 && b64CharsetMapping[char] {
count++
} else {
if count > threshold {
substringsCount++
}
count = 0
}
}
if count > threshold {
substringsCount++
}
count = 0
start := 0
substrings := make([]string, 0, substringsCount)
for i, char := range data {
if char < 128 && b64CharsetMapping[char] {
if count == 0 {
start = i
}
count++
} else {
if count > threshold {
substrings = appendB64Substring(data, start, count, substrings)
}
count = 0
}
}
if count > threshold {
substrings = appendB64Substring(data, start, count, substrings)
}
return substrings
}
func appendB64Substring(data []byte, start, count int, substrings []string) []string {
substring := bytes.TrimLeft(data[start:start+count], b64EndChars)
if idx := bytes.IndexByte(bytes.TrimRight(substring, b64EndChars), '='); idx != -1 {
substrings = append(substrings, string(substring[idx+1:]))
} else {
substrings = append(substrings, string(substring))
}
return substrings
}
+7 -23
View File
@@ -90,29 +90,13 @@ func TestBase64_FromChunk(t *testing.T) {
}
}
func BenchmarkFromChunkSmall(b *testing.B) {
func BenchmarkFromChunk(benchmark *testing.B) {
d := Base64{}
data := detectors.MustGetBenchmarkData()["small"]
for n := 0; n < b.N; n++ {
d.FromChunk(&sources.Chunk{Data: data})
}
}
func BenchmarkFromChunkMedium(b *testing.B) {
d := Base64{}
data := detectors.MustGetBenchmarkData()["medium"]
for n := 0; n < b.N; n++ {
d.FromChunk(&sources.Chunk{Data: data})
}
}
func BenchmarkFromChunkLarge(b *testing.B) {
d := Base64{}
data := detectors.MustGetBenchmarkData()["big"]
for n := 0; n < b.N; n++ {
d.FromChunk(&sources.Chunk{Data: data})
for name, data := range detectors.MustGetBenchmarkData() {
benchmark.Run(name, func(b *testing.B) {
for n := 0; n < b.N; n++ {
d.FromChunk(&sources.Chunk{Data: data})
}
})
}
}
-1
View File
@@ -8,7 +8,6 @@ func DefaultDecoders() []Decoder {
return []Decoder{
&UTF8{},
&Base64{},
&UTF16{},
}
}
-68
View File
@@ -1,68 +0,0 @@
package decoders
import (
"bytes"
"encoding/binary"
"fmt"
"unicode/utf8"
"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
)
type UTF16 struct{}
func (d *UTF16) FromChunk(chunk *sources.Chunk) *sources.Chunk {
if chunk == nil || len(chunk.Data) == 0 {
return nil
}
if utf16Data, err := utf16ToUTF8(chunk.Data); err == nil {
chunk.Data = utf16Data
return chunk
}
return nil
}
// utf16ToUTF8 converts a byte slice containing UTF-16 encoded data to a UTF-8 encoded byte slice.
func utf16ToUTF8(b []byte) ([]byte, error) {
endianness, err := guessUTF16Endianness(b)
if err != nil {
return nil, err
}
buf := &bytes.Buffer{}
for i := 0; i < len(b); i += 2 {
r := rune(endianness.Uint16(b[i:]))
if utf8.ValidRune(r) {
buf.WriteRune(r)
}
}
return buf.Bytes(), nil
}
func guessUTF16Endianness(b []byte) (binary.ByteOrder, error) {
if len(b) < 2 || len(b)%2 != 0 {
return nil, fmt.Errorf("input length must be even and at least 2 bytes long")
}
var evenNullBytes, oddNullBytes int
for i := 0; i < len(b); i += 2 {
if b[i] == 0 {
oddNullBytes++
}
if b[i+1] == 0 {
evenNullBytes++
}
}
if evenNullBytes > oddNullBytes {
return binary.LittleEndian, nil
}
if oddNullBytes > evenNullBytes {
return binary.BigEndian, nil
}
return nil, fmt.Errorf("could not determine endianness")
}
-73
View File
@@ -1,73 +0,0 @@
package decoders
import (
"bytes"
"testing"
"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
)
func TestUTF16Decoder(t *testing.T) {
testCases := []struct {
name string
input []byte
expected []byte
expectNil bool
}{
{
name: "Valid UTF-16LE input",
input: []byte{72, 0, 101, 0, 108, 0, 108, 0, 111, 0, 32, 0, 87, 0, 111, 0, 114, 0, 108, 0, 100, 0},
expected: []byte("Hello World"),
expectNil: false,
},
{
name: "Valid UTF-16BE input",
input: []byte{0, 72, 0, 101, 0, 108, 0, 108, 0, 111, 0, 32, 0, 87, 0, 111, 0, 114, 0, 108, 0, 100},
expected: []byte("Hello World"),
expectNil: false,
},
{
name: "Invalid UTF-16 input (it's UTF-8)",
input: []byte("Hello World!"),
expected: nil,
expectNil: true,
},
{
name: "Invalid UTF-16 input (odd length)",
input: []byte{72, 0, 101, 0, 108, 0, 108, 0, 111, 0, 32, 0, 87, 0, 111, 0, 114, 0, 108, 0, 0},
expected: nil,
expectNil: true,
},
}
for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
chunk := &sources.Chunk{Data: tc.input}
decoder := &UTF16{}
decodedChunk := decoder.FromChunk(chunk)
if tc.expectNil {
if decodedChunk != nil {
t.Errorf("Expected nil, got chunk with data: %v", decodedChunk.Data)
}
return
}
if decodedChunk == nil {
t.Errorf("Expected chunk with data, got nil")
return
}
if !bytes.Equal(decodedChunk.Data, tc.expected) {
t.Errorf("Expected decoded data: %v, got: %v", tc.expected, decodedChunk.Data)
}
})
}
}
func BenchmarkUtf16ToUtf8(b *testing.B) {
// Example UTF-16LE encoded data
data := []byte{72, 0, 101, 0, 108, 0, 108, 0, 111, 0, 32, 0, 87, 0, 111, 0, 114, 0, 108, 0, 100, 0}
for n := 0; n < b.N; n++ {
_, _ = utf16ToUTF8(data)
}
}
+9 -20
View File
@@ -26,30 +26,19 @@ func (d *UTF8) FromChunk(chunk *sources.Chunk) *sources.Chunk {
// extacting contigous portions of printable characters that we care
// about from some bytes
func extractSubstrings(b []byte) []byte {
isValidByte := func(c byte) bool {
fields := bytes.FieldsFunc(b, func(r rune) bool {
// https://www.rapidtables.com/code/text/ascii-table.html
// split on anything that is not ascii space through tilde
return c > 31 && c < 127
}
return !(r > 31 && r < 127)
})
field := make([]byte, len(b))
fieldLen := 0
buf := &bytes.Buffer{}
for i, c := range b {
if isValidByte(c) {
field[fieldLen] = c
fieldLen++
} else {
if fieldLen > 5 {
buf.Write(field[:fieldLen])
}
fieldLen = 0
}
if i == len(b)-1 && fieldLen > 5 {
buf.Write(field[:fieldLen])
keep := [][]byte{}
for _, field := range fields {
// Remove fields shorter than 6 characters.
if bts := bytes.TrimSpace(field); len(bts) > 5 {
keep = append(keep, bts)
}
}
return buf.Bytes()
return bytes.Join(keep, []byte("\n"))
}
-1
View File
@@ -4,7 +4,6 @@ import (
"testing"
"github.com/kylelemons/godebug/pretty"
"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
)
+3 -2
View File
@@ -3,9 +3,9 @@ package alibaba
import (
"context"
"crypto/hmac"
"crypto/rand"
"crypto/sha1"
"encoding/base64"
"math/rand"
"net/http"
"net/url"
"regexp"
@@ -38,9 +38,10 @@ func (s Scanner) Keywords() []string {
}
func randString(n int) string {
rand.Seed(time.Now().UnixNano())
const alphanum = "0123456789abcdefghijklmnopqrstuvwxyz"
var bytes = make([]byte, n)
_, _ = rand.Read(bytes)
rand.Read(bytes)
for i, b := range bytes {
bytes[i] = alphanum[b%byte(len(alphanum))]
}
-1
View File
@@ -96,7 +96,6 @@ func TestAlibaba_FromChunk(t *testing.T) {
t.Fatalf("no raw secret present: \n %+v", got[i])
}
got[i].Raw = nil
got[i].RawV2 = nil
}
if diff := pretty.Compare(got, tt.want); diff != "" {
t.Errorf("Alibaba.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+20 -20
View File
@@ -56,11 +56,11 @@ func TestAWS_FromChunk(t *testing.T) {
{
DetectorType: detectorspb.DetectorType_AWS,
Verified: true,
Redacted: "AKIASP2TPHJSQH3FJRUX",
Redacted: "AKIAWARWQKZNHMZBLY4I",
ExtraData: map[string]string{
"account": "171436882533",
"arn": "arn:aws:iam::171436882533:user/canarytokens.com@@4dxkh0pdeop3bzu9zx5wob793",
"user_id": "AIDASP2TPHJSUFRSTTZX4",
"account": "413504919130",
"arn": "arn:aws:iam::413504919130:root",
"user_id": "413504919130",
},
},
},
@@ -78,7 +78,7 @@ func TestAWS_FromChunk(t *testing.T) {
{
DetectorType: detectorspb.DetectorType_AWS,
Verified: false,
Redacted: "AKIASP2TPHJSQH3FJRUX",
Redacted: "AKIAWARWQKZNHMZBLY4I",
ExtraData: nil,
},
},
@@ -106,18 +106,18 @@ func TestAWS_FromChunk(t *testing.T) {
want: []detectors.Result{
{
DetectorType: detectorspb.DetectorType_AWS,
Verified: false,
Redacted: "AKIASP2TPHJSQH3FJXYZ",
Verified: true,
Redacted: id,
ExtraData: map[string]string{
"account": "413504919130",
"arn": "arn:aws:iam::413504919130:root",
"user_id": "413504919130",
},
},
{
DetectorType: detectorspb.DetectorType_AWS,
Verified: true,
Redacted: "AKIASP2TPHJSQH3FJRUX",
ExtraData: map[string]string{
"account": "171436882533",
"arn": "arn:aws:iam::171436882533:user/canarytokens.com@@4dxkh0pdeop3bzu9zx5wob793",
"user_id": "AIDASP2TPHJSUFRSTTZX4",
},
Verified: false,
Redacted: inactiveID,
},
},
wantErr: false,
@@ -145,11 +145,11 @@ func TestAWS_FromChunk(t *testing.T) {
{
DetectorType: detectorspb.DetectorType_AWS,
Verified: true,
Redacted: "AKIASP2TPHJSQH3FJRUX",
Redacted: id,
ExtraData: map[string]string{
"account": "171436882533",
"arn": "arn:aws:iam::171436882533:user/canarytokens.com@@4dxkh0pdeop3bzu9zx5wob793",
"user_id": "AIDASP2TPHJSUFRSTTZX4",
"account": "413504919130",
"arn": "arn:aws:iam::413504919130:root",
"user_id": "413504919130",
},
},
{
@@ -172,7 +172,7 @@ func TestAWS_FromChunk(t *testing.T) {
{
DetectorType: detectorspb.DetectorType_AWS,
Verified: false,
Redacted: "AKIASP2TPHJSQH3FJRUX",
Redacted: "AKIAWARWQKZNHMZBLY4I",
},
},
wantErr: false,
@@ -181,7 +181,7 @@ func TestAWS_FromChunk(t *testing.T) {
name: "skipped",
s: scanner{
skipIDs: map[string]struct{}{
"AKIASP2TPHJSQH3FJRUX": {},
"AKIAWARWQKZNHMZBLY4I": {},
},
},
args: args{
-1
View File
@@ -54,7 +54,6 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
s := detectors.Result{
DetectorType: detectorspb.DetectorType_Azure,
Raw: []byte(clientSecret[2]),
RawV2: []byte(clientID[2] + clientSecret[2] + tenantID[2]),
Redacted: clientID[2],
}
+2 -1
View File
@@ -2,6 +2,7 @@ package blocknative
import (
"context"
"fmt"
"io"
"net/http"
"regexp"
@@ -52,7 +53,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
if err != nil {
continue
}
req.Header.Add("Authorization", resMatch)
req.Header.Add("Authorization", fmt.Sprintf("%s", resMatch))
res, err := client.Do(req)
if err == nil {
defer res.Body.Close()
+1 -1
View File
@@ -56,7 +56,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
}
if verify {
req, err := http.NewRequestWithContext(ctx, "GET", "https://www.browserstack.com/automate/plan.json", nil)
req, err := http.NewRequestWithContext(ctx, "GET", "https://api-cloud.browserstack.com/app-automate/espresso/v2/apps", nil)
if err != nil {
continue
}
+4 -3
View File
@@ -2,6 +2,7 @@ package coinmarketcap
import (
"context"
"fmt"
"net/http"
"regexp"
"strings"
@@ -20,13 +21,13 @@ var (
client = common.SaneHttpClient()
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"coinmarketcap", "CMC"}) + `\b([0-9Aa-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})\b`)
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"coinmarketcap","CMC"}) + `\b([0-9Aa-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})\b`)
)
// Keywords are used for efficiently pre-filtering chunks.
// Use identifiers in the secret preferably, or the provider name.
func (s Scanner) Keywords() []string {
return []string{"coinmarketcap", "CMC"}
return []string{"coinmarketcap","CMC"}
}
// FromData will find and optionally verify Coinmarketcap secrets in a given set of bytes.
@@ -52,7 +53,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
continue
}
req.Header.Add("Accept", "application/json")
req.Header.Add("X-CMC_PRO_API_KEY", resMatch)
req.Header.Add("X-CMC_PRO_API_KEY", fmt.Sprintf("%s", resMatch))
res, err := client.Do(req)
if err == nil {
defer res.Body.Close()
@@ -19,7 +19,7 @@ var _ detectors.Detector = (*Scanner)(nil)
var (
client = common.SaneHttpClient()
keyPat = regexp.MustCompile(`\b(CFPAT-[a-zA-Z0-9_\-]{43})\b`)
keyPat = regexp.MustCompile(`\b([CFPAT\-a-zA-Z-0-9]{49})\b`)
)
// Keywords are used for efficiently pre-filtering chunks.
@@ -51,7 +51,6 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
s1 := detectors.Result{
DetectorType: detectorspb.DetectorType_DatadogToken,
Raw: []byte(resMatch),
RawV2: []byte(resMatch + resApiMatch),
}
if verify {
-9
View File
@@ -31,18 +31,9 @@ type Versioner interface {
Version() int
}
// EndpointCustomizer is an optional interface that a detector can implement to
// support verifying against user-supplied endpoints.
type EndpointCustomizer interface {
SetEndpoints(...string) error
DefaultEndpoint() string
}
type Result struct {
// DetectorType is the type of Detector.
DetectorType detectorspb.DetectorType
// DetectorName is the name of the Detector. Used for custom detectors.
DetectorName string
// DecoderType is the type of Decoder.
DecoderType detectorspb.DecoderType
Verified bool
-36
View File
@@ -1,36 +0,0 @@
package detectors
import (
"fmt"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
)
// EndpointSetter implements a sensible default for the SetEndpoints function
// of the EndpointCustomizer interface. A detector can embed this struct to
// gain the functionality.
type EndpointSetter struct {
endpoints []string
}
func (e *EndpointSetter) SetEndpoints(endpoints ...string) error {
if len(endpoints) == 0 {
return fmt.Errorf("at least one endpoint required")
}
deduped := make([]string, 0, len(endpoints))
for _, endpoint := range endpoints {
common.AddStringSliceItem(endpoint, &deduped)
}
e.endpoints = deduped
return nil
}
func (e *EndpointSetter) Endpoints(defaultEndpoint string) []string {
// The only valid time len(e.endpoints) == 0 is when EndpointSetter is
// initializetd to its default state. That means SetEndpoints was never
// called and we should use the default.
if len(e.endpoints) == 0 {
return []string{defaultEndpoint}
}
return e.endpoints
}
-16
View File
@@ -1,16 +0,0 @@
package detectors
import (
"testing"
"github.com/stretchr/testify/assert"
)
func TestEmbeddedEndpointSetter(t *testing.T) {
type Scanner struct{ EndpointSetter }
var s Scanner
assert.Equal(t, []string{"baz"}, s.Endpoints("baz"))
assert.NoError(t, s.SetEndpoints("foo", "bar"))
assert.Error(t, s.SetEndpoints())
assert.Equal(t, []string{"foo", "bar"}, s.Endpoints("baz"))
}
+1 -3
View File
@@ -79,16 +79,14 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
raw = []byte(key)
}
credBytes, _ := json.Marshal(creds)
s := detectors.Result{
DetectorType: detectorspb.DetectorType_GCP,
Raw: raw,
RawV2: credBytes,
Redacted: creds.ClientEmail,
}
if verify {
credBytes, _ := json.Marshal(creds)
credentials, err := google.CredentialsFromJSON(ctx, credBytes, "https://www.googleapis.com/auth/cloud-platform")
if err != nil {
continue
+24 -28
View File
@@ -13,15 +13,13 @@ import (
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
)
type Scanner struct{ detectors.EndpointSetter }
type Scanner struct{}
// Ensure the Scanner satisfies the interfaces at compile time.
var _ detectors.Detector = (*Scanner)(nil)
var _ detectors.Versioner = (*Scanner)(nil)
var _ detectors.EndpointCustomizer = (*Scanner)(nil)
func (Scanner) Version() int { return 2 }
func (Scanner) DefaultEndpoint() string { return "https://api.github.com" }
func (s Scanner) Version() int { return 2 }
var (
// Oauth token
@@ -31,7 +29,7 @@ var (
// https://github.blog/changelog/2022-10-18-introducing-fine-grained-personal-access-tokens/
keyPat = regexp.MustCompile(`\b((?:ghp|gho|ghu|ghs|ghr|github_pat)_[a-zA-Z0-9_]{36,255})\b`)
// TODO: Oauth2 client_id and client_secret
//TODO: Oauth2 client_id and client_secret
// https://developer.github.com/v3/#oauth2-keysecret
)
@@ -73,29 +71,27 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
if verify {
client := common.SaneHttpClient()
// https://developer.github.com/v3/users/#get-the-authenticated-user
for _, url := range s.Endpoints(s.DefaultEndpoint()) {
req, err := http.NewRequestWithContext(ctx, "GET", fmt.Sprintf("%s/user", url), nil)
if err != nil {
continue
}
req.Header.Add("Content-Type", "application/json; charset=utf-8")
req.Header.Add("Authorization", fmt.Sprintf("token %s", token))
res, err := client.Do(req)
if err == nil {
if res.StatusCode >= 200 && res.StatusCode < 300 {
var userResponse userRes
err = json.NewDecoder(res.Body).Decode(&userResponse)
res.Body.Close()
if err == nil {
s1.Verified = true
s1.ExtraData = map[string]string{
"username": userResponse.Login,
"url": userResponse.UserURL,
"account_type": userResponse.Type,
"site_admin": fmt.Sprintf("%t", userResponse.SiteAdmin),
"name": userResponse.Name,
"company": userResponse.Company,
}
req, err := http.NewRequestWithContext(ctx, "GET", "https://api.github.com/user", nil)
if err != nil {
continue
}
req.Header.Add("Content-Type", "application/json; charset=utf-8")
req.Header.Add("Authorization", fmt.Sprintf("token %s", token))
res, err := client.Do(req)
if err == nil {
if res.StatusCode >= 200 && res.StatusCode < 300 {
var userResponse userRes
err = json.NewDecoder(res.Body).Decode(&userResponse)
res.Body.Close()
if err == nil {
s1.Verified = true
s1.ExtraData = map[string]string{
"username": userResponse.Login,
"url": userResponse.UserURL,
"account_type": userResponse.Type,
"site_admin": fmt.Sprintf("%t", userResponse.SiteAdmin),
"name": userResponse.Name,
"company": userResponse.Company,
}
}
}
+19 -23
View File
@@ -12,15 +12,13 @@ import (
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
)
type Scanner struct{ detectors.EndpointSetter }
type Scanner struct{}
// Ensure the Scanner satisfies the interfaces at compile time.
var _ detectors.Detector = (*Scanner)(nil)
var _ detectors.Versioner = (*Scanner)(nil)
var _ detectors.EndpointCustomizer = (*Scanner)(nil)
func (Scanner) Version() int { return 1 }
func (Scanner) DefaultEndpoint() string { return "https://api.github.com" }
func (s Scanner) Version() int { return 1 }
var (
// Oauth token
@@ -65,7 +63,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
continue
}
s1 := detectors.Result{
s := detectors.Result{
DetectorType: detectorspb.DetectorType_Github,
Raw: []byte(token),
}
@@ -73,32 +71,30 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
if verify {
client := common.SaneHttpClient()
// https://developer.github.com/v3/users/#get-the-authenticated-user
for _, url := range s.Endpoints(s.DefaultEndpoint()) {
req, err := http.NewRequestWithContext(ctx, "GET", fmt.Sprintf("%s/user", url), nil)
if err != nil {
continue
}
req.Header.Add("Content-Type", "application/json; charset=utf-8")
req.Header.Add("Authorization", fmt.Sprintf("token %s", token))
res, err := client.Do(req)
if err == nil {
if res.StatusCode >= 200 && res.StatusCode < 300 {
var userResponse userRes
err = json.NewDecoder(res.Body).Decode(&userResponse)
res.Body.Close()
if err == nil {
s1.Verified = true
}
req, err := http.NewRequestWithContext(ctx, "GET", "https://api.github.com/user", nil)
if err != nil {
continue
}
req.Header.Add("Content-Type", "application/json; charset=utf-8")
req.Header.Add("Authorization", fmt.Sprintf("token %s", token))
res, err := client.Do(req)
if err == nil {
if res.StatusCode >= 200 && res.StatusCode < 300 {
var userResponse userRes
err = json.NewDecoder(res.Body).Decode(&userResponse)
res.Body.Close()
if err == nil {
s.Verified = true
}
}
}
}
if !s1.Verified && detectors.IsKnownFalsePositive(token, detectors.DefaultFalsePositives, true) {
if !s.Verified && detectors.IsKnownFalsePositive(token, detectors.DefaultFalsePositives, true) {
continue
}
results = append(results, s1)
results = append(results, s)
}
return results, nil
+17 -20
View File
@@ -12,15 +12,13 @@ import (
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
)
type Scanner struct{ detectors.EndpointSetter }
type Scanner struct{}
// Ensure the Scanner satisfies the interfaces at compile time.
var _ detectors.Detector = (*Scanner)(nil)
var _ detectors.Versioner = (*Scanner)(nil)
var _ detectors.EndpointCustomizer = (*Scanner)(nil)
func (Scanner) Version() int { return 1 }
func (Scanner) DefaultEndpoint() string { return "https://gitlab.com" }
func (s Scanner) Version() int { return 1 }
var (
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"gitlab"}) + `\b((?:glpat|)[a-zA-Z0-9\-=_]{20,22})\b`)
@@ -58,25 +56,24 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
// they all grant access to different parts of the API. I couldn't find an endpoint that every
// one of these scopes has access to, so we just check an example endpoint for each scope. If any
// of them contain data, we know we have a valid key, but if they all fail, we don't
baseURL := "https://gitlab.com/api/v4"
client := common.SaneHttpClient()
for _, baseURL := range s.Endpoints(s.DefaultEndpoint()) {
// test `read_user` scope
req, err := http.NewRequestWithContext(ctx, "GET", baseURL+"/api/v4/user", nil)
if err != nil {
continue
}
req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", resMatch))
res, err := client.Do(req)
if err == nil {
res.Body.Close() // The request body is unused.
// test `read_user` scope
req, err := http.NewRequestWithContext(ctx, "GET", baseURL+"/user", nil)
if err != nil {
continue
}
req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", resMatch))
res, err := client.Do(req)
if err == nil {
res.Body.Close() // The request body is unused.
// 200 means good key and has `read_user` scope
// 403 means good key but not the right scope
// 401 is bad key
if res.StatusCode == http.StatusOK || res.StatusCode == http.StatusForbidden {
secret.Verified = true
}
// 200 means good key and has `read_user` scope
// 403 means good key but not the right scope
// 401 is bad key
if res.StatusCode == http.StatusOK || res.StatusCode == http.StatusForbidden {
secret.Verified = true
}
}
}
+20 -22
View File
@@ -11,15 +11,13 @@ import (
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
)
type Scanner struct{ detectors.EndpointSetter }
type Scanner struct{}
// Ensure the Scanner satisfies the interfaces at compile time.
var _ detectors.Detector = (*Scanner)(nil)
var _ detectors.Versioner = (*Scanner)(nil)
var _ detectors.EndpointCustomizer = (*Scanner)(nil)
func (Scanner) Version() int { return 2 }
func (Scanner) DefaultEndpoint() string { return "https://gitlab.com" }
func (*Scanner) Version() int { return 2 }
var (
keyPat = regexp.MustCompile(`\b(glpat-[a-zA-Z0-9\-=_]{20,22})\b`)
@@ -52,27 +50,27 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
// they all grant access to different parts of the API. I couldn't find an endpoint that every
// one of these scopes has access to, so we just check an example endpoint for each scope. If any
// of them contain data, we know we have a valid key, but if they all fail, we don't
client := common.SaneHttpClient()
for _, baseURL := range s.Endpoints(s.DefaultEndpoint()) {
// test `read_user` scope
req, err := http.NewRequestWithContext(ctx, "GET", baseURL+"/api/v4/user", nil)
if err != nil {
continue
}
req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", match[1]))
res, err := client.Do(req)
if err == nil {
res.Body.Close() // The request body is unused.
baseURL := "https://gitlab.com/api/v4"
// 200 means good key and has `read_user` scope
// 403 means good key but not the right scope
// 401 is bad key
if res.StatusCode == http.StatusOK || res.StatusCode == http.StatusForbidden {
secret.Verified = true
}
client := common.SaneHttpClient()
// test `read_user` scope
req, err := http.NewRequestWithContext(ctx, "GET", baseURL+"/user", nil)
if err != nil {
continue
}
req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", match[1]))
res, err := client.Do(req)
if err == nil {
res.Body.Close() // The request body is unused.
// 200 means good key and has `read_user` scope
// 403 means good key but not the right scope
// 401 is bad key
if res.StatusCode == http.StatusOK || res.StatusCode == http.StatusForbidden {
secret.Verified = true
}
}
}
if !secret.Verified && detectors.IsKnownFalsePositive(string(secret.Raw), detectors.DefaultFalsePositives, true) {
+1 -1
View File
@@ -21,7 +21,7 @@ var (
client = common.SaneHttpClient()
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"ipstack"}) + `\b([a-fA-F0-9]{32})\b`)
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"ipstack"}) + `\b([a-fA-f0-9]{32})\b`)
)
// Keywords are used for efficiently pre-filtering chunks.
+1 -38
View File
@@ -4,7 +4,6 @@ import (
"context"
"database/sql"
"errors"
"fmt"
"regexp"
"strings"
"time"
@@ -13,35 +12,7 @@ import (
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
)
type Scanner struct {
ignorePatterns []regexp.Regexp
}
func New(opts ...func(*Scanner)) *Scanner {
scanner := &Scanner{
ignorePatterns: []regexp.Regexp{},
}
for _, opt := range opts {
opt(scanner)
}
return scanner
}
func WithIgnorePattern(ignoreStrings []string) func(*Scanner) {
return func(s *Scanner) {
ignorePatterns := []regexp.Regexp{}
for _, ignoreString := range ignoreStrings {
ignorePattern, err := regexp.Compile(ignoreString)
if err != nil {
panic(fmt.Sprintf("%s is not a valid regex, error received: %v", ignoreString, err))
}
ignorePatterns = append(ignorePatterns, *ignorePattern)
}
s.ignorePatterns = ignorePatterns
}
}
type Scanner struct{}
// Ensure the Scanner satisfies the interface at compile time.
var _ detectors.Detector = (*Scanner)(nil)
@@ -61,15 +32,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
dataStr := string(data)
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
matchLoop:
for _, match := range matches {
if len(s.ignorePatterns) != 0 {
for _, ignore := range s.ignorePatterns {
if ignore.MatchString(match[0]) {
continue matchLoop
}
}
}
jdbcConn := match[0]
s := detectors.Result{
-51
View File
@@ -156,57 +156,6 @@ func TestJdbc_FromChunk(t *testing.T) {
}
}
func TestJdbc_FromDataWithIgnorePattern(t *testing.T) {
type args struct {
ctx context.Context
data []byte
verify bool
}
tests := []struct {
name string
args args
want []detectors.Result
ignorePatterns []string
wantErr bool
}{
{
name: "not found",
args: args{
ctx: context.Background(),
data: []byte("jdbc:sqlite::secretpattern:"),
verify: false,
},
want: nil,
ignorePatterns: []string{
".*secretpattern.*",
},
wantErr: false,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
s := New(WithIgnorePattern(tt.ignorePatterns))
got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
if (err != nil) != tt.wantErr {
t.Errorf("Jdbc.FromDataWithConfig() error = %v, wantErr %v", err, tt.wantErr)
return
}
if os.Getenv("FORCE_PASS_DIFF") == "true" {
return
}
for i := range got {
if len(got[i].Raw) == 0 {
t.Fatal("no raw secret present")
}
got[i].Raw = nil
}
if diff := pretty.Compare(got, tt.want); diff != "" {
t.Errorf("Jdbc.FromDataWithConfig() %s diff: (-got +want)\n%s", tt.name, diff)
}
})
}
}
func BenchmarkFromData(benchmark *testing.B) {
ctx := context.Background()
s := Scanner{}
+3 -13
View File
@@ -24,12 +24,7 @@ var (
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
tokenPat = regexp.MustCompile(detectors.PrefixRegex([]string{"jira"}) + `\b([a-zA-Z-0-9]{24})\b`)
domainPat = regexp.MustCompile(detectors.PrefixRegex([]string{"jira"}) + `\b([a-zA-Z-0-9]{5,24}\.[a-zA-Z-0-9]{3,16}\.[a-zA-Z-0-9]{3,16})\b`)
emailPat = regexp.MustCompile(detectors.PrefixRegex([]string{"jira"}) + `\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Z|a-z]{2,}\b`)
)
const (
failedAuth = "AUTHENTICATED_FAILED"
loginReasonHeaderKey = "X-Seraph-LoginReason"
emailPat = regexp.MustCompile(detectors.PrefixRegex([]string{"jira"}) + `\b([a-zA-Z-0-9]{5,24}\@[a-zA-Z-0-9]{3,16}\.com)\b`)
)
// Keywords are used for efficiently pre-filtering chunks.
@@ -47,7 +42,6 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
emails := emailPat.FindAllStringSubmatch(dataStr, -1)
for _, email := range emails {
email = strings.Split(email[0], " ")
if len(email) != 2 {
continue
}
@@ -66,10 +60,10 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
s1 := detectors.Result{
DetectorType: detectorspb.DetectorType_JiraToken,
Raw: []byte(resToken),
RawV2: []byte(fmt.Sprintf("%s:%s:%s", resEmail, resToken, resDomain)),
}
if verify {
data := fmt.Sprintf("%s:%s", resEmail, resToken)
sEnc := b64.StdEncoding.EncodeToString([]byte(data))
req, err := http.NewRequestWithContext(ctx, "GET", "https://"+resDomain+"/rest/api/3/dashboard", nil)
@@ -81,11 +75,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
res, err := client.Do(req)
if err == nil {
defer res.Body.Close()
// If the request is successful and the login reason is not failed authentication, then the token is valid.
// This is because Jira returns a 200 status code even if the token is invalid.
// Jira returns a default dashboard page.
if (res.StatusCode >= 200 && res.StatusCode < 300) && res.Header.Get(loginReasonHeaderKey) != failedAuth {
if res.StatusCode >= 200 && res.StatusCode < 300 {
s1.Verified = true
}
}
@@ -97,7 +97,6 @@ func TestJiraToken_FromChunk(t *testing.T) {
t.Fatalf("no raw secret present: \n %+v", got[i])
}
got[i].Raw = nil
got[i].RawV2 = nil
}
if diff := pretty.Compare(got, tt.want); diff != "" {
t.Errorf("JiraToken.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+2 -1
View File
@@ -2,6 +2,7 @@ package moralis
import (
"context"
"fmt"
"net/http"
"regexp"
"strings"
@@ -52,7 +53,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
continue
}
req.Header.Add("Accept", "application/json")
req.Header.Add("X-API-Key", resMatch)
req.Header.Add("X-API-Key", fmt.Sprintf("%s", resMatch))
res, err := client.Do(req)
if err == nil {
defer res.Body.Close()
-1
View File
@@ -41,7 +41,6 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
s := detectors.Result{
DetectorType: detectorspb.DetectorType_Okta,
Raw: []byte(token),
RawV2: []byte(fmt.Sprintf("%s:%s", domain, token)),
}
if verify {
-1
View File
@@ -50,7 +50,6 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
s := detectors.Result{
DetectorType: detectorspb.DetectorType_OneLogin,
Raw: []byte(clientID[1]),
RawV2: []byte(fmt.Sprintf("%s:%s", clientID[1], clientSecret[1])),
Redacted: clientID[1],
}
+12 -3
View File
@@ -50,7 +50,7 @@ func TestOpenAI_FromChunk(t *testing.T) {
want: []detectors.Result{
{
DetectorType: detectorspb.DetectorType_OpenAI,
Redacted: "sk-...gOPc",
Redacted: "sk-...xxxx",
Verified: false,
},
},
@@ -68,7 +68,17 @@ func TestOpenAI_FromChunk(t *testing.T) {
{
DetectorType: detectorspb.DetectorType_OpenAI,
Verified: true,
Redacted: "sk-...gOPb",
Redacted: "sk-...xxxx",
ExtraData: map[string]string{
"description": "",
"id": "",
"is_default": "",
"is_personal": "",
"role": "",
"title": "",
"total_orgs": "",
"user": "",
},
},
},
wantErr: false,
@@ -98,7 +108,6 @@ func TestOpenAI_FromChunk(t *testing.T) {
t.Fatal("no raw secret present")
}
got[i].Raw = nil
got[i].ExtraData = nil
}
if diff := pretty.Compare(got, tt.want); diff != "" {
t.Errorf("OpenAI.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
-79
View File
@@ -1,79 +0,0 @@
package percy
import (
"context"
"fmt"
"net/http"
"regexp"
"strings"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
)
type Scanner struct{}
// Ensure the Scanner satisfies the interface at compile time.
var _ detectors.Detector = (*Scanner)(nil)
var (
client = common.SaneHttpClient()
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"percy"}) + `\bPERCY_TOKEN=([0-9Aa-f]{64})\b`)
)
// Keywords are used for efficiently pre-filtering chunks.
// Use identifiers in the secret preferably, or the provider name.
func (s Scanner) Keywords() []string {
return []string{"percy"}
}
// FromData will find and optionally verify Percy secrets in a given set of bytes.
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
dataStr := string(data)
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
for _, match := range matches {
if len(match) != 2 {
continue
}
resMatch := strings.TrimSpace(match[1])
s1 := detectors.Result{
DetectorType: detectorspb.DetectorType_Percy,
Raw: []byte(resMatch),
}
if verify {
req, err := http.NewRequestWithContext(ctx, "GET", "https://percy.io/api/v1/projects", nil)
if err != nil {
continue
}
req.Header.Add("Accept", "application/vnd.percy+json; version=3")
req.Header.Add("Authorization", fmt.Sprintf("Token %s", resMatch))
res, err := client.Do(req)
if err == nil {
defer res.Body.Close()
if res.StatusCode >= 200 && res.StatusCode < 300 {
s1.Verified = true
} else {
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
continue
}
}
}
}
results = append(results, s1)
}
return results, nil
}
func (s Scanner) Type() detectorspb.DetectorType {
return detectorspb.DetectorType_Percy
}
-119
View File
@@ -1,119 +0,0 @@
//go:build detectors
// +build detectors
package percy
import (
"context"
"fmt"
"testing"
"time"
"github.com/kylelemons/godebug/pretty"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
)
func TestPercy_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
secret := testSecrets.MustGetField("PERCY_VERIFIED")
inactiveSecret := testSecrets.MustGetField("PERCY_INACTIVE")
type args struct {
ctx context.Context
data []byte
verify bool
}
tests := []struct {
name string
s Scanner
args args
want []detectors.Result
wantErr bool
}{
{
name: "found, verified",
s: Scanner{},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a percy secret %s within", secret)),
verify: true,
},
want: []detectors.Result{
{
DetectorType: detectorspb.DetectorType_Percy,
Verified: true,
},
},
wantErr: false,
},
{
name: "found, unverified",
s: Scanner{},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a percy secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
verify: true,
},
want: []detectors.Result{
{
DetectorType: detectorspb.DetectorType_Percy,
Verified: false,
},
},
wantErr: false,
},
{
name: "not found",
s: Scanner{},
args: args{
ctx: context.Background(),
data: []byte("You cannot find the secret within"),
verify: true,
},
want: nil,
wantErr: false,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
s := Scanner{}
got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
if (err != nil) != tt.wantErr {
t.Errorf("Percy.FromData() error = %v, wantErr %v", err, tt.wantErr)
return
}
for i := range got {
if len(got[i].Raw) == 0 {
t.Fatalf("no raw secret present: \n %+v", got[i])
}
got[i].Raw = nil
}
if diff := pretty.Compare(got, tt.want); diff != "" {
t.Errorf("Percy.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
}
})
}
}
func BenchmarkFromData(benchmark *testing.B) {
ctx := context.Background()
s := Scanner{}
for name, data := range detectors.MustGetBenchmarkData() {
benchmark.Run(name, func(b *testing.B) {
for n := 0; n < b.N; n++ {
_, err := s.FromData(ctx, false, data)
if err != nil {
b.Fatal(err)
}
}
})
}
}
+1 -1
View File
@@ -24,7 +24,7 @@ var (
// Keywords are used for efficiently pre-filtering chunks.
// Use identifiers in the secret preferably, or the provider name.
func (s Scanner) Keywords() []string {
return []string{"sql", "database", "Data Source"}
return []string{"sqlserver"}
}
// FromData will find and optionally verify SpotifyKey secrets in a given set of bytes.
+5 -62
View File
@@ -5,15 +5,14 @@ package sqlserver
import (
"context"
"errors"
"fmt"
"testing"
"github.com/denisenkom/go-mssqldb/msdsn"
"github.com/google/go-cmp/cmp"
"github.com/google/go-cmp/cmp/cmpopts"
"testing"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
)
@@ -76,59 +75,6 @@ func TestSQLServer_FromChunk(t *testing.T) {
}
},
},
{
name: "not found, in XML, missing password param (pwd is not valid)",
s: Scanner{},
args: args{
ctx: context.Background(),
data: []byte(`<add name="Sample2" value="SERVER=server_name;DATABASE=database_name;user=user_name;pwd=plaintextpassword;encrypt=true;Timeout=120;MultipleActiveResultSets=True;" />`),
verify: true,
},
want: nil,
wantErr: false,
mockFunc: func() {
ping = func(config msdsn.Config) (bool, error) {
return true, nil
}
},
},
{
name: "found, verified, in XML",
s: Scanner{},
args: args{
ctx: context.Background(),
data: []byte(`<add name="test db" value="SERVER=server_name;DATABASE=testdb;user=username;password=badpassword;encrypt=true;Timeout=120;MultipleActiveResultSets=True;" />`),
verify: true,
},
want: []detectors.Result{
{
DetectorType: detectorspb.DetectorType_SQLServer,
Verified: true,
},
},
wantErr: false,
mockFunc: func() {
ping = func(config msdsn.Config) (bool, error) {
if config.Host != "server_name" {
return false, errors.New("invalid host")
}
if config.User != "username" {
return false, errors.New("invalid database")
}
if config.Password != "badpassword" {
return false, errors.New("invalid password")
}
if config.Database != "testdb" {
return false, errors.New("invalid database")
}
return true, nil
}
},
},
{
name: "not found",
s: Scanner{},
@@ -173,16 +119,13 @@ func TestSQLServer_FromChunk(t *testing.T) {
func TestSQLServer_pattern(t *testing.T) {
if !pattern.Match([]byte(`builder.Services.AddDbContext<Database>(optionsBuilder => optionsBuilder.UseSqlServer("Server=localhost;Initial Catalog=master;User ID=sa;Password=P@ssw0rd!;Persist Security Info=true;MultipleActiveResultSets=true;"));`)) {
t.Errorf("SQLServer.pattern: did not find connection string from Program.cs")
t.Errorf("SQLServer.pattern: did not catched connection string from Program.cs")
}
if !pattern.Match([]byte(`{"ConnectionStrings": {"Demo": "Server=localhost;Initial Catalog=master;User ID=sa;Password=P@ssw0rd!;Persist Security Info=true;MultipleActiveResultSets=true;"}}`)) {
t.Errorf("SQLServer.pattern: did not find connection string from appsettings.json")
t.Errorf("SQLServer.pattern: did not catched connection string from appsettings.json")
}
if !pattern.Match([]byte(`CONNECTION_STRING: Server=localhost;Initial Catalog=master;User ID=sa;Password=P@ssw0rd!;Persist Security Info=true;MultipleActiveResultSets=true`)) {
t.Errorf("SQLServer.pattern: did not find connection string from .env")
}
if !pattern.Match([]byte(`<add name="Sample2" value="SERVER=server_name;DATABASE=database_name;user=user_name;pwd=plaintextpassword;encrypt=true;Timeout=120;MultipleActiveResultSets=True;" />`)) {
t.Errorf("SQLServer.pattern: did not find connection string in xml format")
t.Errorf("SQLServer.pattern: did not catched connection string from .env")
}
}
+5 -4
View File
@@ -20,7 +20,7 @@ type Scanner struct{}
var _ detectors.Detector = (*Scanner)(nil)
var (
// possibly always `sq0csp` for secret
//possibly always `sq0csp` for secret
// and `sq0idb` for app
keyPat = regexp.MustCompile(`[\w\-]*sq0i[a-z]{2}-[0-9A-Za-z\-_]{22,43}`)
secPat = regexp.MustCompile(`[\w\-]*sq0c[a-z]{2}-[0-9A-Za-z\-_]{40,50}`)
@@ -39,6 +39,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
matches := keyPat.FindAllString(dataStr, -1)
secMatches := secPat.FindAllString(dataStr, -1)
for _, match := range matches {
for _, secMatch := range secMatches {
if detectors.IsKnownFalsePositive(strings.ToLower(secMatch), detectors.DefaultFalsePositives, true) {
@@ -46,7 +47,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
}
s := detectors.Result{
DetectorType: detectorspb.DetectorType_SquareApp,
DetectorType: detectorspb.DetectorType_Square,
Raw: []byte(match),
Redacted: match,
}
@@ -71,7 +72,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
req.Header.Add("Content-Type", "application/json")
// unclear if this version needs to be set or matters, seems to work without, but docs want it
// req.Header.Add("Square-Version", "2020-08-12")
//req.Header.Add("Square-Version", "2020-08-12")
res, err := client.Do(req)
if err == nil {
res.Body.Close() // The request body is unused.
@@ -94,5 +95,5 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
}
func (s Scanner) Type() detectorspb.DetectorType {
return detectorspb.DetectorType_SquareApp
return detectorspb.DetectorType_Square
}
@@ -1,75 +0,0 @@
package tineswebhook
import (
"context"
"net/http"
"regexp"
"strings"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
)
type Scanner struct{}
// Ensure the Scanner satisfies the interface at compile time.
var _ detectors.Detector = (*Scanner)(nil)
var (
client = common.SaneHttpClient()
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
keyPat = regexp.MustCompile(`(https://[\w-]+\.tines\.com/webhook/[a-z0-9]{32}/[a-z0-9]{32})`)
)
// Keywords are used for efficiently pre-filtering chunks.
// Use identifiers in the secret preferably, or the provider name.
func (s Scanner) Keywords() []string {
return []string{"tines.com"}
}
// FromData will find and optionally verify TinesWebhook secrets in a given set of bytes.
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
dataStr := string(data)
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
for _, match := range matches {
if len(match) != 2 {
continue
}
resMatch := strings.TrimSpace(match[1])
s1 := detectors.Result{
DetectorType: detectorspb.DetectorType_TinesWebhook,
Raw: []byte(resMatch),
}
if verify {
payload := strings.NewReader(``)
req, err := http.NewRequestWithContext(ctx, "GET", resMatch, payload)
if err != nil {
continue
}
res, err := client.Do(req)
if err == nil {
defer res.Body.Close()
if err != nil {
continue
}
if res.StatusCode >= 200 && res.StatusCode < 300 {
s1.Verified = true
}
}
}
results = append(results, s1)
}
return results, nil
}
func (s Scanner) Type() detectorspb.DetectorType {
return detectorspb.DetectorType_TinesWebhook
}
@@ -1,119 +0,0 @@
//go:build detectors
// +build detectors
package tineswebhook
import (
"context"
"fmt"
"testing"
"time"
"github.com/kylelemons/godebug/pretty"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
)
func TestTinesWebhook_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
secret := testSecrets.MustGetField("TINESWEBHOOK_TOKEN")
inactiveSecret := testSecrets.MustGetField("TINESWEBHOOK_INACTIVE")
type args struct {
ctx context.Context
data []byte
verify bool
}
tests := []struct {
name string
s Scanner
args args
want []detectors.Result
wantErr bool
}{
{
name: "found, verified",
s: Scanner{},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a tineswebhook secret %s within", secret)),
verify: true,
},
want: []detectors.Result{
{
DetectorType: detectorspb.DetectorType_TinesWebhook,
Verified: true,
},
},
wantErr: false,
},
{
name: "found, unverified",
s: Scanner{},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a tineswebhook secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
verify: true,
},
want: []detectors.Result{
{
DetectorType: detectorspb.DetectorType_TinesWebhook,
Verified: false,
},
},
wantErr: false,
},
{
name: "not found",
s: Scanner{},
args: args{
ctx: context.Background(),
data: []byte("You cannot find the secret within"),
verify: true,
},
want: nil,
wantErr: false,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
s := Scanner{}
got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
if (err != nil) != tt.wantErr {
t.Errorf("TinesWebhook.FromData() error = %v, wantErr %v", err, tt.wantErr)
return
}
for i := range got {
if len(got[i].Raw) == 0 {
t.Fatalf("no raw secret present: \n %+v", got[i])
}
got[i].Raw = nil
}
if diff := pretty.Compare(got, tt.want); diff != "" {
t.Errorf("TinesWebhook.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
}
})
}
}
func BenchmarkFromData(benchmark *testing.B) {
ctx := context.Background()
s := Scanner{}
for name, data := range detectors.MustGetBenchmarkData() {
benchmark.Run(name, func(b *testing.B) {
for n := 0; n < b.N; n++ {
_, err := s.FromData(ctx, false, data)
if err != nil {
b.Fatal(err)
}
}
})
}
}
-3
View File
@@ -66,15 +66,12 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
}
rawURL, _ := url.Parse(urlMatch)
rawURLStr := rawURL.String()
// Removing the path causes possible deduplication issues if some paths have basic auth and some do not.
rawURL.Path = ""
redact := strings.TrimSpace(strings.Replace(rawURL.String(), password, "********", -1))
s := detectors.Result{
DetectorType: detectorspb.DetectorType_URI,
Raw: []byte(rawURL.String()),
RawV2: []byte(rawURLStr),
Redacted: redact,
}
+3 -4
View File
@@ -38,7 +38,7 @@ func TestURI_FromChunk(t *testing.T) {
{
DetectorType: detectorspb.DetectorType_URI,
Verified: false,
Redacted: "https://user:********@www.httpwatch.com",
Redacted: "https://user:****@www.httpwatch.com",
},
},
wantErr: false,
@@ -55,7 +55,7 @@ func TestURI_FromChunk(t *testing.T) {
{
DetectorType: detectorspb.DetectorType_URI,
Verified: true,
Redacted: "https://httpwatch:********@www.httpwatch.com",
Redacted: "https://httpwatch:****@www.httpwatch.com",
},
},
wantErr: false,
@@ -72,7 +72,7 @@ func TestURI_FromChunk(t *testing.T) {
{
DetectorType: detectorspb.DetectorType_URI,
Verified: true,
Redacted: "https://httpwatch:********@www.httpwatch.com",
Redacted: "https://httpwatch:****@www.httpwatch.com",
},
},
wantErr: false,
@@ -101,7 +101,6 @@ func TestURI_FromChunk(t *testing.T) {
// }
for i := range got {
got[i].Raw = nil
got[i].RawV2 = nil
}
if diff := pretty.Compare(got, tt.want); diff != "" {
t.Errorf("URI.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+7 -23
View File
@@ -15,7 +15,6 @@ import (
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/airship"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/airtableapikey"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/airvisual"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/alchemy"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/alconost"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/alegra"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/aletheiaapi"
@@ -75,7 +74,6 @@ import (
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/blablabus"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/blazemeter"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/blitapp"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/blocknative"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/blogger"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/bombbomb"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/boostnote"
@@ -84,7 +82,6 @@ import (
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/brandfetch"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/browserstack"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/browshot"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/bscscan"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/buddyns"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/bugherd"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/bugsnag"
@@ -144,7 +141,6 @@ import (
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/coinbase"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/coinlayer"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/coinlib"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/coinmarketcap"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/collect2"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/column"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/commercejs"
@@ -216,7 +212,6 @@ import (
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/elasticemail"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/enablex"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/enigma"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/etherscan"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/ethplorer"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/etsyapikey"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/everhour"
@@ -323,7 +318,6 @@ import (
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/imagekit"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/imagga"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/impala"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/infura"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/insightly"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/instabot"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/integromat"
@@ -413,7 +407,6 @@ import (
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/monkeylearn"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/moonclerk"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/moosend"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/moralis"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/mrticktock"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/mux"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/myfreshworks"
@@ -477,7 +470,6 @@ import (
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/pdfshift"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/peopledatalabs"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/pepipost"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/percy"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/pinata"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/pipedream"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/pipedrive"
@@ -658,7 +650,6 @@ import (
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/tiingo"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/timecamp"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/timezoneapi"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/tineswebhook"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/tmetric"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/todoist"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/tokeet"
@@ -740,7 +731,13 @@ import (
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/zipcodebase"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/zonkafeedback"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/zulipchat"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/etherscan"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/infura"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/alchemy"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/blocknative"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/moralis"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/bscscan"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/coinmarketcap"
)
func DefaultDetectors() []detectors.Detector {
@@ -1507,18 +1504,5 @@ func DefaultDetectors() []detectors.Detector {
moralis.Scanner{},
bscscan.Scanner{},
coinmarketcap.Scanner{},
percy.Scanner{},
tineswebhook.Scanner{},
}
}
func DefaultDetectorTypesImplementing[T any]() map[detectorspb.DetectorType]struct{} {
out := make(map[detectorspb.DetectorType]struct{})
for _, detector := range DefaultDetectors() {
if _, ok := detector.(T); ok {
out[detector.Type()] = struct{}{}
}
}
return out
}
-50
View File
@@ -1,50 +0,0 @@
package engine
import (
"testing"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
)
func TestDefaultDetectorsHaveUniqueVersions(t *testing.T) {
detectorTypeToVersions := make(map[detectorspb.DetectorType]map[int]struct{})
addVersion := func(versions map[int]struct{}, version int) map[int]struct{} {
if versions == nil {
versions = make(map[int]struct{})
}
versions[version] = struct{}{}
return versions
}
// Loop through all our default detectors and find the ones that
// implement Versioner. Of those, check each version number is unique.
for _, detector := range DefaultDetectors() {
v, ok := detector.(detectors.Versioner)
if !ok {
continue
}
version := v.Version()
key := detector.Type()
if set, ok := detectorTypeToVersions[key]; ok && set != nil {
if _, ok := set[version]; ok {
t.Errorf("detector %q has duplicate version: %d", detectorspb.DetectorType_name[int32(key)], version)
}
}
detectorTypeToVersions[key] = addVersion(detectorTypeToVersions[key], version)
}
}
func TestDefaultDetectorTypesImplementing(t *testing.T) {
isVersioner := DefaultDetectorTypesImplementing[detectors.Versioner]()
for _, detector := range DefaultDetectors() {
_, expectedOk := detector.(detectors.Versioner)
_, gotOk := isVersioner[detector.Type()]
if expectedOk == gotOk {
continue
}
t.Errorf(
"detector %q doesn't match expected",
detectorspb.DetectorType_name[int32(detector.Type())],
)
}
}
+12 -31
View File
@@ -13,7 +13,6 @@ import (
"google.golang.org/protobuf/proto"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/config"
"github.com/trufflesecurity/trufflehog/v3/pkg/context"
"github.com/trufflesecurity/trufflehog/v3/pkg/decoders"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
@@ -150,29 +149,14 @@ func Start(ctx context.Context, options ...EngineOption) *Engine {
})
e.prefilter = builder.Build(keywords)
ctx.Logger().Info("loaded decoders", "count", len(e.decoders))
ctx.Logger().Info("loaded detectors",
ctx.Logger().V(2).Info("loaded decoders", "count", len(e.decoders))
ctx.Logger().V(2).Info("loaded detectors",
"total", len(e.detectors[true])+len(e.detectors[false]),
"verification_enabled", len(e.detectors[true]),
"verification_disabled", len(e.detectors[false]),
)
// Sanity check detectors for duplicate configuration. Only log in case
// a detector has been configured in a way that isn't represented by
// the DetectorID (type and version).
{
dets := append(e.detectors[true], e.detectors[false]...)
seenDetectors := make(map[config.DetectorID]struct{}, len(dets))
for _, det := range dets {
id := config.GetDetectorID(det)
if _, ok := seenDetectors[id]; ok {
ctx.Logger().Info("possible duplicate detector configured", "detector", id)
}
seenDetectors[id] = struct{}{}
}
}
// Start the workers.
// start the workers
for i := 0; i < e.concurrency; i++ {
e.workersWg.Add(1)
go func() {
@@ -254,8 +238,6 @@ func (e *Engine) detectorWorker(ctx context.Context) {
decoderType = detectorspb.DecoderType_PLAIN
case *decoders.Base64:
decoderType = detectorspb.DecoderType_BASE64
case *decoders.UTF16:
decoderType = detectorspb.DecoderType_UTF16
default:
ctx.Logger().Info("unknown decoder type", "type", reflect.TypeOf(decoder).String())
decoderType = detectorspb.DecoderType_UNKNOWN
@@ -265,9 +247,11 @@ func (e *Engine) detectorWorker(ctx context.Context) {
continue
}
// build a map of all keywords that were matched in the chunk
for _, m := range e.prefilter.FindAll(string(decoded.Data)) {
matchedKeywords[strings.ToLower(string(decoded.Data[m.Start():m.End()]))] = struct{}{}
dataLower := strings.ToLower(string(decoded.Data))
matches := e.prefilter.FindAll(dataLower)
for _, m := range matches {
matchedKeywords[dataLower[m.Start():m.End()]] = struct{}{}
}
for verify, detectorsSet := range e.detectors {
@@ -341,9 +325,9 @@ func (e *Engine) detectorWorker(ctx context.Context) {
}
}
// lineNumberSupportedSources is a list of sources that support line numbers.
// It is stored this way because slice consts are not supported.
func lineNumberSupportedSources() []sourcespb.SourceType {
// gitSources is a list of sources that utilize the Git source. It is stored this way because slice consts are not
// supported.
func gitSources() []sourcespb.SourceType {
return []sourcespb.SourceType{
sourcespb.SourceType_SOURCE_TYPE_GIT,
sourcespb.SourceType_SOURCE_TYPE_GITHUB,
@@ -352,13 +336,12 @@ func lineNumberSupportedSources() []sourcespb.SourceType {
sourcespb.SourceType_SOURCE_TYPE_GERRIT,
sourcespb.SourceType_SOURCE_TYPE_GITHUB_UNAUTHENTICATED_ORG,
sourcespb.SourceType_SOURCE_TYPE_PUBLIC_GIT,
sourcespb.SourceType_SOURCE_TYPE_FILESYSTEM,
}
}
// SupportsLineNumbers determines if a line number can be found for a source type.
func SupportsLineNumbers(sourceType sourcespb.SourceType) bool {
for _, i := range lineNumberSupportedSources() {
for _, i := range gitSources() {
if i == sourceType {
return true
}
@@ -392,8 +375,6 @@ func FragmentFirstLine(chunk *sources.Chunk) (int64, *int64) {
fragmentStart = &metadata.Bitbucket.Line
case *source_metadatapb.MetaData_Gerrit:
fragmentStart = &metadata.Gerrit.Line
case *source_metadatapb.MetaData_Filesystem:
fragmentStart = &metadata.Filesystem.Line
default:
return 0, nil
}
+1 -9
View File
@@ -22,12 +22,7 @@ func (e *Engine) ScanGit(ctx context.Context, c sources.GitConfig) error {
git.ScanOptionLogOptions(logOptions),
}
options := &gogit.PlainOpenOptions{
DetectDotGit: true,
EnableDotGitCommonDir: true,
}
repo, err := gogit.PlainOpenWithOptions(c.RepoPath, options)
repo, err := gogit.PlainOpenWithOptions(c.RepoPath, &gogit.PlainOpenOptions{DetectDotGit: true})
if err != nil {
return fmt.Errorf("could not open repo: %s: %w", c.RepoPath, err)
}
@@ -41,9 +36,6 @@ func (e *Engine) ScanGit(ctx context.Context, c sources.GitConfig) error {
if c.HeadRef != "" {
opts = append(opts, git.ScanOptionHeadCommit(c.HeadRef))
}
if c.ExcludeGlobs != nil {
opts = append(opts, git.ScanOptionExcludeGlobs(c.ExcludeGlobs))
}
scanOptions := git.NewScanOptions(opts...)
gitSource := git.NewGit(sourcespb.SourceType_SOURCE_TYPE_GIT, 0, 0, "trufflehog - git", true, runtime.NumCPU(),
+7 -17
View File
@@ -22,27 +22,17 @@ func (e *Engine) ScanS3(ctx context.Context, c sources.S3Config) error {
Credential: &sourcespb.S3_Unauthenticated{},
}
if c.CloudCred {
if len(c.Key) > 0 || len(c.Secret) > 0 || len(c.SessionToken) > 0 {
return fmt.Errorf("cannot use cloud environment and static credentials together")
if len(c.Key) > 0 || len(c.Secret) > 0 {
return fmt.Errorf("cannot use cloud credentials and basic auth together")
}
connection.Credential = &sourcespb.S3_CloudEnvironment{}
}
if len(c.Key) > 0 && len(c.Secret) > 0 {
if len(c.SessionToken) > 0 {
connection.Credential = &sourcespb.S3_SessionToken{
SessionToken: &credentialspb.AWSSessionTokenSecret{
Key: c.Key,
Secret: c.Secret,
SessionToken: c.SessionToken,
},
}
} else {
connection.Credential = &sourcespb.S3_AccessKey{
AccessKey: &credentialspb.KeySecret{
Key: c.Key,
Secret: c.Secret,
},
}
connection.Credential = &sourcespb.S3_AccessKey{
AccessKey: &credentialspb.KeySecret{
Key: c.Key,
Secret: c.Secret,
},
}
}
if len(c.Buckets) > 0 {
+11 -28
View File
@@ -33,7 +33,6 @@ type Commit struct {
Date time.Time
Message strings.Builder
Diffs []Diff
Size int // in bytes
}
// Diff contains the info about a file diff in a commit.
@@ -117,8 +116,8 @@ func (c1 *Commit) Equal(c2 *Commit) bool {
}
// RepoPath parses the output of the `git log` command for the `source` path.
func (c *Parser) RepoPath(ctx context.Context, source string, head string, abbreviatedLog bool, excludedGlobs []string) (chan Commit, error) {
args := []string{"-C", source, "log", "-p", "--full-history", "--date=format:%a %b %d %H:%M:%S %Y %z"}
func (c *Parser) RepoPath(ctx context.Context, source string, head string, abbreviatedLog bool) (chan Commit, error) {
args := []string{"-C", source, "log", "-p", "-U5", "--full-history", "--date=format:%a %b %d %H:%M:%S %Y %z"}
if abbreviatedLog {
args = append(args, "--diff-filter=AM")
}
@@ -127,11 +126,9 @@ func (c *Parser) RepoPath(ctx context.Context, source string, head string, abbre
} else {
args = append(args, "--all")
}
for _, glob := range excludedGlobs {
args = append(args, "--", ".", fmt.Sprintf(":(exclude)%s", glob))
}
cmd := exec.Command("git", args...)
absPath, err := filepath.Abs(source)
if err == nil {
cmd.Env = append(cmd.Env, fmt.Sprintf("GIT_DIR=%s", filepath.Join(absPath, ".git")))
@@ -143,7 +140,7 @@ func (c *Parser) RepoPath(ctx context.Context, source string, head string, abbre
// Unstaged parses the output of the `git diff` command for the `source` path.
func (c *Parser) Unstaged(ctx context.Context, source string) (chan Commit, error) {
// Provide the --cached flag to diff to get the diff of the staged changes.
args := []string{"-C", source, "diff", "-p", "--cached", "--full-history", "--diff-filter=AM", "--date=format:%a %b %d %H:%M:%S %Y %z", "HEAD"}
args := []string{"-C", source, "diff", "-p", "-U5", "--cached", "--full-history", "--diff-filter=AM", "--date=format:%a %b %d %H:%M:%S %Y %z", "HEAD"}
cmd := exec.Command("git", args...)
@@ -181,7 +178,7 @@ func (c *Parser) executeCommand(ctx context.Context, cmd *exec.Cmd) (chan Commit
}()
go func() {
c.FromReader(ctx, stdOut, commitChan)
c.fromReader(ctx, stdOut, commitChan)
if err := cmd.Wait(); err != nil {
ctx.Logger().V(2).Info("Error waiting for git command to complete.", "error", err)
}
@@ -190,14 +187,10 @@ func (c *Parser) executeCommand(ctx context.Context, cmd *exec.Cmd) (chan Commit
return commitChan, nil
}
func (c *Parser) FromReader(ctx context.Context, stdOut io.Reader, commitChan chan Commit) {
func (c *Parser) fromReader(ctx context.Context, stdOut io.Reader, commitChan chan Commit) {
outReader := bufio.NewReader(stdOut)
var (
currentCommit *Commit
currentDiff *Diff
recentlyPassedAuthor bool
totalLogSize int
)
var currentCommit *Commit
var currentDiff *Diff
defer common.RecoverWithExit(ctx)
defer close(commitChan)
@@ -214,12 +207,10 @@ func (c *Parser) FromReader(ctx context.Context, stdOut io.Reader, commitChan ch
// If there is a currentDiff, add it to currentCommit.
if currentDiff != nil && currentDiff.Content.Len() > 0 {
currentCommit.Diffs = append(currentCommit.Diffs, *currentDiff)
currentCommit.Size += currentDiff.Content.Len()
}
// If there is a currentCommit, send it to the channel.
if currentCommit != nil {
commitChan <- *currentCommit
totalLogSize += currentCommit.Size
}
// Create a new currentDiff and currentCommit
currentDiff = &Diff{}
@@ -232,7 +223,6 @@ func (c *Parser) FromReader(ctx context.Context, stdOut io.Reader, commitChan ch
}
case isAuthorLine(line):
currentCommit.Author = strings.TrimRight(string(line[8:]), "\n")
recentlyPassedAuthor = true
case isDateLine(line):
date, err := time.Parse(c.dateFormat, strings.TrimSpace(string(line[6:])))
if err != nil {
@@ -254,7 +244,6 @@ func (c *Parser) FromReader(ctx context.Context, stdOut io.Reader, commitChan ch
if totalSize > c.maxCommitSize {
oldCommit := currentCommit
commitChan <- *currentCommit
totalLogSize += currentCommit.Size
currentCommit = &Commit{
Hash: currentCommit.Hash,
Author: currentCommit.Author,
@@ -270,7 +259,6 @@ func (c *Parser) FromReader(ctx context.Context, stdOut io.Reader, commitChan ch
case isModeLine(line):
// NoOp
case isIndexLine(line):
recentlyPassedAuthor = false
// NoOp
case isPlusFileLine(line):
currentDiff.PathB = strings.TrimRight(strings.TrimRight(string(line[6:]), "\n"), "\t") // Trim the newline and tab characters. https://github.com/trufflesecurity/trufflehog/issues/1060
@@ -280,7 +268,7 @@ func (c *Parser) FromReader(ctx context.Context, stdOut io.Reader, commitChan ch
currentDiff.Content.Write(line[1:])
case isMinusDiffLine(line):
// NoOp. We only care about additions.
case (isMessageLine(line) && recentlyPassedAuthor):
case isMessageLine(line):
currentCommit.Message.Write(line[4:])
case isContextDiffLine(line):
currentDiff.Content.Write([]byte("\n"))
@@ -313,20 +301,15 @@ func (c *Parser) FromReader(ctx context.Context, stdOut io.Reader, commitChan ch
break
}
}
cleanupParse(currentCommit, currentDiff, commitChan, &totalLogSize)
ctx.Logger().V(2).Info("finished parsing git log.", "total_log_size", totalLogSize)
cleanupParse(currentCommit, currentDiff, commitChan)
}
func cleanupParse(currentCommit *Commit, currentDiff *Diff, commitChan chan Commit, totalLogSize *int) {
func cleanupParse(currentCommit *Commit, currentDiff *Diff, commitChan chan Commit) {
if currentDiff != nil && currentDiff.Content.Len() > 0 {
currentCommit.Diffs = append(currentCommit.Diffs, *currentDiff)
}
if currentCommit != nil {
commitChan <- *currentCommit
if totalLogSize != nil {
*totalLogSize += currentCommit.Size
}
}
}
+4 -37
View File
@@ -2,7 +2,6 @@ package gitparse
import (
"bytes"
"fmt"
"strings"
"testing"
"time"
@@ -110,7 +109,7 @@ func TestSingleCommitSingleDiff(t *testing.T) {
},
}
go func() {
parser.FromReader(context.TODO(), r, commitChan)
parser.fromReader(context.TODO(), r, commitChan)
}()
i := 0
for commit := range commitChan {
@@ -169,7 +168,7 @@ func TestMultiCommitContextDiff(t *testing.T) {
},
}
go func() {
NewParser().FromReader(context.TODO(), r, commitChan)
NewParser().fromReader(context.TODO(), r, commitChan)
}()
i := 0
for commit := range commitChan {
@@ -199,7 +198,7 @@ func TestMaxDiffSize(t *testing.T) {
commitChan := make(chan Commit)
go func() {
parser.FromReader(context.TODO(), bigReader, commitChan)
parser.fromReader(context.TODO(), bigReader, commitChan)
}()
commit := <-commitChan
@@ -217,7 +216,7 @@ func TestMaxCommitSize(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Duration(1)*time.Second)
defer cancel()
go func() {
parser.FromReader(ctx, &commitText, commitChan)
parser.fromReader(ctx, &commitText, commitChan)
}()
commitCount := 0
for range commitChan {
@@ -229,38 +228,6 @@ func TestMaxCommitSize(t *testing.T) {
}
func TestMessageParsing(t *testing.T) {
r := bytes.NewReader([]byte(fmt.Sprintf(singleCommitTabbedContent, ` `)))
commitChan := make(chan Commit)
parser := NewParser()
expectedMessage := singleCommitSingleDiffMessage
go func() {
parser.FromReader(context.TODO(), r, commitChan)
}()
for commit := range commitChan {
if commit.Message.String() != expectedMessage {
t.Errorf("Message does not match. Got:\n%s\nexpected:\n%s", commit.Message.String(), expectedMessage)
}
}
}
const singleCommitTabbedContent = `commit 70001020fab32b1fcf2f1f0e5c66424eae649826 (HEAD -> master, origin/master, origin/HEAD)
Author: Dustin Decker <humanatcomputer@gmail.com>
Date: Mon Mar 15 23:27:16 2021 -0700
Update aws
diff --git a/aws b/aws
index 2ee133b..12b4843 100644
--- a/aws
+++ b/aws
@@ -1,7 +1,5 @@
+[default]
%saws_access_key_id = AKIAXYZDQCEN4B6JSJQI
+aws_secret_access_key = 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
`
const singleCommitSingleDiff = `commit 70001020fab32b1fcf2f1f0e5c66424eae649826 (HEAD -> master, origin/master, origin/HEAD)
Author: Dustin Decker <humanatcomputer@gmail.com>
Date: Mon Mar 15 23:27:16 2021 -0700
+6 -7
View File
@@ -9,7 +9,6 @@ import (
"time"
"github.com/mholt/archiver/v4"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
logContext "github.com/trufflesecurity/trufflehog/v3/pkg/context"
)
@@ -92,6 +91,12 @@ func (d *Archive) openArchive(ctx context.Context, depth int, reader io.Reader,
return err
}
switch archive := format.(type) {
case archiver.Extractor:
err := archive.Extract(context.WithValue(ctx, depthKey, depth+1), reader, nil, d.extractorHandler(archiveChan))
if err != nil {
return err
}
return nil
case archiver.Decompressor:
compReader, err := archive.OpenReader(reader)
if err != nil {
@@ -103,12 +108,6 @@ func (d *Archive) openArchive(ctx context.Context, depth int, reader io.Reader,
}
newReader := bytes.NewReader(fileBytes)
return d.openArchive(ctx, depth+1, newReader, archiveChan)
case archiver.Extractor:
err := archive.Extract(context.WithValue(ctx, depthKey, depth+1), reader, nil, d.extractorHandler(archiveChan))
if err != nil {
return err
}
return nil
}
return fmt.Errorf("Unknown archive type: %s", format.Name())
}
-72
View File
@@ -1,72 +0,0 @@
package output
import (
"crypto/sha256"
"encoding/hex"
"fmt"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
)
var dedupeCache = make(map[string]struct{})
func PrintGitHubActionsOutput(r *detectors.ResultWithMetadata) error {
out := gitHubActionsOutputFormat{
DetectorType: r.Result.DetectorType.String(),
DecoderType: r.Result.DecoderType.String(),
Verified: r.Result.Verified,
}
meta, err := structToMap(r.SourceMetadata.Data)
if err != nil {
return fmt.Errorf("could not marshal result: %w", err)
}
for _, data := range meta {
for k, v := range data {
if k == "line" {
if line, ok := v.(float64); ok {
out.StartLine = int64(line)
}
}
if k == "file" {
if filename, ok := v.(string); ok {
out.Filename = filename
}
}
}
}
verifiedStatus := "unverified"
if out.Verified {
verifiedStatus = "verified"
}
key := fmt.Sprintf("%s:%s:%s:%s:%d", out.DecoderType, out.DetectorType, verifiedStatus, out.Filename, out.StartLine)
h := sha256.New()
h.Write([]byte(key))
key = hex.EncodeToString(h.Sum(nil))
if _, ok := dedupeCache[key]; ok {
return nil
}
dedupeCache[key] = struct{}{}
message := fmt.Sprintf("Found %s %s result 🐷🔑\n", verifiedStatus, out.DetectorType)
if r.Result.DecoderType != detectorspb.DecoderType_PLAIN {
message = fmt.Sprintf("Found %s %s result with %s encoding 🐷🔑\n", verifiedStatus, out.DetectorType, out.DecoderType)
}
fmt.Printf("::warning file=%s,line=%d,endLine=%d::%s",
out.Filename, out.StartLine, out.StartLine, message)
return nil
}
type gitHubActionsOutputFormat struct {
DetectorType,
DecoderType string
Verified bool
StartLine int64
Filename string
}
-4
View File
@@ -29,9 +29,6 @@ func PrintJSON(r *detectors.ResultWithMetadata) error {
Verified bool
// Raw contains the raw secret data.
Raw string
// RawV2 contains the raw secret identifier that is a combination of both the ID and the secret.
// This is used for secrets that are multi part and could have the same ID. Ex: AWS credentials
RawV2 string
// Redacted contains the redacted version of the raw secret identification data for display purposes.
// A secret ID should be used if available.
Redacted string
@@ -47,7 +44,6 @@ func PrintJSON(r *detectors.ResultWithMetadata) error {
DecoderName: r.DecoderType.String(),
Verified: r.Verified,
Raw: string(r.Raw),
RawV2: string(r.RawV2),
Redacted: r.Redacted,
ExtraData: r.ExtraData,
StructuredData: r.StructuredData,
+3 -8
View File
@@ -66,14 +66,9 @@ func ConvertToLegacyJSON(r *detectors.ResultWithMetadata, repoPath string) (*Leg
return nil, fmt.Errorf("legacy JSON output can not be used with this source: %s", r.SourceName)
}
options := &gogit.PlainOpenOptions{
DetectDotGit: true,
EnableDotGitCommonDir: true,
}
// The repo will be needed to gather info needed for the legacy output that
// isn't included in the new output format.
repo, err := gogit.PlainOpenWithOptions(repoPath, options)
// The repo will be needed to gather info needed for the legacy output that isn't included in the new
// output format.
repo, err := gogit.PlainOpenWithOptions(repoPath, &gogit.PlainOpenOptions{DetectDotGit: true})
if err != nil {
return nil, fmt.Errorf("could not open repo %q: %w", repoPath, err)
}
+77 -170
View File
@@ -386,7 +386,6 @@ type Oauth2 struct {
RefreshToken string `protobuf:"bytes,1,opt,name=refresh_token,json=refreshToken,proto3" json:"refresh_token,omitempty"`
ClientId string `protobuf:"bytes,2,opt,name=client_id,json=clientId,proto3" json:"client_id,omitempty"`
ClientSecret string `protobuf:"bytes,3,opt,name=client_secret,json=clientSecret,proto3" json:"client_secret,omitempty"`
AccessToken string `protobuf:"bytes,4,opt,name=access_token,json=accessToken,proto3" json:"access_token,omitempty"`
}
func (x *Oauth2) Reset() {
@@ -442,13 +441,6 @@ func (x *Oauth2) GetClientSecret() string {
return ""
}
func (x *Oauth2) GetAccessToken() string {
if x != nil {
return x.AccessToken
}
return ""
}
type KeySecret struct {
state protoimpl.MessageState
sizeCache protoimpl.SizeCache
@@ -504,69 +496,6 @@ func (x *KeySecret) GetSecret() string {
return ""
}
type AWSSessionTokenSecret struct {
state protoimpl.MessageState
sizeCache protoimpl.SizeCache
unknownFields protoimpl.UnknownFields
Key string `protobuf:"bytes,1,opt,name=key,proto3" json:"key,omitempty"`
Secret string `protobuf:"bytes,2,opt,name=secret,proto3" json:"secret,omitempty"`
SessionToken string `protobuf:"bytes,3,opt,name=sessionToken,proto3" json:"sessionToken,omitempty"`
}
func (x *AWSSessionTokenSecret) Reset() {
*x = AWSSessionTokenSecret{}
if protoimpl.UnsafeEnabled {
mi := &file_credentials_proto_msgTypes[9]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
}
func (x *AWSSessionTokenSecret) String() string {
return protoimpl.X.MessageStringOf(x)
}
func (*AWSSessionTokenSecret) ProtoMessage() {}
func (x *AWSSessionTokenSecret) ProtoReflect() protoreflect.Message {
mi := &file_credentials_proto_msgTypes[9]
if protoimpl.UnsafeEnabled && x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
ms.StoreMessageInfo(mi)
}
return ms
}
return mi.MessageOf(x)
}
// Deprecated: Use AWSSessionTokenSecret.ProtoReflect.Descriptor instead.
func (*AWSSessionTokenSecret) Descriptor() ([]byte, []int) {
return file_credentials_proto_rawDescGZIP(), []int{9}
}
func (x *AWSSessionTokenSecret) GetKey() string {
if x != nil {
return x.Key
}
return ""
}
func (x *AWSSessionTokenSecret) GetSecret() string {
if x != nil {
return x.Secret
}
return ""
}
func (x *AWSSessionTokenSecret) GetSessionToken() string {
if x != nil {
return x.SessionToken
}
return ""
}
type AWS struct {
state protoimpl.MessageState
sizeCache protoimpl.SizeCache
@@ -580,7 +509,7 @@ type AWS struct {
func (x *AWS) Reset() {
*x = AWS{}
if protoimpl.UnsafeEnabled {
mi := &file_credentials_proto_msgTypes[10]
mi := &file_credentials_proto_msgTypes[9]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
@@ -593,7 +522,7 @@ func (x *AWS) String() string {
func (*AWS) ProtoMessage() {}
func (x *AWS) ProtoReflect() protoreflect.Message {
mi := &file_credentials_proto_msgTypes[10]
mi := &file_credentials_proto_msgTypes[9]
if protoimpl.UnsafeEnabled && x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
@@ -606,7 +535,7 @@ func (x *AWS) ProtoReflect() protoreflect.Message {
// Deprecated: Use AWS.ProtoReflect.Descriptor instead.
func (*AWS) Descriptor() ([]byte, []int) {
return file_credentials_proto_rawDescGZIP(), []int{10}
return file_credentials_proto_rawDescGZIP(), []int{9}
}
func (x *AWS) GetKey() string {
@@ -643,7 +572,7 @@ type SES struct {
func (x *SES) Reset() {
*x = SES{}
if protoimpl.UnsafeEnabled {
mi := &file_credentials_proto_msgTypes[11]
mi := &file_credentials_proto_msgTypes[10]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
@@ -656,7 +585,7 @@ func (x *SES) String() string {
func (*SES) ProtoMessage() {}
func (x *SES) ProtoReflect() protoreflect.Message {
mi := &file_credentials_proto_msgTypes[11]
mi := &file_credentials_proto_msgTypes[10]
if protoimpl.UnsafeEnabled && x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
@@ -669,7 +598,7 @@ func (x *SES) ProtoReflect() protoreflect.Message {
// Deprecated: Use SES.ProtoReflect.Descriptor instead.
func (*SES) Descriptor() ([]byte, []int) {
return file_credentials_proto_rawDescGZIP(), []int{11}
return file_credentials_proto_rawDescGZIP(), []int{10}
}
func (x *SES) GetCreds() *AWS {
@@ -706,7 +635,7 @@ type GitHubApp struct {
func (x *GitHubApp) Reset() {
*x = GitHubApp{}
if protoimpl.UnsafeEnabled {
mi := &file_credentials_proto_msgTypes[12]
mi := &file_credentials_proto_msgTypes[11]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
@@ -719,7 +648,7 @@ func (x *GitHubApp) String() string {
func (*GitHubApp) ProtoMessage() {}
func (x *GitHubApp) ProtoReflect() protoreflect.Message {
mi := &file_credentials_proto_msgTypes[12]
mi := &file_credentials_proto_msgTypes[11]
if protoimpl.UnsafeEnabled && x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
@@ -732,7 +661,7 @@ func (x *GitHubApp) ProtoReflect() protoreflect.Message {
// Deprecated: Use GitHubApp.ProtoReflect.Descriptor instead.
func (*GitHubApp) Descriptor() ([]byte, []int) {
return file_credentials_proto_rawDescGZIP(), []int{12}
return file_credentials_proto_rawDescGZIP(), []int{11}
}
func (x *GitHubApp) GetPrivateKey() string {
@@ -769,7 +698,7 @@ type SlackTokens struct {
func (x *SlackTokens) Reset() {
*x = SlackTokens{}
if protoimpl.UnsafeEnabled {
mi := &file_credentials_proto_msgTypes[13]
mi := &file_credentials_proto_msgTypes[12]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
@@ -782,7 +711,7 @@ func (x *SlackTokens) String() string {
func (*SlackTokens) ProtoMessage() {}
func (x *SlackTokens) ProtoReflect() protoreflect.Message {
mi := &file_credentials_proto_msgTypes[13]
mi := &file_credentials_proto_msgTypes[12]
if protoimpl.UnsafeEnabled && x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
@@ -795,7 +724,7 @@ func (x *SlackTokens) ProtoReflect() protoreflect.Message {
// Deprecated: Use SlackTokens.ProtoReflect.Descriptor instead.
func (*SlackTokens) Descriptor() ([]byte, []int) {
return file_credentials_proto_rawDescGZIP(), []int{13}
return file_credentials_proto_rawDescGZIP(), []int{12}
}
func (x *SlackTokens) GetAppToken() string {
@@ -852,56 +781,47 @@ var file_credentials_proto_rawDesc = []byte{
0x63, 0x61, 0x74, 0x65, 0x50, 0x61, 0x74, 0x68, 0x12, 0x31, 0x0a, 0x14, 0x63, 0x65, 0x72, 0x74,
0x69, 0x66, 0x69, 0x63, 0x61, 0x74, 0x65, 0x5f, 0x70, 0x61, 0x73, 0x73, 0x77, 0x6f, 0x72, 0x64,
0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x13, 0x63, 0x65, 0x72, 0x74, 0x69, 0x66, 0x69, 0x63,
0x61, 0x74, 0x65, 0x50, 0x61, 0x73, 0x73, 0x77, 0x6f, 0x72, 0x64, 0x22, 0x92, 0x01, 0x0a, 0x06,
0x4f, 0x61, 0x75, 0x74, 0x68, 0x32, 0x12, 0x23, 0x0a, 0x0d, 0x72, 0x65, 0x66, 0x72, 0x65, 0x73,
0x68, 0x5f, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0c, 0x72,
0x65, 0x66, 0x72, 0x65, 0x73, 0x68, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x12, 0x1b, 0x0a, 0x09, 0x63,
0x6c, 0x69, 0x65, 0x6e, 0x74, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08,
0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x49, 0x64, 0x12, 0x23, 0x0a, 0x0d, 0x63, 0x6c, 0x69, 0x65,
0x6e, 0x74, 0x5f, 0x73, 0x65, 0x63, 0x72, 0x65, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52,
0x0c, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x53, 0x65, 0x63, 0x72, 0x65, 0x74, 0x12, 0x21, 0x0a,
0x0c, 0x61, 0x63, 0x63, 0x65, 0x73, 0x73, 0x5f, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x18, 0x04, 0x20,
0x01, 0x28, 0x09, 0x52, 0x0b, 0x61, 0x63, 0x63, 0x65, 0x73, 0x73, 0x54, 0x6f, 0x6b, 0x65, 0x6e,
0x22, 0x35, 0x0a, 0x09, 0x4b, 0x65, 0x79, 0x53, 0x65, 0x63, 0x72, 0x65, 0x74, 0x12, 0x10, 0x0a,
0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12,
0x16, 0x0a, 0x06, 0x73, 0x65, 0x63, 0x72, 0x65, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52,
0x06, 0x73, 0x65, 0x63, 0x72, 0x65, 0x74, 0x22, 0x65, 0x0a, 0x15, 0x41, 0x57, 0x53, 0x53, 0x65,
0x73, 0x73, 0x69, 0x6f, 0x6e, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x53, 0x65, 0x63, 0x72, 0x65, 0x74,
0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b,
0x65, 0x79, 0x12, 0x16, 0x0a, 0x06, 0x73, 0x65, 0x63, 0x72, 0x65, 0x74, 0x18, 0x02, 0x20, 0x01,
0x28, 0x09, 0x52, 0x06, 0x73, 0x65, 0x63, 0x72, 0x65, 0x74, 0x12, 0x22, 0x0a, 0x0c, 0x73, 0x65,
0x73, 0x73, 0x69, 0x6f, 0x6e, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09,
0x52, 0x0c, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x22, 0x47,
0x0a, 0x03, 0x41, 0x57, 0x53, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01,
0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x16, 0x0a, 0x06, 0x73, 0x65, 0x63, 0x72, 0x65,
0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x73, 0x65, 0x63, 0x72, 0x65, 0x74, 0x12,
0x16, 0x0a, 0x06, 0x72, 0x65, 0x67, 0x69, 0x6f, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52,
0x06, 0x72, 0x65, 0x67, 0x69, 0x6f, 0x6e, 0x22, 0x65, 0x0a, 0x03, 0x53, 0x45, 0x53, 0x12, 0x26,
0x0a, 0x05, 0x63, 0x72, 0x65, 0x64, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x10, 0x2e,
0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x73, 0x2e, 0x41, 0x57, 0x53, 0x52,
0x05, 0x63, 0x72, 0x65, 0x64, 0x73, 0x12, 0x16, 0x0a, 0x06, 0x73, 0x65, 0x6e, 0x64, 0x65, 0x72,
0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x73, 0x65, 0x6e, 0x64, 0x65, 0x72, 0x12, 0x1e,
0x0a, 0x0a, 0x72, 0x65, 0x63, 0x69, 0x70, 0x69, 0x65, 0x6e, 0x74, 0x73, 0x18, 0x03, 0x20, 0x03,
0x28, 0x09, 0x52, 0x0a, 0x72, 0x65, 0x63, 0x69, 0x70, 0x69, 0x65, 0x6e, 0x74, 0x73, 0x22, 0x6c,
0x0a, 0x09, 0x47, 0x69, 0x74, 0x48, 0x75, 0x62, 0x41, 0x70, 0x70, 0x12, 0x1f, 0x0a, 0x0b, 0x70,
0x72, 0x69, 0x76, 0x61, 0x74, 0x65, 0x5f, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09,
0x52, 0x0a, 0x70, 0x72, 0x69, 0x76, 0x61, 0x74, 0x65, 0x4b, 0x65, 0x79, 0x12, 0x27, 0x0a, 0x0f,
0x69, 0x6e, 0x73, 0x74, 0x61, 0x6c, 0x6c, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x69, 0x64, 0x18,
0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0e, 0x69, 0x6e, 0x73, 0x74, 0x61, 0x6c, 0x6c, 0x61, 0x74,
0x69, 0x6f, 0x6e, 0x49, 0x64, 0x12, 0x15, 0x0a, 0x06, 0x61, 0x70, 0x70, 0x5f, 0x69, 0x64, 0x18,
0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x61, 0x70, 0x70, 0x49, 0x64, 0x22, 0x6a, 0x0a, 0x0b,
0x53, 0x6c, 0x61, 0x63, 0x6b, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x73, 0x12, 0x1b, 0x0a, 0x09, 0x61,
0x70, 0x70, 0x5f, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08,
0x61, 0x70, 0x70, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x12, 0x1b, 0x0a, 0x09, 0x62, 0x6f, 0x74, 0x5f,
0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x62, 0x6f, 0x74,
0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x12, 0x21, 0x0a, 0x0c, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x5f,
0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x63, 0x6c, 0x69,
0x65, 0x6e, 0x74, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x42, 0x3f, 0x5a, 0x3d, 0x67, 0x69, 0x74, 0x68,
0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x74, 0x72, 0x75, 0x66, 0x66, 0x6c, 0x65, 0x73, 0x65,
0x63, 0x75, 0x72, 0x69, 0x74, 0x79, 0x2f, 0x74, 0x72, 0x75, 0x66, 0x66, 0x6c, 0x65, 0x68, 0x6f,
0x67, 0x2f, 0x76, 0x33, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x70, 0x62, 0x2f, 0x63, 0x72, 0x65, 0x64,
0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x73, 0x70, 0x62, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f,
0x33,
0x61, 0x74, 0x65, 0x50, 0x61, 0x73, 0x73, 0x77, 0x6f, 0x72, 0x64, 0x22, 0x6f, 0x0a, 0x06, 0x4f,
0x61, 0x75, 0x74, 0x68, 0x32, 0x12, 0x23, 0x0a, 0x0d, 0x72, 0x65, 0x66, 0x72, 0x65, 0x73, 0x68,
0x5f, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0c, 0x72, 0x65,
0x66, 0x72, 0x65, 0x73, 0x68, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x12, 0x1b, 0x0a, 0x09, 0x63, 0x6c,
0x69, 0x65, 0x6e, 0x74, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x63,
0x6c, 0x69, 0x65, 0x6e, 0x74, 0x49, 0x64, 0x12, 0x23, 0x0a, 0x0d, 0x63, 0x6c, 0x69, 0x65, 0x6e,
0x74, 0x5f, 0x73, 0x65, 0x63, 0x72, 0x65, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0c,
0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x53, 0x65, 0x63, 0x72, 0x65, 0x74, 0x22, 0x35, 0x0a, 0x09,
0x4b, 0x65, 0x79, 0x53, 0x65, 0x63, 0x72, 0x65, 0x74, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79,
0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x16, 0x0a, 0x06, 0x73,
0x65, 0x63, 0x72, 0x65, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x73, 0x65, 0x63,
0x72, 0x65, 0x74, 0x22, 0x47, 0x0a, 0x03, 0x41, 0x57, 0x53, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65,
0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x16, 0x0a, 0x06,
0x73, 0x65, 0x63, 0x72, 0x65, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x73, 0x65,
0x63, 0x72, 0x65, 0x74, 0x12, 0x16, 0x0a, 0x06, 0x72, 0x65, 0x67, 0x69, 0x6f, 0x6e, 0x18, 0x03,
0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x72, 0x65, 0x67, 0x69, 0x6f, 0x6e, 0x22, 0x65, 0x0a, 0x03,
0x53, 0x45, 0x53, 0x12, 0x26, 0x0a, 0x05, 0x63, 0x72, 0x65, 0x64, 0x73, 0x18, 0x01, 0x20, 0x01,
0x28, 0x0b, 0x32, 0x10, 0x2e, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x73,
0x2e, 0x41, 0x57, 0x53, 0x52, 0x05, 0x63, 0x72, 0x65, 0x64, 0x73, 0x12, 0x16, 0x0a, 0x06, 0x73,
0x65, 0x6e, 0x64, 0x65, 0x72, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x73, 0x65, 0x6e,
0x64, 0x65, 0x72, 0x12, 0x1e, 0x0a, 0x0a, 0x72, 0x65, 0x63, 0x69, 0x70, 0x69, 0x65, 0x6e, 0x74,
0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0a, 0x72, 0x65, 0x63, 0x69, 0x70, 0x69, 0x65,
0x6e, 0x74, 0x73, 0x22, 0x6c, 0x0a, 0x09, 0x47, 0x69, 0x74, 0x48, 0x75, 0x62, 0x41, 0x70, 0x70,
0x12, 0x1f, 0x0a, 0x0b, 0x70, 0x72, 0x69, 0x76, 0x61, 0x74, 0x65, 0x5f, 0x6b, 0x65, 0x79, 0x18,
0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x70, 0x72, 0x69, 0x76, 0x61, 0x74, 0x65, 0x4b, 0x65,
0x79, 0x12, 0x27, 0x0a, 0x0f, 0x69, 0x6e, 0x73, 0x74, 0x61, 0x6c, 0x6c, 0x61, 0x74, 0x69, 0x6f,
0x6e, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0e, 0x69, 0x6e, 0x73, 0x74,
0x61, 0x6c, 0x6c, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x64, 0x12, 0x15, 0x0a, 0x06, 0x61, 0x70,
0x70, 0x5f, 0x69, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x61, 0x70, 0x70, 0x49,
0x64, 0x22, 0x6a, 0x0a, 0x0b, 0x53, 0x6c, 0x61, 0x63, 0x6b, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x73,
0x12, 0x1b, 0x0a, 0x09, 0x61, 0x70, 0x70, 0x5f, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x18, 0x01, 0x20,
0x01, 0x28, 0x09, 0x52, 0x08, 0x61, 0x70, 0x70, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x12, 0x1b, 0x0a,
0x09, 0x62, 0x6f, 0x74, 0x5f, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09,
0x52, 0x08, 0x62, 0x6f, 0x74, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x12, 0x21, 0x0a, 0x0c, 0x63, 0x6c,
0x69, 0x65, 0x6e, 0x74, 0x5f, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09,
0x52, 0x0b, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x42, 0x3f, 0x5a,
0x3d, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x74, 0x72, 0x75, 0x66,
0x66, 0x6c, 0x65, 0x73, 0x65, 0x63, 0x75, 0x72, 0x69, 0x74, 0x79, 0x2f, 0x74, 0x72, 0x75, 0x66,
0x66, 0x6c, 0x65, 0x68, 0x6f, 0x67, 0x2f, 0x76, 0x33, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x70, 0x62,
0x2f, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x73, 0x70, 0x62, 0x62, 0x06,
0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
}
var (
@@ -916,30 +836,29 @@ func file_credentials_proto_rawDescGZIP() []byte {
return file_credentials_proto_rawDescData
}
var file_credentials_proto_msgTypes = make([]protoimpl.MessageInfo, 14)
var file_credentials_proto_msgTypes = make([]protoimpl.MessageInfo, 13)
var file_credentials_proto_goTypes = []interface{}{
(*Unauthenticated)(nil), // 0: credentials.Unauthenticated
(*SSHAuth)(nil), // 1: credentials.SSHAuth
(*CloudEnvironment)(nil), // 2: credentials.CloudEnvironment
(*BasicAuth)(nil), // 3: credentials.BasicAuth
(*Header)(nil), // 4: credentials.Header
(*ClientCredentials)(nil), // 5: credentials.ClientCredentials
(*ClientCertificate)(nil), // 6: credentials.ClientCertificate
(*Oauth2)(nil), // 7: credentials.Oauth2
(*KeySecret)(nil), // 8: credentials.KeySecret
(*AWSSessionTokenSecret)(nil), // 9: credentials.AWSSessionTokenSecret
(*AWS)(nil), // 10: credentials.AWS
(*SES)(nil), // 11: credentials.SES
(*GitHubApp)(nil), // 12: credentials.GitHubApp
(*SlackTokens)(nil), // 13: credentials.SlackTokens
(*Unauthenticated)(nil), // 0: credentials.Unauthenticated
(*SSHAuth)(nil), // 1: credentials.SSHAuth
(*CloudEnvironment)(nil), // 2: credentials.CloudEnvironment
(*BasicAuth)(nil), // 3: credentials.BasicAuth
(*Header)(nil), // 4: credentials.Header
(*ClientCredentials)(nil), // 5: credentials.ClientCredentials
(*ClientCertificate)(nil), // 6: credentials.ClientCertificate
(*Oauth2)(nil), // 7: credentials.Oauth2
(*KeySecret)(nil), // 8: credentials.KeySecret
(*AWS)(nil), // 9: credentials.AWS
(*SES)(nil), // 10: credentials.SES
(*GitHubApp)(nil), // 11: credentials.GitHubApp
(*SlackTokens)(nil), // 12: credentials.SlackTokens
}
var file_credentials_proto_depIdxs = []int32{
10, // 0: credentials.SES.creds:type_name -> credentials.AWS
1, // [1:1] is the sub-list for method output_type
1, // [1:1] is the sub-list for method input_type
1, // [1:1] is the sub-list for extension type_name
1, // [1:1] is the sub-list for extension extendee
0, // [0:1] is the sub-list for field type_name
9, // 0: credentials.SES.creds:type_name -> credentials.AWS
1, // [1:1] is the sub-list for method output_type
1, // [1:1] is the sub-list for method input_type
1, // [1:1] is the sub-list for extension type_name
1, // [1:1] is the sub-list for extension extendee
0, // [0:1] is the sub-list for field type_name
}
func init() { file_credentials_proto_init() }
@@ -1057,18 +976,6 @@ func file_credentials_proto_init() {
}
}
file_credentials_proto_msgTypes[9].Exporter = func(v interface{}, i int) interface{} {
switch v := v.(*AWSSessionTokenSecret); i {
case 0:
return &v.state
case 1:
return &v.sizeCache
case 2:
return &v.unknownFields
default:
return nil
}
}
file_credentials_proto_msgTypes[10].Exporter = func(v interface{}, i int) interface{} {
switch v := v.(*AWS); i {
case 0:
return &v.state
@@ -1080,7 +987,7 @@ func file_credentials_proto_init() {
return nil
}
}
file_credentials_proto_msgTypes[11].Exporter = func(v interface{}, i int) interface{} {
file_credentials_proto_msgTypes[10].Exporter = func(v interface{}, i int) interface{} {
switch v := v.(*SES); i {
case 0:
return &v.state
@@ -1092,7 +999,7 @@ func file_credentials_proto_init() {
return nil
}
}
file_credentials_proto_msgTypes[12].Exporter = func(v interface{}, i int) interface{} {
file_credentials_proto_msgTypes[11].Exporter = func(v interface{}, i int) interface{} {
switch v := v.(*GitHubApp); i {
case 0:
return &v.state
@@ -1104,7 +1011,7 @@ func file_credentials_proto_init() {
return nil
}
}
file_credentials_proto_msgTypes[13].Exporter = func(v interface{}, i int) interface{} {
file_credentials_proto_msgTypes[12].Exporter = func(v interface{}, i int) interface{} {
switch v := v.(*SlackTokens); i {
case 0:
return &v.state
@@ -1123,7 +1030,7 @@ func file_credentials_proto_init() {
GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
RawDescriptor: file_credentials_proto_rawDesc,
NumEnums: 0,
NumMessages: 14,
NumMessages: 13,
NumExtensions: 0,
NumServices: 0,
},
@@ -783,8 +783,6 @@ func (m *Oauth2) validate(all bool) error {
// no validation rules for ClientSecret
// no validation rules for AccessToken
if len(errors) > 0 {
return Oauth2MultiError(errors)
}
@@ -965,114 +963,6 @@ var _ interface {
ErrorName() string
} = KeySecretValidationError{}
// Validate checks the field values on AWSSessionTokenSecret with the rules
// defined in the proto definition for this message. If any rules are
// violated, the first error encountered is returned, or nil if there are no violations.
func (m *AWSSessionTokenSecret) Validate() error {
return m.validate(false)
}
// ValidateAll checks the field values on AWSSessionTokenSecret with the rules
// defined in the proto definition for this message. If any rules are
// violated, the result is a list of violation errors wrapped in
// AWSSessionTokenSecretMultiError, or nil if none found.
func (m *AWSSessionTokenSecret) ValidateAll() error {
return m.validate(true)
}
func (m *AWSSessionTokenSecret) validate(all bool) error {
if m == nil {
return nil
}
var errors []error
// no validation rules for Key
// no validation rules for Secret
// no validation rules for SessionToken
if len(errors) > 0 {
return AWSSessionTokenSecretMultiError(errors)
}
return nil
}
// AWSSessionTokenSecretMultiError is an error wrapping multiple validation
// errors returned by AWSSessionTokenSecret.ValidateAll() if the designated
// constraints aren't met.
type AWSSessionTokenSecretMultiError []error
// Error returns a concatenation of all the error messages it wraps.
func (m AWSSessionTokenSecretMultiError) Error() string {
var msgs []string
for _, err := range m {
msgs = append(msgs, err.Error())
}
return strings.Join(msgs, "; ")
}
// AllErrors returns a list of validation violation errors.
func (m AWSSessionTokenSecretMultiError) AllErrors() []error { return m }
// AWSSessionTokenSecretValidationError is the validation error returned by
// AWSSessionTokenSecret.Validate if the designated constraints aren't met.
type AWSSessionTokenSecretValidationError struct {
field string
reason string
cause error
key bool
}
// Field function returns field value.
func (e AWSSessionTokenSecretValidationError) Field() string { return e.field }
// Reason function returns reason value.
func (e AWSSessionTokenSecretValidationError) Reason() string { return e.reason }
// Cause function returns cause value.
func (e AWSSessionTokenSecretValidationError) Cause() error { return e.cause }
// Key function returns key value.
func (e AWSSessionTokenSecretValidationError) Key() bool { return e.key }
// ErrorName returns error name.
func (e AWSSessionTokenSecretValidationError) ErrorName() string {
return "AWSSessionTokenSecretValidationError"
}
// Error satisfies the builtin error interface
func (e AWSSessionTokenSecretValidationError) Error() string {
cause := ""
if e.cause != nil {
cause = fmt.Sprintf(" | caused by: %v", e.cause)
}
key := ""
if e.key {
key = "key for "
}
return fmt.Sprintf(
"invalid %sAWSSessionTokenSecret.%s: %s%s",
key,
e.field,
e.reason,
cause)
}
var _ error = AWSSessionTokenSecretValidationError{}
var _ interface {
Field() string
Reason() string
Key() bool
Cause() error
ErrorName() string
} = AWSSessionTokenSecretValidationError{}
// Validate checks the field values on AWS with the rules defined in the proto
// definition for this message. If any rules are violated, the first error
// encountered is returned, or nil if there are no violations.
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff
@@ -852,8 +852,6 @@ func (m *Filesystem) validate(all bool) error {
// no validation rules for Email
// no validation rules for Line
if len(errors) > 0 {
return FilesystemMultiError(errors)
}
@@ -1269,6 +1267,117 @@ var _ interface {
ErrorName() string
} = GitlabValidationError{}
// Validate checks the field values on GoogleDrive with the rules defined in
// the proto definition for this message. If any rules are violated, the first
// error encountered is returned, or nil if there are no violations.
func (m *GoogleDrive) Validate() error {
return m.validate(false)
}
// ValidateAll checks the field values on GoogleDrive with the rules defined in
// the proto definition for this message. If any rules are violated, the
// result is a list of violation errors wrapped in GoogleDriveMultiError, or
// nil if none found.
func (m *GoogleDrive) ValidateAll() error {
return m.validate(true)
}
func (m *GoogleDrive) validate(all bool) error {
if m == nil {
return nil
}
var errors []error
// no validation rules for File
// no validation rules for Link
// no validation rules for Email
// no validation rules for Timestamp
// no validation rules for Shared
// no validation rules for LastModifiedBy
if len(errors) > 0 {
return GoogleDriveMultiError(errors)
}
return nil
}
// GoogleDriveMultiError is an error wrapping multiple validation errors
// returned by GoogleDrive.ValidateAll() if the designated constraints aren't met.
type GoogleDriveMultiError []error
// Error returns a concatenation of all the error messages it wraps.
func (m GoogleDriveMultiError) Error() string {
var msgs []string
for _, err := range m {
msgs = append(msgs, err.Error())
}
return strings.Join(msgs, "; ")
}
// AllErrors returns a list of validation violation errors.
func (m GoogleDriveMultiError) AllErrors() []error { return m }
// GoogleDriveValidationError is the validation error returned by
// GoogleDrive.Validate if the designated constraints aren't met.
type GoogleDriveValidationError struct {
field string
reason string
cause error
key bool
}
// Field function returns field value.
func (e GoogleDriveValidationError) Field() string { return e.field }
// Reason function returns reason value.
func (e GoogleDriveValidationError) Reason() string { return e.reason }
// Cause function returns cause value.
func (e GoogleDriveValidationError) Cause() error { return e.cause }
// Key function returns key value.
func (e GoogleDriveValidationError) Key() bool { return e.key }
// ErrorName returns error name.
func (e GoogleDriveValidationError) ErrorName() string { return "GoogleDriveValidationError" }
// Error satisfies the builtin error interface
func (e GoogleDriveValidationError) Error() string {
cause := ""
if e.cause != nil {
cause = fmt.Sprintf(" | caused by: %v", e.cause)
}
key := ""
if e.key {
key = "key for "
}
return fmt.Sprintf(
"invalid %sGoogleDrive.%s: %s%s",
key,
e.field,
e.reason,
cause)
}
var _ error = GoogleDriveValidationError{}
var _ interface {
Field() string
Reason() string
Key() bool
Cause() error
ErrorName() string
} = GoogleDriveValidationError{}
// Validate checks the field values on GCS with the rules defined in the proto
// definition for this message. If any rules are violated, the first error
// encountered is returned, or nil if there are no violations.
@@ -2284,10 +2393,6 @@ func (m *Teams) validate(all bool) error {
// no validation rules for Location
// no validation rules for TeamName
// no validation rules for TeamId
if len(errors) > 0 {
return TeamsMultiError(errors)
}
@@ -2723,230 +2828,6 @@ var _ interface {
ErrorName() string
} = PublicEventMonitoringValidationError{}
// Validate checks the field values on SharePoint with the rules defined in the
// proto definition for this message. If any rules are violated, the first
// error encountered is returned, or nil if there are no violations.
func (m *SharePoint) Validate() error {
return m.validate(false)
}
// ValidateAll checks the field values on SharePoint with the rules defined in
// the proto definition for this message. If any rules are violated, the
// result is a list of violation errors wrapped in SharePointMultiError, or
// nil if none found.
func (m *SharePoint) ValidateAll() error {
return m.validate(true)
}
func (m *SharePoint) validate(all bool) error {
if m == nil {
return nil
}
var errors []error
// no validation rules for Link
// no validation rules for Timestamp
// no validation rules for Author
// no validation rules for Title
// no validation rules for Views
// no validation rules for Docid
// no validation rules for Email
if len(errors) > 0 {
return SharePointMultiError(errors)
}
return nil
}
// SharePointMultiError is an error wrapping multiple validation errors
// returned by SharePoint.ValidateAll() if the designated constraints aren't met.
type SharePointMultiError []error
// Error returns a concatenation of all the error messages it wraps.
func (m SharePointMultiError) Error() string {
var msgs []string
for _, err := range m {
msgs = append(msgs, err.Error())
}
return strings.Join(msgs, "; ")
}
// AllErrors returns a list of validation violation errors.
func (m SharePointMultiError) AllErrors() []error { return m }
// SharePointValidationError is the validation error returned by
// SharePoint.Validate if the designated constraints aren't met.
type SharePointValidationError struct {
field string
reason string
cause error
key bool
}
// Field function returns field value.
func (e SharePointValidationError) Field() string { return e.field }
// Reason function returns reason value.
func (e SharePointValidationError) Reason() string { return e.reason }
// Cause function returns cause value.
func (e SharePointValidationError) Cause() error { return e.cause }
// Key function returns key value.
func (e SharePointValidationError) Key() bool { return e.key }
// ErrorName returns error name.
func (e SharePointValidationError) ErrorName() string { return "SharePointValidationError" }
// Error satisfies the builtin error interface
func (e SharePointValidationError) Error() string {
cause := ""
if e.cause != nil {
cause = fmt.Sprintf(" | caused by: %v", e.cause)
}
key := ""
if e.key {
key = "key for "
}
return fmt.Sprintf(
"invalid %sSharePoint.%s: %s%s",
key,
e.field,
e.reason,
cause)
}
var _ error = SharePointValidationError{}
var _ interface {
Field() string
Reason() string
Key() bool
Cause() error
ErrorName() string
} = SharePointValidationError{}
// Validate checks the field values on GoogleDrive with the rules defined in
// the proto definition for this message. If any rules are violated, the first
// error encountered is returned, or nil if there are no violations.
func (m *GoogleDrive) Validate() error {
return m.validate(false)
}
// ValidateAll checks the field values on GoogleDrive with the rules defined in
// the proto definition for this message. If any rules are violated, the
// result is a list of violation errors wrapped in GoogleDriveMultiError, or
// nil if none found.
func (m *GoogleDrive) ValidateAll() error {
return m.validate(true)
}
func (m *GoogleDrive) validate(all bool) error {
if m == nil {
return nil
}
var errors []error
// no validation rules for File
// no validation rules for Link
// no validation rules for Email
// no validation rules for Timestamp
// no validation rules for Shared
// no validation rules for LastModifiedBy
if len(errors) > 0 {
return GoogleDriveMultiError(errors)
}
return nil
}
// GoogleDriveMultiError is an error wrapping multiple validation errors
// returned by GoogleDrive.ValidateAll() if the designated constraints aren't met.
type GoogleDriveMultiError []error
// Error returns a concatenation of all the error messages it wraps.
func (m GoogleDriveMultiError) Error() string {
var msgs []string
for _, err := range m {
msgs = append(msgs, err.Error())
}
return strings.Join(msgs, "; ")
}
// AllErrors returns a list of validation violation errors.
func (m GoogleDriveMultiError) AllErrors() []error { return m }
// GoogleDriveValidationError is the validation error returned by
// GoogleDrive.Validate if the designated constraints aren't met.
type GoogleDriveValidationError struct {
field string
reason string
cause error
key bool
}
// Field function returns field value.
func (e GoogleDriveValidationError) Field() string { return e.field }
// Reason function returns reason value.
func (e GoogleDriveValidationError) Reason() string { return e.reason }
// Cause function returns cause value.
func (e GoogleDriveValidationError) Cause() error { return e.cause }
// Key function returns key value.
func (e GoogleDriveValidationError) Key() bool { return e.key }
// ErrorName returns error name.
func (e GoogleDriveValidationError) ErrorName() string { return "GoogleDriveValidationError" }
// Error satisfies the builtin error interface
func (e GoogleDriveValidationError) Error() string {
cause := ""
if e.cause != nil {
cause = fmt.Sprintf(" | caused by: %v", e.cause)
}
key := ""
if e.key {
key = "key for "
}
return fmt.Sprintf(
"invalid %sGoogleDrive.%s: %s%s",
key,
e.field,
e.reason,
cause)
}
var _ error = GoogleDriveValidationError{}
var _ interface {
Field() string
Reason() string
Key() bool
Cause() error
ErrorName() string
} = GoogleDriveValidationError{}
// Validate checks the field values on MetaData with the rules defined in the
// proto definition for this message. If any rules are violated, the first
// error encountered is returned, or nil if there are no violations.
@@ -3715,68 +3596,6 @@ func (m *MetaData) validate(all bool) error {
}
}
case *MetaData_Sharepoint:
if all {
switch v := interface{}(m.GetSharepoint()).(type) {
case interface{ ValidateAll() error }:
if err := v.ValidateAll(); err != nil {
errors = append(errors, MetaDataValidationError{
field: "Sharepoint",
reason: "embedded message failed validation",
cause: err,
})
}
case interface{ Validate() error }:
if err := v.Validate(); err != nil {
errors = append(errors, MetaDataValidationError{
field: "Sharepoint",
reason: "embedded message failed validation",
cause: err,
})
}
}
} else if v, ok := interface{}(m.GetSharepoint()).(interface{ Validate() error }); ok {
if err := v.Validate(); err != nil {
return MetaDataValidationError{
field: "Sharepoint",
reason: "embedded message failed validation",
cause: err,
}
}
}
case *MetaData_GoogleDrive:
if all {
switch v := interface{}(m.GetGoogleDrive()).(type) {
case interface{ ValidateAll() error }:
if err := v.ValidateAll(); err != nil {
errors = append(errors, MetaDataValidationError{
field: "GoogleDrive",
reason: "embedded message failed validation",
cause: err,
})
}
case interface{ Validate() error }:
if err := v.Validate(); err != nil {
errors = append(errors, MetaDataValidationError{
field: "GoogleDrive",
reason: "embedded message failed validation",
cause: err,
})
}
}
} else if v, ok := interface{}(m.GetGoogleDrive()).(interface{ Validate() error }); ok {
if err := v.Validate(); err != nil {
return MetaDataValidationError{
field: "GoogleDrive",
reason: "embedded message failed validation",
cause: err,
}
}
}
}
if len(errors) > 0 {
File diff suppressed because it is too large Load Diff
+2 -231
View File
@@ -1319,37 +1319,6 @@ func (m *GCS) validate(all bool) error {
case *GCS_ServiceAccountFile:
// no validation rules for ServiceAccountFile
case *GCS_Oauth:
if all {
switch v := interface{}(m.GetOauth()).(type) {
case interface{ ValidateAll() error }:
if err := v.ValidateAll(); err != nil {
errors = append(errors, GCSValidationError{
field: "Oauth",
reason: "embedded message failed validation",
cause: err,
})
}
case interface{ Validate() error }:
if err := v.Validate(); err != nil {
errors = append(errors, GCSValidationError{
field: "Oauth",
reason: "embedded message failed validation",
cause: err,
})
}
}
} else if v, ok := interface{}(m.GetOauth()).(interface{ Validate() error }); ok {
if err := v.Validate(); err != nil {
return GCSValidationError{
field: "Oauth",
reason: "embedded message failed validation",
cause: err,
}
}
}
}
if len(errors) > 0 {
@@ -2601,8 +2570,6 @@ func (m *S3) validate(all bool) error {
var errors []error
// no validation rules for MaxObjectSize
switch m.Credential.(type) {
case *S3_AccessKey:
@@ -2698,37 +2665,6 @@ func (m *S3) validate(all bool) error {
}
}
case *S3_SessionToken:
if all {
switch v := interface{}(m.GetSessionToken()).(type) {
case interface{ ValidateAll() error }:
if err := v.ValidateAll(); err != nil {
errors = append(errors, S3ValidationError{
field: "SessionToken",
reason: "embedded message failed validation",
cause: err,
})
}
case interface{ Validate() error }:
if err := v.Validate(); err != nil {
errors = append(errors, S3ValidationError{
field: "SessionToken",
reason: "embedded message failed validation",
cause: err,
})
}
}
} else if v, ok := interface{}(m.GetSessionToken()).(interface{ Validate() error }); ok {
if err := v.Validate(); err != nil {
return S3ValidationError{
field: "SessionToken",
reason: "embedded message failed validation",
cause: err,
}
}
}
}
if len(errors) > 0 {
@@ -3545,6 +3481,8 @@ func (m *Teams) validate(all bool) error {
errors = append(errors, err)
}
// no validation rules for TeamId
switch m.Credential.(type) {
case *Teams_Token:
@@ -3581,37 +3519,6 @@ func (m *Teams) validate(all bool) error {
}
}
case *Teams_Oauth:
if all {
switch v := interface{}(m.GetOauth()).(type) {
case interface{ ValidateAll() error }:
if err := v.ValidateAll(); err != nil {
errors = append(errors, TeamsValidationError{
field: "Oauth",
reason: "embedded message failed validation",
cause: err,
})
}
case interface{ Validate() error }:
if err := v.Validate(); err != nil {
errors = append(errors, TeamsValidationError{
field: "Oauth",
reason: "embedded message failed validation",
cause: err,
})
}
}
} else if v, ok := interface{}(m.GetOauth()).(interface{ Validate() error }); ok {
if err := v.Validate(); err != nil {
return TeamsValidationError{
field: "Oauth",
reason: "embedded message failed validation",
cause: err,
}
}
}
}
if len(errors) > 0 {
@@ -4252,139 +4159,3 @@ var _ interface {
Cause() error
ErrorName() string
} = SlackRealtimeValidationError{}
// Validate checks the field values on Sharepoint with the rules defined in the
// proto definition for this message. If any rules are violated, the first
// error encountered is returned, or nil if there are no violations.
func (m *Sharepoint) Validate() error {
return m.validate(false)
}
// ValidateAll checks the field values on Sharepoint with the rules defined in
// the proto definition for this message. If any rules are violated, the
// result is a list of violation errors wrapped in SharepointMultiError, or
// nil if none found.
func (m *Sharepoint) ValidateAll() error {
return m.validate(true)
}
func (m *Sharepoint) validate(all bool) error {
if m == nil {
return nil
}
var errors []error
// no validation rules for SiteUrl
switch m.Credential.(type) {
case *Sharepoint_Oauth:
if all {
switch v := interface{}(m.GetOauth()).(type) {
case interface{ ValidateAll() error }:
if err := v.ValidateAll(); err != nil {
errors = append(errors, SharepointValidationError{
field: "Oauth",
reason: "embedded message failed validation",
cause: err,
})
}
case interface{ Validate() error }:
if err := v.Validate(); err != nil {
errors = append(errors, SharepointValidationError{
field: "Oauth",
reason: "embedded message failed validation",
cause: err,
})
}
}
} else if v, ok := interface{}(m.GetOauth()).(interface{ Validate() error }); ok {
if err := v.Validate(); err != nil {
return SharepointValidationError{
field: "Oauth",
reason: "embedded message failed validation",
cause: err,
}
}
}
}
if len(errors) > 0 {
return SharepointMultiError(errors)
}
return nil
}
// SharepointMultiError is an error wrapping multiple validation errors
// returned by Sharepoint.ValidateAll() if the designated constraints aren't met.
type SharepointMultiError []error
// Error returns a concatenation of all the error messages it wraps.
func (m SharepointMultiError) Error() string {
var msgs []string
for _, err := range m {
msgs = append(msgs, err.Error())
}
return strings.Join(msgs, "; ")
}
// AllErrors returns a list of validation violation errors.
func (m SharepointMultiError) AllErrors() []error { return m }
// SharepointValidationError is the validation error returned by
// Sharepoint.Validate if the designated constraints aren't met.
type SharepointValidationError struct {
field string
reason string
cause error
key bool
}
// Field function returns field value.
func (e SharepointValidationError) Field() string { return e.field }
// Reason function returns reason value.
func (e SharepointValidationError) Reason() string { return e.reason }
// Cause function returns cause value.
func (e SharepointValidationError) Cause() error { return e.cause }
// Key function returns key value.
func (e SharepointValidationError) Key() bool { return e.key }
// ErrorName returns error name.
func (e SharepointValidationError) ErrorName() string { return "SharepointValidationError" }
// Error satisfies the builtin error interface
func (e SharepointValidationError) Error() string {
cause := ""
if e.cause != nil {
cause = fmt.Sprintf(" | caused by: %v", e.cause)
}
key := ""
if e.key {
key = "key for "
}
return fmt.Sprintf(
"invalid %sSharepoint.%s: %s%s",
key,
e.field,
e.reason,
cause)
}
var _ error = SharepointValidationError{}
var _ interface {
Field() string
Reason() string
Key() bool
Cause() error
ErrorName() string
} = SharepointValidationError{}
+6 -6
View File
@@ -50,10 +50,12 @@ func TestSource_Scan(t *testing.T) {
wantSourceMetadata: &source_metadatapb.MetaData{
Data: &source_metadatapb.MetaData_Circleci{
Circleci: &source_metadatapb.CircleCI{
VcsType: "github",
Username: "dustin-decker",
Repository: "circle-ci",
BuildStep: "Spin up environment",
VcsType: "github",
Username: "dustin-decker",
Repository: "circle-ci",
BuildNumber: 2,
BuildStep: "Spin up environment",
Link: "https://app.circleci.com/pipelines/github/dustin-decker/circle-ci/2",
},
},
},
@@ -83,8 +85,6 @@ func TestSource_Scan(t *testing.T) {
}
}()
gotChunk := <-chunksCh
gotChunk.SourceMetadata.Data.(*source_metadatapb.MetaData_Circleci).Circleci.BuildNumber = 0 // override this because we need to periodically re-run the builds
gotChunk.SourceMetadata.Data.(*source_metadatapb.MetaData_Circleci).Circleci.Link = "" // override this because we need to periodically re-run the builds
if diff := pretty.Compare(gotChunk.SourceMetadata, tt.wantSourceMetadata); diff != "" {
t.Errorf("Source.Chunks() %s diff: (-got +want)\n%s", tt.name, diff)
}
+226 -156
View File
@@ -1,34 +1,42 @@
package gcs
import (
"encoding/json"
"fmt"
"io"
"net/http"
"os"
"strconv"
"strings"
"sort"
"sync"
"cloud.google.com/go/storage"
diskbufferreader "github.com/bill-rich/disk-buffer-reader"
"github.com/go-errors/errors"
"github.com/go-logr/logr"
"golang.org/x/oauth2"
"golang.org/x/oauth2/endpoints"
"google.golang.org/protobuf/proto"
"google.golang.org/protobuf/types/known/anypb"
"github.com/trufflesecurity/trufflehog/v3/pkg/cache"
"github.com/trufflesecurity/trufflehog/v3/pkg/cache/memory"
"github.com/trufflesecurity/trufflehog/v3/pkg/context"
"github.com/trufflesecurity/trufflehog/v3/pkg/handlers"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/credentialspb"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/source_metadatapb"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
)
const defaultCachePersistIncrement = 2500
// Source represents a GCS source.
type Source struct {
name string
jobId int64
sourceId int64
concurrency int
verify bool
gcsManager objectManager
stats *attributes
log logr.Logger
chunksCh chan *sources.Chunk
mu sync.Mutex
sources.Progress // not safe for concurrent access.
}
// Ensure the Source satisfies the interface at compile time.
var _ sources.Source = (*Source)(nil)
@@ -49,56 +57,6 @@ func (s *Source) JobID() int64 {
return s.jobId
}
// Source represents a GCS source.
type Source struct {
name string
jobId int64
sourceId int64
concurrency int
verify bool
gcsManager objectManager
stats *attributes
log logr.Logger
chunksCh chan *sources.Chunk
mu sync.Mutex
sources.Progress // progress is not thread safe
}
// persistableCache is a wrapper around cache.Cache that allows
// for the persistence of the cache contents in the Progress of the source
// at given increments.
type persistableCache struct {
persistIncrement int
cache.Cache
*sources.Progress
}
func newPersistableCache(increment int, cache cache.Cache, p *sources.Progress) *persistableCache {
return &persistableCache{
persistIncrement: increment,
Cache: cache,
Progress: p,
}
}
// Set overrides the cache Set method of the cache to enable the persistence
// of the cache contents the Progress of the source at given increments.
func (c *persistableCache) Set(key, val string) {
c.Cache.Set(key, val)
if ok, contents := c.shouldPersist(); ok {
c.Progress.EncodedResumeInfo = contents
}
}
func (c *persistableCache) shouldPersist() (bool, string) {
if c.Count()%c.persistIncrement != 0 {
return false, ""
}
return true, c.Contents()
}
// Init returns an initialized GCS source.
func (s *Source) Init(aCtx context.Context, name string, id int64, sourceID int64, verify bool, connection *anypb.Any, concurrency int) error {
s.log = aCtx.Logger()
@@ -115,25 +73,6 @@ func (s *Source) Init(aCtx context.Context, name string, id int64, sourceID int6
return errors.WrapPrefix(err, "error unmarshalling connection", 0)
}
gcsManager, err := configureGCSManager(aCtx, &conn, concurrency)
if err != nil {
return err
}
s.gcsManager = gcsManager
s.log.V(2).Info("enumerating buckets and objects")
if err := s.enumerate(aCtx); err != nil {
return fmt.Errorf("error enumerating buckets and objects: %w", err)
}
return nil
}
func configureGCSManager(aCtx context.Context, conn *sourcespb.GCS, concurrency int) (*gcsManager, error) {
if conn == nil {
return nil, fmt.Errorf("GCS connection is nil, cannot configure GCS manager")
}
var gcsManagerAuthOption gcsManagerOption
switch conn.Credential.(type) {
@@ -142,7 +81,7 @@ func configureGCSManager(aCtx context.Context, conn *sourcespb.GCS, concurrency
case *sourcespb.GCS_ServiceAccountFile:
b, err := os.ReadFile(conn.GetServiceAccountFile())
if err != nil {
return nil, fmt.Errorf("error reading GCS JSON Service Account file: %w", err)
return fmt.Errorf("error reading GCS JSON Service Account file: %w", err)
}
gcsManagerAuthOption = withJSONServiceAccount(aCtx, b)
case *sourcespb.GCS_JsonServiceAccount:
@@ -151,60 +90,40 @@ func configureGCSManager(aCtx context.Context, conn *sourcespb.GCS, concurrency
gcsManagerAuthOption = withDefaultADC(aCtx)
case *sourcespb.GCS_Unauthenticated:
gcsManagerAuthOption = withoutAuthentication()
case *sourcespb.GCS_Oauth:
client, err := oauth2Client(aCtx, conn.GetOauth())
if err != nil {
return nil, fmt.Errorf("error creating oauth2 client: %w", err)
}
gcsManagerAuthOption = withHTTPClient(aCtx, client)
default:
return nil, fmt.Errorf("unknown GCS authentication type: %T", conn.Credential)
return fmt.Errorf("unknown GCS authentication type: %T", conn.Credential)
}
resume, err := setResumeBucketOffset(s.Progress.EncodedResumeInfo)
if err != nil {
return fmt.Errorf("error setting resume info: %w", err)
}
gcsManagerOpts := []gcsManagerOption{
withConcurrency(concurrency),
withBucketOffsets(resume),
withMaxObjectSize(conn.MaxObjectSize),
gcsManagerAuthOption,
}
if setGCSManagerBucketOptions(conn) != nil {
gcsManagerOpts = append(gcsManagerOpts, setGCSManagerBucketOptions(conn))
if setGCSManagerBucketOptions(&conn) != nil {
gcsManagerOpts = append(gcsManagerOpts, setGCSManagerBucketOptions(&conn))
}
if setGCSManagerObjectOptions(conn) != nil {
gcsManagerOpts = append(gcsManagerOpts, setGCSManagerObjectOptions(conn))
if setGCSManagerObjectOptions(&conn) != nil {
gcsManagerOpts = append(gcsManagerOpts, setGCSManagerObjectOptions(&conn))
}
gcsManager, err := newGCSManager(conn.ProjectId, gcsManagerOpts...)
if err != nil {
return nil, fmt.Errorf("error creating GCS manager: %w", err)
if s.gcsManager, err = newGCSManager(conn.ProjectId, gcsManagerOpts...); err != nil {
return fmt.Errorf("error creating GCS manager: %w", err)
}
return gcsManager, nil
}
func oauth2Client(ctx context.Context, creds *credentialspb.Oauth2) (*http.Client, error) {
if creds == nil {
return nil, fmt.Errorf("oauth2 credentials are required")
}
if creds.GetClientId() == "" || creds.GetRefreshToken() == "" || creds.GetAccessToken() == "" {
return nil, fmt.Errorf("oauth2 credentials are incomplete, client_id, refresh_token, and access_token are required")
if len(resume) > 0 {
s.log.V(2).Info("resuming from previous run")
return nil
}
conf := &oauth2.Config{
ClientID: creds.GetClientId(),
Scopes: []string{storage.ScopeReadOnly},
Endpoint: oauth2.Endpoint{
AuthURL: endpoints.Google.AuthURL,
TokenURL: endpoints.Google.TokenURL,
},
}
tok := &oauth2.Token{
AccessToken: creds.GetAccessToken(),
RefreshToken: creds.GetRefreshToken(),
}
return conf.Client(ctx, tok), nil
s.log.V(2).Info("new run, enumerating buckets and objects")
return s.enumerate(aCtx)
}
func setGCSManagerBucketOptions(conn *sourcespb.GCS) gcsManagerOption {
@@ -228,8 +147,148 @@ func setGCSManagerOptions(include, exclude []string, includeFn, excludeFn func([
return nil
}
// enumerate all the objects and buckets in the source.
// This will be used to calculate progress.
type progressInfo struct {
processedBucketCnt,
totalBucketsCnt,
processedObjectsCnt,
totalObjectsCnt int
// bucketObjects tracks processing and processed objects for each bucket.
bucketObjects map[string]*objectsProgress
}
// objectsProgress keeps track of the progress of processing objects in a bucket.
// It is marshalled/unmarshalled to/from a string as part of the Source's Progress.
type objectsProgress struct {
// IsBuckedProcessed is true if all objects in the bucket have been processed.
IsBucketProcessed bool
// ProcessedCnt is the number of objects that have been processed.
ProcessedCnt,
// TotalCnt is the total number of objects in the bucket.
TotalCnt int
// Processed is the last object that was fully processed.
Processed string
// Processing are the objects that were not fully processed.
Processing map[string]struct{}
}
func newObjectsProgress(cnt int) *objectsProgress {
return &objectsProgress{TotalCnt: cnt}
}
// newProgressInfo constructs a progressInfo from the Source's Progress information.
func newProgressInfo(ctx context.Context, s *sources.Progress) (*progressInfo, error) {
totalObjCnt := s.SectionsRemaining
processedObjCnt := s.SectionsCompleted
info := &progressInfo{
totalObjectsCnt: int(totalObjCnt),
processedObjectsCnt: int(processedObjCnt),
}
encodeResume := s.EncodedResumeInfo
if encodeResume == "" {
info.bucketObjects = make(map[string]*objectsProgress)
return info, nil
}
var m map[string]*objectsProgress
if err := json.Unmarshal([]byte(encodeResume), &m); err != nil {
ctx.Logger().V(1).Info("error unmarshalling progress info", "error", err)
return info, nil
}
info.bucketObjects = m
return info, nil
}
type progressStateFn func(string, string, *progressInfo)
func (p *progressInfo) setProcessStatus(obj object, fn progressStateFn) {
fn(obj.bucket, obj.name, p)
}
func setProcessingBucketObject(bucket, obj string, progress *progressInfo) {
if progress.bucketObjects[bucket].Processing == nil {
progress.bucketObjects[bucket].Processing = make(map[string]struct{})
}
progress.processing(bucket, obj)
}
func (p *progressInfo) processing(bkt, obj string) {
p.bucketObjects[bkt].Processing[obj] = struct{}{}
}
func setProcessedBucketObject(bucket, obj string, progress *progressInfo) {
progress.processed(bucket, obj)
}
func (p *progressInfo) processed(bkt, obj string) {
// Remove the object from the processing list and increment the processed count.
delete(p.bucketObjects[bkt].Processing, obj)
p.bucketObjects[bkt].ProcessedCnt++
// If all objects in the bucket have been processed, mark the bucket as processed.
if p.bucketObjects[bkt].ProcessedCnt == p.bucketObjects[bkt].TotalCnt {
p.bucketObjects[bkt].IsBucketProcessed = true
p.processedBucketCnt++
}
p.processedObjectsCnt++
// If the object is not greater (lexicographically) than the last processed object, we can skip it.
// This ensures we keep the greatest object name as the last processed object.
if obj < p.bucketObjects[bkt].Processed {
return
}
p.bucketObjects[bkt].Processed = obj
}
func setResumeBucketOffset(s string) (map[string]offsetInfo, error) {
resumeInfo := progressInfo{}
if s != "" {
if err := json.Unmarshal([]byte(s), &resumeInfo.bucketObjects); err != nil {
return nil, fmt.Errorf("error unmarshalling resume info: %w", err)
}
}
return calcBktOffset(resumeInfo.bucketObjects)
}
// In order to calculate the bucket offset, we need to know the last object
// that was processed for each bucket, as well as any objects that were processing.
// If the bucket was fully processed, we don't need to set an offset.
// If there were no objects processing, we can just use the last object that was processed.
// If there were objects processing, we need to find the first object (lexicographically)
// that was processing, as that is the next object that needs to be processed.
func calcBktOffset(resumeInfo map[string]*objectsProgress) (map[string]offsetInfo, error) {
bucketOffset := make(map[string]offsetInfo, len(resumeInfo))
for objName, progress := range resumeInfo {
info := offsetInfo{}
if progress.IsBucketProcessed {
info.isBucketProcessed = true
bucketOffset[objName] = info
continue
}
if len(progress.Processing) == 0 {
info.lastProcessedObject = progress.Processed
bucketOffset[objName] = info
continue
}
processing := make([]string, 0, len(progress.Processing))
for k := range progress.Processing {
processing = append(processing, k)
}
sort.Strings(processing)
info.lastProcessedObject = processing[0]
bucketOffset[objName] = info
}
return bucketOffset, nil
}
// enumerate all the objects and buckets in the source and use the results to
// set the progress information. This will be used track progression of the scan,
// and to resume the scan if it is interrupted.
func (s *Source) enumerate(ctx context.Context) error {
stats, err := s.gcsManager.attributes(ctx)
if err != nil {
@@ -237,19 +296,32 @@ func (s *Source) enumerate(ctx context.Context) error {
}
s.stats = stats
return nil
info := progressInfo{
totalBucketsCnt: int(s.stats.numBuckets),
totalObjectsCnt: int(s.stats.numObjects),
}
progress := make(map[string]*objectsProgress, len(s.stats.bucketObjects))
for k, v := range s.stats.bucketObjects {
progress[k] = newObjectsProgress(int(v))
}
info.bucketObjects = progress
return s.setProgress(ctx, &info, fmt.Sprintf("enumerated %d buckets, %d objects", s.stats.numBuckets, s.stats.numObjects))
}
// Chunks emits chunks of bytes over a channel.
func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) error {
persistableCache := s.setupCache(ctx)
objectCh, err := s.gcsManager.listObjects(ctx)
if err != nil {
return fmt.Errorf("error listing objects: %w", err)
}
s.chunksCh = chunksChan
s.Progress.Message = "starting to process objects..."
progress, err := newProgressInfo(ctx, &s.Progress)
if err != nil {
return fmt.Errorf("error constructing progress info: %w", err)
}
var wg sync.WaitGroup
for obj := range objectCh {
@@ -260,57 +332,55 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) err
continue
}
if persistableCache.Exists(o.name) {
ctx.Logger().V(5).Info("skipping object, object already processed", "name", o.name)
if err := s.startProcessing(ctx, progress, o); err != nil {
ctx.Logger().V(1).Info("GCS source error starting to process objects", "error", err)
continue
}
wg.Add(1)
go func(obj object) {
go func(obj object, resume *progressInfo) {
defer wg.Done()
if err := s.processObject(ctx, o); err != nil {
ctx.Logger().V(1).Info("error setting start progress progress", "name", o.name, "error", err)
return
}
s.setProgress(ctx, o.md5, o.name, persistableCache)
}(o)
if err := s.endProcessing(ctx, resume, o); err != nil {
ctx.Logger().V(1).Info("error setting end progress progress", "name", o.name, "error", err)
}
}(o, progress)
}
wg.Wait()
s.completeProgress(ctx)
ctx.Logger().Info("GCS source finished processing", "name", s.name)
return nil
}
func (s *Source) setupCache(ctx context.Context) *persistableCache {
var c cache.Cache
if s.Progress.EncodedResumeInfo != "" {
c = memory.NewWithData(ctx, strings.Split(s.Progress.EncodedResumeInfo, ","))
} else {
c = memory.New()
}
// TODO (ahrav): Make this configurable via conn.
persistCache := newPersistableCache(defaultCachePersistIncrement, c, &s.Progress)
return persistCache
}
func (s *Source) setProgress(ctx context.Context, md5, objName string, cache cache.Cache) {
func (s *Source) startProcessing(ctx context.Context, progress *progressInfo, o object) error {
s.mu.Lock()
defer s.mu.Unlock()
ctx.Logger().V(5).Info("setting progress for object", "object-name", objName)
s.SectionsCompleted++
cache.Set(md5, md5)
s.Progress.SectionsRemaining = int32(s.stats.numObjects)
s.Progress.PercentComplete = int64(float64(s.SectionsCompleted) / float64(s.stats.numObjects) * 100)
progress.setProcessStatus(o, setProcessingBucketObject)
return s.setProgress(ctx, progress, fmt.Sprintf("GCS source beginning to process object %s in bucket %s", o.name, o.bucket))
}
func (s *Source) completeProgress(ctx context.Context) {
msg := fmt.Sprintf("GCS source finished processing %d objects", s.stats.numObjects)
ctx.Logger().Info(msg)
s.Progress.Message = msg
func (s *Source) endProcessing(ctx context.Context, progress *progressInfo, o object) error {
s.mu.Lock()
defer s.mu.Unlock()
progress.setProcessStatus(o, setProcessedBucketObject)
return s.setProgress(ctx, progress, fmt.Sprintf("GCS source finished processing object %s in bucket %s", o.name, o.bucket))
}
func (s *Source) setProgress(ctx context.Context, progress *progressInfo, msg string) error {
ctx.Logger().V(5).Info(msg)
b, err := json.Marshal(progress.bucketObjects)
if err != nil {
return fmt.Errorf("error marshalling resume info: %w", err)
}
s.SetProgressComplete(progress.processedObjectsCnt, progress.totalObjectsCnt, msg, string(b))
return nil
}
func (s *Source) processObject(ctx context.Context, o object) error {
@@ -328,7 +398,7 @@ func (s *Source) processObject(ctx context.Context, o object) error {
Email: o.owner,
ContentType: o.contentType,
Acls: o.acl,
CreatedAt: strconv.FormatInt(o.createdAt.Unix(), 10), // Unix time as string
CreatedAt: o.createdAt.String(),
UpdatedAt: o.updatedAt.String(),
},
},
+5 -23
View File
@@ -2,10 +2,8 @@ package gcs
import (
aCtx "context"
"encoding/hex"
"fmt"
"io"
"net/http"
"runtime"
"sync"
"sync/atomic"
@@ -29,7 +27,9 @@ const (
maxObjectSizeLimit = 50 * 1024 * 1024 // 50MB
)
var defaultConcurrency = runtime.NumCPU()
var (
defaultConcurrency = runtime.NumCPU()
)
type objectManager interface {
listObjects(context.Context) (chan io.Reader, error)
@@ -119,19 +119,6 @@ func (s *attributes) setBucketCnt(bkt string, cnt uint64) {
type gcsManagerOption func(*gcsManager) error
// withHTTPClient uses the provided HTTP client when creating a new GCS client.
func withHTTPClient(ctx context.Context, httpClient *http.Client) gcsManagerOption {
client, err := storage.NewClient(ctx, option.WithHTTPClient(httpClient), option.WithScopes(storage.ScopeReadOnly))
return func(m *gcsManager) error {
if err != nil {
return err
}
m.client = client
return nil
}
}
// withAPIKey uses the provided API key when creating a new GCS client.
// This can ONLY be used for public buckets.
func withAPIKey(ctx context.Context, apiKey string) gcsManagerOption {
@@ -294,7 +281,7 @@ func withConcurrency(concurrency int) gcsManagerOption {
// withMaxObjectSize sets the maximum size of objects that will be scanned.
// If not set, set to a negative number, or set larger than 1GB,
// the default value of 50MB will be used.
// the default value of 500MB will be used.
func withMaxObjectSize(maxObjectSize int64) gcsManagerOption {
return func(m *gcsManager) error {
if maxObjectSize <= 0 || maxObjectSize > maxObjectSizeLimit {
@@ -370,7 +357,6 @@ type object struct {
contentType string
owner string
link string
md5 string
// acl represents an ACLEntities.
// https://pkg.go.dev/cloud.google.com/go/storage#ACLEntity
acl []string
@@ -397,10 +383,6 @@ func (g *gcsManager) enumerate(ctx context.Context, bkts []bucket) (*attributes,
logger.V(5).Info("enumerating buckets", "numBuckets", len(bkts))
stats := newStats(len(bkts))
defer func(start time.Time) {
logger.V(5).Info("finished enumerating buckets", "duration-seconds", fmt.Sprintf("%.1f", time.Since(start).Seconds()), "num-buckets", len(bkts), "num-objects", stats.numObjects)
}(time.Now())
for _, bkt := range bkts {
bkt := bkt
g.workerPool.Go(func() error {
@@ -444,6 +426,7 @@ func (g *gcsManager) enumerate(ctx context.Context, bkts []bucket) (*attributes,
_ = g.workerPool.Wait()
g.attr = stats
logger.V(5).Info("finished enumerating buckets", "num-objects", stats.numObjects, "num-buckets", stats.numBuckets)
return stats, nil
}
@@ -648,7 +631,6 @@ func (g *gcsManager) constructObject(ctx context.Context, obj *storage.ObjectHan
o.contentType = attrs.ContentType
o.owner = attrs.Owner
o.link = attrs.MediaLink
o.md5 = hex.EncodeToString(attrs.MD5)
o.createdAt = attrs.Created
o.updatedAt = attrs.Updated
o.acl = objectACLs(attrs.ACL)
-23
View File
@@ -2,7 +2,6 @@ package gcs
import (
"fmt"
"net/http"
"sort"
"testing"
"time"
@@ -329,12 +328,6 @@ func TestNewGcsManager(t *testing.T) {
maxObjectSize: defaultMaxObjectSize,
},
},
{
name: "new gcs manager, with HTTP client",
projID: testProjectID,
opts: []gcsManagerOption{withHTTPClient(ctx, &http.Client{})},
want: &gcsManager{projectID: testProjectID, concurrency: defaultConcurrency, maxObjectSize: defaultMaxObjectSize},
},
}
for _, tc := range testCases {
@@ -480,7 +473,6 @@ func TestGCSManagerListObjects(t *testing.T) {
size: 150,
link: "https://storage.googleapis.com/download/storage/v1/b/test-bkt-th/o/aws1.txt?generation=1677870994890594&alt=media",
acl: []string{},
md5: "7a4df45f839cd594ac236b19524cec92",
},
{
name: "moar2.txt",
@@ -489,7 +481,6 @@ func TestGCSManagerListObjects(t *testing.T) {
size: 12,
link: "https://storage.googleapis.com/download/storage/v1/b/test-bkt-th/o/moar2.txt?generation=1677871000378542&alt=media",
acl: []string{},
md5: "e59ff97941044f85df5297e1c302d260",
},
{
name: "aws3.txt",
@@ -498,7 +489,6 @@ func TestGCSManagerListObjects(t *testing.T) {
size: 150,
link: "https://storage.googleapis.com/download/storage/v1/b/test-bkt-th2/o/aws3.txt?generation=1677871022489611&alt=media",
acl: []string{},
md5: "26eb03378a0c8e99ee0cecf9155ace81",
},
{
name: "moar.txt",
@@ -507,7 +497,6 @@ func TestGCSManagerListObjects(t *testing.T) {
size: 6,
link: "https://storage.googleapis.com/download/storage/v1/b/test-bkt-th3/o/moar.txt?generation=1677871042896804&alt=media",
acl: []string{},
md5: "09f7e02f1290be211da707a266f153b3",
},
{
name: "AMAZON_FASHION_5.json",
@@ -516,7 +505,6 @@ func TestGCSManagerListObjects(t *testing.T) {
size: 1413469,
link: "https://storage.googleapis.com/download/storage/v1/b/test-bkt-th4/o/AMAZON_FASHION_5.json?generation=1677871063457469&alt=media",
acl: []string{},
md5: "5a1d8f483aa34695256379050d42a2b7",
},
},
wantNumBkt: 4,
@@ -534,7 +522,6 @@ func TestGCSManagerListObjects(t *testing.T) {
size: 150,
link: "https://storage.googleapis.com/download/storage/v1/b/test-bkt-th/o/aws1.txt?generation=1677870994890594&alt=media",
acl: []string{},
md5: "7a4df45f839cd594ac236b19524cec92",
},
{
name: "moar2.txt",
@@ -543,7 +530,6 @@ func TestGCSManagerListObjects(t *testing.T) {
size: 12,
link: "https://storage.googleapis.com/download/storage/v1/b/test-bkt-th/o/moar2.txt?generation=1677871000378542&alt=media",
acl: []string{},
md5: "e59ff97941044f85df5297e1c302d260",
},
},
wantNumBkt: 1,
@@ -561,7 +547,6 @@ func TestGCSManagerListObjects(t *testing.T) {
size: 6,
link: "https://storage.googleapis.com/download/storage/v1/b/test-bkt-th3/o/moar.txt?generation=1677871042896804&alt=media",
acl: []string{},
md5: "09f7e02f1290be211da707a266f153b3",
},
{
name: "AMAZON_FASHION_5.json",
@@ -570,7 +555,6 @@ func TestGCSManagerListObjects(t *testing.T) {
size: 1413469,
link: "https://storage.googleapis.com/download/storage/v1/b/test-bkt-th4/o/AMAZON_FASHION_5.json?generation=1677871063457469&alt=media",
acl: []string{},
md5: "5a1d8f483aa34695256379050d42a2b7",
},
},
wantNumBkt: 2,
@@ -592,7 +576,6 @@ func TestGCSManagerListObjects(t *testing.T) {
size: 150,
link: "https://storage.googleapis.com/download/storage/v1/b/test-bkt-th2/o/aws3.txt?generation=1677871022489611&alt=media",
acl: []string{},
md5: "26eb03378a0c8e99ee0cecf9155ace81",
},
{
name: "moar.txt",
@@ -601,7 +584,6 @@ func TestGCSManagerListObjects(t *testing.T) {
size: 6,
link: "https://storage.googleapis.com/download/storage/v1/b/test-bkt-th3/o/moar.txt?generation=1677871042896804&alt=media",
acl: []string{},
md5: "09f7e02f1290be211da707a266f153b3",
},
{
name: "AMAZON_FASHION_5.json",
@@ -610,7 +592,6 @@ func TestGCSManagerListObjects(t *testing.T) {
size: 1413469,
link: "https://storage.googleapis.com/download/storage/v1/b/test-bkt-th4/o/AMAZON_FASHION_5.json?generation=1677871063457469&alt=media",
acl: []string{},
md5: "5a1d8f483aa34695256379050d42a2b7",
},
},
wantNumBkt: 4, // We still list objects in all buckets.
@@ -632,7 +613,6 @@ func TestGCSManagerListObjects(t *testing.T) {
size: 150,
link: "https://storage.googleapis.com/download/storage/v1/b/test-bkt-th/o/aws1.txt?generation=1677870994890594&alt=media",
acl: []string{},
md5: "7a4df45f839cd594ac236b19524cec92",
},
},
wantNumBkt: 1,
@@ -654,7 +634,6 @@ func TestGCSManagerListObjects(t *testing.T) {
size: 150,
link: "https://storage.googleapis.com/download/storage/v1/b/test-bkt-th/o/aws1.txt?generation=1677870994890594&alt=media",
acl: []string{},
md5: "7a4df45f839cd594ac236b19524cec92",
},
},
wantNumBkt: 4,
@@ -677,7 +656,6 @@ func TestGCSManagerListObjects(t *testing.T) {
size: 12,
link: "https://storage.googleapis.com/download/storage/v1/b/test-bkt-th/o/moar2.txt?generation=1677871000378542&alt=media",
acl: []string{},
md5: "e59ff97941044f85df5297e1c302d260",
},
},
wantNumBkt: 1,
@@ -765,7 +743,6 @@ func TestGCSManagerListObjects_Resuming(t *testing.T) {
size: 12,
link: "https://storage.googleapis.com/download/storage/v1/b/test-bkt-th/o/moar2.txt?generation=1677871000378542&alt=media",
acl: []string{},
md5: "e59ff97941044f85df5297e1c302d260",
},
},
wantNumBkt: 1,
+105 -147
View File
@@ -1,14 +1,11 @@
package gcs
import (
"encoding/json"
"fmt"
"io"
"net/http"
"sort"
"strconv"
"strings"
"testing"
"time"
"github.com/google/go-cmp/cmp"
"github.com/google/go-cmp/cmp/cmpopts"
@@ -16,7 +13,6 @@ import (
"google.golang.org/protobuf/types/known/anypb"
"github.com/trufflesecurity/trufflehog/v3/pkg/context"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/credentialspb"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/source_metadatapb"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
@@ -35,7 +31,7 @@ func TestSourceInit(t *testing.T) {
source, conn := createTestSource(&sourcespb.GCS{
ProjectId: testProjectID,
IncludeBuckets: []string{
publicBucket,
"bucket1",
},
ExcludeBuckets: []string{
perfTestBucketGlob,
@@ -43,15 +39,22 @@ func TestSourceInit(t *testing.T) {
ExcludeObjects: []string{
"object1",
},
Credential: &sourcespb.GCS_Unauthenticated{},
Credential: &sourcespb.GCS_ApiKey{
ApiKey: testAPIKey,
},
})
err := source.Init(context.Background(), "test", 1, 1, true, conn, 8)
// Set the EncodeResumeInfo to prevent enumeration.
b, err := json.Marshal(map[string]objectsProgress{"other": {}})
assert.Nil(t, err)
source.Progress.EncodedResumeInfo = string(b)
err = source.Init(context.Background(), "test", 1, 1, true, conn, 8)
assert.Nil(t, err)
assert.NotNil(t, source.gcsManager)
}
func TestConfigureGCSManager(t *testing.T) {
func TestSourceInit_Conn(t *testing.T) {
testCases := []struct {
name string
conn *sourcespb.GCS
@@ -74,6 +77,7 @@ func TestConfigureGCSManager(t *testing.T) {
want: &gcsManager{
projectID: testProjectID,
excludeBuckets: map[string]struct{}{perfTestBucketGlob: {}},
hasEnumerated: true,
},
},
{
@@ -91,6 +95,7 @@ func TestConfigureGCSManager(t *testing.T) {
want: &gcsManager{
projectID: testProjectID,
includeBuckets: map[string]struct{}{"bucket1": {}},
hasEnumerated: true,
},
},
{
@@ -112,20 +117,27 @@ func TestConfigureGCSManager(t *testing.T) {
projectID: testProjectID,
includeObjects: map[string]struct{}{"object1": {}},
excludeBuckets: map[string]struct{}{perfTestBucketGlob: {}},
hasEnumerated: true,
},
},
}
for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
ctx := context.Background()
got, err := configureGCSManager(ctx, tc.conn, 8)
source, conn := createTestSource(tc.conn)
// Set the EncodeResumeInfo to prevent enumeration.
b, err := json.Marshal(map[string]objectsProgress{"other": {}})
assert.Nil(t, err)
source.Progress.EncodedResumeInfo = string(b)
err = source.Init(context.Background(), "test", 1, 1, true, conn, 8)
if err != nil && !tc.wantErr {
t.Errorf("source.configureGCSManager() error = %v", err)
t.Errorf("source.Init() got: %v, want: %v", err, nil)
return
}
if !tc.wantErr {
if diff := cmp.Diff(tc.want, got,
if diff := cmp.Diff(tc.want, source.gcsManager,
cmp.AllowUnexported(gcsManager{}),
cmpopts.IgnoreFields(gcsManager{}, "client", "workerPool", "concurrency", "buckets", "maxObjectSize", "attr"),
); diff != "" {
@@ -136,78 +148,67 @@ func TestConfigureGCSManager(t *testing.T) {
}
}
func TestSourceOauth2Client(t *testing.T) {
func TestSourceInit_Progress(t *testing.T) {
complete := map[string]objectsProgress{testBucket: {
Processed: object2,
IsBucketProcessed: true,
ProcessedCnt: 1,
TotalCnt: 1,
}}
completeEncodeResumeInfo, err := json.Marshal(complete)
assert.Nil(t, err)
processing := map[string]objectsProgress{
testBucket: {
IsBucketProcessed: false,
ProcessedCnt: 1,
Processed: object2,
Processing: map[string]struct{}{object1: {}},
TotalCnt: 2,
},
}
processingEncodeResumeInfo, err := json.Marshal(processing)
assert.Nil(t, err)
testCases := []struct {
name string
creds *credentialspb.Oauth2
want *http.Client
wantErr bool
name string
conn *sourcespb.GCS
encodeResumeInfo string
gcsManagerBuckets map[string]bucket
}{
{
name: "valid creds",
creds: &credentialspb.Oauth2{
RefreshToken: "some-refresh-token",
ClientId: "some-client-id",
AccessToken: "some-access-token",
},
want: &http.Client{},
name: "no progress",
conn: &sourcespb.GCS{ProjectId: testProjectID, Credential: &sourcespb.GCS_Adc{}, ExcludeBuckets: []string{perfTestBucketGlob}},
gcsManagerBuckets: map[string]bucket{},
},
{
name: "invalid creds, nil creds",
wantErr: true,
name: "progress set, no processing objects in any buckets",
conn: &sourcespb.GCS{ProjectId: testProjectID, Credential: &sourcespb.GCS_Adc{}},
encodeResumeInfo: string(completeEncodeResumeInfo),
gcsManagerBuckets: map[string]bucket{testBucket: {name: testBucket, startOffset: "", shouldInclude: false}},
},
{
name: "invalid creds, empty refresh token",
creds: &credentialspb.Oauth2{
RefreshToken: "",
AccessToken: "some-access-token",
ClientId: "some-client-id",
},
wantErr: true,
},
{
name: "invalid creds, empty client id",
creds: &credentialspb.Oauth2{
RefreshToken: "some-refresh-token",
AccessToken: "some-access-token",
ClientId: "",
},
wantErr: true,
},
{
name: "invalid creds, empty access token",
creds: &credentialspb.Oauth2{
AccessToken: "",
RefreshToken: "some-refresh-token",
ClientId: "some-client-id",
},
wantErr: true,
name: "progress set, processing objects in some buckets",
conn: &sourcespb.GCS{ProjectId: testProjectID, Credential: &sourcespb.GCS_Adc{}},
encodeResumeInfo: string(processingEncodeResumeInfo),
gcsManagerBuckets: map[string]bucket{testBucket: {name: testBucket, startOffset: object1, shouldInclude: true}},
},
}
for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
ctx := context.Background()
got, err := oauth2Client(ctx, tc.creds)
if (err != nil) != tc.wantErr {
t.Errorf("source.oauth2Client() error = %v", err)
}
source, conn := createTestSource(tc.conn)
source.Progress.EncodedResumeInfo = tc.encodeResumeInfo
if !tc.wantErr {
if diff := cmp.Diff(tc.want, got,
cmpopts.IgnoreFields(http.Client{}, "Transport"),
); diff != "" {
t.Errorf("source.oauth2Client() diff: (-want +got)\n%s", diff)
}
}
err := source.Init(context.Background(), "test", 1, 1, true, conn, 8)
assert.Nil(t, err)
assert.Equal(t, tc.gcsManagerBuckets, source.gcsManager.(*gcsManager).buckets)
})
}
}
type mockObjectManager struct {
// numObjects is the number of objects to return in the listObjects call.
numObjects int
wantErr bool
wantErr bool
}
func (m *mockObjectManager) attributes(_ context.Context) (*attributes, error) {
@@ -216,7 +217,7 @@ func (m *mockObjectManager) attributes(_ context.Context) (*attributes, error) {
}
return &attributes{
numObjects: uint64(m.numObjects),
numObjects: 5,
numBuckets: 1,
bucketObjects: map[string]uint64{testBucket: 5},
}, nil
@@ -246,7 +247,7 @@ func (m *mockObjectManager) listObjects(context.Context) (chan io.Reader, error)
go func() {
defer close(ch)
// Add 5 objects to the channel.
for i := 0; i < m.numObjects; i++ {
for i := 0; i < 5; i++ {
ch <- createTestObject(i)
}
}()
@@ -263,9 +264,7 @@ func createTestObject(id int) object {
link: fmt.Sprintf("https://storage.googleapis.com/%s/%s", testBucket, fmt.Sprintf("object%d", id)),
acl: []string{"authenticatedUsers"},
size: 42,
md5: fmt.Sprintf("md5hash%d", id),
Reader: &mockReader{data: []byte(fmt.Sprintf("hello world %d", id))},
createdAt: time.Date(2020, 1, 1, 0, 0, 0, 0, time.UTC),
}
}
@@ -285,7 +284,6 @@ func createTestSourceChunk(id int) *sources.Chunk {
Email: "testman@test.com",
Link: fmt.Sprintf("https://storage.googleapis.com/%s/%s", testBucket, fmt.Sprintf("object%d", id)),
Acls: []string{"authenticatedUsers"},
CreatedAt: "1577836800",
},
},
},
@@ -297,9 +295,8 @@ func TestSourceChunks_ListObjects(t *testing.T) {
chunksCh := make(chan *sources.Chunk, 1)
source := &Source{
gcsManager: &mockObjectManager{numObjects: 5},
gcsManager: &mockObjectManager{},
chunksCh: chunksCh,
Progress: sources.Progress{},
}
err := source.enumerate(ctx)
@@ -334,17 +331,16 @@ func TestSourceChunks_ListObjects(t *testing.T) {
return got[i].SourceMetadata.GetGcs().Filename < got[j].SourceMetadata.GetGcs().Filename
})
for i, c := range got {
assert.Equal(t, want[i].SourceMetadata.GetGcs().Filename, c.SourceMetadata.GetGcs().Filename)
assert.Equal(t, want[i].Data, c.Data)
assert.Equal(t, want[i].SourceMetadata.GetGcs().CreatedAt, c.SourceMetadata.GetGcs().CreatedAt)
for _, c := range got {
assert.Equal(t, c.SourceMetadata.GetGcs().Filename, c.SourceMetadata.GetGcs().Filename)
assert.Equal(t, c.Data, c.Data)
}
}
func TestSourceInit_Enumerate(t *testing.T) {
ctx := context.Background()
source := &Source{gcsManager: &mockObjectManager{numObjects: 5}}
source := &Source{gcsManager: &mockObjectManager{}}
err := source.enumerate(ctx)
assert.Nil(t, err)
@@ -353,6 +349,11 @@ func TestSourceInit_Enumerate(t *testing.T) {
assert.Equal(t, uint64(5), source.stats.numObjects)
assert.Equal(t, uint32(1), source.stats.numBuckets)
assert.Equal(t, uint64(5), source.stats.bucketObjects[testBucket])
// Ensure progress is initialized correctly.
p := map[string]*objectsProgress{testBucket: {IsBucketProcessed: false, ProcessedCnt: 0, TotalCnt: 5}}
b, err := json.Marshal(p)
assert.Nil(t, err)
assert.Equal(t, string(b), source.Progress.EncodedResumeInfo)
}
func TestSourceChunks_ListObjects_Error(t *testing.T) {
@@ -369,12 +370,11 @@ func TestSourceChunks_ListObjects_Error(t *testing.T) {
func TestSourceChunks_ProgressSet(t *testing.T) {
ctx := context.Background()
chunksCh := make(chan *sources.Chunk, 1)
source := &Source{
gcsManager: &mockObjectManager{numObjects: defaultCachePersistIncrement},
chunksCh: chunksCh,
Progress: sources.Progress{},
}
source := &Source{
gcsManager: &mockObjectManager{},
chunksCh: chunksCh,
}
err := source.enumerate(ctx)
assert.Nil(t, err)
@@ -384,81 +384,39 @@ func TestSourceChunks_ProgressSet(t *testing.T) {
assert.Nil(t, err)
}()
want := make([]*sources.Chunk, 0, defaultCachePersistIncrement)
for i := 0; i < defaultCachePersistIncrement; i++ {
want := make([]*sources.Chunk, 0, 5)
for i := 0; i < 5; i++ {
want = append(want, createTestSourceChunk(i))
}
got := make([]*sources.Chunk, 0, defaultCachePersistIncrement)
got := make([]*sources.Chunk, 0, 5)
for ch := range chunksCh {
got = append(got, ch)
}
// Ensure we get 2500 objects back.
// Ensure we get 5 objects back.
assert.Equal(t, len(want), len(got))
processing := map[string]struct{}{"object0": {}, "object1": {}, "object2": {}, "object3": {}, "object4": {}}
for i := 0; i < 5; i++ {
delete(processing, fmt.Sprintf("object%d", i))
}
// Test that the resume progress is set.
var progress strings.Builder
for i := range got {
progress.WriteString(fmt.Sprintf("md5hash%d", i))
// Add a comma if not the last element.
if i != len(got)-1 {
progress.WriteString(",")
}
// The processed values should be the greatest object name lexicographically.
resume := map[string]objectsProgress{
testBucket: {
Processed: "object4",
Processing: processing,
IsBucketProcessed: true,
ProcessedCnt: 5,
TotalCnt: 5,
},
}
encodeResume := strings.Split(source.Progress.EncodedResumeInfo, ",")
sort.Slice(encodeResume, func(i, j int) bool {
numI, _ := strconv.Atoi(strings.TrimPrefix(encodeResume[i], "md5hash"))
numJ, _ := strconv.Atoi(strings.TrimPrefix(encodeResume[j], "md5hash"))
return numI < numJ
})
assert.Equal(t, progress.String(), strings.Join(encodeResume, ","))
assert.Equal(t, int32(defaultCachePersistIncrement), source.Progress.SectionsCompleted)
assert.Equal(t, int64(100), source.Progress.PercentComplete)
assert.Equal(t, fmt.Sprintf("GCS source finished processing %d objects", defaultCachePersistIncrement), source.Progress.Message)
}
func TestSource_CachePersistence(t *testing.T) {
ctx := context.Background()
wantObjCnt := 4 // ensure we have less objects than the cache increment
mockObjManager := &mockObjectManager{numObjects: wantObjCnt}
chunksCh := make(chan *sources.Chunk, 1)
source := &Source{
gcsManager: mockObjManager,
chunksCh: chunksCh,
Progress: sources.Progress{},
}
err := source.enumerate(ctx)
b, err := json.Marshal(resume)
assert.Nil(t, err)
go func() {
defer close(chunksCh)
err := source.Chunks(ctx, chunksCh)
assert.Nil(t, err)
}()
want := make([]*sources.Chunk, 0, wantObjCnt)
for i := 0; i < wantObjCnt; i++ {
want = append(want, createTestSourceChunk(i))
}
got := make([]*sources.Chunk, 0, wantObjCnt)
for ch := range chunksCh {
got = append(got, ch)
}
// Ensure we get 4 objects back.
assert.Equal(t, len(want), len(got))
// Test that the resume progress is empty.
// The cache should not have been persisted.
assert.Equal(t, "", source.Progress.EncodedResumeInfo)
assert.Equal(t, int32(wantObjCnt), source.Progress.SectionsCompleted)
assert.Equal(t, string(b), source.Progress.EncodedResumeInfo)
assert.Equal(t, int32(5), source.Progress.SectionsCompleted)
assert.Equal(t, int64(100), source.Progress.PercentComplete)
assert.Equal(t, fmt.Sprintf("GCS source finished processing %d objects", wantObjCnt), source.Progress.Message)
}
+11 -19
View File
@@ -86,6 +86,7 @@ func (s *Source) JobID() int64 {
// Init returns an initialized GitHub source.
func (s *Source) Init(aCtx context.Context, name string, jobId, sourceId int64, verify bool, connection *anypb.Any, concurrency int) error {
s.name = name
s.sourceId = sourceId
s.jobId = jobId
@@ -227,11 +228,7 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) err
}
func RepoFromPath(path string) (*git.Repository, error) {
options := &git.PlainOpenOptions{
DetectDotGit: true,
EnableDotGitCommonDir: true,
}
return git.PlainOpenWithOptions(path, options)
return git.PlainOpen(path)
}
func CleanOnError(err *error, path string) {
@@ -298,15 +295,11 @@ func CloneRepo(ctx context.Context, userInfo *url.Userinfo, gitUrl string, args
return "", nil, errors.New("clone command exited with no output")
}
if cloneCmd.ProcessState != nil && cloneCmd.ProcessState.ExitCode() != 0 {
logger.V(1).Info("git clone failed", "output", string(output), "error", err)
logger.V(1).Info("git clone failed", "output", output, "error", err)
return "", nil, fmt.Errorf("could not clone repo: %s, %w", safeUrl, err)
}
options := &git.PlainOpenOptions{
DetectDotGit: true,
EnableDotGitCommonDir: true,
}
repo, err := git.PlainOpenWithOptions(clonePath, options)
repo, err := git.PlainOpen(clonePath)
if err != nil {
return "", nil, fmt.Errorf("could not open cloned repo: %w", err)
}
@@ -345,7 +338,7 @@ func (s *Git) ScanCommits(ctx context.Context, repo *git.Repository, path string
return err
}
commitChan, err := gitparse.NewParser().RepoPath(ctx, path, scanOptions.HeadHash, scanOptions.BaseHash == "", scanOptions.ExcludeGlobs)
commitChan, err := gitparse.NewParser().RepoPath(ctx, path, scanOptions.HeadHash, scanOptions.BaseHash == "")
if err != nil {
return err
}
@@ -385,7 +378,7 @@ func (s *Git) ScanCommits(ctx context.Context, repo *git.Repository, path string
var email, hash, when string
email = commit.Author
hash = commit.Hash
when = commit.Date.UTC().Format("2006-01-02 15:04:05 -0700")
when = commit.Date.Format("2006-01-02 15:04:05 -0700")
// Handle binary files by reading the entire file rather than using the diff.
if diff.IsBinary {
@@ -493,7 +486,7 @@ func (s *Git) ScanStaged(ctx context.Context, repo *git.Repository, path string,
}
var depth int64
reachedBase := false
var reachedBase = false
ctx.Logger().V(1).Info("scanning staged changes", "path", path)
for commit := range commitChan {
@@ -527,7 +520,7 @@ func (s *Git) ScanStaged(ctx context.Context, repo *git.Repository, path string,
var email, hash, when string
email = commit.Author
hash = commit.Hash
when = commit.Date.UTC().Format("2006-01-02 15:04:05 -0700")
when = commit.Date.Format("2006-01-02 15:04:05 -0700")
// Handle binary files by reading the entire file rather than using the diff.
if diff.IsBinary {
@@ -567,8 +560,7 @@ func (s *Git) ScanRepo(ctx context.Context, repo *git.Repository, repoPath strin
if err := normalizeConfig(scanOptions, repo); err != nil {
return err
}
start := time.Now().Unix()
start := time.Now().UnixNano()
if err := s.ScanCommits(ctx, repo, repoPath, scanOptions, chunksChan); err != nil {
return err
}
@@ -584,8 +576,8 @@ func (s *Git) ScanRepo(ctx context.Context, repo *git.Repository, repoPath strin
repoUrl = getSafeRemoteURL(repo, remotes[0].Config().Name)
}
scanTime := time.Now().Unix() - start
ctx.Logger().V(1).Info("scanning git repo complete", "repo", repoUrl, "path", repoPath, "time_seconds", scanTime)
scanTime := time.Now().UnixNano() - start
ctx.Logger().V(1).Info("scanning git repo complete", "Repo", repoUrl, "path", repoPath, "time", scanTime)
return nil
}
+1 -1
View File
@@ -226,7 +226,7 @@ func TestSource_Chunks_Integration(t *testing.T) {
"ce62d79908803153ef6e145e042d3e80488ef747-bump": {B: []byte("\n")},
// Normally we might expect to see this commit, and we may in the future.
// But at the moment we're ignoring any commit unless it contains at least one non-space character.
"27fbead3bf883cdb7de9d7825ed401f28f9398f1-slack": {B: []byte("\n\n\nyup, just did that\n\ngithub_lol: \"ffc7e0f9400fb6300167009e42d2f842cd7956e2\"\n\noh, goodness. there's another one!\n")},
"27fbead3bf883cdb7de9d7825ed401f28f9398f1-slack": {B: []byte("\n\n\n\nyup, just did that\n\ngithub_lol: \"ffc7e0f9400fb6300167009e42d2f842cd7956e2\"\n\noh, goodness. there's another one!\n")},
"8afb0ecd4998b1179e428db5ebbcdc8221214432-slack": {B: []byte("oops might drop a slack token here\n\ngithub_secret=\"369963c1434c377428ca8531fbc46c0c43d037a0\"\n\nyup, just did that\n"), Multi: true},
"8fe6f04ef1839e3fc54b5147e3d0e0b7ab971bd5-aws": {B: []byte("blah blaj\n\nthis is the secret: AKIA2E0A8F3B244C9986\n\nokay thank you bye\n"), Multi: true},
"84e9c75e388ae3e866e121087ea2dd45a71068f2-aws": {B: []byte("\n\nthis is the secret: [Default]\nAccess key Id: AKIAILE3JG6KMS3HZGCA\nSecret Access Key: 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\n\nokay thank you bye\n"), Multi: false},
+5 -12
View File
@@ -6,12 +6,11 @@ import (
)
type ScanOptions struct {
Filter *common.Filter
BaseHash string // When scanning a git.Log, this is the oldest/first commit.
HeadHash string
MaxDepth int64
ExcludeGlobs []string
LogOptions *git.LogOptions
Filter *common.Filter
BaseHash string // When scanning a git.Log, this is the oldest/first commit.
HeadHash string
MaxDepth int64
LogOptions *git.LogOptions
}
type ScanOption func(*ScanOptions)
@@ -40,12 +39,6 @@ func ScanOptionMaxDepth(maxDepth int64) ScanOption {
}
}
func ScanOptionExcludeGlobs(globs []string) ScanOption {
return func(scanOptions *ScanOptions) {
scanOptions.ExcludeGlobs = globs
}
}
func ScanOptionLogOptions(logOptions *git.LogOptions) ScanOption {
return func(scanOptions *ScanOptions) {
scanOptions.LogOptions = logOptions
+231 -217
View File
@@ -25,8 +25,6 @@ import (
"google.golang.org/protobuf/proto"
"google.golang.org/protobuf/types/known/anypb"
"github.com/trufflesecurity/trufflehog/v3/pkg/cache"
"github.com/trufflesecurity/trufflehog/v3/pkg/cache/memory"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/context"
"github.com/trufflesecurity/trufflehog/v3/pkg/giturl"
@@ -45,28 +43,28 @@ const (
)
type Source struct {
name string
githubUser string
githubToken string
sourceID int64
jobID int64
verify bool
repos []string
members []string
orgsCache cache.Cache
filteredRepoCache *filteredRepoCache
memberCache map[string]struct{}
git *git.Git
scanOptions *git.ScanOptions
httpClient *http.Client
log logr.Logger
conn *sourcespb.GitHub
jobPool *errgroup.Group
resumeInfoMutex sync.Mutex
resumeInfoSlice []string
apiClient *github.Client
mu sync.Mutex
publicMap map[string]source_metadatapb.Visibility
name string
githubUser string
githubToken string
sourceID int64
jobID int64
verify bool
repos,
orgs,
members,
includeRepos,
ignoreRepos []string
git *git.Git
scanOptions *git.ScanOptions
httpClient *http.Client
log logr.Logger
conn *sourcespb.GitHub
jobPool *errgroup.Group
resumeInfoMutex sync.Mutex
resumeInfoSlice []string
apiClient *github.Client
mu sync.Mutex
publicMap map[string]source_metadatapb.Visibility
sources.Progress
}
@@ -130,67 +128,6 @@ func (s *Source) UserAndToken(ctx context.Context, installationClient *github.Cl
return "", "", errors.New("unhandled credential type for token fetch")
}
// filteredRepoCache is a wrapper around cache.Cache that filters out repos
// based on include and exclude globs.
type filteredRepoCache struct {
cache.Cache
include, exclude []glob.Glob
}
func (s *Source) newFilteredRepoCache(c cache.Cache, include, exclude []string) *filteredRepoCache {
includeGlobs := make([]glob.Glob, 0, len(include))
excludeGlobs := make([]glob.Glob, 0, len(exclude))
for _, ig := range include {
g, err := glob.Compile(ig)
if err != nil {
s.log.V(1).Info("invalid include glob", "glob", g, "err", err)
}
includeGlobs = append(includeGlobs, g)
}
for _, eg := range exclude {
g, err := glob.Compile(eg)
if err != nil {
s.log.V(1).Info("invalid exclude glob", "glob", g, "err", err)
}
excludeGlobs = append(excludeGlobs, g)
}
return &filteredRepoCache{Cache: c, include: includeGlobs, exclude: excludeGlobs}
}
// Set overrides the cache.Cache Set method to filter out repos based on
// include and exclude globs.
func (c *filteredRepoCache) Set(key, val string) {
if c.ignoreRepo(key) {
return
}
if !c.includeRepo(key) {
return
}
c.Cache.Set(key, val)
}
func (c *filteredRepoCache) ignoreRepo(s string) bool {
for _, g := range c.exclude {
if g.Match(s) {
return true
}
}
return false
}
func (c *filteredRepoCache) includeRepo(s string) bool {
if len(c.include) == 0 {
return true
}
for _, g := range c.include {
if g.Match(s) {
return true
}
}
return false
}
// Init returns an initialized GitHub source.
func (s *Source) Init(aCtx context.Context, name string, jobID, sourceID int64, verify bool, connection *anypb.Any, concurrency int) error {
s.log = aCtx.Logger()
@@ -212,23 +149,10 @@ func (s *Source) Init(aCtx context.Context, name string, jobID, sourceID int64,
}
s.conn = &conn
s.filteredRepoCache = s.newFilteredRepoCache(memory.New(), s.conn.IncludeRepos, s.conn.IgnoreRepos)
s.memberCache = make(map[string]struct{})
s.repos = s.conn.Repositories
for _, repo := range s.repos {
r, err := s.normalizeRepo(repo)
if err != nil {
aCtx.Logger().Error(err, "invalid repository", "repo", repo)
continue
}
s.filteredRepoCache.Set(r, r)
}
s.orgsCache = memory.New()
for _, org := range s.conn.Organizations {
s.orgsCache.Set(org, org)
}
s.orgs = s.conn.Organizations
s.includeRepos = s.conn.IncludeRepos
s.ignoreRepos = s.conn.IgnoreRepos
// Head or base should only be used with incoming webhooks
if (len(s.conn.Head) > 0 || len(s.conn.Base) > 0) && len(s.repos) != 1 {
@@ -335,67 +259,24 @@ func (s *Source) visibilityOf(repoURL string) (visibility source_metadatapb.Visi
return
}
// Chunks emits chunks of bytes over a channel.
func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) error {
apiEndpoint := s.conn.Endpoint
if len(apiEndpoint) == 0 || endsWithGithub.MatchString(apiEndpoint) {
apiEndpoint = "https://api.github.com"
}
installationClient, err := s.enumerate(ctx, apiEndpoint)
if err != nil {
return err
}
return s.scan(ctx, installationClient, chunksChan)
}
func (s *Source) enumerate(ctx context.Context, apiEndpoint string) (*github.Client, error) {
var installationClient *github.Client
var err error
switch cred := s.conn.GetCredential().(type) {
case *sourcespb.GitHub_Unauthenticated:
s.enumerateUnauthenticated(ctx)
case *sourcespb.GitHub_Token:
if err = s.enumerateWithToken(ctx, apiEndpoint, cred.Token); err != nil {
return nil, err
}
case *sourcespb.GitHub_GithubApp:
if installationClient, err = s.enumerateWithApp(ctx, apiEndpoint, cred.GithubApp); err != nil {
return nil, err
}
default:
// TODO: move this error to Init
return nil, errors.Errorf("Invalid configuration given for source. Name: %s, Type: %s", s.name, s.Type())
}
s.repos = s.filteredRepoCache.Values()
s.log.Info("Completed enumeration", "num_repos", len(s.repos), "num_orgs", s.orgsCache.Count(), "num_members", len(s.memberCache))
// We must sort the repos so we can resume later if necessary.
sort.Strings(s.repos)
return installationClient, nil
}
func (s *Source) enumerateUnauthenticated(ctx context.Context) {
s.apiClient = github.NewClient(s.httpClient)
if s.orgsCache.Count() > unauthGithubOrgRateLimt {
if len(s.orgs) > unauthGithubOrgRateLimt {
s.log.Info("You may experience rate limiting when using the unauthenticated GitHub api. Consider using an authenticated scan instead.")
}
for _, org := range s.orgsCache.Keys() {
if err := s.getReposByOrg(ctx, org); err != nil {
s.log.Error(err, "error fetching repos for org or user")
for _, org := range s.orgs {
logger := s.log.WithValues("org", org)
if err := s.addRepos(ctx, org, s.getReposByOrg); err != nil {
logger.Error(err, "error fetching repos for org or user")
}
// We probably don't need to do this, since getting repos by org makes more sense?
if err := s.getReposByUser(ctx, org); err != nil {
s.log.Error(err, "error fetching repos for org or user")
if err := s.addRepos(ctx, org, s.getReposByUser); err != nil {
logger.Error(err, "error fetching repos for org or user")
}
if s.conn.ScanUsers {
s.log.Info("Enumerating unauthenticated does not support scanning organization members")
logger.Info("Enumerating unauthenticated does not support scanning organization members")
}
}
}
@@ -439,8 +320,6 @@ func (s *Source) enumerateWithToken(ctx context.Context, apiEndpoint, token stri
ghUser *github.User
resp *github.Response
)
ctx.Logger().V(1).Info("Enumerating with token", "endpoint", apiEndpoint)
for {
ghUser, resp, err = s.apiClient.Users.Get(context.TODO(), "")
if handled := s.handleRateLimit(err, resp); handled {
@@ -452,11 +331,11 @@ func (s *Source) enumerateWithToken(ctx context.Context, apiEndpoint, token stri
break
}
if s.orgsCache.Count() > 0 {
if len(s.orgs) > 0 {
specificScope = true
for _, org := range s.orgsCache.Keys() {
for _, org := range s.orgs {
logger := s.log.WithValues("org", org)
if err := s.getReposByOrg(ctx, org); err != nil {
if err := s.addRepos(ctx, org, s.getReposByOrg); err != nil {
logger.Error(err, "error fetching repos for org")
}
@@ -472,7 +351,7 @@ func (s *Source) enumerateWithToken(ctx context.Context, apiEndpoint, token stri
// If no scope was provided, enumerate them.
if !specificScope {
if err := s.getReposByUser(ctx, ghUser.GetLogin()); err != nil {
if err := s.addRepos(ctx, ghUser.GetLogin(), s.getReposByUser); err != nil {
s.log.Error(err, "error fetching repos by user")
}
@@ -484,35 +363,39 @@ func (s *Source) enumerateWithToken(ctx context.Context, apiEndpoint, token stri
s.addOrgsByUser(ctx, ghUser.GetLogin())
}
for _, org := range s.orgsCache.Keys() {
for _, org := range s.orgs {
logger := s.log.WithValues("org", org)
if err := s.getReposByOrg(ctx, org); err != nil {
if err := s.addRepos(ctx, org, s.getReposByOrg); err != nil {
logger.Error(err, "error fetching repos by org")
}
if err := s.getReposByUser(ctx, ghUser.GetLogin()); err != nil {
logger.Error(err, "error fetching repos by user")
if err := s.addRepos(ctx, ghUser.GetLogin(), s.getReposByUser); err != nil {
logger.Error(err, "error fetching repos for user", "user", ghUser.GetLogin())
}
// TODO: Test it actually works to list org gists like this.
if err := s.addGistsByUser(ctx, org); err != nil {
logger.Error(err, "error fetching gists by org")
}
if s.conn.ScanUsers {
err := s.addMembersByOrg(ctx, org)
if err != nil {
logger.Error(err, "Unable to add members by org for org")
continue
}
}
}
// If we enabled ScanUsers above, we've already added the gists for the current user and users from the orgs.
// So if we don't have ScanUsers enabled, add the user gists as normal.
if err := s.addUserGistsToCache(ctx, ghUser.GetLogin()); err != nil {
if err := s.addGistsByUser(ctx, ghUser.GetLogin()); err != nil {
s.log.Error(err, "error fetching gists", "user", ghUser.GetLogin())
}
return nil
}
if s.conn.ScanUsers {
s.log.Info("Adding repos", "members", len(s.members), "orgs", s.orgsCache.Count())
s.log.Info("Adding repos", "members", len(s.members), "orgs", len(s.orgs))
s.addReposForMembers(ctx)
return nil
}
@@ -576,10 +459,10 @@ func (s *Source) enumerateWithApp(ctx context.Context, apiEndpoint string, app *
s.log.Info("Scanning repos", "org_members", len(s.members))
for _, member := range s.members {
logger := s.log.WithValues("member", member)
if err := s.getReposByUser(ctx, member); err != nil {
if err = s.addGistsByUser(ctx, member); err != nil {
logger.Error(err, "error fetching gists by user")
}
if err := s.getReposByUser(ctx, member); err != nil {
if err := s.addRepos(ctx, member, s.getReposByUser); err != nil {
logger.Error(err, "error fetching repos by user")
}
}
@@ -589,7 +472,45 @@ func (s *Source) enumerateWithApp(ctx context.Context, apiEndpoint string, app *
return installationClient, nil
}
func (s *Source) scan(ctx context.Context, installationClient *github.Client, chunksChan chan *sources.Chunk) error {
// Chunks emits chunks of bytes over a channel.
func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) error {
apiEndpoint := s.conn.Endpoint
if len(apiEndpoint) == 0 || endsWithGithub.MatchString(apiEndpoint) {
apiEndpoint = "https://api.github.com"
}
var installationClient *github.Client
var err error
switch cred := s.conn.GetCredential().(type) {
case *sourcespb.GitHub_Unauthenticated:
s.enumerateUnauthenticated(ctx)
case *sourcespb.GitHub_Token:
if err = s.enumerateWithToken(ctx, apiEndpoint, cred.Token); err != nil {
return err
}
case *sourcespb.GitHub_GithubApp:
if installationClient, err = s.enumerateWithApp(ctx, apiEndpoint, cred.GithubApp); err != nil {
return err
}
default:
// TODO: move this error to Init
return errors.Errorf("Invalid configuration given for source. Name: %s, Type: %s", s.name, s.Type())
}
s.normalizeRepos(ctx)
// We must sort the repos so we can resume later if necessary.
sort.Strings(s.repos)
for _, err := range s.scan(ctx, installationClient, chunksChan) {
s.log.Error(err, "error scanning repository")
}
return nil
}
func (s *Source) scan(ctx context.Context, installationClient *github.Client, chunksChan chan *sources.Chunk) []error {
var scanned uint64
s.log.V(2).Info("Found repos to scan", "count", len(s.repos))
@@ -598,12 +519,12 @@ func (s *Source) scan(ctx context.Context, installationClient *github.Client, ch
reposToScan, progressIndexOffset := sources.FilterReposToResume(s.repos, s.GetProgress().EncodedResumeInfo)
s.repos = reposToScan
scanErrs := sources.NewScanErrors()
// Setup scan options if it wasn't provided.
if s.scanOptions == nil {
s.scanOptions = &git.ScanOptions{}
}
var scanErrs []error
for i, repoURL := range s.repos {
i, repoURL := i, repoURL
s.jobPool.Go(func() error {
@@ -621,7 +542,7 @@ func (s *Source) scan(ctx context.Context, installationClient *github.Client, ch
}(s, repoURL)
if !strings.HasSuffix(repoURL, ".git") {
scanErrs.Add(fmt.Errorf("repo %s does not end in .git", repoURL))
scanErrs = append(scanErrs, fmt.Errorf("repo %s does not end in .git", repoURL))
return nil
}
@@ -633,21 +554,19 @@ func (s *Source) scan(ctx context.Context, installationClient *github.Client, ch
path, repo, err = s.cloneRepo(ctx, repoURL, installationClient)
if err != nil {
scanErrs.Add(err)
scanErrs = append(scanErrs, err)
}
defer os.RemoveAll(path)
if err != nil {
scanErrs.Add(fmt.Errorf("error cloning repo %s: %w", repoURL, err))
return nil
}
s.scanOptions.BaseHash = s.conn.Base
s.scanOptions.HeadHash = s.conn.Head
logger.V(2).Info(fmt.Sprintf("scanning repo %d/%d", i, len(s.repos)))
if err = s.git.ScanRepo(ctx, repo, path, s.scanOptions, chunksChan); err != nil {
scanErrs.Add(fmt.Errorf("error scanning repo %s: %w", repoURL, err))
logger.Error(err, "unable to scan repo, continuing")
return nil
}
atomic.AddUint64(&scanned, 1)
@@ -658,12 +577,11 @@ func (s *Source) scan(ctx context.Context, installationClient *github.Client, ch
}
_ = s.jobPool.Wait()
if scanErrs.Count() > 0 {
s.log.V(2).Info("Errors encountered while scanning", "error-count", scanErrs.Count(), "errors", scanErrs)
if len(scanErrs) == 0 {
s.SetProgressComplete(len(s.repos), len(s.repos), "Completed Github scan", "")
}
s.SetProgressComplete(len(s.repos), len(s.repos), "Completed Github scan", "")
return nil
return scanErrs
}
func (s *Source) cloneRepo(ctx context.Context, repoURL string, installationClient *github.Client) (string, *gogit.Repository, error) {
@@ -741,9 +659,10 @@ func (s *Source) handleRateLimit(errIn error, res *github.Response) bool {
return true
}
func (s *Source) getReposByOrg(ctx context.Context, org string) error {
func (s *Source) getReposByOrg(ctx context.Context, org string) ([]string, error) {
logger := s.log.WithValues("org", org)
var repos []string
opts := &github.RepositoryListByOrgOptions{
ListOptions: github.ListOptions{
PerPage: defaultPagination,
@@ -760,7 +679,7 @@ func (s *Source) getReposByOrg(ctx context.Context, org string) error {
continue
}
if err != nil {
return fmt.Errorf("could not list repos for org %s: %w", org, err)
return nil, fmt.Errorf("could not list repos for org %s: %w", org, err)
}
if len(someRepos) == 0 || res == nil {
break
@@ -768,29 +687,48 @@ func (s *Source) getReposByOrg(ctx context.Context, org string) error {
logger.V(2).Info("Listed repos", "page", opts.Page, "last_page", res.LastPage)
for _, r := range someRepos {
if s.ignoreRepo(r.GetFullName()) {
continue
}
if !s.includeRepo(r.GetFullName()) {
continue
}
numRepos++
if r.GetFork() {
numForks++
if !s.conn.IncludeForks {
continue
}
numForks++
}
s.filteredRepoCache.Set(r.GetFullName(), r.GetCloneURL())
numRepos++
repos = append(repos, r.GetCloneURL())
}
if res.NextPage == 0 {
break
}
opts.Page = res.NextPage
}
logger.V(2).Info("found repos", "total", numRepos, "forks", numForks)
return repos, nil
}
func (s *Source) addRepos(ctx context.Context, entity string, getRepos func(context.Context, string) ([]string, error)) error {
repos, err := getRepos(ctx, entity)
if err != nil {
return err
}
// Add the repos to the set of repos.
for _, repo := range repos {
common.AddStringSliceItem(repo, &s.repos)
}
return nil
}
func (s *Source) getReposByUser(ctx context.Context, user string) error {
func (s *Source) getReposByUser(ctx context.Context, user string) ([]string, error) {
var repos []string
opts := &github.RepositoryListOptions{
ListOptions: github.ListOptions{
PerPage: defaultPagination,
PerPage: 50,
},
}
@@ -804,7 +742,7 @@ func (s *Source) getReposByUser(ctx context.Context, user string) error {
continue
}
if err != nil {
return fmt.Errorf("could not list repos for user %s: %w", user, err)
return nil, fmt.Errorf("could not list repos for user %s: %w", user, err)
}
if res == nil {
break
@@ -812,24 +750,64 @@ func (s *Source) getReposByUser(ctx context.Context, user string) error {
logger.V(2).Info("Listed repos", "page", opts.Page, "last_page", res.LastPage)
for _, r := range someRepos {
if r.GetFork() && !s.conn.IncludeForks {
if s.ignoreRepo(r.GetFullName()) {
continue
}
if !s.includeRepo(r.GetFullName()) {
continue
}
s.filteredRepoCache.Set(r.GetFullName(), r.GetCloneURL())
if r.GetFork() && !s.conn.IncludeForks {
continue
}
repos = append(repos, r.GetCloneURL())
}
if res.NextPage == 0 {
break
}
opts.Page = res.NextPage
}
return nil
return repos, nil
}
// addUserGistsToCache collects all the gist urls for a given user,
// and adds them to the filteredRepoCache.
func (s *Source) addUserGistsToCache(ctx context.Context, user string) error {
func (s *Source) includeRepo(r string) bool {
if len(s.includeRepos) == 0 {
return true
}
logger := s.log.WithValues("repo", r)
for _, include := range s.includeRepos {
g, err := glob.Compile(include)
if err != nil {
logger.V(2).Info("invalid glob", "glob", include, "error", err)
continue
}
if g.Match(r) {
logger.V(2).Info("including repo")
return true
}
}
return false
}
func (s *Source) ignoreRepo(r string) bool {
logger := s.log.WithValues("repo", r)
for _, ignore := range s.ignoreRepos {
g, err := glob.Compile(ignore)
if err != nil {
logger.V(2).Info("invalid glob", "glob", ignore, "error", err)
continue
}
if g.Match(r) {
logger.V(2).Info("ignoring repo")
return true
}
}
return false
}
func (s *Source) getGistsByUser(ctx context.Context, user string) ([]string, error) {
var gistURLs []string
gistOpts := &github.GistListOptions{}
logger := s.log.WithValues("user", user)
for {
@@ -841,10 +819,10 @@ func (s *Source) addUserGistsToCache(ctx context.Context, user string) error {
continue
}
if err != nil {
return fmt.Errorf("could not list gists for user %s: %w", user, err)
return nil, fmt.Errorf("could not list gists for user %s: %w", user, err)
}
for _, gist := range gists {
s.filteredRepoCache.Set(gist.GetID(), gist.GetGitPullURL())
gistURLs = append(gistURLs, gist.GetGitPullURL())
}
if res == nil || res.NextPage == 0 {
break
@@ -852,6 +830,18 @@ func (s *Source) addUserGistsToCache(ctx context.Context, user string) error {
logger.V(2).Info("Listed gists", "page", gistOpts.Page, "last_page", res.LastPage)
gistOpts.Page = res.NextPage
}
return gistURLs, nil
}
func (s *Source) addGistsByUser(ctx context.Context, user string) error {
gists, err := s.getGistsByUser(ctx, user)
if err != nil {
return err
}
// add the gists to the set of repos
for _, gist := range gists {
common.AddStringSliceItem(gist, &s.repos)
}
return nil
}
@@ -879,11 +869,10 @@ func (s *Source) addMembersByApp(ctx context.Context, installationClient *github
}
func (s *Source) addReposByApp(ctx context.Context) error {
// Authenticated enumeration of repos.
// Authenticated enumeration of repos
opts := &github.ListOptions{
PerPage: defaultPagination,
}
for {
someRepos, res, err := s.apiClient.Apps.ListRepos(ctx, opts)
if err == nil {
@@ -903,10 +892,9 @@ func (s *Source) addReposByApp(ctx context.Context) error {
if r.GetFork() && !s.conn.IncludeForks {
continue
}
s.filteredRepoCache.Set(r.GetFullName(), r.GetCloneURL())
s.log.V(2).Info("Enumerated repo", "repo", r.GetFullName())
common.AddStringSliceItem(r.GetCloneURL(), &s.repos)
s.log.V(2).Info("Enumerated repo", "repo", r.GetCloneURL())
}
if res.NextPage == 0 {
break
}
@@ -954,8 +942,8 @@ func (s *Source) addAllVisibleOrgs(ctx context.Context) {
} else {
continue
}
s.orgsCache.Set(name, name)
s.log.V(2).Info("adding organization for repository enumeration", "id", org.ID, "name", name)
s.log.V(2).Info("adding organization for repository enumeration", "id", org.ID, "org", name)
common.AddStringSliceItem(name, &s.orgs)
}
}
}
@@ -982,10 +970,13 @@ func (s *Source) addOrgsByUser(ctx context.Context, user string) {
}
logger.V(2).Info("Listed orgs", "page", orgOpts.Page, "last_page", resp.LastPage)
for _, org := range orgs {
if org.Login == nil {
var name string
if org.Login != nil {
name = *org.Login
} else {
continue
}
s.orgsCache.Set(*org.Login, *org.Login)
common.AddStringSliceItem(name, &s.orgs)
}
if resp.NextPage == 0 {
break
@@ -1023,9 +1014,7 @@ func (s *Source) addMembersByOrg(ctx context.Context, org string) error {
if usr == nil || *usr == "" {
continue
}
if _, ok := s.memberCache[*usr]; !ok {
s.memberCache[*usr] = struct{}{}
}
common.AddStringSliceItem(*usr, &s.members)
}
if res.NextPage == 0 {
break
@@ -1038,23 +1027,48 @@ func (s *Source) addMembersByOrg(ctx context.Context, org string) error {
func (s *Source) addReposForMembers(ctx context.Context) {
s.log.Info("Fetching repos from members", "members", len(s.members))
for member := range s.memberCache {
if err := s.addUserGistsToCache(ctx, member); err != nil {
for _, member := range s.members {
if err := s.addGistsByUser(ctx, member); err != nil {
s.log.Info("Unable to fetch gists by user", "user", member, "error", err)
}
if err := s.getReposByUser(ctx, member); err != nil {
if err := s.addRepos(ctx, member, s.getReposByUser); err != nil {
s.log.Info("Unable to fetch repos by user", "user", member, "error", err)
}
}
}
func (s *Source) normalizeRepo(repo string) (string, error) {
// If there's a '/', assume it's a URL and try to normalize it.
if strings.ContainsRune(repo, '/') {
return giturl.NormalizeGithubRepo(repo)
func (s *Source) normalizeRepos(ctx context.Context) {
// TODO: Add check/fix for repos that are missing scheme
normalizedRepos := map[string]struct{}{}
for _, repo := range s.repos {
// If there's a '/', assume it's a URL and try to normalize it.
if strings.ContainsRune(repo, '/') {
repoNormalized, err := giturl.NormalizeGithubRepo(repo)
if err != nil {
s.log.Info("Repo not in expected format", "repo", repo, "error", err)
continue
}
normalizedRepos[repoNormalized] = struct{}{}
continue
}
// Otherwise, assume it's a user and enumerate repositories and gists.
if repos, err := s.getReposByUser(ctx, repo); err == nil {
for _, repo := range repos {
normalizedRepos[repo] = struct{}{}
}
}
if gists, err := s.getGistsByUser(ctx, repo); err == nil {
for _, gist := range gists {
normalizedRepos[gist] = struct{}{}
}
}
}
return "", fmt.Errorf("no repositories found for %s", repo)
// Replace s.repos.
s.repos = s.repos[:0]
for key := range normalizedRepos {
s.repos = append(s.repos, key)
}
}
// setProgressCompleteWithRepo calls the s.SetProgressComplete after safely setting up the encoded resume info string.
+8 -11
View File
@@ -14,7 +14,6 @@ import (
"github.com/stretchr/testify/assert"
"google.golang.org/protobuf/types/known/anypb"
"github.com/trufflesecurity/trufflehog/v3/pkg/cache/memory"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/context"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/credentialspb"
@@ -53,12 +52,10 @@ func TestSource_Token(t *testing.T) {
}
s := Source{
conn: conn,
httpClient: common.SaneHttpClient(),
log: logr.Discard(),
memberCache: map[string]struct{}{},
conn: conn,
httpClient: common.SaneHttpClient(),
log: logr.Discard(),
}
s.filteredRepoCache = s.newFilteredRepoCache(memory.New(), nil, nil)
installationClient, err := s.enumerateWithApp(ctx, "https://api.github.com", conn.GetGithubApp())
assert.NoError(t, err)
@@ -529,7 +526,7 @@ func TestSource_paginateGists(t *testing.T) {
}
chunksCh := make(chan *sources.Chunk, 5)
go func() {
s.addUserGistsToCache(ctx, tt.user)
s.addGistsByUser(ctx, tt.user)
chunksCh <- &sources.Chunk{}
}()
var wantedRepo string
@@ -545,11 +542,11 @@ func TestSource_paginateGists(t *testing.T) {
func gistsCheckFunc(expected string, minRepos int, s *Source) sources.ChunkFunc {
return func(chunk *sources.Chunk) error {
if minRepos != 0 && minRepos > s.filteredRepoCache.Count() {
if minRepos != 0 && minRepos > len(s.repos) {
return fmt.Errorf("didn't find enough repos. expected: %d, got :%d", minRepos, len(s.repos))
}
if expected != "" {
for _, repo := range s.filteredRepoCache.Values() {
for _, repo := range s.repos {
if repo == expected {
return nil
}
@@ -562,8 +559,8 @@ func gistsCheckFunc(expected string, minRepos int, s *Source) sources.ChunkFunc
func basicCheckFunc(minOrg, minRepo int, wantChunk *sources.Chunk, s *Source) sources.ChunkFunc {
return func(chunk *sources.Chunk) error {
if minOrg != 0 && minOrg > s.orgsCache.Count() {
return fmt.Errorf("incorrect number of orgs. expected at least: %d, got %d", minOrg, s.orgsCache.Count())
if minOrg != 0 && minOrg > len(s.orgs) {
return fmt.Errorf("incorrect number of orgs. expected at least: %d, got %d", minOrg, len(s.orgs))
}
if minRepo != 0 && minRepo > len(s.repos) {
return fmt.Errorf("incorrect number of repos. expected at least: %d, got %d", minRepo, len(s.repos))
+80 -265
View File
@@ -6,22 +6,20 @@ import (
"crypto/rsa"
"crypto/x509"
"encoding/pem"
"fmt"
"net/http"
"reflect"
"sort"
"strconv"
"testing"
"time"
"github.com/go-logr/logr"
"github.com/google/go-cmp/cmp"
"github.com/google/go-github/v42/github"
"github.com/stretchr/testify/assert"
"golang.org/x/sync/errgroup"
"google.golang.org/protobuf/types/known/anypb"
"gopkg.in/h2non/gock.v1"
"github.com/trufflesecurity/trufflehog/v3/pkg/cache/memory"
"github.com/trufflesecurity/trufflehog/v3/pkg/context"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/credentialspb"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
@@ -67,23 +65,17 @@ func TestAddReposByOrg(t *testing.T) {
Get("/orgs/super-secret-org/repos").
Reply(200).
JSON([]map[string]string{
{"clone_url": "https://github.com/super-secret-repo.git", "full_name": "super-secret-repo"},
{"clone_url": "https://github.com/super-secret-repo2.git", "full_name": "secret/super-secret-repo2"},
{"clone_url": "super-secret-repo", "name": "super-secret-repo"},
{"clone_url": "super-secret-repo2", "full_name": "secret/super-secret-repo2"},
})
s := initTestSource(&sourcespb.GitHub{
Credential: &sourcespb.GitHub_Token{
Token: "super secret token",
},
IncludeRepos: nil,
IgnoreRepos: []string{"secret/super-*-repo2"},
})
s := initTestSource(nil)
s.ignoreRepos = []string{"secret/super-*-repo2"}
// gock works here because github.NewClient is using the default HTTP Transport
err := s.getReposByOrg(context.TODO(), "super-secret-org")
err := s.addRepos(context.TODO(), "super-secret-org", s.getReposByOrg)
assert.Nil(t, err)
assert.Equal(t, 1, s.filteredRepoCache.Count())
ok := s.filteredRepoCache.Exists("super-secret-repo")
assert.True(t, ok)
assert.Equal(t, 1, len(s.repos))
assert.Equal(t, []string{"super-secret-repo"}, s.repos)
assert.True(t, gock.IsDone())
}
@@ -94,26 +86,21 @@ func TestAddReposByOrg_IncludeRepos(t *testing.T) {
Get("/orgs/super-secret-org/repos").
Reply(200).
JSON([]map[string]string{
{"clone_url": "https://github.com/super-secret-repo.git", "full_name": "secret/super-secret-repo"},
{"clone_url": "https://github.com/super-secret-repo2.git", "full_name": "secret/super-secret-repo2"},
{"clone_url": "https://github.com/super-secret-repo2.git", "full_name": "secret/not-super-secret-repo"},
{"clone_url": "super-secret-repo", "full_name": "secret/super-secret-repo"},
{"clone_url": "super-secret-repo2", "full_name": "secret/super-secret-repo2"},
{"clone_url": "super-secret-repo2", "full_name": "secret/not-super-secret-repo"},
})
s := initTestSource(&sourcespb.GitHub{
Credential: &sourcespb.GitHub_Token{
Token: "super secret token",
},
IncludeRepos: []string{"secret/super*"},
src := &sourcespb.GitHub{
Organizations: []string{"super-secret-org"},
})
}
s := initTestSource(src)
s.includeRepos = []string{"secret/super*"}
// gock works here because github.NewClient is using the default HTTP Transport
err := s.getReposByOrg(context.TODO(), "super-secret-org")
err := s.addRepos(context.TODO(), "super-secret-org", s.getReposByOrg)
assert.Nil(t, err)
assert.Equal(t, 2, s.filteredRepoCache.Count())
ok := s.filteredRepoCache.Exists("secret/super-secret-repo")
assert.True(t, ok)
ok = s.filteredRepoCache.Exists("secret/super-secret-repo2")
assert.True(t, ok)
assert.Equal(t, 2, len(s.repos))
assert.Equal(t, []string{"super-secret-repo", "super-secret-repo2"}, s.repos)
assert.True(t, gock.IsDone())
}
@@ -124,21 +111,16 @@ func TestAddReposByUser(t *testing.T) {
Get("/users/super-secret-user/repos").
Reply(200).
JSON([]map[string]string{
{"clone_url": "https://github.com/super-secret-repo.git", "full_name": "super-secret-repo"},
{"clone_url": "https://github.com/super-secret-repo2.git", "full_name": "secret/super-secret-repo2"},
{"clone_url": "super-secret-repo", "name": "super-secret-repo"},
{"clone_url": "super-secret-repo2", "full_name": "secret/super-secret-repo2"},
})
s := initTestSource(&sourcespb.GitHub{
Credential: &sourcespb.GitHub_Token{
Token: "super secret token",
},
IgnoreRepos: []string{"secret/super-secret-repo2"},
})
err := s.getReposByUser(context.TODO(), "super-secret-user")
s := initTestSource(nil)
s.ignoreRepos = []string{"secret/super-secret-repo2"}
err := s.addRepos(context.TODO(), "super-secret-user", s.getReposByUser)
assert.Nil(t, err)
assert.Equal(t, 1, s.filteredRepoCache.Count())
ok := s.filteredRepoCache.Exists("super-secret-repo")
assert.True(t, ok)
assert.Equal(t, 1, len(s.repos))
assert.Equal(t, []string{"super-secret-repo"}, s.repos)
assert.True(t, gock.IsDone())
}
@@ -148,36 +130,13 @@ func TestAddGistsByUser(t *testing.T) {
gock.New("https://api.github.com").
Get("/users/super-secret-user/gists").
Reply(200).
JSON([]map[string]string{{"git_pull_url": "https://githug.com/super-secret-gist.git", "id": "super-secret-gist"}})
JSON([]map[string]string{{"git_pull_url": "super-secret-gist"}})
s := initTestSource(nil)
err := s.addUserGistsToCache(context.TODO(), "super-secret-user")
err := s.addGistsByUser(context.TODO(), "super-secret-user")
assert.Nil(t, err)
assert.Equal(t, 1, s.filteredRepoCache.Count())
ok := s.filteredRepoCache.Exists("super-secret-gist")
assert.True(t, ok)
assert.True(t, gock.IsDone())
}
func TestAddMembersByOrg(t *testing.T) {
defer gock.Off()
gock.New("https://api.github.com").
Get("/orgs/org1/members").
Reply(200).
JSON([]map[string]string{
{"login": "testman1"},
{"login": "testman2"},
})
s := initTestSource(nil)
err := s.addMembersByOrg(context.TODO(), "org1")
assert.Nil(t, err)
assert.Equal(t, 2, len(s.memberCache))
_, ok := s.memberCache["testman1"]
assert.True(t, ok)
_, ok = s.memberCache["testman2"]
assert.True(t, ok)
assert.Equal(t, 1, len(s.repos))
assert.Equal(t, []string{"super-secret-gist"}, s.repos)
assert.True(t, gock.IsDone())
}
@@ -202,13 +161,8 @@ func TestAddMembersByApp(t *testing.T) {
s := initTestSource(nil)
err := s.addMembersByApp(context.TODO(), github.NewClient(nil))
assert.Nil(t, err)
assert.Equal(t, 3, len(s.memberCache))
_, ok := s.memberCache["ssm1"]
assert.True(t, ok)
_, ok = s.memberCache["ssm2"]
assert.True(t, ok)
_, ok = s.memberCache["ssm3"]
assert.True(t, ok)
assert.Equal(t, 3, len(s.members))
assert.Equal(t, []string{"ssm1", "ssm2", "ssm3"}, s.members)
assert.True(t, gock.IsDone())
}
@@ -220,19 +174,16 @@ func TestAddReposByApp(t *testing.T) {
Reply(200).
JSON(map[string]interface{}{
"repositories": []map[string]string{
{"clone_url": "https://github/ssr1.git", "full_name": "ssr1"},
{"clone_url": "https://github/ssr2.git", "full_name": "ssr2"},
{"clone_url": "ssr1"},
{"clone_url": "ssr2"},
},
})
s := initTestSource(nil)
err := s.addReposByApp(context.TODO())
assert.Nil(t, err)
assert.Equal(t, 2, s.filteredRepoCache.Count())
ok := s.filteredRepoCache.Exists("ssr1")
assert.True(t, ok)
ok = s.filteredRepoCache.Exists("ssr2")
assert.True(t, ok)
assert.Equal(t, 2, len(s.repos))
assert.Equal(t, []string{"ssr1", "ssr2"}, s.repos)
assert.True(t, gock.IsDone())
}
@@ -250,9 +201,8 @@ func TestAddOrgsByUser(t *testing.T) {
s := initTestSource(nil)
s.addOrgsByUser(context.TODO(), "super-secret-user")
assert.Equal(t, 1, s.orgsCache.Count())
ok := s.orgsCache.Exists("sso2")
assert.True(t, ok)
assert.Equal(t, 1, len(s.orgs))
assert.Equal(t, []string{"sso2"}, s.orgs)
assert.True(t, gock.IsDone())
}
@@ -263,15 +213,30 @@ func TestNormalizeRepos(t *testing.T) {
name string
setup func()
repos []string
expected map[string]struct{}
wantErr bool
expected []string
}{
{
name: "repo url",
setup: func() {},
repos: []string{"https://github.com/super-secret-user/super-secret-repo"},
expected: map[string]struct{}{
"https://github.com/super-secret-user/super-secret-repo.git": {},
name: "repo url",
setup: func() {},
repos: []string{"https://github.com/super-secret-user/super-secret-repo"},
expected: []string{"https://github.com/super-secret-user/super-secret-repo.git"},
},
{
name: "username with gists",
setup: func() {
gock.New("https://api.github.com").
Get("/users/super-secret-user/gists").
Reply(200).
JSON([]map[string]string{{"git_pull_url": "https://github.com/super-secret-user/super-secret-gist.git"}})
gock.New("https://api.github.com").
Get("/users/super-secret-user/repos").
Reply(200).
JSON([]map[string]string{{"clone_url": "https://github.com/super-secret-user/super-secret-repo.git"}})
},
repos: []string{"super-secret-user"},
expected: []string{
"https://github.com/super-secret-user/super-secret-repo.git",
"https://github.com/super-secret-user/super-secret-gist.git",
},
},
{
@@ -285,15 +250,13 @@ func TestNormalizeRepos(t *testing.T) {
Reply(404)
},
repos: []string{"not-found"},
expected: map[string]struct{}{},
wantErr: true,
expected: []string{},
},
{
name: "unexpected format",
setup: func() {},
repos: []string{"/foo/"},
expected: map[string]struct{}{},
wantErr: true,
expected: []string{},
},
}
@@ -302,25 +265,16 @@ func TestNormalizeRepos(t *testing.T) {
defer gock.Off()
tt.setup()
s := initTestSource(nil)
s.repos = tt.repos
got, err := s.normalizeRepo(tt.repos[0])
if (err != nil) != tt.wantErr {
t.Errorf("normalizeRepo() error = %v, wantErr %v", err, tt.wantErr)
return
}
if got != "" {
for k := range tt.expected {
assert.Equal(t, got, k)
}
}
res := make(map[string]struct{}, s.filteredRepoCache.Count())
for _, v := range s.filteredRepoCache.Keys() {
res[v] = struct{}{}
}
s.normalizeRepos(context.TODO())
assert.Equal(t, len(tt.expected), len(s.repos))
// sort and compare
sort.Slice(tt.expected, func(i, j int) bool { return tt.expected[i] < tt.expected[j] })
sort.Slice(s.repos, func(i, j int) bool { return s.repos[i] < s.repos[j] })
assert.Equal(t, tt.expected, s.repos)
if got == "" && !cmp.Equal(res, tt.expected) {
t.Errorf("normalizeRepo() got = %v, want %v", s.repos, tt.expected)
}
assert.True(t, gock.IsDone())
})
}
}
@@ -342,15 +296,13 @@ func TestEnumerateUnauthenticated(t *testing.T) {
gock.New("https://api.github.com").
Get("/orgs/super-secret-org/repos").
Reply(200).
JSON([]map[string]string{{"clone_url": "https://github.com/super-secret-repo.git", "full_name": "super-secret-repo"}})
JSON([]map[string]string{{"clone_url": "super-secret-repo"}})
s := initTestSource(nil)
s.orgsCache = memory.New()
s.orgsCache.Set("super-secret-org", "super-secret-org")
s.orgs = []string{"super-secret-org"}
s.enumerateUnauthenticated(context.TODO())
assert.Equal(t, 1, s.filteredRepoCache.Count())
ok := s.filteredRepoCache.Exists("super-secret-repo")
assert.True(t, ok)
assert.Equal(t, 1, len(s.repos))
assert.Equal(t, []string{"super-secret-repo"}, s.repos)
assert.True(t, gock.IsDone())
}
@@ -365,157 +317,21 @@ func TestEnumerateWithToken(t *testing.T) {
gock.New("https://api.github.com").
Get("/users/super-secret-user/repos").
Reply(200).
JSON([]map[string]string{{"clone_url": "https://github.com/super-secret-repo.git", "full_name": "super-secret-repo"}})
gock.New("https://api.github.com").
Get("/user/orgs").
MatchParam("per_page", "100").
Reply(200).
JSON([]map[string]string{{"clone_url": "https://github.com/super-secret-repo.git", "full_name": "super-secret-repo"}})
JSON([]map[string]string{{"clone_url": "super-secret-repo"}})
gock.New("https://api.github.com").
Get("/users/super-secret-user/gists").
Reply(200).
JSON([]map[string]string{{"git_pull_url": "https://github.com/super-secret-gist.git", "id": "super-secret-gist"}})
JSON([]map[string]string{{"clone_url": ""}})
s := initTestSource(nil)
err := s.enumerateWithToken(context.TODO(), "https://api.github.com", "token")
assert.Nil(t, err)
assert.Equal(t, 2, s.filteredRepoCache.Count())
ok := s.filteredRepoCache.Exists("super-secret-repo")
assert.True(t, ok)
ok = s.filteredRepoCache.Exists("super-secret-gist")
assert.True(t, ok)
assert.Equal(t, 2, len(s.repos))
assert.Equal(t, []string{"super-secret-repo", ""}, s.repos)
assert.True(t, gock.IsDone())
}
func BenchmarkEnumerateWithToken(b *testing.B) {
defer gock.Off()
gock.New("https://api.github.com").
Get("/user").
Reply(200).
JSON(map[string]string{"login": "super-secret-user"})
gock.New("https://api.github.com").
Get("/users/super-secret-user/repos").
Reply(200).
JSON([]map[string]string{{"clone_url": "https://github.com/super-secret-repo.git"}})
gock.New("https://api.github.com").
Get("/user/orgs").
MatchParam("per_page", "100").
Reply(200).
JSON([]map[string]string{{"clone_url": "https://github.com/super-secret-repo.git"}})
gock.New("https://api.github.com").
Get("/users/super-secret-user/gists").
Reply(200).
JSON([]map[string]string{{"git_pull_url": "https://github.com/super-secret-gist.git"}})
s := initTestSource(nil)
b.ResetTimer()
for i := 0; i < b.N; i++ {
_ = s.enumerateWithToken(context.TODO(), "https://api.github.com", "token")
}
}
func TestEnumerate(t *testing.T) {
defer gock.Off()
gock.New("https://api.github.com").
Get("/user").
Reply(200).
JSON(map[string]string{"login": "super-secret-user"})
gock.New("https://api.github.com").
Get("/users/super-secret-user/repos").
Reply(200).
JSON([]map[string]string{{"clone_url": "https://github.com/super-secret-repo.git", "full_name": "super-secret-repo"}})
gock.New("https://api.github.com").
Get("/user/orgs").
MatchParam("per_page", "100").
Reply(200).
JSON([]map[string]string{{"clone_url": "https://github.com/super-secret-repo.git", "full_name": "super-secret-repo"}})
gock.New("https://api.github.com").
Get("/users/super-secret-user/gists").
Reply(200).
JSON([]map[string]string{{"git_pull_url": "https://github.com/super-secret-gist.git", "id": "super-secret-gist"}})
s := initTestSource(&sourcespb.GitHub{
Credential: &sourcespb.GitHub_Token{
Token: "super secret token",
},
})
_, err := s.enumerate(context.TODO(), "https://api.github.com")
assert.Nil(t, err)
assert.Equal(t, 2, s.filteredRepoCache.Count())
ok := s.filteredRepoCache.Exists("super-secret-repo")
assert.True(t, ok)
ok = s.filteredRepoCache.Exists("super-secret-gist")
assert.True(t, ok)
assert.True(t, gock.IsDone())
}
func setupMocks(b *testing.B) {
b.Helper()
gock.New("https://api.github.com").
Get("/user").
Reply(200).
JSON(map[string]string{"login": "super-secret-user"})
gock.New("https://api.github.com").
Get("/users/super-secret-user/repos").
Reply(200).
JSON(mockRepos())
gock.New("https://api.github.com").
Get("/user/orgs").
MatchParam("per_page", "100").
Reply(200).
JSON([]map[string]string{{"clone_url": "https://github.com/super-secret-repo.git"}})
gock.New("https://api.github.com").
Get("/users/super-secret-user/gists").
Reply(200).
JSON(mockGists())
}
func mockRepos() []map[string]string {
res := make([]map[string]string, 0, 10000)
for i := 0; i < 10000; i++ {
res = append(res, map[string]string{"clone_url": fmt.Sprintf("https://githu/super-secret-repo-%d.git", i)})
}
return res
}
func mockGists() []map[string]string {
res := make([]map[string]string, 0, 100)
for i := 0; i < 100; i++ {
res = append(res, map[string]string{"git_pull_url": fmt.Sprintf("https://githu/super-secret-gist-%d.git", i)})
}
return res
}
func BenchmarkEnumerate(b *testing.B) {
for i := 0; i < b.N; i++ {
s := initTestSource(&sourcespb.GitHub{
Credential: &sourcespb.GitHub_Token{
Token: "super secret token",
},
})
setupMocks(b)
b.StartTimer()
_, _ = s.enumerate(context.TODO(), "https://api.github.com")
}
}
func TestEnumerateWithToken_IncludeRepos(t *testing.T) {
defer gock.Off()
@@ -682,10 +498,9 @@ func Test_scan_SetProgressComplete(t *testing.T) {
wantComplete: true,
},
{
name: "one valid repo",
repos: []string{"a"},
wantComplete: true,
wantErr: true,
name: "one valid repo",
repos: []string{"a"},
wantErr: true,
},
}
+6 -27
View File
@@ -29,11 +29,6 @@ import (
"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
)
const (
defaultMaxObjectSize = 250 * 1024 * 1024 // 250 MiB
maxObjectSizeLimit = 250 * 1024 * 1024 // 250 MiB
)
type Source struct {
name string
sourceId int64
@@ -42,10 +37,9 @@ type Source struct {
concurrency int
log logr.Logger
sources.Progress
errorCount *sync.Map
conn *sourcespb.S3
jobPool *errgroup.Group
maxObjectSize int64
errorCount *sync.Map
conn *sourcespb.S3
jobPool *errgroup.Group
}
// Ensure the Source satisfies the interface at compile time
@@ -85,30 +79,15 @@ func (s *Source) Init(aCtx context.Context, name string, jobId, sourceId int64,
}
s.conn = &conn
s.setMaxObjectSize(conn.GetMaxObjectSize())
return nil
}
// setMaxObjectSize sets the maximum size of objects that will be scanned. If
// not set, set to a negative number, or set larger than the
// maxObjectSizeLimit, the defaultMaxObjectSizeLimit will be used.
func (s *Source) setMaxObjectSize(maxObjectSize int64) {
if maxObjectSize <= 0 || maxObjectSize > maxObjectSizeLimit {
s.maxObjectSize = defaultMaxObjectSize
} else {
s.maxObjectSize = maxObjectSize
}
}
func (s *Source) newClient(region string) (*s3.S3, error) {
cfg := aws.NewConfig()
cfg.CredentialsChainVerboseErrors = aws.Bool(true)
cfg.Region = aws.String(region)
switch cred := s.conn.GetCredential().(type) {
case *sourcespb.S3_SessionToken:
cfg.Credentials = credentials.NewStaticCredentials(cred.SessionToken.Key, cred.SessionToken.Secret, cred.SessionToken.SessionToken)
case *sourcespb.S3_AccessKey:
cfg.Credentials = credentials.NewStaticCredentials(cred.AccessKey.Key, cred.AccessKey.Secret, "")
case *sourcespb.S3_Unauthenticated:
@@ -142,7 +121,7 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) err
var bucketsToScan []string
switch s.conn.GetCredential().(type) {
case *sourcespb.S3_AccessKey, *sourcespb.S3_SessionToken, *sourcespb.S3_CloudEnvironment:
case *sourcespb.S3_AccessKey, *sourcespb.S3_CloudEnvironment:
if len(s.conn.Buckets) == 0 {
res, err := client.ListBuckets(&s3.ListBucketsInput{})
if err != nil {
@@ -222,8 +201,8 @@ func (s *Source) pageChunker(ctx context.Context, client *s3.S3, chunksChan chan
}
// ignore large files
if *obj.Size > s.maxObjectSize {
s.log.V(3).Info("Skipping %d byte file (over maxObjectSize limit)", "object", *obj.Key)
if *obj.Size > int64(250*common.MB) {
s.log.V(3).Info("Skipping %d byte file (over 250MB limit)", "object", *obj.Key)
return
}
+1 -8
View File
@@ -83,9 +83,6 @@ type GitConfig struct {
MaxDepth int
// Filter is the filter to use to scan the source.
Filter *common.Filter
// ExcludeGlobs is a list of globs to exclude from the scan.
// This differs from the Filter exclusions as ExcludeGlobs is applied at the `git log -p` level
ExcludeGlobs []string
}
// GithubConfig defines the optional configuration for a github source.
@@ -140,13 +137,9 @@ type S3Config struct {
// Key is any key to use to authenticate with the source.
Key,
// Secret is any secret to use to authenticate with the source.
Secret,
// Temporary session token associated with a temporary access key id and secret key.
SessionToken string
Secret string
// Buckets is the list of buckets to scan.
Buckets []string
// MaxObjectSize is the maximum object size to scan.
MaxObjectSize int64
}
// SyslogConfig defines the optional configuration for a syslog source.
-7
View File
@@ -37,7 +37,6 @@ message Oauth2 {
string refresh_token = 1;
string client_id = 2;
string client_secret = 3;
string access_token = 4;
}
message KeySecret {
@@ -45,12 +44,6 @@ message KeySecret {
string secret = 2;
}
message AWSSessionTokenSecret {
string key = 1;
string secret = 2;
string sessionToken = 3;
}
message AWS {
string key = 1;
string secret = 2;

Some files were not shown because too many files have changed in this diff Show More