Compare commits
54 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| ba75163d88 | |||
| 1edefe4f73 | |||
| 6affc903e1 | |||
| 5afc7a32ca | |||
| b66bd9544c | |||
| 62b2195502 | |||
| 1a1b2ca51a | |||
| 590115bca4 | |||
| 9b01c3d71c | |||
| 16f885b2fc | |||
| 7ac4be481c | |||
| f2a528e32b | |||
| b9ae7014b0 | |||
| efa404942a | |||
| a8c89c59b9 | |||
| 47d5ddebf2 | |||
| 63eaccd208 | |||
| f9ea22f72b | |||
| d2676618c0 | |||
| 22876f8381 | |||
| dbcb888063 | |||
| bbc3be3b6c | |||
| b4329e0825 | |||
| 1155ee2736 | |||
| 31d17c4f93 | |||
| 72b6a9ec6b | |||
| be4d0bcb41 | |||
| 91340c1e00 | |||
| de540652cb | |||
| 90a07f0352 | |||
| 72b3fa31a3 | |||
| 3f84a6700e | |||
| fdeccf06a0 | |||
| 70cdff915b | |||
| 2c32b14df1 | |||
| e53a72abd2 | |||
| 62ce9bac8b | |||
| e68ace5338 | |||
| 3aa5369608 | |||
| 8d66fde6de | |||
| 2a9f34962d | |||
| f6512ac4ca | |||
| bf581ae9f9 | |||
| d3e7c5acc2 | |||
| abb131e502 | |||
| b28a2b9177 | |||
| afe708519b | |||
| c9e6086644 | |||
| 000065b225 | |||
| 0a949d7131 | |||
| 2e4b17d3f4 | |||
| 78bbb89a30 | |||
| 6695cf1dce | |||
| 4aa91337e5 |
@@ -0,0 +1,46 @@
|
||||
name: detector test aggregation
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
schedule:
|
||||
- cron: "0 8 * * *"
|
||||
|
||||
jobs:
|
||||
test-detectors:
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
actions: "read"
|
||||
contents: "read"
|
||||
id-token: "write"
|
||||
steps:
|
||||
- name: Install Go
|
||||
uses: actions/setup-go@v4
|
||||
with:
|
||||
go-version: "1.21"
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v3
|
||||
- id: "auth"
|
||||
uses: "google-github-actions/auth@v1"
|
||||
with:
|
||||
workload_identity_provider: "projects/811013774421/locations/global/workloadIdentityPools/github-pool/providers/github-provider"
|
||||
service_account: "github-ci-external@trufflehog-testing.iam.gserviceaccount.com"
|
||||
- name: Test
|
||||
run: make test-integration
|
||||
- name: Set up gotestsum
|
||||
run: |
|
||||
go install gotest.tools/gotestsum@latest
|
||||
mkdir -p tmp/test-results
|
||||
- name: Test
|
||||
run: |
|
||||
CGO_ENABLED=1 gotestsum --junitfile tmp/test-results/test.xml --raw-command -- go test -json -tags=detectors -timeout=15m $(go list ./... | grep pkg/detectors)
|
||||
- name: Upload test results to BuildPulse for flaky test detection
|
||||
if: ${{ !cancelled() }} # Run this step even when the tests fail. Skip if the workflow is cancelled.
|
||||
uses: buildpulse/buildpulse-action@main
|
||||
with:
|
||||
account: 79229934
|
||||
repository: 694446374
|
||||
path: |
|
||||
tmp/test-results/*.xml
|
||||
key: ${{ secrets.BUILDPULSE_DETECTORS_ACCESS_KEY_ID }}
|
||||
secret: ${{ secrets.BUILDPULSE_DETECTORS_SECRET_ACCESS_KEY }}
|
||||
tags: detectors
|
||||
+48
-49
@@ -1,4 +1,3 @@
|
||||
|
||||
name: Test
|
||||
|
||||
on:
|
||||
@@ -14,57 +13,57 @@ jobs:
|
||||
if: ${{ ! github.event.pull_request.head.repo.fork }}
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
actions: 'read'
|
||||
contents: 'read'
|
||||
id-token: 'write'
|
||||
actions: "read"
|
||||
contents: "read"
|
||||
id-token: "write"
|
||||
steps:
|
||||
- name: Install Go
|
||||
uses: actions/setup-go@v4
|
||||
with:
|
||||
go-version: '1.21'
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v3
|
||||
- id: 'auth'
|
||||
uses: 'google-github-actions/auth@v1'
|
||||
with:
|
||||
workload_identity_provider: 'projects/811013774421/locations/global/workloadIdentityPools/github-pool/providers/github-provider'
|
||||
service_account: 'github-ci-external@trufflehog-testing.iam.gserviceaccount.com'
|
||||
- name: Test
|
||||
run: make test-integration
|
||||
test-detectors:
|
||||
if: ${{ ! github.event.pull_request.head.repo.fork }}
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
actions: 'read'
|
||||
contents: 'read'
|
||||
id-token: 'write'
|
||||
steps:
|
||||
- name: Install Go
|
||||
uses: actions/setup-go@v4
|
||||
with:
|
||||
go-version: '1.21'
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v3
|
||||
- id: 'auth'
|
||||
uses: 'google-github-actions/auth@v1'
|
||||
with:
|
||||
workload_identity_provider: 'projects/811013774421/locations/global/workloadIdentityPools/github-pool/providers/github-provider'
|
||||
service_account: 'github-ci-external@trufflehog-testing.iam.gserviceaccount.com'
|
||||
- name: Test
|
||||
run: make test-detectors
|
||||
continue-on-error: true
|
||||
- name: Install Go
|
||||
uses: actions/setup-go@v4
|
||||
with:
|
||||
go-version: "1.21"
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v3
|
||||
- id: "auth"
|
||||
uses: "google-github-actions/auth@v1"
|
||||
with:
|
||||
workload_identity_provider: "projects/811013774421/locations/global/workloadIdentityPools/github-pool/providers/github-provider"
|
||||
service_account: "github-ci-external@trufflehog-testing.iam.gserviceaccount.com"
|
||||
- name: Set up gotestsum
|
||||
run: |
|
||||
go install gotest.tools/gotestsum@latest
|
||||
mkdir -p tmp/test-results
|
||||
- name: Test
|
||||
run: |
|
||||
CGO_ENABLED=1 gotestsum --junitfile tmp/test-results/test.xml --raw-command -- go test -json -tags=sources $(go list ./... | grep -v /vendor/ | grep -v pkg/detectors)
|
||||
if: ${{ success() || failure() }} # always run this step, even if there were previous errors
|
||||
- name: Upload test results to BuildPulse for flaky test detection
|
||||
if: ${{ !cancelled() }} # Run this step even when the tests fail. Skip if the workflow is cancelled.
|
||||
uses: buildpulse/buildpulse-action@main
|
||||
with:
|
||||
account: 79229934
|
||||
repository: 77726177
|
||||
path: |
|
||||
tmp/test-results/*.xml
|
||||
key: ${{ secrets.BUILDPULSE_ACCESS_KEY_ID }}
|
||||
secret: ${{ secrets.BUILDPULSE_SECRET_ACCESS_KEY }}
|
||||
tags: integration
|
||||
- name: Annotate test results
|
||||
uses: mikepenz/action-junit-report@v3
|
||||
if: success() || failure() # always run even if the previous step fails
|
||||
with:
|
||||
report_paths: "tmp/test-results/*.xml"
|
||||
test-community:
|
||||
if: ${{ github.event.pull_request.head.repo.fork }}
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
actions: 'read'
|
||||
contents: 'read'
|
||||
actions: "read"
|
||||
contents: "read"
|
||||
steps:
|
||||
- name: Install Go
|
||||
uses: actions/setup-go@v4
|
||||
with:
|
||||
go-version: '1.21'
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v3
|
||||
- name: Test
|
||||
run: make test-community
|
||||
- name: Install Go
|
||||
uses: actions/setup-go@v4
|
||||
with:
|
||||
go-version: "1.21"
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v3
|
||||
- name: Test
|
||||
run: make test-community
|
||||
|
||||
@@ -110,37 +110,61 @@ Expected output:
|
||||
...
|
||||
```
|
||||
|
||||
## 4: Scan an S3 bucket for verified keys
|
||||
## 4: Scan a GitHub Repo's Issues and PR Comments
|
||||
|
||||
Command:
|
||||
|
||||
```bash
|
||||
trufflehog github --repo https://github.com/trufflesecurity/test_keys --only-verified --issue-comments --pr-comments
|
||||
```
|
||||
|
||||
Expected output:
|
||||
```
|
||||
...
|
||||
2023-09-22T18:22:10Z info-0 trufflehog scanning github pull request comments {"repository": "https://github.com/trufflesecurity/test_keys.git"}
|
||||
Found verified result 🐷🔑
|
||||
Detector Type: AWS
|
||||
Decoder Type: PLAIN
|
||||
Raw result: AKIAQYLPMN5HLU6OP3FX
|
||||
Line: 1
|
||||
Link: https://github.com/trufflesecurity/test_keys/issues/3#issuecomment-1731842331
|
||||
Repository: test_keys
|
||||
Timestamp: 2023-09-22 18:06:26 +0000 UTC
|
||||
...
|
||||
```
|
||||
|
||||
|
||||
## 5: Scan an S3 bucket for verified keys
|
||||
|
||||
```bash
|
||||
trufflehog s3 --bucket=<bucket name> --only-verified
|
||||
```
|
||||
|
||||
## 5: Scan S3 buckets using IAM Roles
|
||||
## 6: Scan S3 buckets using IAM Roles
|
||||
|
||||
```bash
|
||||
trufflehog s3 --role-arn=<iam role arn>
|
||||
```
|
||||
|
||||
## 6: Scan a Github Repo using SSH authentication in docker
|
||||
## 7: Scan a Github Repo using SSH authentication in docker
|
||||
|
||||
```bash
|
||||
docker run --rm -v "$HOME/.ssh:/root/.ssh:ro" trufflesecurity/trufflehog:latest git ssh://github.com/trufflesecurity/test_keys
|
||||
```
|
||||
|
||||
## 7: Scan individual files or directories
|
||||
## 8: Scan individual files or directories
|
||||
|
||||
```bash
|
||||
trufflehog filesystem path/to/file1.txt path/to/file2.txt path/to/dir
|
||||
```
|
||||
|
||||
## 8: Scan GCS buckets for verified secrets.
|
||||
## 9: Scan GCS buckets for verified secrets.
|
||||
|
||||
```bash
|
||||
trufflehog gcs --project-id=<project-ID> --cloud-environment --only-verified
|
||||
```
|
||||
|
||||
## 9: Scan a Docker image for verified secrets.
|
||||
## 10: Scan a Docker image for verified secrets.
|
||||
|
||||
Use the `--image` flag multiple times to scan multiple images.
|
||||
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
module github.com/trufflesecurity/trufflehog/v3
|
||||
|
||||
go 1.20
|
||||
go 1.21
|
||||
|
||||
replace github.com/jpillora/overseer => github.com/trufflesecurity/overseer v1.1.7-custom5
|
||||
|
||||
@@ -9,8 +9,9 @@ replace github.com/couchbase/gocbcore/v10 => github.com/couchbase/gocbcore/v10 v
|
||||
|
||||
require (
|
||||
cloud.google.com/go/secretmanager v1.11.1
|
||||
cloud.google.com/go/storage v1.31.0
|
||||
cloud.google.com/go/storage v1.33.0
|
||||
github.com/Azure/go-autorest/autorest/azure/auth v0.5.11
|
||||
github.com/AzureAD/microsoft-authentication-library-for-go v0.5.1
|
||||
github.com/BobuSumisu/aho-corasick v1.0.3
|
||||
github.com/TheZeroSlave/zapsentry v1.17.0
|
||||
github.com/aws/aws-sdk-go v1.44.83
|
||||
@@ -20,7 +21,7 @@ require (
|
||||
github.com/bitfinexcom/bitfinex-api-go v0.0.0-20210608095005-9e0b26f200fb
|
||||
github.com/bradleyfalzon/ghinstallation/v2 v2.6.0
|
||||
github.com/charmbracelet/bubbles v0.16.1
|
||||
github.com/charmbracelet/bubbletea v0.24.1
|
||||
github.com/charmbracelet/bubbletea v0.24.2
|
||||
github.com/charmbracelet/glamour v0.6.0
|
||||
github.com/charmbracelet/lipgloss v0.7.1
|
||||
github.com/couchbase/gocb/v2 v2.6.3
|
||||
@@ -29,10 +30,10 @@ require (
|
||||
github.com/envoyproxy/protoc-gen-validate v1.0.2
|
||||
github.com/fatih/color v1.15.0
|
||||
github.com/felixge/fgprof v0.9.3
|
||||
github.com/getsentry/sentry-go v0.22.0
|
||||
github.com/getsentry/sentry-go v0.24.1
|
||||
github.com/go-errors/errors v1.4.2
|
||||
github.com/go-git/go-git/v5 v5.8.1
|
||||
github.com/go-ldap/ldap/v3 v3.4.5
|
||||
github.com/go-ldap/ldap/v3 v3.4.6
|
||||
github.com/go-logr/logr v1.2.4
|
||||
github.com/go-logr/zapr v1.2.4
|
||||
github.com/go-redis/redis v6.15.9+incompatible
|
||||
@@ -50,9 +51,11 @@ require (
|
||||
github.com/joho/godotenv v1.5.1
|
||||
github.com/jpillora/overseer v1.1.6
|
||||
github.com/kylelemons/godebug v1.1.0
|
||||
github.com/launchdarkly/go-server-sdk/v6 v6.1.0
|
||||
github.com/lib/pq v1.10.9
|
||||
github.com/lrstanley/bubblezone v0.0.0-20221222153816-e95291e2243e
|
||||
github.com/mattn/go-isatty v0.0.18
|
||||
github.com/marusama/semaphore/v2 v2.5.0
|
||||
github.com/mattn/go-isatty v0.0.19
|
||||
github.com/mattn/go-sqlite3 v1.14.17
|
||||
github.com/mholt/archiver/v4 v4.0.0-alpha.8
|
||||
github.com/muesli/reflow v0.3.0
|
||||
@@ -69,12 +72,12 @@ require (
|
||||
go.mongodb.org/mongo-driver v1.12.0
|
||||
go.uber.org/mock v0.2.0
|
||||
go.uber.org/zap v1.24.0
|
||||
golang.org/x/crypto v0.12.0
|
||||
golang.org/x/crypto v0.13.0
|
||||
golang.org/x/exp v0.0.0-20230206171751-46f607a40771
|
||||
golang.org/x/oauth2 v0.10.0
|
||||
golang.org/x/oauth2 v0.12.0
|
||||
golang.org/x/sync v0.3.0
|
||||
golang.org/x/text v0.12.0
|
||||
google.golang.org/api v0.131.0
|
||||
golang.org/x/text v0.13.0
|
||||
google.golang.org/api v0.132.0
|
||||
google.golang.org/protobuf v1.31.0
|
||||
gopkg.in/alecthomas/kingpin.v2 v2.2.6
|
||||
gopkg.in/h2non/gock.v1 v1.1.2
|
||||
@@ -82,7 +85,7 @@ require (
|
||||
)
|
||||
|
||||
require (
|
||||
cloud.google.com/go v0.110.2 // indirect
|
||||
cloud.google.com/go v0.110.4 // indirect
|
||||
cloud.google.com/go/compute v1.20.1 // indirect
|
||||
cloud.google.com/go/compute/metadata v0.2.3 // indirect
|
||||
cloud.google.com/go/iam v1.1.0 // indirect
|
||||
@@ -150,7 +153,7 @@ require (
|
||||
github.com/emirpasic/gods v1.18.1 // indirect
|
||||
github.com/form3tech-oss/jwt-go v3.2.5+incompatible // indirect
|
||||
github.com/gabriel-vasile/mimetype v1.4.2 // indirect
|
||||
github.com/go-asn1-ber/asn1-ber v1.5.4 // indirect
|
||||
github.com/go-asn1-ber/asn1-ber v1.5.5 // indirect
|
||||
github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect
|
||||
github.com/go-git/go-billy/v5 v5.4.1 // indirect
|
||||
github.com/go-ole/go-ole v1.2.6 // indirect
|
||||
@@ -167,11 +170,11 @@ require (
|
||||
github.com/google/go-querystring v1.1.0 // indirect
|
||||
github.com/google/pprof v0.0.0-20211214055906-6f57359322fd // indirect
|
||||
github.com/google/s2a-go v0.1.4 // indirect
|
||||
github.com/google/uuid v1.3.0 // indirect
|
||||
github.com/google/uuid v1.3.1 // indirect
|
||||
github.com/googleapis/enterprise-certificate-proxy v0.2.5 // indirect
|
||||
github.com/gorilla/css v1.0.0 // indirect
|
||||
github.com/gsterjov/go-libsecret v0.0.0-20161001094733-a6f4afe4910c // indirect
|
||||
github.com/gregjones/httpcache v0.0.0-20171119193500-2bcd89a1743f // indirect
|
||||
github.com/gsterjov/go-libsecret v0.0.0-20161001094733-a6f4afe4910c // indirect
|
||||
github.com/h2non/parth v0.0.0-20190131123155-b4df798d6542 // indirect
|
||||
github.com/hashicorp/errwrap v1.0.0 // indirect
|
||||
github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
|
||||
@@ -192,7 +195,6 @@ require (
|
||||
github.com/launchdarkly/go-sdk-events/v2 v2.0.1 // indirect
|
||||
github.com/launchdarkly/go-semver v1.0.2 // indirect
|
||||
github.com/launchdarkly/go-server-sdk-evaluation/v2 v2.0.2 // indirect
|
||||
github.com/launchdarkly/go-server-sdk/v6 v6.1.0 // indirect
|
||||
github.com/lucasb-eyer/go-colorful v1.2.0 // indirect
|
||||
github.com/mailru/easyjson v0.7.7 // indirect
|
||||
github.com/mattn/go-colorable v0.1.13 // indirect
|
||||
@@ -203,7 +205,7 @@ require (
|
||||
github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8 // indirect
|
||||
github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3 // indirect
|
||||
github.com/mitchellh/go-homedir v1.1.0 // indirect
|
||||
github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe // indirect
|
||||
github.com/montanaflynn/stats v0.6.6 // indirect
|
||||
github.com/mtibben/percent v0.2.1 // indirect
|
||||
github.com/muesli/ansi v0.0.0-20211031195517-c9f0611b6c70 // indirect
|
||||
github.com/muesli/cancelreader v0.2.2 // indirect
|
||||
@@ -244,16 +246,16 @@ require (
|
||||
go.uber.org/multierr v1.6.0 // indirect
|
||||
go4.org v0.0.0-20200411211856-f5505b9728dd // indirect
|
||||
golang.org/x/mod v0.12.0 // indirect
|
||||
golang.org/x/net v0.14.0 // indirect
|
||||
golang.org/x/sys v0.11.0 // indirect
|
||||
golang.org/x/term v0.11.0 // indirect
|
||||
golang.org/x/net v0.15.0 // indirect
|
||||
golang.org/x/sys v0.12.0 // indirect
|
||||
golang.org/x/term v0.12.0 // indirect
|
||||
golang.org/x/time v0.3.0 // indirect
|
||||
golang.org/x/tools v0.12.0 // indirect
|
||||
golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect
|
||||
google.golang.org/appengine v1.6.7 // indirect
|
||||
google.golang.org/genproto v0.0.0-20230629202037-9506855d4529 // indirect
|
||||
google.golang.org/genproto/googleapis/api v0.0.0-20230629202037-9506855d4529 // indirect
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20230706204954-ccb25ca9f130 // indirect
|
||||
google.golang.org/genproto v0.0.0-20230706204954-ccb25ca9f130 // indirect
|
||||
google.golang.org/genproto/googleapis/api v0.0.0-20230706204954-ccb25ca9f130 // indirect
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20230711160842-782d3b101e98 // indirect
|
||||
google.golang.org/grpc v1.56.2 // indirect
|
||||
gopkg.in/warnings.v0 v0.1.2 // indirect
|
||||
gopkg.in/yaml.v2 v2.4.0 // indirect
|
||||
|
||||
@@ -7,8 +7,8 @@ cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTj
|
||||
cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0=
|
||||
cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To=
|
||||
cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M=
|
||||
cloud.google.com/go v0.110.2 h1:sdFPBr6xG9/wkBbfhmUz/JmZC7X6LavQgcrVINrKiVA=
|
||||
cloud.google.com/go v0.110.2/go.mod h1:k04UEeEtb6ZBRTv3dZz4CeJC3jKGxyhl0sAiVVquxiw=
|
||||
cloud.google.com/go v0.110.4 h1:1JYyxKMN9hd5dR2MYTPWkGUgcoxVVhg0LKNKEo0qvmk=
|
||||
cloud.google.com/go v0.110.4/go.mod h1:+EYjdK8e5RME/VY/qLCAtuyALQ9q67dvuum8i+H5xsI=
|
||||
cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o=
|
||||
cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE=
|
||||
cloud.google.com/go/compute v1.20.1 h1:6aKEtlUiwEpJzM001l0yFkpXmUVXaN8W+fbkb2AZNbg=
|
||||
@@ -25,8 +25,8 @@ cloud.google.com/go/secretmanager v1.11.1 h1:cLTCwAjFh9fKvU6F13Y4L9vPcx9yiWPyWXE
|
||||
cloud.google.com/go/secretmanager v1.11.1/go.mod h1:znq9JlXgTNdBeQk9TBW/FnR/W4uChEKGeqQWAJ8SXFw=
|
||||
cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw=
|
||||
cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos=
|
||||
cloud.google.com/go/storage v1.31.0 h1:+S3LjjEN2zZ+L5hOwj4+1OkGCsLVe0NzpXKQ1pSdTCI=
|
||||
cloud.google.com/go/storage v1.31.0/go.mod h1:81ams1PrhW16L4kF7qg+4mTq7SRs5HsbDTM0bWvrwJ0=
|
||||
cloud.google.com/go/storage v1.33.0 h1:PVrDOkIC8qQVa1P3SXGpQvfuJhN2LHOoyZvWs8D2X5M=
|
||||
cloud.google.com/go/storage v1.33.0/go.mod h1:Hhh/dogNRGca7IWv1RC2YqEn0c0G77ctA/OxflYkiD8=
|
||||
dario.cat/mergo v1.0.0 h1:AGCNq9Evsj31mOgNPcLyXc+4PNABt905YmuqPYYpBWk=
|
||||
dario.cat/mergo v1.0.0/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk=
|
||||
dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
|
||||
@@ -39,6 +39,7 @@ github.com/Azure/azure-sdk-for-go/sdk/azcore v1.4.0 h1:rTnT/Jrcm+figWlYz4Ixzt0SJ
|
||||
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.4.0/go.mod h1:ON4tFdPTwRcgWEaVDrN3584Ef+b7GgSJaXxe5fW9t4M=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/azidentity v0.11.0/go.mod h1:HcM1YX14R7CJcghJGOYCgdezslRSVzqwLf/q+4Y2r/0=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.1.0 h1:QkAcEIAKbNL4KoFr4SathZPhDhF4mVwpBMFlYjyAqy8=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.1.0/go.mod h1:bhXu1AjYL+wutSL/kpSq6s7733q2Rb0yuot9Zgfqa/0=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/internal v0.7.0/go.mod h1:yqy467j36fJxcRV2TzfVZ1pCb5vxm4BtZPUdYWe/Xo8=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/internal v1.1.2 h1:+5VZ72z0Qan5Bog5C+ZkgSqUbeVUd9wgtHOrIKuc5b8=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/internal v1.1.2/go.mod h1:eWRD7oawr1Mu1sLCawqVc0CUiF43ia3qQMxLscsKQ9w=
|
||||
@@ -65,6 +66,7 @@ github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBp
|
||||
github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 h1:mFRzDkZVAjdal+s7s0MwaRv9igoPqLRdzOLzw/8Xvq8=
|
||||
github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358/go.mod h1:chxPXzSsl7ZWRAuOIE23GDNzjWuZquvFlgA8xmpunjU=
|
||||
github.com/AzureAD/microsoft-authentication-library-for-go v0.5.1 h1:BWe8a+f/t+7KY7zH2mqygeUD0t8hNFXe08p1Pb3/jKE=
|
||||
github.com/AzureAD/microsoft-authentication-library-for-go v0.5.1/go.mod h1:Vt9sXTKwMyGcOxSmLDMnGPgqsUg7m8pe215qMLrDXw4=
|
||||
github.com/BobuSumisu/aho-corasick v1.0.3 h1:uuf+JHwU9CHP2Vx+wAy6jcksJThhJS9ehR8a+4nPE9g=
|
||||
github.com/BobuSumisu/aho-corasick v1.0.3/go.mod h1:hm4jLcvZKI2vRF2WDU1N4p/jpWtpOzp3nLmi9AzX/XE=
|
||||
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
|
||||
@@ -93,12 +95,14 @@ github.com/alexbrainman/sspi v0.0.0-20210105120005-909beea2cc74/go.mod h1:cEWa1L
|
||||
github.com/andybalholm/brotli v1.0.5 h1:8uQZIdzKmjc/iuPu7O2ioW48L81FgatrcpfFmiq/cCs=
|
||||
github.com/andybalholm/brotli v1.0.5/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig=
|
||||
github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be h1:9AeTilPcZAjCFIImctFaOjnTIavg87rW78vTPkQqLI8=
|
||||
github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be/go.mod h1:ySMOLuWl6zY27l47sB3qLNK6tF2fkHG55UZxx8oIVo4=
|
||||
github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY=
|
||||
github.com/apache/arrow/go/v12 v12.0.1 h1:JsR2+hzYYjgSUkBSaahpqCetqZMr76djX80fF/DiJbg=
|
||||
github.com/apache/arrow/go/v12 v12.0.1/go.mod h1:weuTY7JvTG/HDPtMQxEUp7pU73vkLWMLpY67QwZ/WWw=
|
||||
github.com/apache/thrift v0.16.0 h1:qEy6UW60iVOlUy+b9ZR0d5WzUWYGOo4HfopoyBaNmoY=
|
||||
github.com/apache/thrift v0.16.0/go.mod h1:PHK3hniurgQaNMZYaCLEqXKsYK8upmhPbmdP2FXSqgU=
|
||||
github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio=
|
||||
github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs=
|
||||
github.com/atotto/clipboard v0.1.4 h1:EH0zSVneZPSuFR11BlR9YppQTVDbh5+16AmcJi4g1z4=
|
||||
github.com/atotto/clipboard v0.1.4/go.mod h1:ZY9tmq7sm5xIbd9bOK4onWV4S6X0u6GY7Vn0Yu86PYI=
|
||||
github.com/aws/aws-sdk-go v1.44.83 h1:7+Rtc2Eio6EKUNoZeMV/IVxzVrY5oBQcNPtCcgIHYJA=
|
||||
@@ -174,8 +178,8 @@ github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj
|
||||
github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
|
||||
github.com/charmbracelet/bubbles v0.16.1 h1:6uzpAAaT9ZqKssntbvZMlksWHruQLNxg49H5WdeuYSY=
|
||||
github.com/charmbracelet/bubbles v0.16.1/go.mod h1:2QCp9LFlEsBQMvIYERr7Ww2H2bA7xen1idUDIzm/+Xc=
|
||||
github.com/charmbracelet/bubbletea v0.24.1 h1:LpdYfnu+Qc6XtvMz6d/6rRY71yttHTP5HtrjMgWvixc=
|
||||
github.com/charmbracelet/bubbletea v0.24.1/go.mod h1:rK3g/2+T8vOSEkNHvtq40umJpeVYDn6bLaqbgzhL/hg=
|
||||
github.com/charmbracelet/bubbletea v0.24.2 h1:uaQIKx9Ai6Gdh5zpTbGiWpytMU+CfsPp06RaW2cx/SY=
|
||||
github.com/charmbracelet/bubbletea v0.24.2/go.mod h1:XdrNrV4J8GiyshTtx3DNuYkR1FDaJmO3l2nejekbsgg=
|
||||
github.com/charmbracelet/glamour v0.6.0 h1:wi8fse3Y7nfcabbbDuwolqTqMQPMnVPeZhDM273bISc=
|
||||
github.com/charmbracelet/glamour v0.6.0/go.mod h1:taqWV4swIMMbWALc0m7AfE9JkPSU8om2538k9ITBxOc=
|
||||
github.com/charmbracelet/lipgloss v0.7.1 h1:17WMwi7N1b1rVWOjMT+rCh7sQkvDU75B2hbZpc5Kc1E=
|
||||
@@ -237,6 +241,7 @@ github.com/dsnet/golib v0.0.0-20171103203638-1ea166775780/go.mod h1:Lj+Z9rebOhdf
|
||||
github.com/dvsekhvalnov/jose2go v1.5.0 h1:3j8ya4Z4kMCwT5nXIKFSV84YS+HdqSSO0VsTQxaLAeM=
|
||||
github.com/dvsekhvalnov/jose2go v1.5.0/go.mod h1:QsHjhyTlD/lAVqn/NSbVZmSCGeDehTB/mPZadG+mhXU=
|
||||
github.com/elazarl/goproxy v0.0.0-20221015165544-a0805db90819 h1:RIB4cRk+lBqKK3Oy0r2gRX4ui7tuhiZq2SuTtTCi0/0=
|
||||
github.com/elazarl/goproxy v0.0.0-20221015165544-a0805db90819/go.mod h1:Ro8st/ElPeALwNFlcTpWmkr6IoMFfkjXAvTHpevnDsM=
|
||||
github.com/emirpasic/gods v1.18.1 h1:FXtiHYKDGKCW2KzwZKx0iC0PQmdlorYgdFG9jPXJ1Bc=
|
||||
github.com/emirpasic/gods v1.18.1/go.mod h1:8tpGGwCnJ5H4r6BWwaV6OrWmMoPhUl5jm/FMNAnJvWQ=
|
||||
github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
|
||||
@@ -258,12 +263,13 @@ github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWo
|
||||
github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
|
||||
github.com/gabriel-vasile/mimetype v1.4.2 h1:w5qFW6JKBz9Y393Y4q372O9A7cUSequkh1Q7OhCmWKU=
|
||||
github.com/gabriel-vasile/mimetype v1.4.2/go.mod h1:zApsH/mKG4w07erKIaJPFiX0Tsq9BFQgN3qGY5GnNgA=
|
||||
github.com/getsentry/sentry-go v0.22.0 h1:XNX9zKbv7baSEI65l+H1GEJgSeIC1c7EN5kluWaP6dM=
|
||||
github.com/getsentry/sentry-go v0.22.0/go.mod h1:lc76E2QywIyW8WuBnwl8Lc4bkmQH4+w1gwTf25trprY=
|
||||
github.com/getsentry/sentry-go v0.24.1 h1:W6/0GyTy8J6ge6lVCc94WB6Gx2ZuLrgopnn9w8Hiwuk=
|
||||
github.com/getsentry/sentry-go v0.24.1/go.mod h1:lc76E2QywIyW8WuBnwl8Lc4bkmQH4+w1gwTf25trprY=
|
||||
github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
|
||||
github.com/gliderlabs/ssh v0.3.5 h1:OcaySEmAQJgyYcArR+gGGTHCyE7nvhEMTlYY+Dp8CpY=
|
||||
github.com/go-asn1-ber/asn1-ber v1.5.4 h1:vXT6d/FNDiELJnLb6hGNa309LMsrCoYFvpwHDF0+Y1A=
|
||||
github.com/go-asn1-ber/asn1-ber v1.5.4/go.mod h1:hEBeB/ic+5LoWskz+yKT7vGhhPYkProFKoKdwZRWMe0=
|
||||
github.com/gliderlabs/ssh v0.3.5/go.mod h1:8XB4KraRrX39qHhT6yxPsHedjA08I/uBVwj4xC+/+z4=
|
||||
github.com/go-asn1-ber/asn1-ber v1.5.5 h1:MNHlNMBDgEKD4TcKr36vQN68BA00aDfjIt3/bD50WnA=
|
||||
github.com/go-asn1-ber/asn1-ber v1.5.5/go.mod h1:hEBeB/ic+5LoWskz+yKT7vGhhPYkProFKoKdwZRWMe0=
|
||||
github.com/go-errors/errors v1.4.2 h1:J6MZopCL4uSllY1OfXM374weqZFFItUbrImctkmUxIA=
|
||||
github.com/go-errors/errors v1.4.2/go.mod h1:sIVyrIiJhuEF+Pj9Ebtd6P/rEYROXFi3BopGUQ5a5Og=
|
||||
github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 h1:+zs/tPmkDkHx3U66DAb0lQFJrpS6731Oaa12ikc+DiI=
|
||||
@@ -271,12 +277,13 @@ github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376/go.mod h1:an3vInlBmS
|
||||
github.com/go-git/go-billy/v5 v5.4.1 h1:Uwp5tDRkPr+l/TnbHOQzp+tmJfLceOlbVucgpTz8ix4=
|
||||
github.com/go-git/go-billy/v5 v5.4.1/go.mod h1:vjbugF6Fz7JIflbVpl1hJsGjSHNltrSw45YK/ukIvQg=
|
||||
github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20230305113008-0c11038e723f h1:Pz0DHeFij3XFhoBRGUDPzSJ+w2UcK5/0JvF8DRI58r8=
|
||||
github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20230305113008-0c11038e723f/go.mod h1:8LHG1a3SRW71ettAD/jW13h8c6AqjVSeL11RAdgaqpo=
|
||||
github.com/go-git/go-git/v5 v5.8.1 h1:Zo79E4p7TRk0xoRgMq0RShiTHGKcKI4+DI6BfJc/Q+A=
|
||||
github.com/go-git/go-git/v5 v5.8.1/go.mod h1:FHFuoD6yGz5OSKEBK+aWN9Oah0q54Jxl0abmj6GnqAo=
|
||||
github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
|
||||
github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
|
||||
github.com/go-ldap/ldap/v3 v3.4.5 h1:ekEKmaDrpvR2yf5Nc/DClsGG9lAmdDixe44mLzlW5r8=
|
||||
github.com/go-ldap/ldap/v3 v3.4.5/go.mod h1:bMGIq3AGbytbaMwf8wdv5Phdxz0FWHTIYMSzyrYgnQs=
|
||||
github.com/go-ldap/ldap/v3 v3.4.6 h1:ert95MdbiG7aWo/oPYp9btL3KJlMPKnP58r09rI8T+A=
|
||||
github.com/go-ldap/ldap/v3 v3.4.6/go.mod h1:IGMQANNtxpsOzj7uUAMjpGBaOVTC4DYyIy8VsTdxmtc=
|
||||
github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ=
|
||||
github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
|
||||
github.com/go-logr/zapr v1.2.4 h1:QHVo+6stLbfJmYGkQ7uGHUCu5hnAFAj6mDe6Ea0SeOo=
|
||||
@@ -297,6 +304,7 @@ github.com/goccy/go-json v0.10.0 h1:mXKd9Qw4NuzShiRlOXKews24ufknHO7gx30lsDyokKA=
|
||||
github.com/goccy/go-json v0.10.0/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I=
|
||||
github.com/godbus/dbus v0.0.0-20190726142602-4481cbc300e2 h1:ZpnhV/YsD2/4cESfV5+Hoeu/iUR3ruzNvZ+yQfO03a0=
|
||||
github.com/godbus/dbus v0.0.0-20190726142602-4481cbc300e2/go.mod h1:bBOAhwG1umN6/6ZUMtDFBMQR8jRg9O75tm9K00oMsK4=
|
||||
github.com/golang-jwt/jwt v3.2.1+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I=
|
||||
github.com/golang-jwt/jwt v3.2.2+incompatible h1:IfV12K8xAKAnZqdXVzCZ+TOjboZ2keLg81eXfW3O+oY=
|
||||
github.com/golang-jwt/jwt v3.2.2+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I=
|
||||
github.com/golang-jwt/jwt/v4 v4.0.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg=
|
||||
@@ -366,6 +374,7 @@ github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17
|
||||
github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPgecwXBIDzw5no=
|
||||
github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
|
||||
github.com/google/martian/v3 v3.3.2 h1:IqNFLAmvJOgVlpdEBiQbDc2EwKW77amAycfTuWKdfvw=
|
||||
github.com/google/martian/v3 v3.3.2/go.mod h1:oBOf6HBosgwRXnUGWUB05QECsc6uvmMiJ3+6W4l/CUk=
|
||||
github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
|
||||
github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
|
||||
github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
|
||||
@@ -374,9 +383,11 @@ github.com/google/pprof v0.0.0-20211214055906-6f57359322fd/go.mod h1:KgnwoLYCZ8I
|
||||
github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
|
||||
github.com/google/s2a-go v0.1.4 h1:1kZ/sQM3srePvKs3tXAvQzo66XfcReoqFpIpIccE7Oc=
|
||||
github.com/google/s2a-go v0.1.4/go.mod h1:Ej+mSEMGRnqRzjc7VtF+jdBwYG5fuJfiZ8ELkjEwM0A=
|
||||
github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
||||
github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
||||
github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I=
|
||||
github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
||||
github.com/google/uuid v1.3.1 h1:KjJaJ9iWZ3jOFZIf1Lqf4laDRCasjl0BCmnEGxkdLb4=
|
||||
github.com/google/uuid v1.3.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
||||
github.com/googleapis/enterprise-certificate-proxy v0.2.5 h1:UR4rDjcgpgEnqpIEvkiqTYKBCKLNmlge2eVjoZfySzM=
|
||||
github.com/googleapis/enterprise-certificate-proxy v0.2.5/go.mod h1:RxW0N9901Cko1VOCW3SXCpWP+mlIEkk2tP7jnHy9a3w=
|
||||
github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
|
||||
@@ -427,6 +438,7 @@ github.com/jpillora/s3 v1.1.4 h1:YCCKDWzb/Ye9EBNd83ATRF/8wPEy0xd43Rezb6u6fzc=
|
||||
github.com/jpillora/s3 v1.1.4/go.mod h1:yedE603V+crlFi1Kl/5vZJaBu9pUzE9wvKegU/lF2zs=
|
||||
github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
|
||||
github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk=
|
||||
github.com/karlseguin/expect v1.0.2-0.20190806010014-778a5f0c6003 h1:vJ0Snvo+SLMY72r5J4sEfkuE7AFbixEP2qRbEcum/wA=
|
||||
github.com/karlseguin/expect v1.0.2-0.20190806010014-778a5f0c6003/go.mod h1:zNBxMY8P21owkeogJELCLeHIt+voOSduHYTFUbwRAV8=
|
||||
github.com/kevinburke/ssh_config v1.2.0 h1:x584FjTGwHzMwvHx18PXxbBVzfnxogHaAReU4gf13a4=
|
||||
github.com/kevinburke/ssh_config v1.2.0/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM=
|
||||
@@ -445,6 +457,7 @@ github.com/klauspost/pgzip v1.2.5/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQ
|
||||
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
|
||||
github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
|
||||
github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
|
||||
github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
|
||||
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
|
||||
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
|
||||
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
|
||||
@@ -467,7 +480,10 @@ github.com/launchdarkly/go-server-sdk-evaluation/v2 v2.0.2 h1:PAM0GvE0nIUBeOkjdi
|
||||
github.com/launchdarkly/go-server-sdk-evaluation/v2 v2.0.2/go.mod h1:Mztipcz+7ZMatXVun3k/IfPa8IOgUnAqiZawtFh2MRg=
|
||||
github.com/launchdarkly/go-server-sdk/v6 v6.1.0 h1:pp/VBIon5JNbtSw7ih7I5MXN9mXHfo6T5xikKVy52dI=
|
||||
github.com/launchdarkly/go-server-sdk/v6 v6.1.0/go.mod h1:3TfS6vNsNksPT4LGeGuGKtT3zTjTbbOsCUK3nrj1neA=
|
||||
github.com/launchdarkly/go-test-helpers/v2 v2.2.0 h1:L3kGILP/6ewikhzhdNkHy1b5y4zs50LueWenVF0sBbs=
|
||||
github.com/launchdarkly/go-test-helpers/v2 v2.2.0/go.mod h1:L7+th5govYp5oKU9iN7To5PgznBuIjBPn+ejqKR0avw=
|
||||
github.com/launchdarkly/go-test-helpers/v3 v3.0.2 h1:rh0085g1rVJM5qIukdaQ8z1XTWZztbJ49vRZuveqiuU=
|
||||
github.com/launchdarkly/go-test-helpers/v3 v3.0.2/go.mod h1:u2ZvJlc/DDJTFrshWW50tWMZHLVYXofuSHUfTU/eIwM=
|
||||
github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw=
|
||||
github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
|
||||
github.com/lrstanley/bubblezone v0.0.0-20221222153816-e95291e2243e h1:XoxHx8K6ZKoMtjzWOMDuM69LCdjDDsTOtTfWGrT/fns=
|
||||
@@ -476,13 +492,15 @@ github.com/lucasb-eyer/go-colorful v1.2.0 h1:1nnpGOrhyZZuNyfu1QjKiUICQ74+3FNCN69
|
||||
github.com/lucasb-eyer/go-colorful v1.2.0/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i7ruzyGqttikkLy0=
|
||||
github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0=
|
||||
github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
|
||||
github.com/marusama/semaphore/v2 v2.5.0 h1:o/1QJD9DBYOWRnDhPwDVAXQn6mQYD0gZaS1Tpx6DJGM=
|
||||
github.com/marusama/semaphore/v2 v2.5.0/go.mod h1:z9nMiNUekt/LTpTUQdpp+4sJeYqUGpwMHfW0Z8V8fnQ=
|
||||
github.com/matryer/is v1.2.0 h1:92UTHpy8CDwaJ08GqLDzhhuixiBUUD1p3AU6PHddz4A=
|
||||
github.com/matryer/is v1.2.0/go.mod h1:2fLPjFQM9rhQ15aVEtbuwhJinnOqrmgXPNdZsdwlWXA=
|
||||
github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=
|
||||
github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg=
|
||||
github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
|
||||
github.com/mattn/go-isatty v0.0.18 h1:DOKFKCQ7FNG2L1rbrmstDN4QVRdS89Nkh85u68Uwp98=
|
||||
github.com/mattn/go-isatty v0.0.18/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
|
||||
github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APPA=
|
||||
github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
|
||||
github.com/mattn/go-localereader v0.0.1 h1:ygSAOl7ZXTx4RdPYinUpg6W99U8jWvWi9Ye2JC/oIi4=
|
||||
github.com/mattn/go-localereader v0.0.1/go.mod h1:8fBrzywKY7BI3czFoHkuzRoWE9C+EiG4R1k4Cjx5p88=
|
||||
github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI=
|
||||
@@ -505,8 +523,9 @@ github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3/go.mod h1:RagcQ7I8Ie
|
||||
github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y=
|
||||
github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
|
||||
github.com/modocache/gover v0.0.0-20171022184752-b58185e213c5/go.mod h1:caMODM3PzxT8aQXRPkAt8xlV/e7d7w8GM5g0fa5F0D8=
|
||||
github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe h1:iruDEfMl2E6fbMZ9s0scYfZQ84/6SPL6zC8ACM2oIL0=
|
||||
github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc=
|
||||
github.com/montanaflynn/stats v0.6.6 h1:Duep6KMIDpY4Yo11iFsvyqJDyfzLF9+sndUKT+v64GQ=
|
||||
github.com/montanaflynn/stats v0.6.6/go.mod h1:etXPPgVO6n31NxCd9KQUMvCM+ve0ruNzt6R8Bnaayow=
|
||||
github.com/mtibben/percent v0.2.1 h1:5gssi8Nqo8QU/r2pynCm+hBQHpkB/uNK7BJCFogWdzs=
|
||||
github.com/mtibben/percent v0.2.1/go.mod h1:KG9uO+SZkUp+VkRHsCdYQV3XSZrrSpR3O9ibNBTZrns=
|
||||
github.com/muesli/ansi v0.0.0-20211031195517-c9f0611b6c70 h1:kMlmsLSbjkikxQJ1IPwaM+7LJ9ltFu/fi8CRzvSnQmA=
|
||||
@@ -548,9 +567,11 @@ github.com/paulbellamy/ratecounter v0.2.0/go.mod h1:Hfx1hDpSGoqxkVVpBi/IlYD7kChl
|
||||
github.com/pierrec/lz4/v4 v4.1.17 h1:kV4Ip+/hUBC+8T6+2EgburRtkE9ef4nbY3f4dFhGjMc=
|
||||
github.com/pierrec/lz4/v4 v4.1.17/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
|
||||
github.com/pingcap/errors v0.11.4 h1:lFuQV/oaUMGcD2tqt+01ROSmJs75VG1ToEOkZIZ4nE4=
|
||||
github.com/pingcap/errors v0.11.4/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8=
|
||||
github.com/pjbgf/sha1cd v0.3.0 h1:4D5XXmUUBUl/xQ6IjCkEAbqXskkq/4O7LmGn0AqMDs4=
|
||||
github.com/pjbgf/sha1cd v0.3.0/go.mod h1:nZ1rrWOcGJ5uZgEEVL1VUM9iRQiZvWdbZjkKyFzPPsI=
|
||||
github.com/pkg/browser v0.0.0-20180916011732-0a3d74bf9ce4/go.mod h1:4OwLy04Bl9Ef3GJJCoec+30X3LQs/0/m4HFRt/2LUSA=
|
||||
github.com/pkg/browser v0.0.0-20210115035449-ce105d075bb4/go.mod h1:N6UoU20jOqggOuDwUaBQpluzLNDqif3kq9z2wpdYEfQ=
|
||||
github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 h1:KoWmjvw+nsYOo29YJK9vDA65RGE3NrOnUtO7a+RF9HU=
|
||||
github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8/go.mod h1:HKlIX3XHQyzLZPlr7++PzdhaXEj94dEiJgZDTsxEqUI=
|
||||
github.com/pkg/diff v0.0.0-20200914180035-5b29258ca4f7 h1:+/+DxvQaYifJ+grD4klzrS5y+KJXldn/2YTl5JG+vZ8=
|
||||
@@ -626,6 +647,7 @@ github.com/ulikunitz/xz v0.5.10/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0o
|
||||
github.com/urfave/cli v1.22.12/go.mod h1:sSBEIC79qR6OvcmsD4U3KABeOTxDqQtdDnaFuUN30b8=
|
||||
github.com/vbatts/tar-split v0.11.3 h1:hLFqsOLQ1SsppQNTMpkpPXClLDfC2A3Zgy9OUU+RVck=
|
||||
github.com/vbatts/tar-split v0.11.3/go.mod h1:9QlHN18E+fEH7RdG+QAJJcuya3rqT7eXSTY7wGrAokY=
|
||||
github.com/wsxiaoys/terminal v0.0.0-20160513160801-0940f3fc43a0 h1:3UeQBvD0TFrlVjOeLOBz+CPAI8dnbqNSVwUwRrkp7vQ=
|
||||
github.com/wsxiaoys/terminal v0.0.0-20160513160801-0940f3fc43a0/go.mod h1:IXCdmsXIht47RaVFLEdVnh1t+pgYtTAhQGj73kz+2DM=
|
||||
github.com/xanzy/go-gitlab v0.88.0 h1:9GHBrxyCUNZZNuAsbJ1NbEH6XAYsKyTn6NfE0wYO5SY=
|
||||
github.com/xanzy/go-gitlab v0.88.0/go.mod h1:5ryv+MnpZStBH8I/77HuQBsMbBGANtVpLWC15qOjWAw=
|
||||
@@ -649,6 +671,7 @@ github.com/yuin/goldmark-emoji v1.0.1/go.mod h1:2w1E6FEWLcDQkoTE+7HU6QF1F6SLlNGj
|
||||
github.com/yusufpapurcu/wmi v1.2.2 h1:KBNDSne4vP5mbSWnJbO+51IMOXJB67QiYCSBrubbPRg=
|
||||
github.com/yusufpapurcu/wmi v1.2.2/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0=
|
||||
github.com/zeebo/assert v1.3.0 h1:g7C04CbJuIDKNPFHmsk4hwZDO5O+kntRxzaUoNXj+IQ=
|
||||
github.com/zeebo/assert v1.3.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0=
|
||||
github.com/zeebo/xxh3 v1.0.2 h1:xZmwmqxHZA8AI603jOQ0tMqmBr9lPeFwGg6d+xy9DC0=
|
||||
github.com/zeebo/xxh3 v1.0.2/go.mod h1:5NWz9Sef7zIDm2JHfFlcQvNekmcEl9ekUZQQKCYaDcA=
|
||||
go.mongodb.org/mongo-driver v1.12.0 h1:aPx33jmn/rQuJXPQLZQ8NtfPQG8CaqgLThFtqRb0PiE=
|
||||
@@ -685,8 +708,8 @@ golang.org/x/crypto v0.0.0-20220314234659-1baeb1ce4c0b/go.mod h1:IxCIyHEi3zRg3s0
|
||||
golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
|
||||
golang.org/x/crypto v0.3.1-0.20221117191849-2c476679df9a/go.mod h1:hebNnKkNXi2UzZN1eVRvBB7co0a+JxK6XbPiWVs/3J4=
|
||||
golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU=
|
||||
golang.org/x/crypto v0.12.0 h1:tFM/ta59kqch6LlvYnPa0yx5a83cL2nHflFhYKvv9Yk=
|
||||
golang.org/x/crypto v0.12.0/go.mod h1:NF0Gs7EO5K4qLn+Ylc+fih8BSTeIjAP05siRnAh98yw=
|
||||
golang.org/x/crypto v0.13.0 h1:mvySKfSWJ+UKUii46M40LOvyWfN0s2U+46/jDd0e6Ck=
|
||||
golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc=
|
||||
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
|
||||
golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
|
||||
golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
|
||||
@@ -751,16 +774,16 @@ golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY=
|
||||
golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
|
||||
golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc=
|
||||
golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg=
|
||||
golang.org/x/net v0.14.0 h1:BONx9s002vGdD9umnlX1Po8vOZmrgH34qlHcD1MfK14=
|
||||
golang.org/x/net v0.14.0/go.mod h1:PpSgVXXLK0OxS0F31C1/tv6XNguvCrnXIDrFMspZIUI=
|
||||
golang.org/x/net v0.15.0 h1:ugBLEUaxABaB5AJqW9enI0ACdci2RUd4eP51NTBvuJ8=
|
||||
golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk=
|
||||
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
|
||||
golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
|
||||
golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
|
||||
golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
|
||||
golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
|
||||
golang.org/x/oauth2 v0.8.0/go.mod h1:yr7u4HXZRm1R1kBWqr/xKNqewf0plRYoB7sla+BCIXE=
|
||||
golang.org/x/oauth2 v0.10.0 h1:zHCpF2Khkwy4mMB4bv0U37YtJdTGW8jI0glAApi0Kh8=
|
||||
golang.org/x/oauth2 v0.10.0/go.mod h1:kTpgurOux7LqtuxjuyZa4Gj2gdezIt/jQtGnNFfypQI=
|
||||
golang.org/x/oauth2 v0.12.0 h1:smVPGxink+n1ZI5pkQa8y6fZT0RW0MgCO5bFpepy4B4=
|
||||
golang.org/x/oauth2 v0.12.0/go.mod h1:A74bZ3aGXgCY0qaIC9Ahg6Lglin4AMAco8cIv9baba4=
|
||||
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
@@ -819,16 +842,16 @@ golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.11.0 h1:eG7RXZHdqOJ1i+0lgLgCpSXAp6M3LYlAo6osgSi0xOM=
|
||||
golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.12.0 h1:CM0HF96J0hcLAwsHPJZjfdNzs0gftsLfgKt57wWHJ0o=
|
||||
golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
|
||||
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
|
||||
golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc=
|
||||
golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
|
||||
golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U=
|
||||
golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo=
|
||||
golang.org/x/term v0.11.0 h1:F9tnn/DA/Im8nCwm+fX+1/eBwi4qFjRT++MhtVC4ZX0=
|
||||
golang.org/x/term v0.11.0/go.mod h1:zC9APTIj3jG3FdV/Ons+XE1riIZXG4aZ4GTHiPZJPIU=
|
||||
golang.org/x/term v0.12.0 h1:/ZfYdc3zq+q02Rv9vGqTeSItdzZTSNDmfTi0mBAuidU=
|
||||
golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU=
|
||||
golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
@@ -841,8 +864,8 @@ golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
|
||||
golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
|
||||
golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
|
||||
golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
|
||||
golang.org/x/text v0.12.0 h1:k+n5B8goJNdU7hSvEtMUz3d1Q6D/XW4COJSJR6fN0mc=
|
||||
golang.org/x/text v0.12.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
|
||||
golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k=
|
||||
golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
|
||||
golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
|
||||
golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
|
||||
golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4=
|
||||
@@ -885,6 +908,7 @@ golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8T
|
||||
golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 h1:H2TDz8ibqkAF6YGhCdN3jS9O0/s90v0rJh3X/OLHEUk=
|
||||
golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8=
|
||||
gonum.org/v1/gonum v0.11.0 h1:f1IJhK4Km5tBJmaiJXtk/PkL4cdVX6J+tGiM187uT5E=
|
||||
gonum.org/v1/gonum v0.11.0/go.mod h1:fSG4YDCxxUZQJ7rKsQrj0gMOg00Il0Z96/qMA4bVQhA=
|
||||
google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE=
|
||||
google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M=
|
||||
google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg=
|
||||
@@ -893,8 +917,8 @@ google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsb
|
||||
google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI=
|
||||
google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI=
|
||||
google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
|
||||
google.golang.org/api v0.131.0 h1:AcgWS2edQ4chVEt/SxgDKubVu/9/idCJy00tBGuGB4M=
|
||||
google.golang.org/api v0.131.0/go.mod h1:7vtkbKv2REjJbxmHSkBTBQ5LUGvPdAqjjvt84XAfhpA=
|
||||
google.golang.org/api v0.132.0 h1:8t2/+qZ26kAOGSmOiHwVycqVaDg7q3JDILrNi/Z6rvc=
|
||||
google.golang.org/api v0.132.0/go.mod h1:AeTBC6GpJnJSRJjktDcPX0QwtS8pGYZOV6MSuSCusw0=
|
||||
google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
|
||||
google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
|
||||
google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
|
||||
@@ -917,12 +941,12 @@ google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvx
|
||||
google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
|
||||
google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
|
||||
google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=
|
||||
google.golang.org/genproto v0.0.0-20230629202037-9506855d4529 h1:9JucMWR7sPvCxUFd6UsOUNmA5kCcWOfORaT3tpAsKQs=
|
||||
google.golang.org/genproto v0.0.0-20230629202037-9506855d4529/go.mod h1:xZnkP7mREFX5MORlOPEzLMr+90PPZQ2QWzrVTWfAq64=
|
||||
google.golang.org/genproto/googleapis/api v0.0.0-20230629202037-9506855d4529 h1:s5YSX+ZH5b5vS9rnpGymvIyMpLRJizowqDlOuyjXnTk=
|
||||
google.golang.org/genproto/googleapis/api v0.0.0-20230629202037-9506855d4529/go.mod h1:vHYtlOoi6TsQ3Uk2yxR7NI5z8uoV+3pZtR4jmHIkRig=
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20230706204954-ccb25ca9f130 h1:2FZP5XuJY9zQyGM5N0rtovnoXjiMUEIUMvw0m9wlpLc=
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20230706204954-ccb25ca9f130/go.mod h1:8mL13HKkDa+IuJ8yruA3ci0q+0vsUz4m//+ottjwS5o=
|
||||
google.golang.org/genproto v0.0.0-20230706204954-ccb25ca9f130 h1:Au6te5hbKUV8pIYWHqOUZ1pva5qK/rwbIhoXEUB9Lu8=
|
||||
google.golang.org/genproto v0.0.0-20230706204954-ccb25ca9f130/go.mod h1:O9kGHb51iE/nOGvQaDUuadVYqovW56s5emA88lQnj6Y=
|
||||
google.golang.org/genproto/googleapis/api v0.0.0-20230706204954-ccb25ca9f130 h1:XVeBY8d/FaK4848myy41HBqnDwvxeV3zMZhwN1TvAMU=
|
||||
google.golang.org/genproto/googleapis/api v0.0.0-20230706204954-ccb25ca9f130/go.mod h1:mPBs5jNgx2GuQGvFwUvVKqtn6HsUw9nP64BedgvqEsQ=
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20230711160842-782d3b101e98 h1:bVf09lpb+OJbByTj913DRJioFFAjf/ZGxEz7MajTp2U=
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20230711160842-782d3b101e98/go.mod h1:TUfxEVdsvPg18p6AslUXFoLdpED4oBnGwyqk3dV1XzM=
|
||||
google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
|
||||
google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
|
||||
google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
|
||||
@@ -980,6 +1004,7 @@ gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C
|
||||
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
|
||||
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
gotest.tools/v3 v3.0.3 h1:4AuOwCGf4lLR9u3YOe2awrHygurzhO/HeQ6laiA6Sx0=
|
||||
gotest.tools/v3 v3.0.3/go.mod h1:Z7Lb0S5l+klDB31fvDQX8ss/FlKDxtlFlw3Oa8Ymbl8=
|
||||
honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
|
||||
honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
|
||||
honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
|
||||
|
||||
@@ -44,6 +44,18 @@ If you think that something should be included outside of these guidelines, plea
|
||||
- Go 1.17+
|
||||
- Make
|
||||
|
||||
### Adding New Token Formats to an Existing Scanner
|
||||
|
||||
In some instances, services will update their token format, requiring a new regex to properly detect secrets in addition to supporting the previous token format. Accomodating this can be done without adding a net-new detector. [We provide a `Versioner` interface](https://github.com/trufflesecurity/trufflehog/blob/e18cfd5e0af1469a9f05b8d5732bcc94c39da49c/pkg/detectors/detectors.go#L30) that can be implemented.
|
||||
|
||||
1. Create a copy of the package and append `_v2` to the package and file names. Ex: `<packagename>/` -> `<packagename>_v2`, `<packagename>.go` -> `<packagename>_v2.go`
|
||||
|
||||
Note: Be sure to update the tests to reference the new secret values in GSM, or the tests will fail.
|
||||
|
||||
2. Implement the `Versioner` interface. [GitHub example implementation.](/pkg/detectors/github_old/github_old.go#L22)
|
||||
|
||||
3. Proceed from step 3 of [Creating a new Secret Scanner](#creating-a-new-secret-scanner)
|
||||
|
||||
### Creating a new Secret Scanner
|
||||
|
||||
1. Identify the Secret Detector name from the [/proto/detectors.proto](/proto/detectors.proto) `DetectorType` enum.
|
||||
@@ -96,7 +108,7 @@ Do not embed test credentials in the test code. Instead, use GCP Secrets Manager
|
||||
Note: `/tmp/s` is a valid path on Linux. You will need to change that for Windows or OSX, otherwise you will see an error. On Windows you will also need to install [WSL](https://docs.microsoft.com/en-us/windows/wsl/install).
|
||||
|
||||
```bash
|
||||
gcloud secrets versions access --project trufflehog-testing --secret detectors3 latest > /tmp/s
|
||||
gcloud secrets versions access --project trufflehog-testing --secret detectors5 latest > /tmp/s
|
||||
```
|
||||
|
||||
2. Add the secret that you need for testing.
|
||||
@@ -113,8 +125,9 @@ Do not embed test credentials in the test code. Instead, use GCP Secrets Manager
|
||||
3. Update the secret version with your modification.
|
||||
|
||||
```bash
|
||||
gcloud secrets versions add --project trufflehog-testing detectors3 --data-file /tmp/s
|
||||
gcloud secrets versions add --project trufflehog-testing detectors5 --data-file /tmp/s
|
||||
```
|
||||
Note: We increment the detectors file name `detectors(n+1)` once the previous one exceeds the max size allowed by GSM (65kb).
|
||||
|
||||
4. Access the secret value as shown in the [example code](pkg/detectors/heroku/heroku_test.go).
|
||||
|
||||
|
||||
@@ -40,6 +40,18 @@ If you think that something should be included outside of these guidelines, plea
|
||||
- Go 1.17+
|
||||
- Make
|
||||
|
||||
### Adding New Token Formats to an Existing Scanner
|
||||
|
||||
In some instances, services will update their token format, requiring a new regex to properly detect secrets in addition to supporting the previous token format. Accomodating this can be done without adding a net-new detector. [We provide a `Versioner` interface](https://github.com/trufflesecurity/trufflehog/blob/e18cfd5e0af1469a9f05b8d5732bcc94c39da49c/pkg/detectors/detectors.go#L30) that can be implemented.
|
||||
|
||||
1. Create a copy of the package and append `_v2` to the package and file names. Ex: `<packagename>/` -> `<packagename>_v2`, `<packagename>.go` -> `<packagename>_v2.go`
|
||||
|
||||
Note: Be sure to update the tests to reference the new secret values in GSM, or the tests will fail.
|
||||
|
||||
2. Implement the `Versioner` interface. [GitHub example implementation.](/pkg/detectors/github_old/github_old.go#L22)
|
||||
|
||||
3. Proceed from step 3 of [Creating a new Secret Scanner](#creating-a-new-secret-scanner)
|
||||
|
||||
### Creating a new Secret Scanner
|
||||
|
||||
1. Identify the Secret Detector name from the [/proto/detectors.proto](/proto/detectors.proto) `DetectorType` enum. If necessary, run `make protos` when adding new ones.
|
||||
|
||||
@@ -105,11 +105,11 @@ var (
|
||||
filesystemScanIncludePaths = filesystemScan.Flag("include-paths", "Path to file with newline separated regexes for files to include in scan.").Short('i').String()
|
||||
filesystemScanExcludePaths = filesystemScan.Flag("exclude-paths", "Path to file with newline separated regexes for files to exclude in scan.").Short('x').String()
|
||||
|
||||
s3Scan = cli.Command("s3", "Find credentials in S3 buckets.")
|
||||
s3ScanKey = s3Scan.Flag("key", "S3 key used to authenticate. Can be provided with environment variable AWS_ACCESS_KEY_ID.").Envar("AWS_ACCESS_KEY_ID").String()
|
||||
s3ScanRoleArns = s3Scan.Flag("role-arn", "Specify the ARN of an IAM role to assume for scanning. You can repeat this flag.").Strings()
|
||||
s3ScanSecret = s3Scan.Flag("secret", "S3 secret used to authenticate. Can be provided with environment variable AWS_SECRET_ACCESS_KEY.").Envar("AWS_SECRET_ACCESS_KEY").String()
|
||||
s3ScanSessionToken = s3Scan.Flag("session-token", "S3 session token used to authenticate temporary credentials. Can be provided with environment variable AWS_SESSION_TOKEN.").Envar("AWS_SESSION_TOKEN").String()
|
||||
s3Scan = cli.Command("s3", "Find credentials in S3 buckets.")
|
||||
s3ScanKey = s3Scan.Flag("key", "S3 key used to authenticate. Can be provided with environment variable AWS_ACCESS_KEY_ID.").Envar("AWS_ACCESS_KEY_ID").String()
|
||||
s3ScanRoleArns = s3Scan.Flag("role-arn", "Specify the ARN of an IAM role to assume for scanning. You can repeat this flag.").Strings()
|
||||
s3ScanSecret = s3Scan.Flag("secret", "S3 secret used to authenticate. Can be provided with environment variable AWS_SECRET_ACCESS_KEY.").Envar("AWS_SECRET_ACCESS_KEY").String()
|
||||
s3ScanSessionToken = s3Scan.Flag("session-token", "S3 session token used to authenticate temporary credentials. Can be provided with environment variable AWS_SESSION_TOKEN.").Envar("AWS_SESSION_TOKEN").String()
|
||||
s3ScanCloudEnv = s3Scan.Flag("cloud-environment", "Use IAM credentials in cloud environment.").Bool()
|
||||
s3ScanBuckets = s3Scan.Flag("bucket", "Name of S3 bucket to scan. You can repeat this flag.").Strings()
|
||||
s3ScanMaxObjectSize = s3Scan.Flag("max-object-size", "Maximum size of objects to scan. Objects larger than this will be skipped. (Byte units eg. 512B, 2KB, 4MB)").Default("250MB").Bytes()
|
||||
@@ -354,6 +354,10 @@ func run(state overseer.State) {
|
||||
printer = new(output.PlainPrinter)
|
||||
}
|
||||
|
||||
if !*jsonLegacy && !*jsonOut {
|
||||
fmt.Fprintf(os.Stderr, "🐷🔑🐷 TruffleHog. Unearth your secrets. 🐷🔑🐷\n\n")
|
||||
}
|
||||
|
||||
e, err := engine.Start(ctx,
|
||||
engine.WithConcurrency(uint8(*concurrency)),
|
||||
engine.WithDecoders(decoders.DefaultDecoders()...),
|
||||
@@ -469,7 +473,7 @@ func run(state overseer.State) {
|
||||
Secret: *s3ScanSecret,
|
||||
SessionToken: *s3ScanSessionToken,
|
||||
Buckets: *s3ScanBuckets,
|
||||
Roles: *s3ScanRoleArns,
|
||||
Roles: *s3ScanRoleArns,
|
||||
CloudCred: *s3ScanCloudEnv,
|
||||
MaxObjectSize: int64(*s3ScanMaxObjectSize),
|
||||
}
|
||||
@@ -525,10 +529,6 @@ func run(state overseer.State) {
|
||||
}
|
||||
}
|
||||
|
||||
if !*jsonLegacy && !*jsonOut {
|
||||
fmt.Fprintf(os.Stderr, "🐷🔑🐷 TruffleHog. Unearth your secrets. 🐷🔑🐷\n\n")
|
||||
}
|
||||
|
||||
// Wait for all workers to finish.
|
||||
if err = e.Finish(ctx); err != nil {
|
||||
logFatal(err, "engine failed to finish execution")
|
||||
|
||||
+130
-110
@@ -118,118 +118,21 @@ func (s scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
}
|
||||
|
||||
if verify {
|
||||
// REQUEST VALUES.
|
||||
method := "GET"
|
||||
service := "sts"
|
||||
host := "sts.amazonaws.com"
|
||||
region := "us-east-1"
|
||||
endpoint := "https://sts.amazonaws.com"
|
||||
now := time.Now().UTC()
|
||||
datestamp := now.Format("20060102")
|
||||
amzDate := now.Format("20060102T150405Z0700")
|
||||
|
||||
req, err := http.NewRequestWithContext(ctx, method, endpoint, nil)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
req.Header.Set("Accept", "application/json")
|
||||
|
||||
// TASK 1: CREATE A CANONICAL REQUEST.
|
||||
// http://docs.aws.amazon.com/general/latest/gr/sigv4-create-canonical-request.html
|
||||
canonicalURI := "/"
|
||||
canonicalHeaders := "host:" + host + "\n"
|
||||
signedHeaders := "host"
|
||||
algorithm := "AWS4-HMAC-SHA256"
|
||||
credentialScope := fmt.Sprintf("%s/%s/%s/aws4_request", datestamp, region, service)
|
||||
|
||||
params := req.URL.Query()
|
||||
params.Add("Action", "GetCallerIdentity")
|
||||
params.Add("Version", "2011-06-15")
|
||||
params.Add("X-Amz-Algorithm", algorithm)
|
||||
params.Add("X-Amz-Credential", resIDMatch+"/"+credentialScope)
|
||||
params.Add("X-Amz-Date", amzDate)
|
||||
params.Add("X-Amz-Expires", "30")
|
||||
params.Add("X-Amz-SignedHeaders", signedHeaders)
|
||||
|
||||
canonicalQuerystring := params.Encode()
|
||||
payloadHash := GetHash("") // empty payload
|
||||
canonicalRequest := method + "\n" + canonicalURI + "\n" + canonicalQuerystring + "\n" + canonicalHeaders + "\n" + signedHeaders + "\n" + payloadHash
|
||||
|
||||
// TASK 2: CREATE THE STRING TO SIGN.
|
||||
stringToSign := algorithm + "\n" + amzDate + "\n" + credentialScope + "\n" + GetHash(canonicalRequest)
|
||||
|
||||
// TASK 3: CALCULATE THE SIGNATURE.
|
||||
// https://docs.aws.amazon.com/general/latest/gr/sigv4-calculate-signature.html
|
||||
hash := GetHMAC([]byte(fmt.Sprintf("AWS4%s", resSecretMatch)), []byte(datestamp))
|
||||
hash = GetHMAC(hash, []byte(region))
|
||||
hash = GetHMAC(hash, []byte(service))
|
||||
hash = GetHMAC(hash, []byte("aws4_request"))
|
||||
|
||||
signature2 := GetHMAC(hash, []byte(stringToSign)) // Get Signature HMAC SHA256
|
||||
signature := hex.EncodeToString(signature2)
|
||||
|
||||
// TASK 4: ADD SIGNING INFORMATION TO THE REQUEST.
|
||||
params.Add("X-Amz-Signature", signature)
|
||||
req.Header.Add("Content-type", "application/x-www-form-urlencoded; charset=utf-8")
|
||||
req.URL.RawQuery = params.Encode()
|
||||
|
||||
client := s.verificationClient
|
||||
if client == nil {
|
||||
client = defaultVerificationClient
|
||||
}
|
||||
res, err := client.Do(req)
|
||||
if err == nil {
|
||||
|
||||
if res.StatusCode >= 200 && res.StatusCode < 300 {
|
||||
identityInfo := identityRes{}
|
||||
err := json.NewDecoder(res.Body).Decode(&identityInfo)
|
||||
if err == nil {
|
||||
s1.Verified = true
|
||||
s1.ExtraData = map[string]string{
|
||||
"account": identityInfo.GetCallerIdentityResponse.GetCallerIdentityResult.Account,
|
||||
"user_id": identityInfo.GetCallerIdentityResponse.GetCallerIdentityResult.UserID,
|
||||
"arn": identityInfo.GetCallerIdentityResponse.GetCallerIdentityResult.Arn,
|
||||
}
|
||||
} else {
|
||||
s1.VerificationError = err
|
||||
}
|
||||
res.Body.Close()
|
||||
} else {
|
||||
// This function will check false positives for common test words, but also it will make sure the key appears "random" enough to be a real key.
|
||||
if detectors.IsKnownFalsePositive(resSecretMatch, detectors.DefaultFalsePositives, true) {
|
||||
continue
|
||||
}
|
||||
|
||||
if res.StatusCode == 403 {
|
||||
var body awsErrorResponseBody
|
||||
err = json.NewDecoder(res.Body).Decode(&body)
|
||||
res.Body.Close()
|
||||
if err == nil {
|
||||
// All instances of the code I've seen in the wild are PascalCased but this check is
|
||||
// case-insensitive out of an abundance of caution
|
||||
if strings.EqualFold(body.Error.Code, "InvalidClientTokenId") {
|
||||
// determinate failure - nothing to do
|
||||
} else {
|
||||
// We see a surprising number of false-negative signature mismatch errors here
|
||||
// (The official SDK somehow elicits even more than just making the request
|
||||
// ourselves)
|
||||
s1.VerificationError = fmt.Errorf("request to %v returned status %d with an unexpected reason (%s: %s)", res.Request.URL, res.StatusCode, body.Error.Code, body.Error.Message)
|
||||
}
|
||||
} else {
|
||||
s1.VerificationError = fmt.Errorf("couldn't parse the sts response body (%v)", err)
|
||||
}
|
||||
} else {
|
||||
s1.VerificationError = fmt.Errorf("request to %v returned unexpected status %d", res.Request.URL, res.StatusCode)
|
||||
}
|
||||
}
|
||||
} else {
|
||||
s1.VerificationError = err
|
||||
}
|
||||
verified, extraData, verificationErr := s.verifyMatch(ctx, resIDMatch, resSecretMatch, true)
|
||||
s1.Verified = verified
|
||||
s1.ExtraData = extraData
|
||||
s1.VerificationError = verificationErr
|
||||
}
|
||||
|
||||
// If the result is unverified and matches something like a git hash, don't include it in the results.
|
||||
if !s1.Verified && falsePositiveSecretCheck.MatchString(resSecretMatch) {
|
||||
continue
|
||||
if !s1.Verified {
|
||||
// Unverified results that contain common test words are probably not secrets
|
||||
if detectors.IsKnownFalsePositive(resSecretMatch, detectors.DefaultFalsePositives, true) {
|
||||
continue
|
||||
}
|
||||
// Unverified results that look like hashes are probably not secrets
|
||||
if falsePositiveSecretCheck.MatchString(resSecretMatch) {
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
results = append(results, s1)
|
||||
@@ -242,6 +145,123 @@ func (s scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
return awsCustomCleanResults(results), nil
|
||||
}
|
||||
|
||||
func (s scanner) verifyMatch(ctx context.Context, resIDMatch, resSecretMatch string, retryOn403 bool) (bool, map[string]string, error) {
|
||||
// REQUEST VALUES.
|
||||
method := "GET"
|
||||
service := "sts"
|
||||
host := "sts.amazonaws.com"
|
||||
region := "us-east-1"
|
||||
endpoint := "https://sts.amazonaws.com"
|
||||
now := time.Now().UTC()
|
||||
datestamp := now.Format("20060102")
|
||||
amzDate := now.Format("20060102T150405Z0700")
|
||||
|
||||
req, err := http.NewRequestWithContext(ctx, method, endpoint, nil)
|
||||
if err != nil {
|
||||
return false, nil, err
|
||||
}
|
||||
req.Header.Set("Accept", "application/json")
|
||||
|
||||
// TASK 1: CREATE A CANONICAL REQUEST.
|
||||
// http://docs.aws.amazon.com/general/latest/gr/sigv4-create-canonical-request.html
|
||||
canonicalURI := "/"
|
||||
canonicalHeaders := "host:" + host + "\n"
|
||||
signedHeaders := "host"
|
||||
algorithm := "AWS4-HMAC-SHA256"
|
||||
credentialScope := fmt.Sprintf("%s/%s/%s/aws4_request", datestamp, region, service)
|
||||
|
||||
params := req.URL.Query()
|
||||
params.Add("Action", "GetCallerIdentity")
|
||||
params.Add("Version", "2011-06-15")
|
||||
params.Add("X-Amz-Algorithm", algorithm)
|
||||
params.Add("X-Amz-Credential", resIDMatch+"/"+credentialScope)
|
||||
params.Add("X-Amz-Date", amzDate)
|
||||
params.Add("X-Amz-Expires", "30")
|
||||
params.Add("X-Amz-SignedHeaders", signedHeaders)
|
||||
|
||||
canonicalQuerystring := params.Encode()
|
||||
payloadHash := GetHash("") // empty payload
|
||||
canonicalRequest := method + "\n" + canonicalURI + "\n" + canonicalQuerystring + "\n" + canonicalHeaders + "\n" + signedHeaders + "\n" + payloadHash
|
||||
|
||||
// TASK 2: CREATE THE STRING TO SIGN.
|
||||
stringToSign := algorithm + "\n" + amzDate + "\n" + credentialScope + "\n" + GetHash(canonicalRequest)
|
||||
|
||||
// TASK 3: CALCULATE THE SIGNATURE.
|
||||
// https://docs.aws.amazon.com/general/latest/gr/sigv4-calculate-signature.html
|
||||
hash := GetHMAC([]byte(fmt.Sprintf("AWS4%s", resSecretMatch)), []byte(datestamp))
|
||||
hash = GetHMAC(hash, []byte(region))
|
||||
hash = GetHMAC(hash, []byte(service))
|
||||
hash = GetHMAC(hash, []byte("aws4_request"))
|
||||
|
||||
signature2 := GetHMAC(hash, []byte(stringToSign)) // Get Signature HMAC SHA256
|
||||
signature := hex.EncodeToString(signature2)
|
||||
|
||||
// TASK 4: ADD SIGNING INFORMATION TO THE REQUEST.
|
||||
params.Add("X-Amz-Signature", signature)
|
||||
req.Header.Add("Content-type", "application/x-www-form-urlencoded; charset=utf-8")
|
||||
req.URL.RawQuery = params.Encode()
|
||||
|
||||
client := s.verificationClient
|
||||
if client == nil {
|
||||
client = defaultVerificationClient
|
||||
}
|
||||
|
||||
res, err := client.Do(req)
|
||||
if err == nil {
|
||||
defer res.Body.Close()
|
||||
if res.StatusCode >= 200 && res.StatusCode < 300 {
|
||||
identityInfo := identityRes{}
|
||||
err := json.NewDecoder(res.Body).Decode(&identityInfo)
|
||||
if err == nil {
|
||||
extraData := map[string]string{
|
||||
"account": identityInfo.GetCallerIdentityResponse.GetCallerIdentityResult.Account,
|
||||
"user_id": identityInfo.GetCallerIdentityResponse.GetCallerIdentityResult.UserID,
|
||||
"arn": identityInfo.GetCallerIdentityResponse.GetCallerIdentityResult.Arn,
|
||||
}
|
||||
return true, extraData, nil
|
||||
} else {
|
||||
return false, nil, err
|
||||
}
|
||||
} else if res.StatusCode == 403 {
|
||||
// Experimentation has indicated that if you make two GetCallerIdentity requests within five seconds that
|
||||
// share a key ID but are signed with different secrets the second one will be rejected with a 403 that
|
||||
// carries a SignatureDoesNotMatch code in its body. This happens even if the second ID-secret pair is
|
||||
// valid. Since this is exactly our access pattern, we need to work around it.
|
||||
//
|
||||
// Fortunately, experimentation has also revealed a workaround: simply resubmit the second request. The
|
||||
// response to the resubmission will be as expected. But there's a caveat: You can't have closed the body of
|
||||
// the response to the original second request, or read to its end, or the resubmission will also yield a
|
||||
// SignatureDoesNotMatch. For this reason, we have to re-request all 403s. We can't re-request only
|
||||
// SignatureDoesNotMatch responses, because we can only tell whether a given 403 is a SignatureDoesNotMatch
|
||||
// after decoding its response body, which requires reading the entire response body, which disables the
|
||||
// workaround.
|
||||
//
|
||||
// We are clearly deep in the guts of AWS implementation details here, so this all might change with no
|
||||
// notice. If you're here because something in this detector broke, you have my condolences.
|
||||
if retryOn403 {
|
||||
return s.verifyMatch(ctx, resIDMatch, resSecretMatch, false)
|
||||
}
|
||||
var body awsErrorResponseBody
|
||||
err = json.NewDecoder(res.Body).Decode(&body)
|
||||
if err == nil {
|
||||
// All instances of the code I've seen in the wild are PascalCased but this check is
|
||||
// case-insensitive out of an abundance of caution
|
||||
if strings.EqualFold(body.Error.Code, "InvalidClientTokenId") {
|
||||
return false, nil, nil
|
||||
} else {
|
||||
return false, nil, fmt.Errorf("request to %v returned status %d with an unexpected reason (%s: %s)", res.Request.URL, res.StatusCode, body.Error.Code, body.Error.Message)
|
||||
}
|
||||
} else {
|
||||
return false, nil, fmt.Errorf("couldn't parse the sts response body (%v)", err)
|
||||
}
|
||||
} else {
|
||||
return false, nil, fmt.Errorf("request to %v returned unexpected status %d", res.Request.URL, res.StatusCode)
|
||||
}
|
||||
} else {
|
||||
return false, nil, err
|
||||
}
|
||||
}
|
||||
|
||||
func awsCustomCleanResults(results []detectors.Result) []detectors.Result {
|
||||
if len(results) == 0 {
|
||||
return results
|
||||
|
||||
@@ -248,6 +248,28 @@ func TestAWS_FromChunk(t *testing.T) {
|
||||
wantErr: false,
|
||||
wantVerificationError: true,
|
||||
},
|
||||
{
|
||||
name: "verified secret checked directly after unverified secret with same key id",
|
||||
s: scanner{},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("%s\n%s\n%s", inactiveSecret, id, secret)),
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_AWS,
|
||||
Verified: true,
|
||||
Redacted: "AKIASP2TPHJSQH3FJRUX",
|
||||
ExtraData: map[string]string{
|
||||
"account": "171436882533",
|
||||
"arn": "arn:aws:iam::171436882533:user/canarytokens.com@@4dxkh0pdeop3bzu9zx5wob793",
|
||||
"user_id": "AIDASP2TPHJSUFRSTTZX4",
|
||||
},
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
|
||||
@@ -66,7 +66,7 @@ type ResultWithMetadata struct {
|
||||
// SourceMetadata contains source-specific contextual information.
|
||||
SourceMetadata *source_metadatapb.MetaData
|
||||
// SourceID is the ID of the source that the API uses to map secrets to specific sources.
|
||||
SourceID int64
|
||||
SourceID sources.SourceID
|
||||
// SourceType is the type of Source.
|
||||
SourceType sourcespb.SourceType
|
||||
// SourceName is the name of the Source.
|
||||
|
||||
@@ -96,7 +96,7 @@ func constructRequest(secret, keyID string) (*http.Request, error) {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
params := map[string]interface{}{
|
||||
params := map[string]any{
|
||||
"request": accountDetail,
|
||||
"nonce": time.Now().UnixNano(),
|
||||
}
|
||||
|
||||
@@ -13,13 +13,18 @@ import (
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
|
||||
)
|
||||
|
||||
type Scanner struct{}
|
||||
type Scanner struct {
|
||||
client *http.Client
|
||||
}
|
||||
|
||||
// Ensure the Scanner satisfies the interface at compile time.
|
||||
var _ detectors.Detector = (*Scanner)(nil)
|
||||
var _ detectors.Versioner = (*Scanner)(nil)
|
||||
|
||||
func (Scanner) Version() int { return 1 }
|
||||
|
||||
var (
|
||||
client = common.SaneHttpClient()
|
||||
defaultClient = common.SaneHttpClient()
|
||||
|
||||
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
|
||||
tokenPat = regexp.MustCompile(detectors.PrefixRegex([]string{"jira"}) + `\b([a-zA-Z-0-9]{24})\b`)
|
||||
@@ -70,6 +75,11 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
}
|
||||
|
||||
if verify {
|
||||
client := s.client
|
||||
if client == nil {
|
||||
client = defaultClient
|
||||
}
|
||||
|
||||
data := fmt.Sprintf("%s:%s", resEmail, resToken)
|
||||
sEnc := b64.StdEncoding.EncodeToString([]byte(data))
|
||||
req, err := http.NewRequestWithContext(ctx, "GET", "https://"+resDomain+"/rest/api/3/dashboard", nil)
|
||||
@@ -85,9 +95,15 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
// If the request is successful and the login reason is not failed authentication, then the token is valid.
|
||||
// This is because Jira returns a 200 status code even if the token is invalid.
|
||||
// Jira returns a default dashboard page.
|
||||
if (res.StatusCode >= 200 && res.StatusCode < 300) && res.Header.Get(loginReasonHeaderKey) != failedAuth {
|
||||
s1.Verified = true
|
||||
if res.StatusCode >= 200 && res.StatusCode < 300 {
|
||||
if res.Header.Get(loginReasonHeaderKey) != failedAuth {
|
||||
s1.Verified = true
|
||||
}
|
||||
} else {
|
||||
s1.VerificationError = fmt.Errorf("unexpected HTTP response status %d", res.StatusCode)
|
||||
}
|
||||
} else {
|
||||
s1.VerificationError = err
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -9,7 +9,8 @@ import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/kylelemons/godebug/pretty"
|
||||
"github.com/google/go-cmp/cmp"
|
||||
"github.com/google/go-cmp/cmp/cmpopts"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
@@ -19,12 +20,12 @@ import (
|
||||
func TestJiraToken_FromChunk(t *testing.T) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
|
||||
defer cancel()
|
||||
testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
|
||||
testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")
|
||||
if err != nil {
|
||||
t.Fatalf("could not get test secrets from GCP: %s", err)
|
||||
}
|
||||
token := testSecrets.MustGetField("JIRA_TOKEN")
|
||||
inactiveToken := testSecrets.MustGetField("JIRA_INACTIVE")
|
||||
secret := testSecrets.MustGetField("JIRA_TOKEN")
|
||||
inactiveSecret := testSecrets.MustGetField("JIRA_INACTIVE")
|
||||
email := testSecrets.MustGetField("JIRA_EMAIL")
|
||||
domain := testSecrets.MustGetField("JIRA_DOMAIN")
|
||||
|
||||
@@ -34,18 +35,19 @@ func TestJiraToken_FromChunk(t *testing.T) {
|
||||
verify bool
|
||||
}
|
||||
tests := []struct {
|
||||
name string
|
||||
s Scanner
|
||||
args args
|
||||
want []detectors.Result
|
||||
wantErr bool
|
||||
name string
|
||||
s Scanner
|
||||
args args
|
||||
want []detectors.Result
|
||||
wantErr bool
|
||||
wantVerificationErr bool
|
||||
}{
|
||||
{
|
||||
name: "found, verified",
|
||||
s: Scanner{},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a jira secret %s within jira %s with jira %s", token, email, domain)),
|
||||
data: []byte(fmt.Sprintf("You can find a jira secret %s within jira %s with jira %s", secret, email, domain)),
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
@@ -54,14 +56,15 @@ func TestJiraToken_FromChunk(t *testing.T) {
|
||||
Verified: true,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
wantErr: false,
|
||||
wantVerificationErr: false,
|
||||
},
|
||||
{
|
||||
name: "found, unverified",
|
||||
s: Scanner{},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a jira secret %s within jira %s but not jira %s valid", inactiveToken, email, domain)), // the secret would satisfy the regex but not pass validation
|
||||
data: []byte(fmt.Sprintf("You can find a jira secret %s within jira %s but not jira %s valid", inactiveSecret, email, domain)), // the secret would satisfy the regex but not pass validation
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
@@ -70,7 +73,8 @@ func TestJiraToken_FromChunk(t *testing.T) {
|
||||
Verified: false,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
wantErr: false,
|
||||
wantVerificationErr: false,
|
||||
},
|
||||
{
|
||||
name: "not found",
|
||||
@@ -80,14 +84,48 @@ func TestJiraToken_FromChunk(t *testing.T) {
|
||||
data: []byte("You cannot find the secret within"),
|
||||
verify: true,
|
||||
},
|
||||
want: nil,
|
||||
wantErr: false,
|
||||
want: nil,
|
||||
wantErr: false,
|
||||
wantVerificationErr: false,
|
||||
},
|
||||
{
|
||||
name: "found, would be verified if not for timeout",
|
||||
s: Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a jira secret %s within jira %s with jira %s", secret, email, domain)),
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_JiraToken,
|
||||
Verified: false,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
wantVerificationErr: true,
|
||||
},
|
||||
{
|
||||
name: "found, verified but unexpected api surface",
|
||||
s: Scanner{client: common.ConstantResponseHttpClient(404, "")},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a jira secret %s within jira %s with jira %s", secret, email, domain)),
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_JiraToken,
|
||||
Verified: false,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
wantVerificationErr: true,
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
s := Scanner{}
|
||||
got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
|
||||
got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
|
||||
if (err != nil) != tt.wantErr {
|
||||
t.Errorf("JiraToken.FromData() error = %v, wantErr %v", err, tt.wantErr)
|
||||
return
|
||||
@@ -96,10 +134,12 @@ func TestJiraToken_FromChunk(t *testing.T) {
|
||||
if len(got[i].Raw) == 0 {
|
||||
t.Fatalf("no raw secret present: \n %+v", got[i])
|
||||
}
|
||||
got[i].Raw = nil
|
||||
got[i].RawV2 = nil
|
||||
if (got[i].VerificationError != nil) != tt.wantVerificationErr {
|
||||
t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError)
|
||||
}
|
||||
}
|
||||
if diff := pretty.Compare(got, tt.want); diff != "" {
|
||||
ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "RawV2", "VerificationError")
|
||||
if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {
|
||||
t.Errorf("JiraToken.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
|
||||
}
|
||||
})
|
||||
|
||||
@@ -0,0 +1,129 @@
|
||||
package jiratoken_v2
|
||||
|
||||
import (
|
||||
"context"
|
||||
b64 "encoding/base64"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
|
||||
)
|
||||
|
||||
type Scanner struct {
|
||||
client *http.Client
|
||||
}
|
||||
|
||||
// Ensure the Scanner satisfies the interface at compile time.
|
||||
var _ detectors.Detector = (*Scanner)(nil)
|
||||
var _ detectors.Versioner = (*Scanner)(nil)
|
||||
|
||||
func (Scanner) Version() int { return 2 }
|
||||
|
||||
var (
|
||||
defaultClient = common.SaneHttpClient()
|
||||
|
||||
// https://support.atlassian.com/atlassian-account/docs/manage-api-tokens-for-your-atlassian-account/
|
||||
// Tokens created after Jan 18 2023 use a variable length
|
||||
tokenPat = regexp.MustCompile(detectors.PrefixRegex([]string{"jira"}) + `\b([A-Za-z0-9+/=_-]+=[A-Za-z0-9]{8})\b`)
|
||||
domainPat = regexp.MustCompile(detectors.PrefixRegex([]string{"jira"}) + `\b([a-zA-Z-0-9]{5,24}\.[a-zA-Z-0-9]{3,16}\.[a-zA-Z-0-9]{3,16})\b`)
|
||||
emailPat = regexp.MustCompile(detectors.PrefixRegex([]string{"jira"}) + `\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Z|a-z]{2,}\b`)
|
||||
)
|
||||
|
||||
const (
|
||||
failedAuth = "AUTHENTICATED_FAILED"
|
||||
loginReasonHeaderKey = "X-Seraph-LoginReason"
|
||||
)
|
||||
|
||||
// Keywords are used for efficiently pre-filtering chunks.
|
||||
// Use identifiers in the secret preferably, or the provider name.
|
||||
func (s Scanner) Keywords() []string {
|
||||
return []string{"jira"}
|
||||
}
|
||||
|
||||
// FromData will find and optionally verify JiraToken secrets in a given set of bytes.
|
||||
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
|
||||
dataStr := string(data)
|
||||
|
||||
tokens := tokenPat.FindAllStringSubmatch(dataStr, -1)
|
||||
domains := domainPat.FindAllStringSubmatch(dataStr, -1)
|
||||
emails := emailPat.FindAllStringSubmatch(dataStr, -1)
|
||||
|
||||
for _, email := range emails {
|
||||
email = strings.Split(email[0], " ")
|
||||
if len(email) != 2 {
|
||||
continue
|
||||
}
|
||||
resEmail := strings.TrimSpace(email[1])
|
||||
for _, token := range tokens {
|
||||
if len(token) != 2 {
|
||||
continue
|
||||
}
|
||||
resToken := strings.TrimSpace(token[1])
|
||||
for _, domain := range domains {
|
||||
if len(domain) != 2 {
|
||||
continue
|
||||
}
|
||||
resDomain := strings.TrimSpace(domain[1])
|
||||
|
||||
s1 := detectors.Result{
|
||||
DetectorType: detectorspb.DetectorType_JiraToken,
|
||||
Raw: []byte(resToken),
|
||||
RawV2: []byte(fmt.Sprintf("%s:%s:%s", resEmail, resToken, resDomain)),
|
||||
}
|
||||
|
||||
if verify {
|
||||
client := s.client
|
||||
if client == nil {
|
||||
client = defaultClient
|
||||
}
|
||||
|
||||
data := fmt.Sprintf("%s:%s", resEmail, resToken)
|
||||
sEnc := b64.StdEncoding.EncodeToString([]byte(data))
|
||||
req, err := http.NewRequestWithContext(ctx, "GET", "https://"+resDomain+"/rest/api/3/dashboard", nil)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
req.Header.Add("Accept", "application/json")
|
||||
req.Header.Add("Authorization", fmt.Sprintf("Basic %s", sEnc))
|
||||
res, err := client.Do(req)
|
||||
if err == nil {
|
||||
defer res.Body.Close()
|
||||
|
||||
// If the request is successful and the login reason is not failed authentication, then the token is valid.
|
||||
// This is because Jira returns a 200 status code even if the token is invalid.
|
||||
// Jira returns a default dashboard page.
|
||||
if res.StatusCode >= 200 && res.StatusCode < 300 {
|
||||
if res.Header.Get(loginReasonHeaderKey) != failedAuth {
|
||||
s1.Verified = true
|
||||
}
|
||||
} else {
|
||||
s1.VerificationError = fmt.Errorf("unexpected HTTP response status %d", res.StatusCode)
|
||||
}
|
||||
} else {
|
||||
s1.VerificationError = err
|
||||
}
|
||||
}
|
||||
|
||||
if !s1.Verified {
|
||||
if detectors.IsKnownFalsePositive(string(s1.Raw), detectors.DefaultFalsePositives, true) {
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
results = append(results, s1)
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
return results, nil
|
||||
}
|
||||
|
||||
func (s Scanner) Type() detectorspb.DetectorType {
|
||||
return detectorspb.DetectorType_JiraToken
|
||||
}
|
||||
@@ -0,0 +1,163 @@
|
||||
//go:build detectors
|
||||
// +build detectors
|
||||
|
||||
package jiratoken_v2
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/google/go-cmp/cmp"
|
||||
"github.com/google/go-cmp/cmp/cmpopts"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
|
||||
)
|
||||
|
||||
func TestJiraToken_FromChunk(t *testing.T) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
|
||||
defer cancel()
|
||||
testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")
|
||||
if err != nil {
|
||||
t.Fatalf("could not get test secrets from GCP: %s", err)
|
||||
}
|
||||
secret := testSecrets.MustGetField("JIRA_V2_TOKEN")
|
||||
inactiveSecret := testSecrets.MustGetField("JIRA_V2_INACTIVE")
|
||||
email := testSecrets.MustGetField("JIRA_V2_EMAIL")
|
||||
domain := testSecrets.MustGetField("JIRA_V2_DOMAIN")
|
||||
|
||||
type args struct {
|
||||
ctx context.Context
|
||||
data []byte
|
||||
verify bool
|
||||
}
|
||||
tests := []struct {
|
||||
name string
|
||||
s Scanner
|
||||
args args
|
||||
want []detectors.Result
|
||||
wantErr bool
|
||||
wantVerificationErr bool
|
||||
}{
|
||||
{
|
||||
name: "found, verified",
|
||||
s: Scanner{},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a jira secret %s within jira %s with jira %s", secret, email, domain)),
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_JiraToken,
|
||||
Verified: true,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
wantVerificationErr: false,
|
||||
},
|
||||
{
|
||||
name: "found, unverified",
|
||||
s: Scanner{},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a jira secret %s within jira %s but not jira %s valid", inactiveSecret, email, domain)), // the secret would satisfy the regex but not pass validation
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_JiraToken,
|
||||
Verified: false,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
wantVerificationErr: false,
|
||||
},
|
||||
{
|
||||
name: "not found",
|
||||
s: Scanner{},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte("You cannot find the secret within"),
|
||||
verify: true,
|
||||
},
|
||||
want: nil,
|
||||
wantErr: false,
|
||||
wantVerificationErr: false,
|
||||
},
|
||||
{
|
||||
name: "found, would be verified if not for timeout",
|
||||
s: Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a jira secret %s within jira %s with jira %s", secret, email, domain)),
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_JiraToken,
|
||||
Verified: false,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
wantVerificationErr: true,
|
||||
},
|
||||
{
|
||||
name: "found, verified but unexpected api surface",
|
||||
s: Scanner{client: common.ConstantResponseHttpClient(404, "")},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a jira secret %s within jira %s with jira %s", secret, email, domain)),
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_JiraToken,
|
||||
Verified: false,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
wantVerificationErr: true,
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
|
||||
if (err != nil) != tt.wantErr {
|
||||
t.Errorf("JiraToken.FromData() error = %v, wantErr %v", err, tt.wantErr)
|
||||
return
|
||||
}
|
||||
for i := range got {
|
||||
if len(got[i].Raw) == 0 {
|
||||
t.Fatalf("no raw secret present: \n %+v", got[i])
|
||||
}
|
||||
if (got[i].VerificationError != nil) != tt.wantVerificationErr {
|
||||
t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError)
|
||||
}
|
||||
}
|
||||
ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "RawV2", "VerificationError")
|
||||
if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {
|
||||
t.Errorf("JiraToken.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkFromData(benchmark *testing.B) {
|
||||
ctx := context.Background()
|
||||
s := Scanner{}
|
||||
for name, data := range detectors.MustGetBenchmarkData() {
|
||||
benchmark.Run(name, func(b *testing.B) {
|
||||
b.ResetTimer()
|
||||
for n := 0; n < b.N; n++ {
|
||||
_, err := s.FromData(ctx, false, data)
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -83,8 +83,8 @@ func (s Scanner) Type() detectorspb.DetectorType {
|
||||
|
||||
type typeRes struct {
|
||||
Types []struct {
|
||||
Name string `json:"name"`
|
||||
Type interface{} `json:"type"`
|
||||
Parameters []interface{} `json:"parameters"`
|
||||
Name string `json:"name"`
|
||||
Type any `json:"type"`
|
||||
Parameters []any `json:"parameters"`
|
||||
} `json:"types"`
|
||||
}
|
||||
|
||||
@@ -21,13 +21,13 @@ var (
|
||||
client = common.SaneHttpClient()
|
||||
|
||||
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
|
||||
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"pagerduty"}) + `\b([a-z]{1}\+[a-zA-Z]{9}\-[a-z]{2}\-[a-z0-9]{5})\b`)
|
||||
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"pagerduty", "pd"}) + `\b([a-zA-Z0-9_+-]{20})\b`)
|
||||
)
|
||||
|
||||
// Keywords are used for efficiently pre-filtering chunks.
|
||||
// Use identifiers in the secret preferably, or the provider name.
|
||||
func (s Scanner) Keywords() []string {
|
||||
return []string{"pagerduty"}
|
||||
return []string{"pagerduty", "pd"}
|
||||
}
|
||||
|
||||
// FromData will find and optionally verify PagerDutyApiKey secrets in a given set of bytes.
|
||||
|
||||
@@ -9,7 +9,7 @@ import (
|
||||
"golang.org/x/crypto/ssh"
|
||||
)
|
||||
|
||||
//go:embed "rockyou-15.txt"
|
||||
//go:embed "list.txt"
|
||||
var rawCrackList []byte
|
||||
var passphrases [][]byte
|
||||
|
||||
@@ -21,17 +21,17 @@ var (
|
||||
ErrUncrackable = errors.New("unable to crack encryption")
|
||||
)
|
||||
|
||||
func crack(in []byte) (interface{}, error) {
|
||||
func crack(in []byte) (any, string, error) {
|
||||
for _, passphrase := range passphrases {
|
||||
parsed, err := ssh.ParseRawPrivateKeyWithPassphrase(in, passphrase)
|
||||
if err != nil {
|
||||
if errors.Is(err, x509.IncorrectPasswordError) {
|
||||
continue
|
||||
} else {
|
||||
return nil, err
|
||||
return nil, "", err
|
||||
}
|
||||
}
|
||||
return parsed, nil
|
||||
return parsed, string(passphrase), nil
|
||||
}
|
||||
return nil, ErrUncrackable
|
||||
return nil, "", ErrUncrackable
|
||||
}
|
||||
|
||||
@@ -56,23 +56,29 @@ Hx7UPVlTK8dyvk1Z+Yw0nrfNClI=
|
||||
|
||||
func Test_crack(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
in []byte
|
||||
wantErr bool
|
||||
name string
|
||||
in []byte
|
||||
wantedPassphrase string
|
||||
wantErr bool
|
||||
}{
|
||||
{
|
||||
name: "crackable",
|
||||
wantErr: false,
|
||||
in: testEncryptedKey,
|
||||
name: "crackable",
|
||||
wantErr: false,
|
||||
in: testEncryptedKey,
|
||||
wantedPassphrase: string(testEncryptedKeyCorrectPassword),
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
_, err := crack(tt.in)
|
||||
_, passphrase, err := crack(tt.in)
|
||||
if (err != nil) != tt.wantErr {
|
||||
t.Errorf("crack() error = %v, wantErr %v", err, tt.wantErr)
|
||||
return
|
||||
}
|
||||
|
||||
if passphrase != string(tt.wantedPassphrase) {
|
||||
t.Errorf("crack() passphrase = %v, want %v", passphrase, string(testEncryptedKeyCorrectPassword))
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -5,10 +5,12 @@ import (
|
||||
"crypto/ed25519"
|
||||
"crypto/rsa"
|
||||
"crypto/sha1"
|
||||
"crypto/sha256"
|
||||
"crypto/x509"
|
||||
"encoding/base64"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"strings"
|
||||
"fmt"
|
||||
|
||||
"golang.org/x/crypto/ssh"
|
||||
)
|
||||
@@ -18,18 +20,8 @@ var (
|
||||
ErrEncryptedKey = errors.New("key is encrypted")
|
||||
)
|
||||
|
||||
func FingerprintPEMKey(in []byte) (string, error) {
|
||||
parsedKey, err := ssh.ParseRawPrivateKey(in)
|
||||
if err != nil && strings.Contains(err.Error(), "private key is passphrase protected") {
|
||||
parsedKey, err = crack(in)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
} else if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
var pubKey interface{}
|
||||
func FingerprintPEMKey(parsedKey any) (string, error) {
|
||||
var pubKey any
|
||||
switch privateKey := parsedKey.(type) {
|
||||
case *rsa.PrivateKey:
|
||||
pubKey = &privateKey.PublicKey
|
||||
@@ -44,6 +36,10 @@ func FingerprintPEMKey(in []byte) (string, error) {
|
||||
return "", ErrNotSupported
|
||||
}
|
||||
|
||||
return fingerprintPublicKey(pubKey)
|
||||
}
|
||||
|
||||
func fingerprintPublicKey(pubKey any) (string, error) {
|
||||
publickeyBytes, err := x509.MarshalPKIXPublicKey(pubKey)
|
||||
if err != nil {
|
||||
return "", err
|
||||
@@ -53,3 +49,8 @@ func FingerprintPEMKey(in []byte) (string, error) {
|
||||
publicKeyFingerprintInHex := hex.EncodeToString(publicKeyFingerprint[:])
|
||||
return publicKeyFingerprintInHex, nil
|
||||
}
|
||||
|
||||
func fingerprintSSHPublicKey(pubKey ssh.PublicKey) string {
|
||||
publicKeyFingerprint := sha256.Sum256(pubKey.Marshal())
|
||||
return fmt.Sprintf("SHA256:%s", base64.RawStdEncoding.EncodeToString(publicKeyFingerprint[:]))
|
||||
}
|
||||
|
||||
@@ -6,11 +6,14 @@ import (
|
||||
"fmt"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/AzureAD/microsoft-authentication-library-for-go/apps/errors"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
|
||||
"golang.org/x/crypto/ssh"
|
||||
)
|
||||
|
||||
type Scanner struct {
|
||||
@@ -53,19 +56,67 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) ([]dete
|
||||
Redacted: token[0:64],
|
||||
}
|
||||
|
||||
fingerprint, err := FingerprintPEMKey([]byte(token))
|
||||
secret.ExtraData = make(map[string]string)
|
||||
|
||||
var passphrase string
|
||||
parsedKey, err := ssh.ParseRawPrivateKey([]byte(token))
|
||||
if err != nil && strings.Contains(err.Error(), "private key is passphrase protected") {
|
||||
secret.ExtraData["encrypted"] = "true"
|
||||
parsedKey, passphrase, err = crack([]byte(token))
|
||||
if err != nil {
|
||||
secret.VerificationError = err
|
||||
continue
|
||||
}
|
||||
if passphrase != "" {
|
||||
secret.ExtraData["cracked_encryption_passphrase"] = "true"
|
||||
}
|
||||
} else if err != nil {
|
||||
// couldn't parse key, probably invalid
|
||||
continue
|
||||
}
|
||||
|
||||
fingerprint, err := FingerprintPEMKey(parsedKey)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
|
||||
if verify {
|
||||
verificationErrors := []string{}
|
||||
data, err := lookupFingerprint(fingerprint, s.IncludeExpired)
|
||||
if err == nil {
|
||||
secret.StructuredData = data
|
||||
if data != nil {
|
||||
secret.Verified = true
|
||||
secret.ExtraData["certificate_urls"] = strings.Join(data.CertificateURLs, ", ")
|
||||
}
|
||||
} else {
|
||||
verificationErrors = append(verificationErrors, err.Error())
|
||||
}
|
||||
|
||||
user, err := verifyGitHubUser(parsedKey)
|
||||
if err != nil && !errors.Is(err, errPermissionDenied) {
|
||||
verificationErrors = append(verificationErrors, err.Error())
|
||||
}
|
||||
if user != nil {
|
||||
secret.Verified = true
|
||||
secret.ExtraData["github_user"] = *user
|
||||
}
|
||||
|
||||
user, err = verifyGitLabUser(parsedKey)
|
||||
if err != nil && !errors.Is(err, errPermissionDenied) {
|
||||
verificationErrors = append(verificationErrors, err.Error())
|
||||
}
|
||||
if user != nil {
|
||||
secret.Verified = true
|
||||
secret.ExtraData["gitlab_user"] = *user
|
||||
}
|
||||
|
||||
if !secret.Verified && len(verificationErrors) > 0 {
|
||||
secret.VerificationError = fmt.Errorf("verification failures: %s", strings.Join(verificationErrors, ", "))
|
||||
}
|
||||
}
|
||||
|
||||
if len(secret.ExtraData) == 0 {
|
||||
secret.ExtraData = nil
|
||||
}
|
||||
|
||||
results = append(results, secret)
|
||||
@@ -74,23 +125,30 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) ([]dete
|
||||
return results, nil
|
||||
}
|
||||
|
||||
func lookupFingerprint(publicKeyFingerprintInHex string, includeExpired bool) (data *detectorspb.StructuredData, err error) {
|
||||
type result struct {
|
||||
CertificateURLs []string
|
||||
GitHubUsername string
|
||||
}
|
||||
|
||||
func lookupFingerprint(publicKeyFingerprintInHex string, includeExpired bool) (*result, error) {
|
||||
req, err := http.NewRequest("GET", fmt.Sprintf("https://keychecker.trufflesecurity.com/fingerprint/%s", publicKeyFingerprintInHex), nil)
|
||||
if err != nil {
|
||||
return
|
||||
return nil, err
|
||||
}
|
||||
res, err := client.Do(req)
|
||||
if err != nil {
|
||||
return
|
||||
return nil, err
|
||||
}
|
||||
defer res.Body.Close()
|
||||
|
||||
results := DriftwoodResult{}
|
||||
err = json.NewDecoder(res.Body).Decode(&results)
|
||||
if err != nil {
|
||||
return
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var data *result
|
||||
|
||||
seen := map[string]struct{}{}
|
||||
for _, r := range results.CertificateResults {
|
||||
if _, ok := seen[r.CertificateFingerprint]; ok {
|
||||
@@ -100,36 +158,13 @@ func lookupFingerprint(publicKeyFingerprintInHex string, includeExpired bool) (d
|
||||
continue
|
||||
}
|
||||
if data == nil {
|
||||
data = &detectorspb.StructuredData{}
|
||||
data = &result{}
|
||||
}
|
||||
if data.TlsPrivateKey == nil {
|
||||
data.TlsPrivateKey = make([]*detectorspb.TlsPrivateKey, 0)
|
||||
}
|
||||
data.TlsPrivateKey = append(data.TlsPrivateKey, &detectorspb.TlsPrivateKey{
|
||||
CertificateFingerprint: r.CertificateFingerprint,
|
||||
ExpirationTimestamp: r.ExpirationTimestamp.Unix(),
|
||||
VerificationUrl: fmt.Sprintf("https://crt.sh/?q=%s", r.CertificateFingerprint),
|
||||
})
|
||||
data.CertificateURLs = append(data.CertificateURLs, fmt.Sprintf("https://crt.sh/?q=%s", r.CertificateFingerprint))
|
||||
seen[r.CertificateFingerprint] = struct{}{}
|
||||
}
|
||||
|
||||
for _, r := range results.GitHubSSHResults {
|
||||
if _, ok := seen[r.Username]; ok {
|
||||
continue
|
||||
}
|
||||
if data == nil {
|
||||
data = &detectorspb.StructuredData{}
|
||||
}
|
||||
if data.GithubSshKey == nil {
|
||||
data.GithubSshKey = make([]*detectorspb.GitHubSSHKey, 0)
|
||||
}
|
||||
data.GithubSshKey = append(data.GithubSshKey, &detectorspb.GitHubSSHKey{
|
||||
User: r.Username,
|
||||
})
|
||||
seen[r.Username] = struct{}{}
|
||||
}
|
||||
|
||||
return
|
||||
return data, nil
|
||||
}
|
||||
|
||||
type DriftwoodResult struct {
|
||||
|
||||
@@ -20,13 +20,15 @@ import (
|
||||
func TestPrivatekey_FromChunk(t *testing.T) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
|
||||
defer cancel()
|
||||
testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")
|
||||
testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors4")
|
||||
if err != nil {
|
||||
t.Fatalf("could not get test secrets from GCP: %s", err)
|
||||
}
|
||||
secretTLS := testSecrets.MustGetField("PRIVATEKEY_TLS")
|
||||
secretGitHub := testSecrets.MustGetField("PRIVATEKEY_GITHUB")
|
||||
secretGitHubEncrypted := testSecrets.MustGetField("PRIVATEKEY_GITHUB_ENCRYPTED")
|
||||
secretInactive := testSecrets.MustGetField("PRIVATEKEY_UNVERIFIED")
|
||||
|
||||
type args struct {
|
||||
ctx context.Context
|
||||
data []byte
|
||||
@@ -75,19 +77,8 @@ func TestPrivatekey_FromChunk(t *testing.T) {
|
||||
DetectorType: detectorspb.DetectorType_PrivateKey,
|
||||
Verified: true,
|
||||
Redacted: "-----BEGIN PRIVATE KEY-----\nMIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgw",
|
||||
StructuredData: &detectorspb.StructuredData{
|
||||
TlsPrivateKey: []*detectorspb.TlsPrivateKey{
|
||||
{
|
||||
CertificateFingerprint: "1e20c40deb44a8539dd3ac3e8c53b72750cb19f9",
|
||||
VerificationUrl: "https://crt.sh/?q=1e20c40deb44a8539dd3ac3e8c53b72750cb19f9",
|
||||
ExpirationTimestamp: 1497337731,
|
||||
},
|
||||
{
|
||||
CertificateFingerprint: "0e9de31fb2ee16465a4d5d93b227d54f870326d1",
|
||||
VerificationUrl: "https://crt.sh/?q=0e9de31fb2ee16465a4d5d93b227d54f870326d1",
|
||||
ExpirationTimestamp: 1528192832,
|
||||
},
|
||||
},
|
||||
ExtraData: map[string]string{
|
||||
"certificate_urls": "https://crt.sh/?q=1e20c40deb44a8539dd3ac3e8c53b72750cb19f9, https://crt.sh/?q=0e9de31fb2ee16465a4d5d93b227d54f870326d1",
|
||||
},
|
||||
},
|
||||
},
|
||||
@@ -106,12 +97,8 @@ func TestPrivatekey_FromChunk(t *testing.T) {
|
||||
DetectorType: detectorspb.DetectorType_PrivateKey,
|
||||
Verified: true,
|
||||
Redacted: "-----BEGIN OPENSSH PRIVATE KEY-----\nb3BlbnNzaC1rZXktdjEAAAAABG5v",
|
||||
StructuredData: &detectorspb.StructuredData{
|
||||
GithubSshKey: []*detectorspb.GitHubSSHKey{
|
||||
{
|
||||
User: "thisisforgithub0",
|
||||
},
|
||||
},
|
||||
ExtraData: map[string]string{
|
||||
"github_user": "sirdetectsalot",
|
||||
},
|
||||
},
|
||||
},
|
||||
@@ -121,47 +108,8 @@ func TestPrivatekey_FromChunk(t *testing.T) {
|
||||
name: "found encrypted GitHub SSH private key, verified",
|
||||
s: Scanner{},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(`-----BEGIN OPENSSH PRIVATE KEY-----
|
||||
b3BlbnNzaC1rZXktdjEAAAAACmFlczI1Ni1jdHIAAAAGYmNyeXB0AAAAGAAAABAjNIZuun
|
||||
xgLkM8KuzfmQuRAAAAEAAAAAEAAAGXAAAAB3NzaC1yc2EAAAADAQABAAABgQDe3Al0EMPz
|
||||
utVNk5DixaYrGMK56RqUoqGBinke6SWVWmqom1lBcJWzor6HlnMRPPr7YCEsJKL4IpuVwu
|
||||
inRa5kdtNTyM7yyQTSR2xXCS0fUItNuq8pUktsH8VUggpMeew8hJv7rFA7tnIg3UXCl6iF
|
||||
OLZKbDA5aa24idpcD8b1I9/RzTOB1fu0of5xd9vgODzGw5JvHQSJ0FaA42aNBMGwrDhDB3
|
||||
sgnRNdWf6NNIh8KpXXMKJADf3klsyn6He8L2bPMp8a4wwys2YB35p5zQ0JURovsdewlOxH
|
||||
NT7eP19eVf4dCreibxUmRUaob5DEoHEk8WrxjKWIYUuLeD6AfcW6oXyRU2Yy8Vrt6SqFl5
|
||||
WAi47VMFTkDZYS/eCvG53q9UBHpCj7Qvb0vSkCZXBvBIhlw193F3PX4WvO1IXsMwvQ1D1X
|
||||
lmomsItbqM0cJyKw6LU18QWiBHvE7BqcphaoL5E08W2ATTSRIMCp6rt4rptM7KyGK8rc6W
|
||||
UYrCnWt6KlCA8AAAWQXk+lVx6bH5itIKKYmQr6cR/5xtZ2GHAxnYtvlW3xnGhU0MHv+lJ2
|
||||
uoWlT2RXE5pdMUQj7rNWAMqkwifSKZs9wBfYeo1TaFDmC3nW7yHSN3XTuO78mPIW5JyvmE
|
||||
Rj5qjsUn7fNmzECoAxnVERhwnF3KqUBEPzIAc6/7v/na9NTiiGaJPco9lvCoPWbVLN08WG
|
||||
SuyU+0x5zc3ebzuPcYqu5/c5nmiGxhALrIhjIS0OV1mtAAFhvdMjMIHOijOzSKVCC7rRk5
|
||||
kG9EMLNvOn/DUVSRHamw5gs2V3V+Zq2g5nYWfgq8aDSTB8XlIzOj1cz3HwfN6pfSNQ/3Qe
|
||||
wOQfWfTWdO+JSL8aoBN5Wg8tDbgmvmbFrINsJfFfSm0wZgcHhC7Ul4U3v4c8PoNdK9HXwi
|
||||
TKKzJ9nxLYb+vDh50cnkseu2gt0KwVpjIorxEqeK755mKPao3JmOMr6uFTQsb+g+ZNgPwl
|
||||
nRHA4Igx+zADFj3twldnKIiRpBQ5J4acur3uQ+saanBTXgul1TiFiUGT2cnz+IiCsdPovg
|
||||
TAMt868W5LmzpfH4Cy54JtaRC4/UuMnkTGbWgutVDnWj2stOAzsQ1YmhH5igUmc94mUL+W
|
||||
8vQDCKpeI8n+quDS9zxTvy4L4H5Iz7OZlh0h6N13BDvCYXKcNF/ugkfxZbu8mZsZQQzXNR
|
||||
wOrEtKoHc4AnXYNzsuHEoEyLyJxGfFRDSTLbyN9wFOS/c0k9Gjte+kQRZjBVGORE5sN6X3
|
||||
akUnTF76RhbEc+LamrwM1h5340bwosRbR8I+UrsQdFfJBEj1ZSyMRJlMkFUNi6blt7bhyx
|
||||
ea+Pm2A614nlYUBjw2KKzzn8N/0H2NpJjIptvDsbrx3BS/rKwOeJwavRrGnIlEzuAag4vx
|
||||
Zb2TPVta45uz7fQP5IBl83b0BJKI5Zv/fniUeLI78W/UsZqb64YQbfRyBzFtI1T/SsCi0B
|
||||
e0EyKMzbxtSceT1Mb8eJiVIq04Xpwez9fIUt5rSedZD8KPq8P6s0cGsR7Qmw6eXZ/dBR/a
|
||||
s5vPhfIUmQawmnwAVuWNRdQQ79jUBSn5M+ZRVVTgEG+vFyvxr/bZqOo1JCoq5BmQhLWGRJ
|
||||
Dk9TolbeFIVFrkuXkcu99a079ux7XSkON64oPzHrcsEzjPA1GPqs9CGBSO16wq/nI3zg+E
|
||||
kcOCaurc9yHJJPwduem0+8WLX3WoGNfQRKurtQze2ppy8KarEtDhDd96sKkhYaqOg3GOX8
|
||||
Yx827L4vuWSJSIqKuO2kH6kOCMUNO16piv0z/8u3CJxOGh9+4FZIop81fiFTKLhV3/gwLm
|
||||
fzFY++KIZrLfZcUjzd80NNEja69F452Eb9HrI5BurN/PznDEi9bzM598Y7beyl4/kd4R2e
|
||||
S7SW9/LOrGw5UgxtiU+kV8nPz1PdgxO4sRlnntSBEwkQBzMkLOpq2h2BuJ2TlMP/TWuwLQ
|
||||
sDkv1Yk1pD0roGmtMzbujnURGxqRJ8gUmuIot4hpfyRSssvnRQQZ3lQCQCwHiE+HJxXWf5
|
||||
c58zOMjW7o21tI8e13uUnbRoQVJM9XYqk1usPXIkYPYL9uOw3AW/Zn+cnDrsXvTK9ZxgGD
|
||||
/90b1BNwVqMlUK+QggHNwl5qD8eoXK5cDvav66te+E+V7FYFQ06w3tytRVz8SjoaiChN02
|
||||
muIjvl6G7Hoj1hObM2t/ZheN1EShS11z868hhS6Mx7GvIdtkXuvdiBYMiBLOshJQxB8Mzx
|
||||
iug9W+Di3upLf0UMC1TqADGphsIHRU7RbmHQ8Rwp7dogswmDfpRSapPt9p0D+6Ad5VBzi3
|
||||
f3BPXj76UBLMEJCrZR1P28vnAA7AyNHaLvMPlWDMG5v3V/UV+ugyFcoBAOyjiQgYST8F3e
|
||||
Hx7UPVlTK8dyvk1Z+Yw0nrfNClI=
|
||||
-----END OPENSSH PRIVATE KEY-----
|
||||
`),
|
||||
ctx: context.Background(),
|
||||
data: []byte(secretGitHubEncrypted),
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
@@ -169,12 +117,10 @@ func TestPrivatekey_FromChunk(t *testing.T) {
|
||||
DetectorType: detectorspb.DetectorType_PrivateKey,
|
||||
Verified: true,
|
||||
Redacted: "-----BEGIN OPENSSH PRIVATE KEY-----\nb3BlbnNzaC1rZXktdjEAAAAACmFl",
|
||||
StructuredData: &detectorspb.StructuredData{
|
||||
GithubSshKey: []*detectorspb.GitHubSSHKey{
|
||||
{
|
||||
User: "thisisforgithub0",
|
||||
},
|
||||
},
|
||||
ExtraData: map[string]string{
|
||||
"github_user": "sirdetectsalot",
|
||||
"encrypted": "true",
|
||||
"cracked_encryption_passphrase": "true",
|
||||
},
|
||||
},
|
||||
},
|
||||
@@ -262,7 +208,7 @@ func Test_lookupFingerprint(t *testing.T) {
|
||||
t.Errorf("lookupFingerprint() error = %v, wantErr %v", err, tt.wantErr)
|
||||
return
|
||||
}
|
||||
if !reflect.DeepEqual(gotFingerprints != nil && len(gotFingerprints.TlsPrivateKey) > 0, tt.wantFingerprints) {
|
||||
if !reflect.DeepEqual(gotFingerprints != nil && len(gotFingerprints.CertificateURLs) > 0, tt.wantFingerprints) {
|
||||
t.Errorf("lookupFingerprint() = %v, want %v", gotFingerprints, tt.wantFingerprints)
|
||||
}
|
||||
})
|
||||
|
||||
@@ -0,0 +1,125 @@
|
||||
package privatekey
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"golang.org/x/crypto/ssh"
|
||||
)
|
||||
|
||||
// https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/githubs-ssh-key-fingerprints
|
||||
var githubFingerprints = map[string]string{
|
||||
"SHA256:uNiVztksCsDhcc0u9e8BujQXVUpKZIDTMczCvj3tD2s": "RSA",
|
||||
"SHA256:br9IjFspm1vxR3iA35FWE+4VTyz1hYVLIE2t1/CeyWQ": "DSA - deprecated",
|
||||
"SHA256:p2QAMXNIC1TJYWeIOttrVc98/R1BUFWu3/LiyKgUfQM": "ECDSA",
|
||||
"SHA256:+DiY3wvvV6TuJJhbpZisF/zLDA0zPMSvHdkr4UvCOqU": "ED25519",
|
||||
}
|
||||
|
||||
// https://docs.gitlab.com/ee/user/gitlab_com/index.html#ssh-host-keys-fingerprints
|
||||
var gitlabFingerprints = map[string]string{
|
||||
"SHA256:HbW3g8zUjNSksFbqTiUWPWg2Bq1x8xdGUrliXFzSnUw": "ECDSA",
|
||||
"SHA256:eUXGGm1YGsMAS7vkcx6JOJdOGHPem5gQp4taiCfCLB8": "ED25519",
|
||||
"SHA256:ROQFvPThGrW4RuWLoL9tq9I9zJ42fK4XywyRtbOz/EQ": "RSA",
|
||||
}
|
||||
|
||||
func firstResponseFromSSH(parsedKey any, username, hostport string) (string, error) {
|
||||
signer, err := ssh.NewSignerFromKey(parsedKey)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
// Verify the server fingerprint to ensure that there is no MITM replay attack
|
||||
config := &ssh.ClientConfig{
|
||||
Timeout: 3 * time.Second,
|
||||
User: username,
|
||||
Auth: []ssh.AuthMethod{
|
||||
ssh.PublicKeys(signer),
|
||||
},
|
||||
HostKeyCallback: func(hostname string, _ net.Addr, key ssh.PublicKey) error {
|
||||
switch hostname {
|
||||
case "github.com:22":
|
||||
fingerprint := fingerprintSSHPublicKey(key)
|
||||
if _, ok := githubFingerprints[fingerprint]; !ok {
|
||||
return fmt.Errorf("unknown host fingerprint for github.com, got %s", fingerprint)
|
||||
}
|
||||
case "gitlab.com:22":
|
||||
fingerprint := fingerprintSSHPublicKey(key)
|
||||
if _, ok := gitlabFingerprints[fingerprint]; !ok {
|
||||
return fmt.Errorf("unknown host fingerprint for gitlab.com, got %s", fingerprint)
|
||||
}
|
||||
default:
|
||||
return errors.New("unknown host in fingerprint db")
|
||||
}
|
||||
return nil
|
||||
},
|
||||
}
|
||||
|
||||
client, err := ssh.Dial("tcp", hostport, config)
|
||||
if err != nil {
|
||||
if strings.Contains(err.Error(), "unable to authenticate") {
|
||||
return "", errPermissionDenied
|
||||
}
|
||||
return "", err
|
||||
}
|
||||
defer client.Close()
|
||||
|
||||
session, err := client.NewSession()
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer session.Close()
|
||||
|
||||
var output bytes.Buffer
|
||||
session.Stderr = &output
|
||||
|
||||
err = session.Shell()
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
_ = session.Wait()
|
||||
|
||||
return output.String(), err
|
||||
}
|
||||
|
||||
var errPermissionDenied = errors.New("permission denied")
|
||||
|
||||
func verifyGitHubUser(parsedKey any) (*string, error) {
|
||||
output, err := firstResponseFromSSH(parsedKey, "git", "github.com:22")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if strings.Contains(output, "Permission denied") {
|
||||
return nil, errPermissionDenied
|
||||
}
|
||||
|
||||
if strings.Contains(output, "successfully authenticated") {
|
||||
username := strings.TrimSuffix(strings.Split(output, " ")[1], "!")
|
||||
return &username, nil
|
||||
}
|
||||
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func verifyGitLabUser(parsedKey any) (*string, error) {
|
||||
output, err := firstResponseFromSSH(parsedKey, "git", "gitlab.com:22")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if strings.Contains(output, "Permission denied") {
|
||||
return nil, errPermissionDenied
|
||||
}
|
||||
|
||||
if strings.Contains(output, "Welcome to GitLab") {
|
||||
split := strings.Split(output, " ")
|
||||
username := strings.TrimPrefix(strings.TrimSuffix(split[len(split)-1], "!"), "@")
|
||||
return &username, nil
|
||||
}
|
||||
|
||||
return nil, nil
|
||||
}
|
||||
@@ -0,0 +1,34 @@
|
||||
package privatekey
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"golang.org/x/crypto/ssh"
|
||||
)
|
||||
|
||||
func TestFirstResponseFromSSH(t *testing.T) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
|
||||
defer cancel()
|
||||
testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors4")
|
||||
if err != nil {
|
||||
t.Fatalf("could not get test secrets from GCP: %s", err)
|
||||
}
|
||||
secretGitHub := testSecrets.MustGetField("PRIVATEKEY_GITHUB")
|
||||
|
||||
parsedKey, err := ssh.ParseRawPrivateKey([]byte(normalize(secretGitHub)))
|
||||
if err != nil {
|
||||
t.Fatalf("could not parse test secret: %s", err)
|
||||
}
|
||||
|
||||
output, err := firstResponseFromSSH(parsedKey, "git", "github.com:22")
|
||||
if err != nil {
|
||||
t.Fail()
|
||||
}
|
||||
|
||||
if len(output) == 0 {
|
||||
t.Fail()
|
||||
}
|
||||
}
|
||||
@@ -47,7 +47,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
}
|
||||
|
||||
if verify {
|
||||
req, err := http.NewRequestWithContext(ctx, "GET", "https://rubygems.org/api/v1/gems", nil)
|
||||
req, err := http.NewRequestWithContext(ctx, "GET", "https://rubygems.org/api/v1/gems.json", nil)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
@@ -56,7 +56,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
res, err := client.Do(req)
|
||||
if err == nil {
|
||||
defer res.Body.Close()
|
||||
if res.StatusCode >= 200 && res.StatusCode < 300 {
|
||||
if res.StatusCode >= 200 && res.StatusCode < 300 || res.StatusCode == http.StatusForbidden {
|
||||
s1.Verified = true
|
||||
} else {
|
||||
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
|
||||
|
||||
@@ -2,6 +2,7 @@ package sendbird
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"regexp"
|
||||
@@ -12,19 +13,25 @@ import (
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
|
||||
)
|
||||
|
||||
type Scanner struct{}
|
||||
type Scanner struct {
|
||||
client *http.Client
|
||||
}
|
||||
|
||||
// Ensure the Scanner satisfies the interface at compile time.
|
||||
var _ detectors.Detector = (*Scanner)(nil)
|
||||
|
||||
var (
|
||||
client = common.SaneHttpClient()
|
||||
defaultClient = common.SaneHttpClient()
|
||||
|
||||
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
|
||||
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"sendbird"}) + `\b([0-9a-f]{40})\b`)
|
||||
appIdPat = regexp.MustCompile(detectors.PrefixRegex([]string{"sendbird"}) + `\b([0-9A-F]{8}-[0-9A-F]{4}-[0-9A-F]{4}-[0-9A-F]{4}-[0-9A-F]{12})\b`)
|
||||
)
|
||||
|
||||
type userResp struct {
|
||||
Code int `json:"code"`
|
||||
}
|
||||
|
||||
// Keywords are used for efficiently pre-filtering chunks.
|
||||
// Use identifiers in the secret preferably, or the provider name.
|
||||
func (s Scanner) Keywords() []string {
|
||||
@@ -61,20 +68,40 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
continue
|
||||
}
|
||||
req.Header.Add("Api-Token", resMatch)
|
||||
|
||||
client := s.client
|
||||
if client == nil {
|
||||
client = defaultClient
|
||||
}
|
||||
|
||||
res, err := client.Do(req)
|
||||
if err == nil {
|
||||
defer res.Body.Close()
|
||||
if res.StatusCode >= 200 && res.StatusCode < 300 {
|
||||
s1.Verified = true
|
||||
} else {
|
||||
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
|
||||
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
|
||||
continue
|
||||
} else if res.StatusCode == 400 { // Sendbird returns 400 for all errors
|
||||
var userResp userResp
|
||||
err := json.NewDecoder(res.Body).Decode(&userResp)
|
||||
if err != nil {
|
||||
s1.VerificationError = fmt.Errorf("error decoding json response body: %w", err)
|
||||
} else if userResp.Code != 400401 {
|
||||
// https://sendbird.com/docs/chat/platform-api/v3/error-codes
|
||||
// Sendbird always includes its own error codes with 400 responses
|
||||
// 400401 (InvalidApiToken) is the only one that indicates a bad token
|
||||
s1.VerificationError = fmt.Errorf("unexpected response code: %d", userResp.Code)
|
||||
}
|
||||
} else {
|
||||
s1.VerificationError = fmt.Errorf("unexpected HTTP response status %d", res.StatusCode)
|
||||
}
|
||||
} else {
|
||||
s1.VerificationError = err
|
||||
}
|
||||
}
|
||||
|
||||
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
|
||||
if !s1.Verified && detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
|
||||
continue
|
||||
}
|
||||
results = append(results, s1)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -9,8 +9,8 @@ import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/kylelemons/godebug/pretty"
|
||||
|
||||
"github.com/google/go-cmp/cmp"
|
||||
"github.com/google/go-cmp/cmp/cmpopts"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
|
||||
@@ -19,7 +19,7 @@ import (
|
||||
func TestSendbird_FromChunk(t *testing.T) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
|
||||
defer cancel()
|
||||
testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
|
||||
testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")
|
||||
if err != nil {
|
||||
t.Fatalf("could not get test secrets from GCP: %s", err)
|
||||
}
|
||||
@@ -33,11 +33,12 @@ func TestSendbird_FromChunk(t *testing.T) {
|
||||
verify bool
|
||||
}
|
||||
tests := []struct {
|
||||
name string
|
||||
s Scanner
|
||||
args args
|
||||
want []detectors.Result
|
||||
wantErr bool
|
||||
name string
|
||||
s Scanner
|
||||
args args
|
||||
want []detectors.Result
|
||||
wantErr bool
|
||||
wantVerificationErr bool
|
||||
}{
|
||||
{
|
||||
name: "found, verified",
|
||||
@@ -53,7 +54,8 @@ func TestSendbird_FromChunk(t *testing.T) {
|
||||
Verified: true,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
wantErr: false,
|
||||
wantVerificationErr: false,
|
||||
},
|
||||
{
|
||||
name: "found, unverified",
|
||||
@@ -69,7 +71,8 @@ func TestSendbird_FromChunk(t *testing.T) {
|
||||
Verified: false,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
wantErr: false,
|
||||
wantVerificationErr: false,
|
||||
},
|
||||
{
|
||||
name: "not found",
|
||||
@@ -79,14 +82,82 @@ func TestSendbird_FromChunk(t *testing.T) {
|
||||
data: []byte("You cannot find the secret within"),
|
||||
verify: true,
|
||||
},
|
||||
want: nil,
|
||||
wantErr: false,
|
||||
want: nil,
|
||||
wantErr: false,
|
||||
wantVerificationErr: false,
|
||||
},
|
||||
{
|
||||
name: "found, would be verified if not for timeout",
|
||||
s: Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a sendbird application ID %s with sendbird secret %s within", appId, secret)),
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_Sendbird,
|
||||
Verified: false,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
wantVerificationErr: true,
|
||||
},
|
||||
{
|
||||
name: "found, verified but unexpected api surface",
|
||||
s: Scanner{client: common.ConstantResponseHttpClient(404, "")},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a sendbird application ID %s with sendbird secret %s within", appId, secret)),
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_Sendbird,
|
||||
Verified: false,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
wantVerificationErr: true,
|
||||
},
|
||||
{
|
||||
name: "found, verified but unexpected error code",
|
||||
s: Scanner{client: common.ConstantResponseHttpClient(400, `{"code":400402}`)},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a sendbird application ID %s with sendbird secret %s within", appId, secret)),
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_Sendbird,
|
||||
Verified: false,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
wantVerificationErr: true,
|
||||
},
|
||||
{
|
||||
name: "found, verified but failed to decode json",
|
||||
s: Scanner{client: common.ConstantResponseHttpClient(400, `{`)},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a sendbird application ID %s with sendbird secret %s within", appId, secret)),
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_Sendbird,
|
||||
Verified: false,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
wantVerificationErr: true,
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
s := Scanner{}
|
||||
got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
|
||||
got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
|
||||
if (err != nil) != tt.wantErr {
|
||||
t.Errorf("Sendbird.FromData() error = %v, wantErr %v", err, tt.wantErr)
|
||||
return
|
||||
@@ -95,9 +166,12 @@ func TestSendbird_FromChunk(t *testing.T) {
|
||||
if len(got[i].Raw) == 0 {
|
||||
t.Fatalf("no raw secret present: \n %+v", got[i])
|
||||
}
|
||||
got[i].Raw = nil
|
||||
if (got[i].VerificationError != nil) != tt.wantVerificationErr {
|
||||
t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError)
|
||||
}
|
||||
}
|
||||
if diff := pretty.Compare(got, tt.want); diff != "" {
|
||||
ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "VerificationError")
|
||||
if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {
|
||||
t.Errorf("Sendbird.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
|
||||
}
|
||||
})
|
||||
|
||||
@@ -2,6 +2,7 @@ package sendbirdorganizationapi
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"strings"
|
||||
@@ -11,13 +12,15 @@ import (
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
|
||||
)
|
||||
|
||||
type Scanner struct{}
|
||||
type Scanner struct {
|
||||
client *http.Client
|
||||
}
|
||||
|
||||
// Ensure the Scanner satisfies the interface at compile time.
|
||||
var _ detectors.Detector = (*Scanner)(nil)
|
||||
|
||||
var (
|
||||
client = common.SaneHttpClient()
|
||||
defaultClient = common.SaneHttpClient()
|
||||
|
||||
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
|
||||
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"sendbird"}) + `\b([0-9a-f]{24})\b`)
|
||||
@@ -47,25 +50,33 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
}
|
||||
|
||||
if verify {
|
||||
client := s.client
|
||||
if client == nil {
|
||||
client = defaultClient
|
||||
}
|
||||
|
||||
req, err := http.NewRequestWithContext(ctx, "GET", "https://gate.sendbird.com/api/v2/applications", nil)
|
||||
if err != nil {
|
||||
continue
|
||||
s1.VerificationError = err
|
||||
}
|
||||
req.Header.Add("SENDBIRDORGANIZATIONAPITOKEN", resMatch)
|
||||
res, err := client.Do(req)
|
||||
if err == nil {
|
||||
if err != nil {
|
||||
s1.VerificationError = err
|
||||
} else {
|
||||
defer res.Body.Close()
|
||||
if res.StatusCode >= 200 && res.StatusCode < 300 {
|
||||
s1.Verified = true
|
||||
} else {
|
||||
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
|
||||
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
|
||||
continue
|
||||
}
|
||||
} else if res.StatusCode != http.StatusForbidden {
|
||||
s1.VerificationError = fmt.Errorf("unexpected HTTP response status %d", res.StatusCode)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
|
||||
if !s1.Verified && detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
|
||||
continue
|
||||
}
|
||||
results = append(results, s1)
|
||||
}
|
||||
|
||||
|
||||
@@ -9,7 +9,8 @@ import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/kylelemons/godebug/pretty"
|
||||
"github.com/google/go-cmp/cmp"
|
||||
"github.com/google/go-cmp/cmp/cmpopts"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
@@ -19,7 +20,7 @@ import (
|
||||
func TestSendbirdOrganizationAPI_FromChunk(t *testing.T) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
|
||||
defer cancel()
|
||||
testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
|
||||
testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")
|
||||
if err != nil {
|
||||
t.Fatalf("could not get test secrets from GCP: %s", err)
|
||||
}
|
||||
@@ -32,11 +33,12 @@ func TestSendbirdOrganizationAPI_FromChunk(t *testing.T) {
|
||||
verify bool
|
||||
}
|
||||
tests := []struct {
|
||||
name string
|
||||
s Scanner
|
||||
args args
|
||||
want []detectors.Result
|
||||
wantErr bool
|
||||
name string
|
||||
s Scanner
|
||||
args args
|
||||
want []detectors.Result
|
||||
wantErr bool
|
||||
wantVerificationErr bool
|
||||
}{
|
||||
{
|
||||
name: "found, verified",
|
||||
@@ -52,7 +54,8 @@ func TestSendbirdOrganizationAPI_FromChunk(t *testing.T) {
|
||||
Verified: true,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
wantErr: false,
|
||||
wantVerificationErr: false,
|
||||
},
|
||||
{
|
||||
name: "found, unverified",
|
||||
@@ -68,7 +71,42 @@ func TestSendbirdOrganizationAPI_FromChunk(t *testing.T) {
|
||||
Verified: false,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
wantErr: false,
|
||||
wantVerificationErr: false,
|
||||
},
|
||||
{
|
||||
name: "found, would be verified if not for timeout",
|
||||
s: Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a sendbird secret %s within", secret)),
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_SendbirdOrganizationAPI,
|
||||
Verified: false,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
wantVerificationErr: true,
|
||||
},
|
||||
{
|
||||
name: "found, verified but unexpected api surface",
|
||||
s: Scanner{client: common.ConstantResponseHttpClient(404, "")},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a sendbird secret %s within", secret)),
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_SendbirdOrganizationAPI,
|
||||
Verified: false,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
wantVerificationErr: true,
|
||||
},
|
||||
{
|
||||
name: "not found",
|
||||
@@ -84,20 +122,23 @@ func TestSendbirdOrganizationAPI_FromChunk(t *testing.T) {
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
s := Scanner{}
|
||||
got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
|
||||
got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
|
||||
if (err != nil) != tt.wantErr {
|
||||
t.Errorf("SendbirdOrganizationAPI.FromData() error = %v, wantErr %v", err, tt.wantErr)
|
||||
return
|
||||
}
|
||||
|
||||
for i := range got {
|
||||
if len(got[i].Raw) == 0 {
|
||||
t.Fatalf("no raw secret present: \n %+v", got[i])
|
||||
}
|
||||
got[i].Raw = nil
|
||||
if (got[i].VerificationError != nil) != tt.wantVerificationErr {
|
||||
t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError)
|
||||
}
|
||||
}
|
||||
if diff := pretty.Compare(got, tt.want); diff != "" {
|
||||
t.Errorf("SendbirdOrganizationAPI.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
|
||||
ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "RawV2", "VerificationError")
|
||||
if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {
|
||||
t.Errorf("Sendbird.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
@@ -13,13 +13,15 @@ import (
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
)
|
||||
|
||||
type Scanner struct{}
|
||||
type Scanner struct {
|
||||
client *http.Client
|
||||
}
|
||||
|
||||
// Ensure the Scanner satisfies the interface at compile time.
|
||||
var _ detectors.Detector = (*Scanner)(nil)
|
||||
|
||||
var (
|
||||
client = common.SaneHttpClient()
|
||||
defaultClient = common.SaneHttpClient()
|
||||
|
||||
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"sendgrid"}) + `(SG\.[\w\-_]{20,24}\.[\w\-_]{39,50})\b`)
|
||||
)
|
||||
@@ -42,7 +44,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
}
|
||||
res := strings.TrimSpace(match[1])
|
||||
|
||||
s := detectors.Result{
|
||||
s1 := detectors.Result{
|
||||
DetectorType: detectorspb.DetectorType_SendGrid,
|
||||
Raw: []byte(res),
|
||||
}
|
||||
@@ -52,6 +54,11 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
// 403 will be issued if the scope is wrong but the key is correct
|
||||
baseURL := "https://api.sendgrid.com/v3/templates"
|
||||
|
||||
client := s.client
|
||||
if client == nil {
|
||||
client = defaultClient
|
||||
}
|
||||
|
||||
// test `read_user` scope
|
||||
req, err := http.NewRequestWithContext(ctx, "GET", baseURL, nil)
|
||||
if err != nil {
|
||||
@@ -67,16 +74,18 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
// 403 means good key but not the right scope
|
||||
// 401 is bad key
|
||||
if res.StatusCode == http.StatusOK || res.StatusCode == http.StatusForbidden {
|
||||
s.Verified = true
|
||||
s1.Verified = true
|
||||
} else if res.StatusCode != http.StatusUnauthorized {
|
||||
s1.VerificationError = fmt.Errorf("unexpected HTTP response status %d", res.StatusCode)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if !s.Verified && detectors.IsKnownFalsePositive(string(s.Raw), detectors.DefaultFalsePositives, true) {
|
||||
if !s1.Verified && detectors.IsKnownFalsePositive(string(s1.Raw), detectors.DefaultFalsePositives, true) {
|
||||
continue
|
||||
}
|
||||
|
||||
results = append(results, s)
|
||||
results = append(results, s1)
|
||||
}
|
||||
|
||||
return
|
||||
|
||||
@@ -9,7 +9,8 @@ import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/kylelemons/godebug/pretty"
|
||||
"github.com/google/go-cmp/cmp"
|
||||
"github.com/google/go-cmp/cmp/cmpopts"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
|
||||
@@ -18,7 +19,7 @@ import (
|
||||
func TestSendgrid_FromChunk(t *testing.T) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
|
||||
defer cancel()
|
||||
testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")
|
||||
testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")
|
||||
if err != nil {
|
||||
t.Fatalf("could not get test secrets from GCP: %s", err)
|
||||
}
|
||||
@@ -30,11 +31,12 @@ func TestSendgrid_FromChunk(t *testing.T) {
|
||||
verify bool
|
||||
}
|
||||
tests := []struct {
|
||||
name string
|
||||
s Scanner
|
||||
args args
|
||||
want []detectors.Result
|
||||
wantErr bool
|
||||
name string
|
||||
s Scanner
|
||||
args args
|
||||
want []detectors.Result
|
||||
wantErr bool
|
||||
wantVerificationErr bool
|
||||
}{
|
||||
{
|
||||
name: "found, verified",
|
||||
@@ -50,7 +52,8 @@ func TestSendgrid_FromChunk(t *testing.T) {
|
||||
Verified: true,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
wantErr: false,
|
||||
wantVerificationErr: false,
|
||||
},
|
||||
{
|
||||
name: "found, unverified",
|
||||
@@ -66,7 +69,8 @@ func TestSendgrid_FromChunk(t *testing.T) {
|
||||
Verified: false,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
wantErr: false,
|
||||
wantVerificationErr: false,
|
||||
},
|
||||
{
|
||||
name: "not found",
|
||||
@@ -76,22 +80,54 @@ func TestSendgrid_FromChunk(t *testing.T) {
|
||||
data: []byte("You cannot find the secret within"),
|
||||
verify: true,
|
||||
},
|
||||
want: nil,
|
||||
wantErr: false,
|
||||
want: nil,
|
||||
wantErr: false,
|
||||
wantVerificationErr: false,
|
||||
},
|
||||
{
|
||||
name: "found, would be verified if not for timeout",
|
||||
s: Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a sendgrid secret %s within", secret)),
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_SendGrid,
|
||||
Verified: false,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
wantVerificationErr: true,
|
||||
},
|
||||
{
|
||||
name: "found, verified but unexpected api surface",
|
||||
s: Scanner{client: common.ConstantResponseHttpClient(404, "")},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a sendgrid secret %s within", secret)),
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_SendGrid,
|
||||
Verified: false,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
wantVerificationErr: true,
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
s := Scanner{}
|
||||
got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
|
||||
got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
|
||||
if (err != nil) != tt.wantErr {
|
||||
t.Errorf("Sendgrid.FromData() error = %v, wantErr %v", err, tt.wantErr)
|
||||
return
|
||||
}
|
||||
for i := range got {
|
||||
got[i].Raw = nil
|
||||
}
|
||||
if diff := pretty.Compare(got, tt.want); diff != "" {
|
||||
ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "VerificationError")
|
||||
if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {
|
||||
t.Errorf("Sendgrid.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
|
||||
}
|
||||
})
|
||||
|
||||
@@ -2,7 +2,10 @@ package sentrytoken
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"strings"
|
||||
@@ -12,16 +15,20 @@ import (
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
|
||||
)
|
||||
|
||||
type Scanner struct{}
|
||||
type Scanner struct {
|
||||
client *http.Client
|
||||
}
|
||||
|
||||
// Ensure the Scanner satisfies the interface at compile time.
|
||||
var _ detectors.Detector = (*Scanner)(nil)
|
||||
|
||||
var (
|
||||
client = common.SaneHttpClient()
|
||||
defaultClient = common.SaneHttpClient()
|
||||
|
||||
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
|
||||
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"sentry"}) + `\b([a-f0-9]{64})\b`)
|
||||
|
||||
errUnauthorized = fmt.Errorf("token unauthorized")
|
||||
)
|
||||
|
||||
// Keywords are used for efficiently pre-filtering chunks.
|
||||
@@ -47,32 +54,81 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
}
|
||||
|
||||
if verify {
|
||||
req, err := http.NewRequestWithContext(ctx, "GET", "https://sentry.io/api/0/projects/", nil)
|
||||
if err != nil {
|
||||
continue
|
||||
client := s.client
|
||||
if client == nil {
|
||||
client = defaultClient
|
||||
}
|
||||
req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", resMatch))
|
||||
// req.Header.Add("Cookie", "sc=g2ceDPcsBwPLGXugHUv0coGBi09TQYKqmtp0u4JYdIvOTFd28yVvn1yiuXgmJJa8; session=.eJwNyE0OwiAQQGGsPwtP4kqoGQa4gUdgZ2CGiaZJCUqjGxOPbnfve7_h29QpHpVSvbw61To9StvEwzre9TkVbkPcr7jN5dPbNl7X1jUt_a4vLpEHgUycQSgFZ0YBRkGxnthxwIyBAgdngcF4YxBssiMjekGv2245_wHyySMG:1m0J5S:FjpRlBVuFx577UQp6GcSGWTq0uk")
|
||||
res, err := client.Do(req)
|
||||
if err == nil {
|
||||
defer res.Body.Close()
|
||||
if res.StatusCode >= 200 && res.StatusCode < 300 {
|
||||
s1.Verified = true
|
||||
} else {
|
||||
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
|
||||
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
|
||||
continue
|
||||
}
|
||||
}
|
||||
isVerified, verificationErr := verifyToken(ctx, client, resMatch)
|
||||
|
||||
switch {
|
||||
case errors.Is(verificationErr, errUnauthorized):
|
||||
s1.Verified = false
|
||||
case isVerified:
|
||||
s1.Verified = true
|
||||
default:
|
||||
s1.VerificationError = verificationErr
|
||||
}
|
||||
}
|
||||
|
||||
if !s1.Verified && detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
|
||||
continue
|
||||
}
|
||||
results = append(results, s1)
|
||||
}
|
||||
|
||||
return results, nil
|
||||
}
|
||||
|
||||
type Response []Project
|
||||
|
||||
type Project struct {
|
||||
Organization Organization `json:"organization"`
|
||||
}
|
||||
|
||||
type Organization struct {
|
||||
ID string `json:"id"`
|
||||
Name string `json:"name"`
|
||||
}
|
||||
|
||||
func verifyToken(ctx context.Context, client *http.Client, token string) (bool, error) {
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, "https://sentry.io/api/0/projects/", nil)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", token))
|
||||
|
||||
res, err := client.Do(req)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
defer res.Body.Close()
|
||||
|
||||
var isVerified bool
|
||||
switch res.StatusCode {
|
||||
case http.StatusOK, http.StatusForbidden:
|
||||
isVerified = true
|
||||
case http.StatusUnauthorized:
|
||||
return false, errUnauthorized
|
||||
default:
|
||||
return false, fmt.Errorf("unexpected HTTP response status %d", res.StatusCode)
|
||||
}
|
||||
|
||||
bytes, readErr := io.ReadAll(res.Body)
|
||||
if readErr != nil {
|
||||
return false, readErr
|
||||
}
|
||||
|
||||
var resp Response
|
||||
if err = json.Unmarshal(bytes, &resp); err != nil {
|
||||
return false, err
|
||||
}
|
||||
if len(resp) == 0 {
|
||||
return false, fmt.Errorf("unexpected response body: %s", string(bytes))
|
||||
}
|
||||
|
||||
return isVerified, err
|
||||
}
|
||||
|
||||
func (s Scanner) Type() detectorspb.DetectorType {
|
||||
return detectorspb.DetectorType_SentryToken
|
||||
}
|
||||
|
||||
@@ -9,7 +9,9 @@ import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/kylelemons/godebug/pretty"
|
||||
"github.com/google/go-cmp/cmp"
|
||||
"github.com/google/go-cmp/cmp/cmpopts"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
@@ -32,11 +34,12 @@ func TestSentryToken_FromChunk(t *testing.T) {
|
||||
verify bool
|
||||
}
|
||||
tests := []struct {
|
||||
name string
|
||||
s Scanner
|
||||
args args
|
||||
want []detectors.Result
|
||||
wantErr bool
|
||||
name string
|
||||
s Scanner
|
||||
args args
|
||||
want []detectors.Result
|
||||
wantErr bool
|
||||
wantVerificationErr bool
|
||||
}{
|
||||
{
|
||||
name: "found, verified",
|
||||
@@ -70,6 +73,90 @@ func TestSentryToken_FromChunk(t *testing.T) {
|
||||
},
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "found, would be verified but for timeout",
|
||||
s: Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a sentry super secret %s within", secret)),
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_SentryToken,
|
||||
Verified: false,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
wantVerificationErr: true,
|
||||
},
|
||||
{
|
||||
name: "found and valid but unexpected api response",
|
||||
s: Scanner{client: common.ConstantResponseHttpClient(500, "")},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a sentry super secret %s within", secret)),
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_SentryToken,
|
||||
Verified: false,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
wantVerificationErr: true,
|
||||
},
|
||||
{
|
||||
name: "found, good key but wrong scope",
|
||||
s: Scanner{client: common.ConstantResponseHttpClient(403, responseBody403)},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a sentry super secret %s within", secret)),
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_SentryToken,
|
||||
Verified: true,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "found, account deactivated",
|
||||
s: Scanner{client: common.ConstantResponseHttpClient(200, reponseAccountDeactivated)},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a sentry super secret %s within", secret)),
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_SentryToken,
|
||||
Verified: false,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
wantVerificationErr: true,
|
||||
},
|
||||
{
|
||||
name: "found, account deactivated",
|
||||
s: Scanner{client: common.ConstantResponseHttpClient(200, responseEnmpty)},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a sentry super secret %s within", secret)),
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_SentryToken,
|
||||
Verified: false,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
wantVerificationErr: true,
|
||||
},
|
||||
{
|
||||
name: "not found",
|
||||
s: Scanner{},
|
||||
@@ -82,27 +169,50 @@ func TestSentryToken_FromChunk(t *testing.T) {
|
||||
wantErr: false,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
s := Scanner{}
|
||||
got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
|
||||
got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
|
||||
if (err != nil) != tt.wantErr {
|
||||
t.Errorf("SentryToken.FromData() error = %v, wantErr %v", err, tt.wantErr)
|
||||
t.Errorf("Gitlab.FromData() error = %v, wantErr %v", err, tt.wantErr)
|
||||
return
|
||||
}
|
||||
for i := range got {
|
||||
if len(got[i].Raw) == 0 {
|
||||
t.Fatalf("no raw secret present: \n %+v", got[i])
|
||||
t.Fatal("no raw secret present")
|
||||
}
|
||||
if (got[i].VerificationError != nil) != tt.wantVerificationErr {
|
||||
t.Fatalf("wantVerificationError = %v, verification error = %v,", tt.wantVerificationErr, got[i].VerificationError)
|
||||
}
|
||||
got[i].Raw = nil
|
||||
}
|
||||
if diff := pretty.Compare(got, tt.want); diff != "" {
|
||||
t.Errorf("SentryToken.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
|
||||
opts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "VerificationError")
|
||||
if diff := cmp.Diff(got, tt.want, opts); diff != "" {
|
||||
t.Errorf("Gitlab.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
const (
|
||||
responseBody403 = `
|
||||
[
|
||||
{
|
||||
"organization": {
|
||||
"id": "911964",
|
||||
"slug": "wigslap",
|
||||
"status": {
|
||||
"id": "active",
|
||||
"name": "active"
|
||||
},
|
||||
"name": "wigslap"
|
||||
}
|
||||
}
|
||||
]
|
||||
`
|
||||
reponseAccountDeactivated = `{"detail": "Authentication credentials were not provided"}`
|
||||
responseEnmpty = `[]`
|
||||
)
|
||||
|
||||
func BenchmarkFromData(benchmark *testing.B) {
|
||||
ctx := context.Background()
|
||||
s := Scanner{}
|
||||
|
||||
@@ -13,13 +13,16 @@ import (
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
)
|
||||
|
||||
type Scanner struct{}
|
||||
type Scanner struct {
|
||||
client *http.Client
|
||||
}
|
||||
|
||||
// Check that the Slack scanner implements the SecretScanner interface at compile time.
|
||||
var _ detectors.Detector = Scanner{}
|
||||
|
||||
var (
|
||||
tokenPats = map[string]*regexp.Regexp{
|
||||
defaultClient = common.SaneHttpClient()
|
||||
tokenPats = map[string]*regexp.Regexp{
|
||||
"Slack Bot Token": regexp.MustCompile(`xoxb\-[0-9]{10,13}\-[0-9]{10,13}[a-zA-Z0-9\-]*`),
|
||||
"Slack User Token": regexp.MustCompile(`xoxp\-[0-9]{10,13}\-[0-9]{10,13}[a-zA-Z0-9\-]*`),
|
||||
"Slack Workspace Access Token": regexp.MustCompile(`xoxa\-[0-9]{10,13}\-[0-9]{10,13}[a-zA-Z0-9\-]*`),
|
||||
@@ -53,34 +56,54 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
tokens := tokenPat.FindAllString(dataStr, -1)
|
||||
|
||||
for _, token := range tokens {
|
||||
s := detectors.Result{
|
||||
s1 := detectors.Result{
|
||||
DetectorType: detectorspb.DetectorType_Slack,
|
||||
Raw: []byte(token),
|
||||
}
|
||||
if verify {
|
||||
client := common.SaneHttpClient()
|
||||
client := s.client
|
||||
if s.client == nil {
|
||||
client = defaultClient
|
||||
}
|
||||
|
||||
req, err := http.NewRequestWithContext(ctx, "POST", verifyURL, nil)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
|
||||
req.Header.Add("Content-Type", "application/json; charset=utf-8")
|
||||
req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", token))
|
||||
|
||||
res, err := client.Do(req)
|
||||
if err == nil {
|
||||
defer res.Body.Close()
|
||||
var authResponse authRes
|
||||
if err := json.NewDecoder(res.Body).Decode(&authResponse); err != nil {
|
||||
continue
|
||||
s1.VerificationError = fmt.Errorf("failed to decode auth response: %w", err)
|
||||
}
|
||||
s.Verified = authResponse.Ok
|
||||
|
||||
if authResponse.Ok {
|
||||
s1.Verified = true
|
||||
// Slack API returns 200 even if the token is invalid. We need to check the error field.
|
||||
} else if authResponse.Error == "invalid_auth" {
|
||||
// The secret is determinately not verified (nothing to do)
|
||||
} else {
|
||||
s1.VerificationError = fmt.Errorf("unexpected error auth response %+v", authResponse.Error)
|
||||
}
|
||||
} else {
|
||||
s1.VerificationError = err
|
||||
}
|
||||
}
|
||||
|
||||
results = append(results, s)
|
||||
if !s1.Verified && detectors.IsKnownFalsePositive(string(s1.Raw), detectors.DefaultFalsePositives, true) {
|
||||
continue
|
||||
}
|
||||
|
||||
results = append(results, s1)
|
||||
}
|
||||
}
|
||||
|
||||
return
|
||||
return results, nil
|
||||
}
|
||||
|
||||
func (s Scanner) Type() detectorspb.DetectorType {
|
||||
|
||||
@@ -9,13 +9,15 @@ import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/kylelemons/godebug/pretty"
|
||||
"github.com/google/go-cmp/cmp"
|
||||
"github.com/google/go-cmp/cmp/cmpopts"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
|
||||
)
|
||||
|
||||
func TestScanner_FromChunk(t *testing.T) {
|
||||
func TestSlack_FromChunk(t *testing.T) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
|
||||
defer cancel()
|
||||
testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")
|
||||
@@ -24,17 +26,28 @@ func TestScanner_FromChunk(t *testing.T) {
|
||||
}
|
||||
secret := testSecrets.MustGetField("SLACK")
|
||||
secretInactive := testSecrets.MustGetField("SLACK_INACTIVE")
|
||||
|
||||
type args struct {
|
||||
ctx context.Context
|
||||
data []byte
|
||||
verify bool
|
||||
}
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
data []byte
|
||||
verify bool
|
||||
wantResults []detectors.Result
|
||||
wantErr bool
|
||||
name string
|
||||
s Scanner
|
||||
args args
|
||||
wantResults []detectors.Result
|
||||
wantErr bool
|
||||
wantVerificationErr bool
|
||||
}{
|
||||
{
|
||||
name: "found, verified",
|
||||
data: []byte(fmt.Sprintf("You can find a slack secret %s within", secret)),
|
||||
verify: true,
|
||||
name: "found, verified",
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a slack secret %s within", secret)),
|
||||
verify: true,
|
||||
},
|
||||
wantResults: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_Slack,
|
||||
@@ -44,9 +57,12 @@ func TestScanner_FromChunk(t *testing.T) {
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "found but unverified",
|
||||
data: []byte(fmt.Sprintf("You can find a slack secret %s within", secretInactive)),
|
||||
verify: true,
|
||||
name: "found but unverified",
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a slack secret %s within", secretInactive)),
|
||||
verify: true,
|
||||
},
|
||||
wantResults: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_Slack,
|
||||
@@ -55,23 +71,62 @@ func TestScanner_FromChunk(t *testing.T) {
|
||||
},
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "found, would be verified if not for timeout",
|
||||
s: Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a slack secret %s within", secret)),
|
||||
verify: true,
|
||||
},
|
||||
wantResults: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_Slack,
|
||||
Verified: false,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
wantVerificationErr: true,
|
||||
},
|
||||
{
|
||||
name: "unexpected auth response",
|
||||
s: Scanner{client: common.ConstantResponseHttpClient(200, `{"ok": false, "error": "unexpected_error"}`)},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a slack secret %s within", secret)),
|
||||
verify: true,
|
||||
},
|
||||
wantResults: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_Slack,
|
||||
Verified: false,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
wantVerificationErr: true,
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
s := Scanner{}
|
||||
got, err := s.FromData(context.Background(), tt.verify, tt.data)
|
||||
got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
|
||||
if (err != nil) != tt.wantErr {
|
||||
t.Errorf("Scanner.FromData) error = %v, wantErr %v", err, tt.wantErr)
|
||||
t.Errorf("Slack.FromData() error = %v, wantErr %v", err, tt.wantErr)
|
||||
return
|
||||
}
|
||||
|
||||
for i := range got {
|
||||
if len(got[i].Raw) == 0 {
|
||||
t.Fatal("no raw secret present")
|
||||
}
|
||||
got[i].Raw = nil
|
||||
|
||||
if (got[i].VerificationError != nil) != tt.wantVerificationErr {
|
||||
t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError)
|
||||
}
|
||||
}
|
||||
if diff := pretty.Compare(got, tt.wantResults); diff != "" {
|
||||
t.Errorf("Scanner.FromData) %s diff: (-got +want)\n%s", tt.name, diff)
|
||||
ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "VerificationError")
|
||||
if diff := cmp.Diff(got, tt.wantResults, ignoreOpts); diff != "" {
|
||||
t.Errorf("Slack.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
package slackwebhook
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
@@ -69,11 +70,10 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
body := string(bodyBytes)
|
||||
|
||||
defer res.Body.Close()
|
||||
|
||||
if res.StatusCode >= 200 && res.StatusCode < 300 || (res.StatusCode == 400 && (strings.Contains(body, "no_text") || strings.Contains(body, "missing_text"))) {
|
||||
if res.StatusCode >= 200 && res.StatusCode < 300 || (res.StatusCode == 400 && (bytes.Equal(bodyBytes, []byte("no_text")) || bytes.Equal(bodyBytes, []byte("missing_text")))) {
|
||||
s1.Verified = true
|
||||
} else if res.StatusCode == 401 || res.StatusCode == 403 {
|
||||
// The secret is determinately not verified (nothing to do)
|
||||
|
||||
@@ -6,14 +6,14 @@ package slackwebhook
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"github.com/google/go-cmp/cmp"
|
||||
"github.com/google/go-cmp/cmp/cmpopts"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
"github.com/google/go-cmp/cmp"
|
||||
"github.com/google/go-cmp/cmp/cmpopts"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
|
||||
)
|
||||
|
||||
@@ -57,6 +57,40 @@ func TestSlackWebhook_FromChunk(t *testing.T) {
|
||||
wantErr: false,
|
||||
wantVerificationErr: false,
|
||||
},
|
||||
{
|
||||
name: "found, verified, 400 no_text",
|
||||
s: Scanner{client: common.ConstantResponseHttpClient(400, "no_text")},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a slackwebhook secret %s within", secret)),
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_SlackWebhook,
|
||||
Verified: true,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
wantVerificationErr: false,
|
||||
},
|
||||
{
|
||||
name: "found, verified, 400 missing_text",
|
||||
s: Scanner{client: common.ConstantResponseHttpClient(400, "missing_text")},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a slackwebhook secret %s within", secret)),
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_SlackWebhook,
|
||||
Verified: true,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
wantVerificationErr: false,
|
||||
},
|
||||
{
|
||||
name: "found, unverified",
|
||||
s: Scanner{},
|
||||
@@ -120,6 +154,23 @@ func TestSlackWebhook_FromChunk(t *testing.T) {
|
||||
wantErr: false,
|
||||
wantVerificationErr: true,
|
||||
},
|
||||
{
|
||||
name: "found, account disabled",
|
||||
s: Scanner{client: common.ConstantResponseHttpClient(400, disabledAccountResponse)},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a slackwebhook secret %s within", secret)),
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_SlackWebhook,
|
||||
Verified: false,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
wantVerificationErr: true,
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
@@ -144,6 +195,8 @@ func TestSlackWebhook_FromChunk(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
const disabledAccountResponse = `missing_text_or_fallback_or_attachments`
|
||||
|
||||
func BenchmarkFromData(benchmark *testing.B) {
|
||||
ctx := context.Background()
|
||||
s := Scanner{}
|
||||
|
||||
@@ -21,13 +21,13 @@ var (
|
||||
client = common.SaneHttpClient()
|
||||
|
||||
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
|
||||
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"sonarcloud"}) + `\b([0-9a-z]{40})\b`)
|
||||
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"sonar"}) + `\b([0-9a-z]{40})\b`)
|
||||
)
|
||||
|
||||
// Keywords are used for efficiently pre-filtering chunks.
|
||||
// Use identifiers in the secret preferably, or the provider name.
|
||||
func (s Scanner) Keywords() []string {
|
||||
return []string{"sonarcloud"}
|
||||
return []string{"sonar"}
|
||||
}
|
||||
|
||||
// FromData will find and optionally verify SonarCloud secrets in a given set of bytes.
|
||||
|
||||
@@ -0,0 +1,94 @@
|
||||
package sourcegraph
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
|
||||
)
|
||||
|
||||
type Scanner struct {
|
||||
client *http.Client
|
||||
}
|
||||
|
||||
// Ensure the Scanner satisfies the interface at compile time.
|
||||
var _ detectors.Detector = (*Scanner)(nil)
|
||||
|
||||
var (
|
||||
defaultClient = common.SaneHttpClient()
|
||||
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
|
||||
keyPat = regexp.MustCompile(`\b(sgp_[a-f0-9]{40})\b`)
|
||||
)
|
||||
|
||||
// Keywords are used for efficiently pre-filtering chunks.
|
||||
// Use identifiers in the secret preferably, or the provider name.
|
||||
func (s Scanner) Keywords() []string {
|
||||
return []string{"sgp_"}
|
||||
}
|
||||
|
||||
// FromData will find and optionally verify Sourcegraph secrets in a given set of bytes.
|
||||
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
|
||||
dataStr := string(data)
|
||||
|
||||
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
|
||||
|
||||
for _, match := range matches {
|
||||
if len(match) != 2 {
|
||||
continue
|
||||
}
|
||||
resMatch := strings.TrimSpace(match[1])
|
||||
|
||||
s1 := detectors.Result{
|
||||
DetectorType: detectorspb.DetectorType_Sourcegraph,
|
||||
Raw: []byte(resMatch),
|
||||
}
|
||||
|
||||
if verify {
|
||||
client := s.client
|
||||
if client == nil {
|
||||
client = defaultClient
|
||||
}
|
||||
payload := strings.NewReader("{\"query\": \"query { currentUser { username } }\"}")
|
||||
|
||||
req, err := http.NewRequestWithContext(ctx, "POST", "https://sourcegraph.com/.api/graphql", payload)
|
||||
req.Header.Add("Authorization", "token "+resMatch)
|
||||
req.Header.Add("Content-Type", "application/json")
|
||||
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
res, err := client.Do(req)
|
||||
if err == nil {
|
||||
defer res.Body.Close()
|
||||
|
||||
if res.StatusCode >= 200 && res.StatusCode < 300 {
|
||||
s1.Verified = true
|
||||
} else if res.StatusCode == 401 {
|
||||
// The secret is determinately not verified (nothing to do)
|
||||
} else {
|
||||
s1.VerificationError = fmt.Errorf("unexpected HTTP response status %d", res.StatusCode)
|
||||
}
|
||||
} else {
|
||||
s1.VerificationError = err
|
||||
}
|
||||
}
|
||||
|
||||
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
|
||||
if !s1.Verified && detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
|
||||
continue
|
||||
}
|
||||
|
||||
results = append(results, s1)
|
||||
}
|
||||
|
||||
return results, nil
|
||||
}
|
||||
|
||||
func (s Scanner) Type() detectorspb.DetectorType {
|
||||
return detectorspb.DetectorType_Sourcegraph
|
||||
}
|
||||
@@ -0,0 +1,161 @@
|
||||
//go:build detectors
|
||||
// +build detectors
|
||||
|
||||
package sourcegraph
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"github.com/google/go-cmp/cmp"
|
||||
"github.com/google/go-cmp/cmp/cmpopts"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
|
||||
)
|
||||
|
||||
func TestSourcegraph_FromChunk(t *testing.T) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
|
||||
defer cancel()
|
||||
testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors4")
|
||||
if err != nil {
|
||||
t.Fatalf("could not get test secrets from GCP: %s", err)
|
||||
}
|
||||
secret := testSecrets.MustGetField("SOURCEGRAPH")
|
||||
inactiveSecret := testSecrets.MustGetField("SOURCEGRAPH_INACTIVE")
|
||||
|
||||
type args struct {
|
||||
ctx context.Context
|
||||
data []byte
|
||||
verify bool
|
||||
}
|
||||
tests := []struct {
|
||||
name string
|
||||
s Scanner
|
||||
args args
|
||||
want []detectors.Result
|
||||
wantErr bool
|
||||
wantVerificationErr bool
|
||||
}{
|
||||
{
|
||||
name: "found, verified",
|
||||
s: Scanner{},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a sourcegraph secret %s within", secret)),
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_Sourcegraph,
|
||||
Verified: true,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
wantVerificationErr: false,
|
||||
},
|
||||
{
|
||||
name: "found, unverified",
|
||||
s: Scanner{},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a sourcegraph secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_Sourcegraph,
|
||||
Verified: false,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
wantVerificationErr: false,
|
||||
},
|
||||
{
|
||||
name: "not found",
|
||||
s: Scanner{},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte("You cannot find the secret within"),
|
||||
verify: true,
|
||||
},
|
||||
want: nil,
|
||||
wantErr: false,
|
||||
wantVerificationErr: false,
|
||||
},
|
||||
{
|
||||
name: "found, would be verified if not for timeout",
|
||||
s: Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a sourcegraph secret %s within", secret)),
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_Sourcegraph,
|
||||
Verified: false,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
wantVerificationErr: true,
|
||||
},
|
||||
{
|
||||
name: "found, verified but unexpected api surface",
|
||||
s: Scanner{client: common.ConstantResponseHttpClient(404, "")},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a sourcegraph secret %s within", secret)),
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_Sourcegraph,
|
||||
Verified: false,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
wantVerificationErr: true,
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
|
||||
if (err != nil) != tt.wantErr {
|
||||
t.Errorf("Sourcegraph.FromData() error = %v, wantErr %v", err, tt.wantErr)
|
||||
return
|
||||
}
|
||||
for i := range got {
|
||||
if len(got[i].Raw) == 0 {
|
||||
t.Fatalf("no raw secret present: \n %+v", got[i])
|
||||
}
|
||||
if (got[i].VerificationError != nil) != tt.wantVerificationErr {
|
||||
t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError)
|
||||
}
|
||||
}
|
||||
ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "VerificationError")
|
||||
if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {
|
||||
t.Errorf("Sourcegraph.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkFromData(benchmark *testing.B) {
|
||||
ctx := context.Background()
|
||||
s := Scanner{}
|
||||
for name, data := range detectors.MustGetBenchmarkData() {
|
||||
benchmark.Run(name, func(b *testing.B) {
|
||||
b.ResetTimer()
|
||||
for n := 0; n < b.N; n++ {
|
||||
_, err := s.FromData(ctx, false, data)
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,85 @@
|
||||
package tailscale
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
|
||||
)
|
||||
|
||||
type Scanner struct{}
|
||||
|
||||
// Ensure the Scanner satisfies the interface at compile time.
|
||||
var _ detectors.Detector = (*Scanner)(nil)
|
||||
|
||||
var (
|
||||
client = common.SaneHttpClient()
|
||||
|
||||
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
|
||||
keyPat = regexp.MustCompile(`\btskey-[a-z]+-[0-9A-Za-z_]+-[0-9A-Za-z_]+\b`)
|
||||
)
|
||||
|
||||
// Keywords are used for efficiently pre-filtering chunks.
|
||||
// Use identifiers in the secret preferably, or the provider name.
|
||||
func (s Scanner) Keywords() []string {
|
||||
return []string{"tskey-", "tskey-api-", "tskey-oauth-"}
|
||||
}
|
||||
|
||||
// FromData will find and optionally verify Tailscaleapi secrets in a given set of bytes.
|
||||
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
|
||||
dataStr := string(data)
|
||||
|
||||
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
|
||||
|
||||
for _, match := range matches {
|
||||
resMatch := strings.TrimSpace(match[0])
|
||||
|
||||
s1 := detectors.Result{
|
||||
DetectorType: detectorspb.DetectorType_Tailscale,
|
||||
Raw: []byte(resMatch),
|
||||
}
|
||||
|
||||
if verify {
|
||||
const u = "https://api.tailscale.com/api/v2/secret-scanning/verify"
|
||||
vals := url.Values{"key": []string{resMatch}}
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodPost, u, strings.NewReader(vals.Encode()))
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
|
||||
res, err := client.Do(req)
|
||||
if err == nil {
|
||||
defer res.Body.Close()
|
||||
switch res.StatusCode {
|
||||
case http.StatusNoContent:
|
||||
s1.Verified = true
|
||||
case http.StatusUnauthorized:
|
||||
// The secret is determinately not verified (nothing to do)
|
||||
default:
|
||||
s1.VerificationError = fmt.Errorf("unexpected HTTP response status %d", res.StatusCode)
|
||||
}
|
||||
} else {
|
||||
s1.VerificationError = err
|
||||
}
|
||||
}
|
||||
|
||||
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
|
||||
if !s1.Verified && detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
|
||||
continue
|
||||
}
|
||||
|
||||
results = append(results, s1)
|
||||
}
|
||||
|
||||
return results, nil
|
||||
}
|
||||
|
||||
func (s Scanner) Type() detectorspb.DetectorType {
|
||||
return detectorspb.DetectorType_Tailscale
|
||||
}
|
||||
@@ -0,0 +1,120 @@
|
||||
//go:build detectors
|
||||
// +build detectors
|
||||
|
||||
package tailscale
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/kylelemons/godebug/pretty"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
|
||||
)
|
||||
|
||||
func TestTailscaleapi_FromChunk(t *testing.T) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
|
||||
defer cancel()
|
||||
testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors4")
|
||||
if err != nil && false {
|
||||
t.Fatalf("could not get test secrets from GCP: %s", err)
|
||||
}
|
||||
secret := testSecrets.MustGetField("TAILSCALEAPI")
|
||||
inactiveSecret := testSecrets.MustGetField("TAILSCALEAPI_INACTIVE")
|
||||
|
||||
type args struct {
|
||||
ctx context.Context
|
||||
data []byte
|
||||
verify bool
|
||||
}
|
||||
tests := []struct {
|
||||
name string
|
||||
s Scanner
|
||||
args args
|
||||
want []detectors.Result
|
||||
wantErr bool
|
||||
}{
|
||||
{
|
||||
name: "found, verified",
|
||||
s: Scanner{},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a tailscaleapi secret %s within", secret)),
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_Tailscale,
|
||||
Verified: true,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "found, unverified",
|
||||
s: Scanner{},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a tailscaleapi secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_Tailscale,
|
||||
Verified: false,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "not found",
|
||||
s: Scanner{},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte("You cannot find the secret within"),
|
||||
verify: true,
|
||||
},
|
||||
want: nil,
|
||||
wantErr: false,
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
s := Scanner{}
|
||||
got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
|
||||
if (err != nil) != tt.wantErr {
|
||||
t.Errorf("Tailscaleapi.FromData() error = %v, wantErr %v", err, tt.wantErr)
|
||||
return
|
||||
}
|
||||
for i := range got {
|
||||
if len(got[i].Raw) == 0 {
|
||||
t.Fatalf("no raw secret present: \n %+v", got[i])
|
||||
}
|
||||
got[i].Raw = nil
|
||||
}
|
||||
if diff := pretty.Compare(got, tt.want); diff != "" {
|
||||
t.Errorf("Tailscaleapi.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkFromData(benchmark *testing.B) {
|
||||
ctx := context.Background()
|
||||
s := Scanner{}
|
||||
for name, data := range detectors.MustGetBenchmarkData() {
|
||||
benchmark.Run(name, func(b *testing.B) {
|
||||
for n := 0; n < b.N; n++ {
|
||||
_, err := s.FromData(ctx, false, data)
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -2,6 +2,7 @@ package twilio
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"regexp"
|
||||
|
||||
@@ -10,12 +11,15 @@ import (
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
|
||||
)
|
||||
|
||||
type Scanner struct{}
|
||||
type Scanner struct {
|
||||
client *http.Client
|
||||
}
|
||||
|
||||
// Ensure the Scanner satisfies the interface at compile time.
|
||||
var _ detectors.Detector = (*Scanner)(nil)
|
||||
|
||||
var (
|
||||
defaultClient = common.SaneHttpClient()
|
||||
identifierPat = regexp.MustCompile(`(?i)sid.{0,20}AC[0-9a-f]{32}`) // Should we have this? Seems restrictive.
|
||||
sidPat = regexp.MustCompile(`\bAC[0-9a-f]{32}\b`)
|
||||
keyPat = regexp.MustCompile(`\b[0-9a-f]{32}\b`)
|
||||
@@ -51,6 +55,11 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
}
|
||||
|
||||
if verify {
|
||||
client = s.client
|
||||
if client == nil {
|
||||
client = defaultClient
|
||||
}
|
||||
|
||||
req, err := http.NewRequestWithContext(
|
||||
ctx, "GET", "https://verify.twilio.com/v2/Services", nil)
|
||||
if err != nil {
|
||||
@@ -65,7 +74,13 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
|
||||
if res.StatusCode >= 200 && res.StatusCode < 300 {
|
||||
s1.Verified = true
|
||||
} else if res.StatusCode == 401 || res.StatusCode == 403 {
|
||||
// The secret is determinately not verified (nothing to do)
|
||||
} else {
|
||||
s1.VerificationError = fmt.Errorf("unexpected HTTP response status %d", res.StatusCode)
|
||||
}
|
||||
} else {
|
||||
s1.VerificationError = err
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -6,11 +6,11 @@ package twilio
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"github.com/google/go-cmp/cmp"
|
||||
"github.com/google/go-cmp/cmp/cmpopts"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/kylelemons/godebug/pretty"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
|
||||
@@ -33,11 +33,12 @@ func TestTwilio_FromChunk(t *testing.T) {
|
||||
verify bool
|
||||
}
|
||||
tests := []struct {
|
||||
name string
|
||||
s Scanner
|
||||
args args
|
||||
want []detectors.Result
|
||||
wantErr bool
|
||||
name string
|
||||
s Scanner
|
||||
args args
|
||||
want []detectors.Result
|
||||
wantErr bool
|
||||
wantVerificationErr bool
|
||||
}{
|
||||
{
|
||||
name: "found, verified",
|
||||
@@ -52,6 +53,7 @@ func TestTwilio_FromChunk(t *testing.T) {
|
||||
DetectorType: detectorspb.DetectorType_Twilio,
|
||||
Verified: true,
|
||||
Redacted: id,
|
||||
RawV2: []byte(id + secret),
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
@@ -69,6 +71,7 @@ func TestTwilio_FromChunk(t *testing.T) {
|
||||
DetectorType: detectorspb.DetectorType_Twilio,
|
||||
Verified: false,
|
||||
Redacted: id,
|
||||
RawV2: []byte(id + secretInactive),
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
@@ -84,23 +87,62 @@ func TestTwilio_FromChunk(t *testing.T) {
|
||||
want: nil,
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "found, would be verified if not for timeout",
|
||||
s: Scanner{client: common.SaneHttpClientTimeOut(100 * time.Microsecond)},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a twillio secret %s within awsId %s", secret, id)),
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_Twilio,
|
||||
Verified: false,
|
||||
Redacted: id,
|
||||
RawV2: []byte(id + secret),
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
wantVerificationErr: true,
|
||||
},
|
||||
{
|
||||
name: "found, verified but unexpected api surface",
|
||||
s: Scanner{client: common.ConstantResponseHttpClient(404, "")},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a twillio secret %s within awsId %s", secret, id)),
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_Twilio,
|
||||
Verified: false,
|
||||
Redacted: id,
|
||||
RawV2: []byte(id + secret),
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
wantVerificationErr: true,
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
s := Scanner{}
|
||||
got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
|
||||
got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
|
||||
if (err != nil) != tt.wantErr {
|
||||
t.Errorf("Twilio.FromData() error = %v, wantErr %v", err, tt.wantErr)
|
||||
return
|
||||
}
|
||||
for i := range got {
|
||||
if len(got[i].Raw) == 0 {
|
||||
t.Fatal("no raw secret present")
|
||||
t.Fatalf("no raw secret present: \n %+v", got[i])
|
||||
}
|
||||
if (got[i].VerificationError != nil) != tt.wantVerificationErr {
|
||||
t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError)
|
||||
}
|
||||
got[i].Raw = nil
|
||||
got[i].RawV2 = nil
|
||||
}
|
||||
if diff := pretty.Compare(got, tt.want); diff != "" {
|
||||
ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "VerificationError")
|
||||
if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {
|
||||
t.Errorf("Twilio.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
|
||||
}
|
||||
})
|
||||
|
||||
+18
-14
@@ -15,6 +15,7 @@ import (
|
||||
|
||||
type Scanner struct {
|
||||
allowKnownTestSites bool
|
||||
client *http.Client
|
||||
}
|
||||
|
||||
// Ensure the Scanner satisfies the interface at compile time.
|
||||
@@ -23,7 +24,7 @@ var _ detectors.Detector = (*Scanner)(nil)
|
||||
var (
|
||||
keyPat = regexp.MustCompile(`\b(?:https?:)?\/\/[\S]{3,50}:([\S]{3,50})@[-.%\w\/:]+\b`)
|
||||
|
||||
client = common.SaneHttpClient()
|
||||
defaultClient = common.SaneHttpClient()
|
||||
)
|
||||
|
||||
// Keywords are used for efficiently pre-filtering chunks.
|
||||
@@ -71,7 +72,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
// Removing the path causes possible deduplication issues if some paths have basic auth and some do not.
|
||||
rawURL.Path = ""
|
||||
|
||||
s := detectors.Result{
|
||||
s1 := detectors.Result{
|
||||
DetectorType: detectorspb.DetectorType_URI,
|
||||
Raw: []byte(rawURL.String()),
|
||||
RawV2: []byte(rawURLStr),
|
||||
@@ -79,27 +80,30 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
}
|
||||
|
||||
if verify {
|
||||
s.Verified = verifyURL(ctx, parsedURL)
|
||||
if s.client == nil {
|
||||
s.client = defaultClient
|
||||
}
|
||||
s1.Verified, s1.VerificationError = verifyURL(ctx, s.client, parsedURL)
|
||||
}
|
||||
|
||||
if !s.Verified {
|
||||
if !s1.Verified {
|
||||
// Skip unverified findings where the password starts with a `$` - it's almost certainly a variable.
|
||||
if strings.HasPrefix(password, "$") {
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
if !s.Verified && detectors.IsKnownFalsePositive(string(s.Raw), detectors.DefaultFalsePositives, false) {
|
||||
if !s1.Verified && !s.allowKnownTestSites && detectors.IsKnownFalsePositive(string(s1.Raw), detectors.DefaultFalsePositives, false) {
|
||||
continue
|
||||
}
|
||||
|
||||
results = append(results, s)
|
||||
results = append(results, s1)
|
||||
}
|
||||
|
||||
return results, nil
|
||||
}
|
||||
|
||||
func verifyURL(ctx context.Context, u *url.URL) bool {
|
||||
func verifyURL(ctx context.Context, client *http.Client, u *url.URL) (bool, error) {
|
||||
// defuse most SSRF payloads
|
||||
u.Path = strings.TrimSuffix(u.Path, "/")
|
||||
u.RawQuery = ""
|
||||
@@ -112,41 +116,41 @@ func verifyURL(ctx context.Context, u *url.URL) bool {
|
||||
|
||||
req, err := http.NewRequest("GET", credentialedURL, nil)
|
||||
if err != nil {
|
||||
return false
|
||||
return false, err
|
||||
}
|
||||
req = req.WithContext(ctx)
|
||||
credentialedRes, err := client.Do(req)
|
||||
if err != nil {
|
||||
return false
|
||||
return false, err
|
||||
}
|
||||
credentialedRes.Body.Close()
|
||||
|
||||
// If the credentialed URL returns a non 2XX code, we can assume it's a false positive.
|
||||
if credentialedRes.StatusCode < 200 || credentialedRes.StatusCode > 299 {
|
||||
return false
|
||||
return false, nil
|
||||
}
|
||||
|
||||
time.Sleep(time.Millisecond * 10)
|
||||
|
||||
req, err = http.NewRequest("GET", nonCredentialedURL, nil)
|
||||
if err != nil {
|
||||
return false
|
||||
return false, err
|
||||
}
|
||||
req = req.WithContext(ctx)
|
||||
nonCredentialedRes, err := client.Do(req)
|
||||
if err != nil {
|
||||
return false
|
||||
return false, err
|
||||
}
|
||||
nonCredentialedRes.Body.Close()
|
||||
|
||||
// If the non-credentialed URL returns a non 400-428 code and basic auth header, we can assume it's verified now.
|
||||
if nonCredentialedRes.StatusCode >= 400 && nonCredentialedRes.StatusCode < 429 {
|
||||
if nonCredentialedRes.Header.Get("WWW-Authenticate") != "" {
|
||||
return true
|
||||
return true, nil
|
||||
}
|
||||
}
|
||||
|
||||
return false
|
||||
return false, nil
|
||||
}
|
||||
|
||||
func (s Scanner) Type() detectorspb.DetectorType {
|
||||
|
||||
@@ -7,8 +7,10 @@ import (
|
||||
"context"
|
||||
"fmt"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/kylelemons/godebug/pretty"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
|
||||
)
|
||||
@@ -20,11 +22,12 @@ func TestURI_FromChunk(t *testing.T) {
|
||||
verify bool
|
||||
}
|
||||
tests := []struct {
|
||||
name string
|
||||
s Scanner
|
||||
args args
|
||||
want []detectors.Result
|
||||
wantErr bool
|
||||
name string
|
||||
s Scanner
|
||||
args args
|
||||
want []detectors.Result
|
||||
wantErr bool
|
||||
wantVerificationErr bool
|
||||
}{
|
||||
{
|
||||
name: "found, unverified, wrong username",
|
||||
@@ -77,6 +80,24 @@ func TestURI_FromChunk(t *testing.T) {
|
||||
},
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "found, verified if not for timeout",
|
||||
s: Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a uri secret %s within", "https://httpwatch:pass@www.httpwatch.com/httpgallery/authentication/authenticatedimage/default.aspx")),
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_URI,
|
||||
Verified: false,
|
||||
Redacted: "https://httpwatch:********@www.httpwatch.com",
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
wantVerificationErr: true,
|
||||
},
|
||||
{
|
||||
name: "bad scheme",
|
||||
s: Scanner{},
|
||||
@@ -101,8 +122,8 @@ func TestURI_FromChunk(t *testing.T) {
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
s := Scanner{allowKnownTestSites: true}
|
||||
got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
|
||||
tt.s.allowKnownTestSites = true
|
||||
got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
|
||||
if (err != nil) != tt.wantErr {
|
||||
t.Errorf("URI.FromData() error = %v, wantErr %v", err, tt.wantErr)
|
||||
return
|
||||
@@ -111,8 +132,13 @@ func TestURI_FromChunk(t *testing.T) {
|
||||
// return
|
||||
// }
|
||||
for i := range got {
|
||||
if (got[i].VerificationError != nil) != tt.wantVerificationErr {
|
||||
t.Errorf("URI.FromData() error = %v, wantVerificationErr %v", got[i].VerificationError, tt.wantErr)
|
||||
return
|
||||
}
|
||||
got[i].Raw = nil
|
||||
got[i].RawV2 = nil
|
||||
got[i].VerificationError = nil
|
||||
}
|
||||
if diff := pretty.Compare(got, tt.want); diff != "" {
|
||||
t.Errorf("URI.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
|
||||
|
||||
+6
-11
@@ -8,7 +8,6 @@ import (
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/context"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/sources/circleci"
|
||||
)
|
||||
|
||||
@@ -27,17 +26,13 @@ func (e *Engine) ScanCircleCI(ctx context.Context, token string) error {
|
||||
return err
|
||||
}
|
||||
|
||||
handle, err := e.sourceManager.Enroll(ctx, "trufflehog - Circle CI", new(circleci.Source).Type(),
|
||||
func(ctx context.Context, jobID, sourceID int64) (sources.Source, error) {
|
||||
circleSource := circleci.Source{}
|
||||
if err := circleSource.Init(ctx, "trufflehog - Circle CI", jobID, sourceID, true, &conn, runtime.NumCPU()); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &circleSource, nil
|
||||
})
|
||||
if err != nil {
|
||||
sourceName := "trufflehog - Circle CI"
|
||||
sourceID, jobID, _ := e.sourceManager.GetIDs(ctx, sourceName, circleci.SourceType)
|
||||
|
||||
circleSource := &circleci.Source{}
|
||||
if err := circleSource.Init(ctx, "trufflehog - Circle CI", jobID, sourceID, true, &conn, runtime.NumCPU()); err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = e.sourceManager.ScheduleRun(ctx, handle)
|
||||
_, err = e.sourceManager.Run(ctx, sourceName, circleSource)
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -5,6 +5,8 @@ import (
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/huggingface"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/salesforce"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/snowflake"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/sourcegraph"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/tailscale"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/trufflehogenterprise"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
@@ -1540,6 +1542,8 @@ func DefaultDetectors() []detectors.Detector {
|
||||
huggingface.Scanner{},
|
||||
trufflehogenterprise.Scanner{},
|
||||
salesforce.Scanner{},
|
||||
sourcegraph.Scanner{},
|
||||
tailscale.Scanner{},
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
+6
-11
@@ -6,23 +6,18 @@ import (
|
||||
"google.golang.org/protobuf/types/known/anypb"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/context"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/sources/docker"
|
||||
)
|
||||
|
||||
// ScanDocker scans a given docker connection.
|
||||
func (e *Engine) ScanDocker(ctx context.Context, conn *anypb.Any) error {
|
||||
handle, err := e.sourceManager.Enroll(ctx, "trufflehog - docker", new(docker.Source).Type(),
|
||||
func(ctx context.Context, jobID, sourceID int64) (sources.Source, error) {
|
||||
dockerSource := docker.Source{}
|
||||
if err := dockerSource.Init(ctx, "trufflehog - docker", jobID, sourceID, true, conn, runtime.NumCPU()); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &dockerSource, nil
|
||||
})
|
||||
if err != nil {
|
||||
sourceName := "trufflehog - docker"
|
||||
sourceID, jobID, _ := e.sourceManager.GetIDs(ctx, sourceName, docker.SourceType)
|
||||
|
||||
dockerSource := &docker.Source{}
|
||||
if err := dockerSource.Init(ctx, sourceName, jobID, sourceID, true, conn, runtime.NumCPU()); err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = e.sourceManager.ScheduleRun(ctx, handle)
|
||||
_, err := e.sourceManager.Run(ctx, sourceName, dockerSource)
|
||||
return err
|
||||
}
|
||||
|
||||
+75
-14
@@ -19,6 +19,7 @@ import (
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/context"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/decoders"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/giturl"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/output"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/source_metadatapb"
|
||||
@@ -239,7 +240,7 @@ func (m *Metrics) getScanDuration() time.Duration {
|
||||
func (e *Engine) DetectorAvgTime() map[string][]time.Duration {
|
||||
logger := context.Background().Logger()
|
||||
avgTime := map[string][]time.Duration{}
|
||||
e.metrics.detectorAvgTime.Range(func(k, v interface{}) bool {
|
||||
e.metrics.detectorAvgTime.Range(func(k, v any) bool {
|
||||
key, ok := k.(string)
|
||||
if !ok {
|
||||
logger.Info("expected detectorAvgTime key to be a string")
|
||||
@@ -532,12 +533,12 @@ func (e *Engine) detectChunk(ctx context.Context, data detectableChunk) {
|
||||
}
|
||||
|
||||
for _, res := range results {
|
||||
e.processResult(data, res)
|
||||
e.processResult(ctx, data, res)
|
||||
}
|
||||
data.wgDoneFn()
|
||||
}
|
||||
|
||||
func (e *Engine) processResult(data detectableChunk, res detectors.Result) {
|
||||
func (e *Engine) processResult(ctx context.Context, data detectableChunk, res detectors.Result) {
|
||||
ignoreLinePresent := false
|
||||
if SupportsLineNumbers(data.chunk.SourceType) {
|
||||
copyChunk := data.chunk
|
||||
@@ -545,8 +546,12 @@ func (e *Engine) processResult(data detectableChunk, res detectors.Result) {
|
||||
if copyMetaData, ok := copyMetaDataClone.(*source_metadatapb.MetaData); ok {
|
||||
copyChunk.SourceMetadata = copyMetaData
|
||||
}
|
||||
fragStart, mdLine := FragmentFirstLine(©Chunk)
|
||||
fragStart, mdLine, link := FragmentFirstLineAndLink(©Chunk)
|
||||
ignoreLinePresent = SetResultLineNumber(©Chunk, &res, fragStart, mdLine)
|
||||
if err := UpdateLink(ctx, copyChunk.SourceMetadata, link, *mdLine); err != nil {
|
||||
ctx.Logger().Error(err, "error setting link")
|
||||
return
|
||||
}
|
||||
data.chunk = copyChunk
|
||||
}
|
||||
if ignoreLinePresent {
|
||||
@@ -565,11 +570,18 @@ func (e *Engine) notifyResults(ctx context.Context) {
|
||||
}
|
||||
atomic.AddUint32(&e.numFoundResults, 1)
|
||||
|
||||
// Dedupe results by comparing the detector type, raw result, and source metadata.
|
||||
// We want to avoid duplicate results with different decoder types, but we also
|
||||
// want to include duplicate results with the same decoder type.
|
||||
// Duplicate results with the same decoder type SHOULD have their own entry in the
|
||||
// results list, this would happen if the same secret is found multiple times.
|
||||
key := fmt.Sprintf("%s%s%s%+v", r.DetectorType.String(), r.Raw, r.RawV2, r.SourceMetadata)
|
||||
if _, ok := e.dedupeCache.Get(key); ok {
|
||||
continue
|
||||
if val, ok := e.dedupeCache.Get(key); ok {
|
||||
if res, ok := val.(detectorspb.DecoderType); ok && res != r.DecoderType {
|
||||
continue
|
||||
}
|
||||
}
|
||||
e.dedupeCache.Add(key, struct{}{})
|
||||
e.dedupeCache.Add(key, r.DecoderType)
|
||||
|
||||
if r.Verified {
|
||||
atomic.AddUint64(&e.metrics.VerifiedSecretsFound, 1)
|
||||
@@ -593,7 +605,8 @@ func SupportsLineNumbers(sourceType sourcespb.SourceType) bool {
|
||||
sourcespb.SourceType_SOURCE_TYPE_GERRIT,
|
||||
sourcespb.SourceType_SOURCE_TYPE_GITHUB_UNAUTHENTICATED_ORG,
|
||||
sourcespb.SourceType_SOURCE_TYPE_PUBLIC_GIT,
|
||||
sourcespb.SourceType_SOURCE_TYPE_FILESYSTEM:
|
||||
sourcespb.SourceType_SOURCE_TYPE_FILESYSTEM,
|
||||
sourcespb.SourceType_SOURCE_TYPE_AZURE_REPOS:
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
@@ -618,27 +631,45 @@ func FragmentLineOffset(chunk *sources.Chunk, result *detectors.Result) (int64,
|
||||
return lineNumber, false
|
||||
}
|
||||
|
||||
// FragmentFirstLine returns the first line number of a fragment along with a pointer to the value to update in the
|
||||
// chunk metadata.
|
||||
func FragmentFirstLine(chunk *sources.Chunk) (int64, *int64) {
|
||||
var fragmentStart *int64
|
||||
// FragmentFirstLineAndLink extracts the first line number and the link from the chunk metadata.
|
||||
// It returns:
|
||||
// - The first line number of the fragment.
|
||||
// - A pointer to the line number, facilitating direct updates.
|
||||
// - The link associated with the fragment. This link may be updated in the chunk metadata
|
||||
// if there's a change in the line number.
|
||||
func FragmentFirstLineAndLink(chunk *sources.Chunk) (int64, *int64, string) {
|
||||
if chunk.SourceMetadata == nil {
|
||||
return 0, nil, ""
|
||||
}
|
||||
|
||||
var (
|
||||
fragmentStart *int64
|
||||
link string
|
||||
)
|
||||
switch metadata := chunk.SourceMetadata.GetData().(type) {
|
||||
case *source_metadatapb.MetaData_Git:
|
||||
fragmentStart = &metadata.Git.Line
|
||||
case *source_metadatapb.MetaData_Github:
|
||||
fragmentStart = &metadata.Github.Line
|
||||
link = metadata.Github.Link
|
||||
case *source_metadatapb.MetaData_Gitlab:
|
||||
fragmentStart = &metadata.Gitlab.Line
|
||||
link = metadata.Gitlab.Link
|
||||
case *source_metadatapb.MetaData_Bitbucket:
|
||||
fragmentStart = &metadata.Bitbucket.Line
|
||||
link = metadata.Bitbucket.Link
|
||||
case *source_metadatapb.MetaData_Gerrit:
|
||||
fragmentStart = &metadata.Gerrit.Line
|
||||
case *source_metadatapb.MetaData_Filesystem:
|
||||
fragmentStart = &metadata.Filesystem.Line
|
||||
link = metadata.Filesystem.Link
|
||||
case *source_metadatapb.MetaData_AzureRepos:
|
||||
fragmentStart = &metadata.AzureRepos.Line
|
||||
link = metadata.AzureRepos.Link
|
||||
default:
|
||||
return 0, nil
|
||||
return 0, nil, ""
|
||||
}
|
||||
return *fragmentStart, fragmentStart
|
||||
return *fragmentStart, fragmentStart, link
|
||||
}
|
||||
|
||||
// SetResultLineNumber sets the line number in the provided result.
|
||||
@@ -647,3 +678,33 @@ func SetResultLineNumber(chunk *sources.Chunk, result *detectors.Result, fragSta
|
||||
*mdLine = fragStart + offset
|
||||
return skip
|
||||
}
|
||||
|
||||
// UpdateLink updates the link of the provided source metadata.
|
||||
func UpdateLink(ctx context.Context, metadata *source_metadatapb.MetaData, link string, line int64) error {
|
||||
if metadata == nil {
|
||||
return fmt.Errorf("metadata is nil when setting the link")
|
||||
}
|
||||
|
||||
if link == "" {
|
||||
ctx.Logger().V(4).Info("link is empty, skipping update")
|
||||
return nil
|
||||
}
|
||||
|
||||
newLink := giturl.UpdateLinkLineNumber(ctx, link, line)
|
||||
|
||||
switch meta := metadata.GetData().(type) {
|
||||
case *source_metadatapb.MetaData_Github:
|
||||
meta.Github.Link = newLink
|
||||
case *source_metadatapb.MetaData_Gitlab:
|
||||
meta.Gitlab.Link = newLink
|
||||
case *source_metadatapb.MetaData_Bitbucket:
|
||||
meta.Bitbucket.Link = newLink
|
||||
case *source_metadatapb.MetaData_Filesystem:
|
||||
meta.Filesystem.Link = newLink
|
||||
case *source_metadatapb.MetaData_AzureRepos:
|
||||
meta.AzureRepos.Link = newLink
|
||||
default:
|
||||
return fmt.Errorf("unsupported metadata type")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
+238
-12
@@ -2,10 +2,16 @@ package engine
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/context"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/decoders"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/source_metadatapb"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
|
||||
)
|
||||
@@ -146,21 +152,27 @@ func TestDefaultDecoders(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestSupportsLineNumbers(t *testing.T) {
|
||||
testCases := []struct {
|
||||
name string
|
||||
input sourcespb.SourceType
|
||||
expected bool
|
||||
tests := []struct {
|
||||
name string
|
||||
sourceType sourcespb.SourceType
|
||||
expectedValue bool
|
||||
}{
|
||||
{"Git source type", sourcespb.SourceType_SOURCE_TYPE_GIT, true},
|
||||
{"Github source type", sourcespb.SourceType_SOURCE_TYPE_GITHUB, true},
|
||||
{"Gitlab source type", sourcespb.SourceType_SOURCE_TYPE_GITLAB, true},
|
||||
{"Git source", sourcespb.SourceType_SOURCE_TYPE_GIT, true},
|
||||
{"Github source", sourcespb.SourceType_SOURCE_TYPE_GITHUB, true},
|
||||
{"Gitlab source", sourcespb.SourceType_SOURCE_TYPE_GITLAB, true},
|
||||
{"Bitbucket source", sourcespb.SourceType_SOURCE_TYPE_BITBUCKET, true},
|
||||
{"Gerrit source", sourcespb.SourceType_SOURCE_TYPE_GERRIT, true},
|
||||
{"Github unauthenticated org source", sourcespb.SourceType_SOURCE_TYPE_GITHUB_UNAUTHENTICATED_ORG, true},
|
||||
{"Public Git source", sourcespb.SourceType_SOURCE_TYPE_PUBLIC_GIT, true},
|
||||
{"Filesystem source", sourcespb.SourceType_SOURCE_TYPE_FILESYSTEM, true},
|
||||
{"Azure Repos source", sourcespb.SourceType_SOURCE_TYPE_AZURE_REPOS, true},
|
||||
{"Unsupported type", sourcespb.SourceType_SOURCE_TYPE_BUILDKITE, false},
|
||||
}
|
||||
|
||||
for _, tc := range testCases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
if result := SupportsLineNumbers(tc.input); result != tc.expected {
|
||||
t.Errorf("Expected %v for input %v, got %v", tc.expected, tc.input, result)
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
result := SupportsLineNumbers(tt.sourceType)
|
||||
assert.Equal(t, tt.expectedValue, result)
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -171,3 +183,217 @@ func BenchmarkSupportsLineNumbersLoop(b *testing.B) {
|
||||
_ = SupportsLineNumbers(sourceType)
|
||||
}
|
||||
}
|
||||
|
||||
// TestEngine_DuplicatSecrets is a test that detects ALL duplicate secrets with the same decoder.
|
||||
func TestEngine_DuplicatSecrets(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
|
||||
absPath, err := filepath.Abs("./testdata")
|
||||
assert.Nil(t, err)
|
||||
|
||||
ctx, cancel := context.WithTimeout(ctx, 10*time.Second)
|
||||
defer cancel()
|
||||
|
||||
e, err := Start(ctx,
|
||||
WithConcurrency(1),
|
||||
WithDecoders(decoders.DefaultDecoders()...),
|
||||
WithDetectors(true, DefaultDetectors()...),
|
||||
WithPrinter(new(discardPrinter)),
|
||||
)
|
||||
assert.Nil(t, err)
|
||||
|
||||
cfg := sources.FilesystemConfig{Paths: []string{absPath}}
|
||||
if err := e.ScanFileSystem(ctx, cfg); err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
// Wait for all the chunks to be processed.
|
||||
assert.Nil(t, e.Finish(ctx))
|
||||
want := uint64(5)
|
||||
assert.Equal(t, want, e.GetMetrics().UnverifiedSecretsFound)
|
||||
}
|
||||
|
||||
func TestFragmentFirstLineAndLink(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
chunk *sources.Chunk
|
||||
expectedLine int64
|
||||
expectedLink string
|
||||
}{
|
||||
{
|
||||
name: "Test Git Metadata",
|
||||
chunk: &sources.Chunk{
|
||||
SourceMetadata: &source_metadatapb.MetaData{
|
||||
Data: &source_metadatapb.MetaData_Git{
|
||||
Git: &source_metadatapb.Git{
|
||||
Line: 10,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
expectedLine: 10,
|
||||
expectedLink: "", // Git doesn't support links
|
||||
},
|
||||
{
|
||||
name: "Test Github Metadata",
|
||||
chunk: &sources.Chunk{
|
||||
SourceMetadata: &source_metadatapb.MetaData{
|
||||
Data: &source_metadatapb.MetaData_Github{
|
||||
Github: &source_metadatapb.Github{
|
||||
Line: 5,
|
||||
Link: "https://example.github.com",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
expectedLine: 5,
|
||||
expectedLink: "https://example.github.com",
|
||||
},
|
||||
{
|
||||
name: "Test Azure Repos Metadata",
|
||||
chunk: &sources.Chunk{
|
||||
SourceMetadata: &source_metadatapb.MetaData{
|
||||
Data: &source_metadatapb.MetaData_AzureRepos{
|
||||
AzureRepos: &source_metadatapb.AzureRepos{
|
||||
Line: 5,
|
||||
Link: "https://example.azure.com",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
expectedLine: 5,
|
||||
expectedLink: "https://example.azure.com",
|
||||
},
|
||||
{
|
||||
name: "Unsupported Type",
|
||||
chunk: &sources.Chunk{},
|
||||
expectedLine: 0,
|
||||
expectedLink: "",
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
line, linePtr, link := FragmentFirstLineAndLink(tt.chunk)
|
||||
assert.Equal(t, tt.expectedLink, link, "Mismatch in link")
|
||||
assert.Equal(t, tt.expectedLine, line, "Mismatch in line")
|
||||
|
||||
if linePtr != nil {
|
||||
assert.Equal(t, tt.expectedLine, *linePtr, "Mismatch in linePtr value")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestSetLink(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
input *source_metadatapb.MetaData
|
||||
link string
|
||||
line int64
|
||||
wantLink string
|
||||
wantErr bool
|
||||
}{
|
||||
{
|
||||
name: "Github link set",
|
||||
input: &source_metadatapb.MetaData{
|
||||
Data: &source_metadatapb.MetaData_Github{
|
||||
Github: &source_metadatapb.Github{},
|
||||
},
|
||||
},
|
||||
link: "https://github.com/example",
|
||||
line: 42,
|
||||
wantLink: "https://github.com/example#L42",
|
||||
},
|
||||
{
|
||||
name: "Gitlab link set",
|
||||
input: &source_metadatapb.MetaData{
|
||||
Data: &source_metadatapb.MetaData_Gitlab{
|
||||
Gitlab: &source_metadatapb.Gitlab{},
|
||||
},
|
||||
},
|
||||
link: "https://gitlab.com/example",
|
||||
line: 10,
|
||||
wantLink: "https://gitlab.com/example#L10",
|
||||
},
|
||||
{
|
||||
name: "Bitbucket link set",
|
||||
input: &source_metadatapb.MetaData{
|
||||
Data: &source_metadatapb.MetaData_Bitbucket{
|
||||
Bitbucket: &source_metadatapb.Bitbucket{},
|
||||
},
|
||||
},
|
||||
link: "https://bitbucket.com/example",
|
||||
line: 8,
|
||||
wantLink: "https://bitbucket.com/example#L8",
|
||||
},
|
||||
{
|
||||
name: "Filesystem link set",
|
||||
input: &source_metadatapb.MetaData{
|
||||
Data: &source_metadatapb.MetaData_Filesystem{
|
||||
Filesystem: &source_metadatapb.Filesystem{},
|
||||
},
|
||||
},
|
||||
link: "file:///path/to/example",
|
||||
line: 3,
|
||||
wantLink: "file:///path/to/example#L3",
|
||||
},
|
||||
{
|
||||
name: "Azure Repos link set",
|
||||
input: &source_metadatapb.MetaData{
|
||||
Data: &source_metadatapb.MetaData_AzureRepos{
|
||||
AzureRepos: &source_metadatapb.AzureRepos{},
|
||||
},
|
||||
},
|
||||
link: "https://dev.azure.com/example",
|
||||
line: 3,
|
||||
wantLink: "https://dev.azure.com/example?line=3",
|
||||
},
|
||||
{
|
||||
name: "Unsupported metadata type",
|
||||
input: &source_metadatapb.MetaData{
|
||||
Data: &source_metadatapb.MetaData_Git{
|
||||
Git: &source_metadatapb.Git{},
|
||||
},
|
||||
},
|
||||
link: "https://git.example.com/link",
|
||||
line: 5,
|
||||
wantErr: true,
|
||||
},
|
||||
{
|
||||
name: "Metadata nil",
|
||||
input: nil,
|
||||
link: "https://some.link",
|
||||
line: 1,
|
||||
wantErr: true,
|
||||
},
|
||||
}
|
||||
|
||||
ctx := context.Background()
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
err := UpdateLink(ctx, tt.input, tt.link, tt.line)
|
||||
if err != nil && !tt.wantErr {
|
||||
t.Errorf("Unexpected error: %v", err)
|
||||
}
|
||||
|
||||
if tt.wantErr {
|
||||
return
|
||||
}
|
||||
|
||||
switch data := tt.input.GetData().(type) {
|
||||
case *source_metadatapb.MetaData_Github:
|
||||
assert.Equal(t, tt.wantLink, data.Github.Link, "Github link mismatch")
|
||||
case *source_metadatapb.MetaData_Gitlab:
|
||||
assert.Equal(t, tt.wantLink, data.Gitlab.Link, "Gitlab link mismatch")
|
||||
case *source_metadatapb.MetaData_Bitbucket:
|
||||
assert.Equal(t, tt.wantLink, data.Bitbucket.Link, "Bitbucket link mismatch")
|
||||
case *source_metadatapb.MetaData_Filesystem:
|
||||
assert.Equal(t, tt.wantLink, data.Filesystem.Link, "Filesystem link mismatch")
|
||||
case *source_metadatapb.MetaData_AzureRepos:
|
||||
assert.Equal(t, tt.wantLink, data.AzureRepos.Link, "Azure Repos link mismatch")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -24,18 +24,14 @@ func (e *Engine) ScanFileSystem(ctx context.Context, c sources.FilesystemConfig)
|
||||
return err
|
||||
}
|
||||
|
||||
handle, err := e.sourceManager.Enroll(ctx, "trufflehog - filesystem", new(filesystem.Source).Type(),
|
||||
func(ctx context.Context, jobID, sourceID int64) (sources.Source, error) {
|
||||
fileSystemSource := filesystem.Source{}
|
||||
fileSystemSource.WithFilter(c.Filter)
|
||||
if err := fileSystemSource.Init(ctx, "trufflehog - filesystem", jobID, sourceID, true, &conn, runtime.NumCPU()); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &fileSystemSource, nil
|
||||
})
|
||||
if err != nil {
|
||||
sourceName := "trufflehog - filesystem"
|
||||
sourceID, jobID, _ := e.sourceManager.GetIDs(ctx, sourceName, filesystem.SourceType)
|
||||
|
||||
fileSystemSource := &filesystem.Source{}
|
||||
fileSystemSource.WithFilter(c.Filter)
|
||||
if err := fileSystemSource.Init(ctx, sourceName, jobID, sourceID, true, &conn, runtime.NumCPU()); err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = e.sourceManager.ScheduleRun(ctx, handle)
|
||||
_, err = e.sourceManager.Run(ctx, sourceName, fileSystemSource)
|
||||
return err
|
||||
}
|
||||
|
||||
+6
-10
@@ -44,18 +44,14 @@ func (e *Engine) ScanGCS(ctx context.Context, c sources.GCSConfig) error {
|
||||
return fmt.Errorf("failed to marshal GCS connection: %w", err)
|
||||
}
|
||||
|
||||
handle, err := e.sourceManager.Enroll(ctx, "trufflehog - gcs", new(gcs.Source).Type(),
|
||||
func(ctx context.Context, jobID, sourceID int64) (sources.Source, error) {
|
||||
gcsSource := gcs.Source{}
|
||||
if err := gcsSource.Init(ctx, "trufflehog - gcs", jobID, sourceID, true, &conn, int(c.Concurrency)); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &gcsSource, nil
|
||||
})
|
||||
if err != nil {
|
||||
sourceName := "trufflehog - gcs"
|
||||
sourceID, jobID, _ := e.sourceManager.GetIDs(ctx, sourceName, gcs.SourceType)
|
||||
|
||||
gcsSource := &gcs.Source{}
|
||||
if err := gcsSource.Init(ctx, sourceName, jobID, sourceID, true, &conn, int(c.Concurrency)); err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = e.sourceManager.ScheduleRun(ctx, handle)
|
||||
_, err = e.sourceManager.Run(ctx, sourceName, gcsSource)
|
||||
return err
|
||||
}
|
||||
|
||||
|
||||
+11
-14
@@ -51,21 +51,18 @@ func (e *Engine) ScanGit(ctx context.Context, c sources.GitConfig) error {
|
||||
return err
|
||||
}
|
||||
|
||||
handle, err := e.sourceManager.Enroll(ctx, "trufflehog - git", new(git.Source).Type(),
|
||||
func(ctx context.Context, jobID, sourceID int64) (sources.Source, error) {
|
||||
gitSource := git.Source{}
|
||||
if err := gitSource.Init(ctx, "trufflehog - git", jobID, sourceID, true, &conn, runtime.NumCPU()); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
gitSource.WithScanOptions(scanOptions)
|
||||
// Don't try to clean up the provided directory. That's handled by the
|
||||
// caller of ScanGit.
|
||||
gitSource.WithPreserveTempDirs(true)
|
||||
return &gitSource, nil
|
||||
})
|
||||
if err != nil {
|
||||
sourceName := "trufflehog - git"
|
||||
sourceID, jobID, _ := e.sourceManager.GetIDs(ctx, sourceName, git.SourceType)
|
||||
|
||||
gitSource := &git.Source{}
|
||||
if err := gitSource.Init(ctx, sourceName, jobID, sourceID, true, &conn, runtime.NumCPU()); err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = e.sourceManager.ScheduleRun(ctx, handle)
|
||||
gitSource.WithScanOptions(scanOptions)
|
||||
// Don't try to clean up the provided directory. That's handled by the
|
||||
// caller of ScanGit.
|
||||
gitSource.WithPreserveTempDirs(true)
|
||||
|
||||
_, err := e.sourceManager.Run(ctx, sourceName, gitSource)
|
||||
return err
|
||||
}
|
||||
|
||||
+7
-11
@@ -47,18 +47,14 @@ func (e *Engine) ScanGitHub(ctx context.Context, c sources.GithubConfig) error {
|
||||
}
|
||||
scanOptions := git.NewScanOptions(opts...)
|
||||
|
||||
handle, err := e.sourceManager.Enroll(ctx, "trufflehog - github", new(github.Source).Type(),
|
||||
func(ctx context.Context, jobID, sourceID int64) (sources.Source, error) {
|
||||
githubSource := github.Source{}
|
||||
if err := githubSource.Init(ctx, "trufflehog - github", jobID, sourceID, true, &conn, c.Concurrency); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
githubSource.WithScanOptions(scanOptions)
|
||||
return &githubSource, nil
|
||||
})
|
||||
if err != nil {
|
||||
sourceName := "trufflehog - github"
|
||||
sourceID, jobID, _ := e.sourceManager.GetIDs(ctx, sourceName, github.SourceType)
|
||||
|
||||
githubSource := &github.Source{}
|
||||
if err := githubSource.Init(ctx, sourceName, jobID, sourceID, true, &conn, c.Concurrency); err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = e.sourceManager.ScheduleRun(ctx, handle)
|
||||
githubSource.WithScanOptions(scanOptions)
|
||||
_, err = e.sourceManager.Run(ctx, sourceName, githubSource)
|
||||
return err
|
||||
}
|
||||
|
||||
+7
-11
@@ -50,18 +50,14 @@ func (e *Engine) ScanGitLab(ctx context.Context, c sources.GitlabConfig) error {
|
||||
return err
|
||||
}
|
||||
|
||||
handle, err := e.sourceManager.Enroll(ctx, "trufflehog - gitlab", new(gitlab.Source).Type(),
|
||||
func(ctx context.Context, jobID, sourceID int64) (sources.Source, error) {
|
||||
gitlabSource := gitlab.Source{}
|
||||
if err := gitlabSource.Init(ctx, "trufflehog - gitlab", jobID, sourceID, true, &conn, runtime.NumCPU()); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
gitlabSource.WithScanOptions(scanOptions)
|
||||
return &gitlabSource, nil
|
||||
})
|
||||
if err != nil {
|
||||
sourceName := "trufflehog - gitlab"
|
||||
sourceID, jobID, _ := e.sourceManager.GetIDs(ctx, sourceName, gitlab.SourceType)
|
||||
|
||||
gitlabSource := &gitlab.Source{}
|
||||
if err := gitlabSource.Init(ctx, sourceName, jobID, sourceID, true, &conn, runtime.NumCPU()); err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = e.sourceManager.ScheduleRun(ctx, handle)
|
||||
gitlabSource.WithScanOptions(scanOptions)
|
||||
_, err = e.sourceManager.Run(ctx, sourceName, gitlabSource)
|
||||
return err
|
||||
}
|
||||
|
||||
+6
-10
@@ -58,17 +58,13 @@ func (e *Engine) ScanS3(ctx context.Context, c sources.S3Config) error {
|
||||
return err
|
||||
}
|
||||
|
||||
handle, err := e.sourceManager.Enroll(ctx, "trufflehog - s3", new(s3.Source).Type(),
|
||||
func(ctx context.Context, jobID, sourceID int64) (sources.Source, error) {
|
||||
s3Source := s3.Source{}
|
||||
if err := s3Source.Init(ctx, "trufflehog - s3", jobID, sourceID, true, &conn, runtime.NumCPU()); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &s3Source, nil
|
||||
})
|
||||
if err != nil {
|
||||
sourceName := "trufflehog - s3"
|
||||
sourceID, jobID, _ := e.sourceManager.GetIDs(ctx, sourceName, s3.SourceType)
|
||||
|
||||
s3Source := &s3.Source{}
|
||||
if err := s3Source.Init(ctx, sourceName, jobID, sourceID, true, &conn, runtime.NumCPU()); err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = e.sourceManager.ScheduleRun(ctx, handle)
|
||||
_, err = e.sourceManager.Run(ctx, sourceName, s3Source)
|
||||
return err
|
||||
}
|
||||
|
||||
+7
-11
@@ -41,18 +41,14 @@ func (e *Engine) ScanSyslog(ctx context.Context, c sources.SyslogConfig) error {
|
||||
return errors.WrapPrefix(err, "error unmarshalling connection", 0)
|
||||
}
|
||||
|
||||
handle, err := e.sourceManager.Enroll(ctx, "trufflehog - syslog", new(syslog.Source).Type(),
|
||||
func(ctx context.Context, jobID, sourceID int64) (sources.Source, error) {
|
||||
syslogSource := syslog.Source{}
|
||||
if err := syslogSource.Init(ctx, "trufflehog - syslog", jobID, sourceID, true, &conn, c.Concurrency); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
syslogSource.InjectConnection(connection)
|
||||
return &syslogSource, nil
|
||||
})
|
||||
if err != nil {
|
||||
sourceName := "trufflehog - syslog"
|
||||
sourceID, jobID, _ := e.sourceManager.GetIDs(ctx, sourceName, syslog.SourceType)
|
||||
syslogSource := &syslog.Source{}
|
||||
if err := syslogSource.Init(ctx, sourceName, jobID, sourceID, true, &conn, c.Concurrency); err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = e.sourceManager.ScheduleRun(ctx, handle)
|
||||
syslogSource.InjectConnection(connection)
|
||||
|
||||
_, err = e.sourceManager.Run(ctx, sourceName, syslogSource)
|
||||
return err
|
||||
}
|
||||
|
||||
Vendored
+6
@@ -0,0 +1,6 @@
|
||||
1 aws AKIAWARWQKZNHMZBLY4I
|
||||
2 secret s6NbZeygUrUdM95K683Lb6IsILWXOJlJ8ZVd1Kw0
|
||||
sentry 27ac84f4bcdb4fca9701f4d6f6f58cd7d96b69c9d9754d40800645a51d668f90
|
||||
sentry 27ac84f4bcdb4fca9701f4d6f6f58cd7d96b69c9d9754d40800645a51d668f90
|
||||
sentry 27ac84f4bcdb4fca9701f4d6f6f58cd7d96b69c9d9754d40800645a51d668f90
|
||||
sentry 27ac84f4bcdb4fca9701f4d6f6f58cd7d96b69c9d9754d40800645a51d668f90
|
||||
+54
-10
@@ -2,10 +2,13 @@ package giturl
|
||||
|
||||
import (
|
||||
"net/url"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/pkg/errors"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/context"
|
||||
)
|
||||
|
||||
type provider string
|
||||
@@ -119,7 +122,18 @@ func GenerateLink(repo, commit, file string, line int64) string {
|
||||
case providerBitbucket:
|
||||
return repo[:len(repo)-4] + "/commits/" + commit
|
||||
|
||||
case providerAzure:
|
||||
baseLink := repo + "/commit/" + commit + "/" + file
|
||||
if line > 0 {
|
||||
baseLink += "?line=" + strconv.FormatInt(line, 10)
|
||||
}
|
||||
return baseLink
|
||||
|
||||
case providerGithub, providerGitlab:
|
||||
// If the provider name isn't one of the cloud defaults, it is probably an on-prem github or gitlab.
|
||||
// So do the same thing.
|
||||
fallthrough
|
||||
default:
|
||||
var baseLink string
|
||||
if file == "" {
|
||||
baseLink = repo[:len(repo)-4] + "/commit/" + commit
|
||||
@@ -130,15 +144,45 @@ func GenerateLink(repo, commit, file string, line int64) string {
|
||||
}
|
||||
}
|
||||
return baseLink
|
||||
|
||||
case providerAzure:
|
||||
baseLink := repo + "?path=" + file + "&version=GB" + commit
|
||||
if line > 0 {
|
||||
baseLink += "&line=" + strconv.FormatInt(line, 10)
|
||||
}
|
||||
return baseLink
|
||||
|
||||
default:
|
||||
return ""
|
||||
}
|
||||
}
|
||||
|
||||
var linePattern = regexp.MustCompile(`L\d+`)
|
||||
|
||||
// UpdateLinkLineNumber updates the line number in a repository link.
|
||||
// Used post-link generation to refine reported issue locations within large scanned blocks.
|
||||
func UpdateLinkLineNumber(ctx context.Context, link string, newLine int64) string {
|
||||
parsedURL, err := url.Parse(link)
|
||||
if err != nil {
|
||||
ctx.Logger().Error(err, "unable to parse link to update line number", "link", link)
|
||||
return link
|
||||
}
|
||||
|
||||
switch determineProvider(link) {
|
||||
case providerBitbucket:
|
||||
// For Bitbucket, it doesn't support line links (based on the GenerateLink function).
|
||||
// So we don't need to change anything.
|
||||
return link
|
||||
|
||||
case providerAzure:
|
||||
// For Azure, line numbers are appended as ?line=<number>.
|
||||
query := parsedURL.Query()
|
||||
query.Set("line", strconv.FormatInt(newLine, 10))
|
||||
parsedURL.RawQuery = query.Encode()
|
||||
|
||||
case providerGithub, providerGitlab:
|
||||
// If the provider name isn't one of the cloud defaults, it is probably an on-prem github or gitlab.
|
||||
// So do the same thing.
|
||||
fallthrough
|
||||
default:
|
||||
// Assumed format: .../blob/<commit>/file.go#L<number>
|
||||
fragment := "L" + strconv.FormatInt(newLine, 10)
|
||||
if linePattern.MatchString(parsedURL.Fragment) {
|
||||
parsedURL.Fragment = linePattern.ReplaceAllString(parsedURL.Fragment, fragment)
|
||||
} else {
|
||||
parsedURL.Fragment += fragment
|
||||
}
|
||||
}
|
||||
|
||||
return parsedURL.String()
|
||||
}
|
||||
|
||||
+103
-2
@@ -4,6 +4,8 @@ import (
|
||||
"testing"
|
||||
|
||||
"github.com/pkg/errors"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/context"
|
||||
)
|
||||
|
||||
func Test_NormalizeOrgRepoURL(t *testing.T) {
|
||||
@@ -148,7 +150,7 @@ func TestGenerateLink(t *testing.T) {
|
||||
commit: "abcdef",
|
||||
file: "main.go",
|
||||
},
|
||||
want: "https://dev.azure.com/org/project/_git/repo?path=main.go&version=GBabcdef",
|
||||
want: "https://dev.azure.com/org/project/_git/repo/commit/abcdef/main.go",
|
||||
},
|
||||
{
|
||||
name: "Azure link gen with line",
|
||||
@@ -158,7 +160,25 @@ func TestGenerateLink(t *testing.T) {
|
||||
file: "main.go",
|
||||
line: int64(20),
|
||||
},
|
||||
want: "https://dev.azure.com/org/project/_git/repo?path=main.go&version=GBabcdef&line=20",
|
||||
want: "https://dev.azure.com/org/project/_git/repo/commit/abcdef/main.go?line=20",
|
||||
},
|
||||
{
|
||||
name: "Unknown provider on-prem instance",
|
||||
args: args{
|
||||
repo: "https://onprem.customdomain.com/org/repo.git",
|
||||
commit: "xyz123",
|
||||
file: "main.go",
|
||||
line: int64(30),
|
||||
},
|
||||
want: "https://onprem.customdomain.com/org/repo/blob/xyz123/main.go#L30",
|
||||
},
|
||||
{
|
||||
name: "Unknown provider on-prem instance - no file",
|
||||
args: args{
|
||||
repo: "https://onprem.customdomain.com/org/repo.git",
|
||||
commit: "xyz123",
|
||||
},
|
||||
want: "https://onprem.customdomain.com/org/repo/commit/xyz123",
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
@@ -169,3 +189,84 @@ func TestGenerateLink(t *testing.T) {
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestUpdateLinkLineNumber(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
type args struct {
|
||||
link string
|
||||
newLine int64
|
||||
}
|
||||
tests := []struct {
|
||||
name string
|
||||
args args
|
||||
want string
|
||||
wantErr bool
|
||||
}{
|
||||
{
|
||||
name: "Update bitbucket, no line number supported",
|
||||
args: args{
|
||||
link: "https://bitbucket.org/org/repo/blob/xyz123/main.go",
|
||||
newLine: int64(10),
|
||||
},
|
||||
want: "https://bitbucket.org/org/repo/blob/xyz123/main.go",
|
||||
},
|
||||
{
|
||||
name: "Update github link with line",
|
||||
args: args{
|
||||
link: "https://github.com/trufflesec-julian/confluence-go-api/blob/047b4a2ba42fc5b6c0bd535c5307434a666db5ec/.gitignore#L4",
|
||||
newLine: int64(10),
|
||||
},
|
||||
want: "https://github.com/trufflesec-julian/confluence-go-api/blob/047b4a2ba42fc5b6c0bd535c5307434a666db5ec/.gitignore#L10",
|
||||
},
|
||||
{
|
||||
name: "Update Azure link with line",
|
||||
args: args{
|
||||
link: "https://dev.azure.com/org/project/_git/repo/commit/abcdef/main.go?line=20",
|
||||
newLine: int64(40),
|
||||
},
|
||||
want: "https://dev.azure.com/org/project/_git/repo/commit/abcdef/main.go?line=40",
|
||||
},
|
||||
{
|
||||
name: "Add line to github link without line",
|
||||
args: args{
|
||||
link: "https://github.com/trufflesec-julian/confluence-go-api/blob/047b4a2ba42fc5b6c0bd535c5307434a666db5ec/.gitignore",
|
||||
newLine: int64(7),
|
||||
},
|
||||
want: "https://github.com/trufflesec-julian/confluence-go-api/blob/047b4a2ba42fc5b6c0bd535c5307434a666db5ec/.gitignore#L7",
|
||||
},
|
||||
{
|
||||
name: "Update Unknown provider on-prem instance with line",
|
||||
args: args{
|
||||
link: "https://onprem.customdomain.com/org/repo/blob/xyz123/main.go#L30",
|
||||
newLine: int64(50),
|
||||
},
|
||||
want: "https://onprem.customdomain.com/org/repo/blob/xyz123/main.go#L50",
|
||||
},
|
||||
{
|
||||
name: "Update Unknown provider on-prem instance without line",
|
||||
args: args{
|
||||
link: "https://onprem.customdomain.com/org/repo/commit/xyz123",
|
||||
newLine: int64(50),
|
||||
},
|
||||
want: "https://onprem.customdomain.com/org/repo/commit/xyz123#L50",
|
||||
},
|
||||
{
|
||||
name: "Invalid link",
|
||||
args: args{
|
||||
link: "definitely not a link",
|
||||
newLine: int64(50),
|
||||
},
|
||||
wantErr: true,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := UpdateLinkLineNumber(context.Background(), tt.args.link, tt.args.newLine)
|
||||
if got != tt.want && !tt.wantErr {
|
||||
t.Errorf("UpdateLinkLineNumber() = %v, want %v", got, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
+56
-33
@@ -17,12 +17,15 @@ import (
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
logContext "github.com/trufflesecurity/trufflehog/v3/pkg/context"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
|
||||
)
|
||||
|
||||
type ctxKey int
|
||||
|
||||
const (
|
||||
depthKey ctxKey = iota
|
||||
|
||||
errMaxArchiveDepthReached = "max archive depth reached"
|
||||
)
|
||||
|
||||
var (
|
||||
@@ -75,57 +78,76 @@ func (a *Archive) FromFile(originalCtx context.Context, data io.Reader) chan []b
|
||||
if errors.Is(err, archiver.ErrNoMatch) {
|
||||
return
|
||||
}
|
||||
logger.V(2).Info("Error unarchiving chunk.")
|
||||
logger.Error(err, "error unarchiving chunk.")
|
||||
}
|
||||
}()
|
||||
return archiveChan
|
||||
}
|
||||
|
||||
type decompressorInfo struct {
|
||||
depth int
|
||||
reader io.Reader
|
||||
archiveChan chan []byte
|
||||
archiver archiver.Decompressor
|
||||
}
|
||||
|
||||
// openArchive takes a reader and extracts the contents up to the maximum depth.
|
||||
func (a *Archive) openArchive(ctx context.Context, depth int, reader io.Reader, archiveChan chan []byte) error {
|
||||
if depth >= maxDepth {
|
||||
return fmt.Errorf("max archive depth reached")
|
||||
return fmt.Errorf(errMaxArchiveDepthReached)
|
||||
}
|
||||
format, reader, err := archiver.Identify("", reader)
|
||||
|
||||
format, arReader, err := archiver.Identify("", reader)
|
||||
if errors.Is(err, archiver.ErrNoMatch) && depth > 0 {
|
||||
return a.handleNonArchiveContent(ctx, arReader, archiveChan)
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
if errors.Is(err, archiver.ErrNoMatch) && depth > 0 {
|
||||
chunkSize := 10 * 1024
|
||||
for {
|
||||
chunk := make([]byte, chunkSize)
|
||||
n, _ := reader.Read(chunk)
|
||||
archiveChan <- chunk
|
||||
if n < chunkSize {
|
||||
break
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
switch archive := format.(type) {
|
||||
case archiver.Decompressor:
|
||||
compReader, err := archive.OpenReader(reader)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fileBytes, err := a.ReadToMax(ctx, compReader)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
newReader := bytes.NewReader(fileBytes)
|
||||
return a.openArchive(ctx, depth+1, newReader, archiveChan)
|
||||
info := decompressorInfo{depth: depth, reader: arReader, archiveChan: archiveChan, archiver: archive}
|
||||
return a.handleDecompressor(ctx, info)
|
||||
case archiver.Extractor:
|
||||
err := archive.Extract(context.WithValue(ctx, depthKey, depth+1), reader, nil, a.extractorHandler(archiveChan))
|
||||
if err != nil {
|
||||
return archive.Extract(context.WithValue(ctx, depthKey, depth+1), reader, nil, a.extractorHandler(archiveChan))
|
||||
default:
|
||||
return fmt.Errorf("unknown archive type: %s", format.Name())
|
||||
}
|
||||
}
|
||||
|
||||
func (a *Archive) handleNonArchiveContent(ctx context.Context, reader io.Reader, archiveChan chan []byte) error {
|
||||
aCtx := logContext.AddLogger(ctx)
|
||||
chunkReader := sources.NewChunkReader()
|
||||
chunkResChan := chunkReader(aCtx, reader)
|
||||
for data := range chunkResChan {
|
||||
if err := data.Error(); err != nil {
|
||||
aCtx.Logger().Error(err, "error reading chunk")
|
||||
continue
|
||||
}
|
||||
if err := common.CancellableWrite(ctx, archiveChan, data.Bytes()); err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
return fmt.Errorf("Unknown archive type: %s", format.Name())
|
||||
return nil
|
||||
}
|
||||
|
||||
func (a *Archive) handleDecompressor(ctx context.Context, info decompressorInfo) error {
|
||||
compReader, err := info.archiver.OpenReader(info.reader)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fileBytes, err := a.ReadToMax(ctx, compReader)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
newReader := bytes.NewReader(fileBytes)
|
||||
return a.openArchive(ctx, info.depth+1, newReader, info.archiveChan)
|
||||
}
|
||||
|
||||
// IsFiletype returns true if the provided reader is an archive.
|
||||
func (a *Archive) IsFiletype(ctx context.Context, reader io.Reader) (io.Reader, bool) {
|
||||
func (a *Archive) IsFiletype(_ context.Context, reader io.Reader) (io.Reader, bool) {
|
||||
format, readerB, err := archiver.Identify("", reader)
|
||||
if err != nil {
|
||||
return readerB, false
|
||||
@@ -135,8 +157,9 @@ func (a *Archive) IsFiletype(ctx context.Context, reader io.Reader) (io.Reader,
|
||||
return readerB, true
|
||||
case archiver.Decompressor:
|
||||
return readerB, true
|
||||
default:
|
||||
return readerB, false
|
||||
}
|
||||
return readerB, false
|
||||
}
|
||||
|
||||
// extractorHandler is applied to each file in an archiver.Extractor file.
|
||||
@@ -180,7 +203,7 @@ func (a *Archive) ReadToMax(ctx context.Context, reader io.Reader) (data []byte,
|
||||
if e, ok := r.(error); ok {
|
||||
err = e
|
||||
} else {
|
||||
err = fmt.Errorf("Panic occurred: %v", r)
|
||||
err = fmt.Errorf("panic occurred: %v", r)
|
||||
}
|
||||
logger.Error(err, "Panic occurred when reading archive")
|
||||
}
|
||||
@@ -283,7 +306,7 @@ func (a *Archive) HandleSpecialized(ctx logContext.Context, reader io.Reader) (i
|
||||
// The caller is responsible for closing the returned reader.
|
||||
func (a *Archive) extractDebContent(ctx logContext.Context, file io.Reader) (io.ReadCloser, error) {
|
||||
if a.currentDepth >= maxDepth {
|
||||
return nil, fmt.Errorf("max archive depth reached")
|
||||
return nil, fmt.Errorf(errMaxArchiveDepthReached)
|
||||
}
|
||||
|
||||
tmpEnv, err := a.createTempEnv(ctx, file)
|
||||
|
||||
@@ -9,6 +9,7 @@ import (
|
||||
"regexp"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
diskbufferreader "github.com/bill-rich/disk-buffer-reader"
|
||||
"github.com/stretchr/testify/assert"
|
||||
@@ -236,3 +237,37 @@ func TestExtractRPMContent(t *testing.T) {
|
||||
expectedLength := 1822720
|
||||
assert.Equal(t, expectedLength, len(string(content)))
|
||||
}
|
||||
|
||||
func TestOpenInvalidArchive(t *testing.T) {
|
||||
reader := strings.NewReader("invalid archive")
|
||||
|
||||
ctx := logContext.AddLogger(context.Background())
|
||||
a := &Archive{}
|
||||
|
||||
archiveChan := make(chan []byte)
|
||||
|
||||
err := a.openArchive(ctx, 0, reader, archiveChan)
|
||||
assert.Error(t, err)
|
||||
}
|
||||
|
||||
func TestNestedDirArchive(t *testing.T) {
|
||||
file, err := os.Open("testdata/dir-archive.zip")
|
||||
assert.Nil(t, err)
|
||||
defer file.Close()
|
||||
|
||||
ctx, cancel := logContext.WithTimeout(logContext.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
sourceChan := make(chan *sources.Chunk, 1)
|
||||
|
||||
go func() {
|
||||
defer close(sourceChan)
|
||||
HandleFile(ctx, file, &sources.Chunk{}, sourceChan)
|
||||
}()
|
||||
|
||||
count := 0
|
||||
want := 4
|
||||
for range sourceChan {
|
||||
count++
|
||||
}
|
||||
assert.Equal(t, want, count)
|
||||
}
|
||||
|
||||
+29
-27
@@ -47,42 +47,44 @@ func HandleFile(ctx context.Context, file io.Reader, chunkSkel *sources.Chunk, c
|
||||
// an io.MultiReader, which is used by the SpecializedHandler.
|
||||
reReader, err := diskbufferreader.New(file)
|
||||
if err != nil {
|
||||
aCtx.Logger().Error(err, "error creating re-reader reader")
|
||||
aCtx.Logger().Error(err, "error creating reusable reader")
|
||||
return false
|
||||
}
|
||||
defer reReader.Close()
|
||||
|
||||
// Check if the handler implements SpecializedHandler and process accordingly.
|
||||
if specialHandler, ok := h.(SpecializedHandler); ok {
|
||||
file, isSpecial, err := specialHandler.HandleSpecialized(aCtx, reReader)
|
||||
if isSpecial {
|
||||
return handleChunks(aCtx, h.FromFile(ctx, file), chunkSkel, chunksChan)
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
aCtx.Logger().Error(err, "error handling file")
|
||||
}
|
||||
if success := processHandler(aCtx, h, reReader, chunkSkel, chunksChan); success {
|
||||
return true
|
||||
}
|
||||
|
||||
if err := reReader.Reset(); err != nil {
|
||||
aCtx.Logger().Error(err, "error resetting re-reader")
|
||||
return false
|
||||
}
|
||||
if _, isType := h.IsFiletype(aCtx, reReader); !isType {
|
||||
continue
|
||||
}
|
||||
|
||||
if err := reReader.Reset(); err != nil {
|
||||
aCtx.Logger().Error(err, "error resetting re-reader")
|
||||
return false
|
||||
}
|
||||
reReader.Stop()
|
||||
return handleChunks(aCtx, h.FromFile(ctx, reReader), chunkSkel, chunksChan)
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
func processHandler(ctx logContext.Context, h Handler, reReader *diskbufferreader.DiskBufferReader, chunkSkel *sources.Chunk, chunksChan chan *sources.Chunk) bool {
|
||||
defer reReader.Close()
|
||||
defer reReader.Stop()
|
||||
|
||||
if specialHandler, ok := h.(SpecializedHandler); ok {
|
||||
file, isSpecial, err := specialHandler.HandleSpecialized(ctx, reReader)
|
||||
if isSpecial {
|
||||
return handleChunks(ctx, h.FromFile(ctx, file), chunkSkel, chunksChan)
|
||||
}
|
||||
if err != nil {
|
||||
ctx.Logger().Error(err, "error handling file")
|
||||
}
|
||||
}
|
||||
|
||||
if _, err := reReader.Seek(0, io.SeekStart); err != nil {
|
||||
ctx.Logger().Error(err, "error seeking to start of file")
|
||||
return false
|
||||
}
|
||||
|
||||
if _, isType := h.IsFiletype(ctx, reReader); !isType {
|
||||
return false
|
||||
}
|
||||
|
||||
return handleChunks(ctx, h.FromFile(ctx, reReader), chunkSkel, chunksChan)
|
||||
}
|
||||
|
||||
func handleChunks(ctx context.Context, handlerChan chan []byte, chunkSkel *sources.Chunk, chunksChan chan *sources.Chunk) bool {
|
||||
for {
|
||||
select {
|
||||
|
||||
BIN
Binary file not shown.
+2
-1
@@ -10,6 +10,7 @@ import (
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/source_metadatapb"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
|
||||
)
|
||||
|
||||
// JSONPrinter is a printer that prints results in JSON format.
|
||||
@@ -20,7 +21,7 @@ func (p *JSONPrinter) Print(_ context.Context, r *detectors.ResultWithMetadata)
|
||||
// SourceMetadata contains source-specific contextual information.
|
||||
SourceMetadata *source_metadatapb.MetaData
|
||||
// SourceID is the ID of the source that the API uses to map secrets to specific sources.
|
||||
SourceID int64
|
||||
SourceID sources.SourceID
|
||||
// SourceType is the type of Source.
|
||||
SourceType sourcespb.SourceType
|
||||
// SourceName is the name of the Source.
|
||||
|
||||
+2
-2
@@ -76,7 +76,7 @@ func (p *PlainPrinter) Print(_ context.Context, r *detectors.ResultWithMetadata)
|
||||
}
|
||||
}
|
||||
|
||||
aggregateData := make(map[string]interface{})
|
||||
aggregateData := make(map[string]any)
|
||||
var aggregateDataKeys []string
|
||||
|
||||
for _, data := range meta {
|
||||
@@ -93,7 +93,7 @@ func (p *PlainPrinter) Print(_ context.Context, r *detectors.ResultWithMetadata)
|
||||
return nil
|
||||
}
|
||||
|
||||
func structToMap(obj interface{}) (m map[string]map[string]interface{}, err error) {
|
||||
func structToMap(obj any) (m map[string]map[string]any, err error) {
|
||||
data, err := json.Marshal(obj)
|
||||
if err != nil {
|
||||
return
|
||||
|
||||
@@ -999,6 +999,8 @@ const (
|
||||
DetectorType_Salesforce DetectorType = 925
|
||||
DetectorType_HuggingFace DetectorType = 926
|
||||
DetectorType_Snowflake DetectorType = 927
|
||||
DetectorType_Sourcegraph DetectorType = 928
|
||||
DetectorType_Tailscale DetectorType = 929
|
||||
)
|
||||
|
||||
// Enum value maps for DetectorType.
|
||||
@@ -1928,6 +1930,8 @@ var (
|
||||
925: "Salesforce",
|
||||
926: "HuggingFace",
|
||||
927: "Snowflake",
|
||||
928: "Sourcegraph",
|
||||
929: "Tailscale",
|
||||
}
|
||||
DetectorType_value = map[string]int32{
|
||||
"Alibaba": 0,
|
||||
@@ -2854,6 +2858,8 @@ var (
|
||||
"Salesforce": 925,
|
||||
"HuggingFace": 926,
|
||||
"Snowflake": 927,
|
||||
"Sourcegraph": 928,
|
||||
"Tailscale": 929,
|
||||
}
|
||||
)
|
||||
|
||||
@@ -3232,7 +3238,7 @@ var file_detectors_proto_rawDesc = []byte{
|
||||
0x44, 0x65, 0x63, 0x6f, 0x64, 0x65, 0x72, 0x54, 0x79, 0x70, 0x65, 0x12, 0x0b, 0x0a, 0x07, 0x55,
|
||||
0x4e, 0x4b, 0x4e, 0x4f, 0x57, 0x4e, 0x10, 0x00, 0x12, 0x09, 0x0a, 0x05, 0x50, 0x4c, 0x41, 0x49,
|
||||
0x4e, 0x10, 0x01, 0x12, 0x0a, 0x0a, 0x06, 0x42, 0x41, 0x53, 0x45, 0x36, 0x34, 0x10, 0x02, 0x12,
|
||||
0x09, 0x0a, 0x05, 0x55, 0x54, 0x46, 0x31, 0x36, 0x10, 0x03, 0x2a, 0x8c, 0x74, 0x0a, 0x0c, 0x44,
|
||||
0x09, 0x0a, 0x05, 0x55, 0x54, 0x46, 0x31, 0x36, 0x10, 0x03, 0x2a, 0xae, 0x74, 0x0a, 0x0c, 0x44,
|
||||
0x65, 0x74, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x54, 0x79, 0x70, 0x65, 0x12, 0x0b, 0x0a, 0x07, 0x41,
|
||||
0x6c, 0x69, 0x62, 0x61, 0x62, 0x61, 0x10, 0x00, 0x12, 0x08, 0x0a, 0x04, 0x41, 0x4d, 0x51, 0x50,
|
||||
0x10, 0x01, 0x12, 0x07, 0x0a, 0x03, 0x41, 0x57, 0x53, 0x10, 0x02, 0x12, 0x09, 0x0a, 0x05, 0x41,
|
||||
@@ -4161,11 +4167,14 @@ var file_detectors_proto_rawDesc = []byte{
|
||||
0x74, 0x68, 0x32, 0x10, 0x9c, 0x07, 0x12, 0x0f, 0x0a, 0x0a, 0x53, 0x61, 0x6c, 0x65, 0x73, 0x66,
|
||||
0x6f, 0x72, 0x63, 0x65, 0x10, 0x9d, 0x07, 0x12, 0x10, 0x0a, 0x0b, 0x48, 0x75, 0x67, 0x67, 0x69,
|
||||
0x6e, 0x67, 0x46, 0x61, 0x63, 0x65, 0x10, 0x9e, 0x07, 0x12, 0x0e, 0x0a, 0x09, 0x53, 0x6e, 0x6f,
|
||||
0x77, 0x66, 0x6c, 0x61, 0x6b, 0x65, 0x10, 0x9f, 0x07, 0x42, 0x3d, 0x5a, 0x3b, 0x67, 0x69, 0x74,
|
||||
0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x74, 0x72, 0x75, 0x66, 0x66, 0x6c, 0x65, 0x73,
|
||||
0x65, 0x63, 0x75, 0x72, 0x69, 0x74, 0x79, 0x2f, 0x74, 0x72, 0x75, 0x66, 0x66, 0x6c, 0x65, 0x68,
|
||||
0x6f, 0x67, 0x2f, 0x76, 0x33, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x70, 0x62, 0x2f, 0x64, 0x65, 0x74,
|
||||
0x65, 0x63, 0x74, 0x6f, 0x72, 0x73, 0x70, 0x62, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
|
||||
0x77, 0x66, 0x6c, 0x61, 0x6b, 0x65, 0x10, 0x9f, 0x07, 0x12, 0x10, 0x0a, 0x0b, 0x53, 0x6f, 0x75,
|
||||
0x72, 0x63, 0x65, 0x67, 0x72, 0x61, 0x70, 0x68, 0x10, 0xa0, 0x07, 0x12, 0x0e, 0x0a, 0x09, 0x54,
|
||||
0x61, 0x69, 0x6c, 0x73, 0x63, 0x61, 0x6c, 0x65, 0x10, 0xa1, 0x07, 0x42, 0x3d, 0x5a, 0x3b, 0x67,
|
||||
0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x74, 0x72, 0x75, 0x66, 0x66, 0x6c,
|
||||
0x65, 0x73, 0x65, 0x63, 0x75, 0x72, 0x69, 0x74, 0x79, 0x2f, 0x74, 0x72, 0x75, 0x66, 0x66, 0x6c,
|
||||
0x65, 0x68, 0x6f, 0x67, 0x2f, 0x76, 0x33, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x70, 0x62, 0x2f, 0x64,
|
||||
0x65, 0x74, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x73, 0x70, 0x62, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74,
|
||||
0x6f, 0x33,
|
||||
}
|
||||
|
||||
var (
|
||||
|
||||
@@ -19,13 +19,17 @@ import (
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
|
||||
)
|
||||
|
||||
const baseURL = "https://circleci.com/api/v1.1/"
|
||||
const (
|
||||
SourceType = sourcespb.SourceType_SOURCE_TYPE_CIRCLECI
|
||||
|
||||
baseURL = "https://circleci.com/api/v1.1/"
|
||||
)
|
||||
|
||||
type Source struct {
|
||||
name string
|
||||
token string
|
||||
sourceId int64
|
||||
jobId int64
|
||||
sourceId sources.SourceID
|
||||
jobId sources.JobID
|
||||
verify bool
|
||||
jobPool *errgroup.Group
|
||||
sources.Progress
|
||||
@@ -40,19 +44,19 @@ var _ sources.SourceUnitUnmarshaller = (*Source)(nil)
|
||||
// Type returns the type of source.
|
||||
// It is used for matching source types in configuration and job input.
|
||||
func (s *Source) Type() sourcespb.SourceType {
|
||||
return sourcespb.SourceType_SOURCE_TYPE_CIRCLECI
|
||||
return SourceType
|
||||
}
|
||||
|
||||
func (s *Source) SourceID() int64 {
|
||||
func (s *Source) SourceID() sources.SourceID {
|
||||
return s.sourceId
|
||||
}
|
||||
|
||||
func (s *Source) JobID() int64 {
|
||||
func (s *Source) JobID() sources.JobID {
|
||||
return s.jobId
|
||||
}
|
||||
|
||||
// Init returns an initialized CircleCI source.
|
||||
func (s *Source) Init(_ context.Context, name string, jobId, sourceId int64, verify bool, connection *anypb.Any, concurrency int) error {
|
||||
func (s *Source) Init(_ context.Context, name string, jobId sources.JobID, sourceId sources.SourceID, verify bool, connection *anypb.Any, concurrency int) error {
|
||||
s.name = name
|
||||
s.sourceId = sourceId
|
||||
s.jobId = jobId
|
||||
@@ -75,7 +79,7 @@ func (s *Source) Init(_ context.Context, name string, jobId, sourceId int64, ver
|
||||
}
|
||||
|
||||
// Chunks emits chunks of bytes over a channel.
|
||||
func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) error {
|
||||
func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk, _ ...sources.ChunkingTarget) error {
|
||||
projects, err := s.projects(ctx)
|
||||
if err != nil {
|
||||
return fmt.Errorf("error getting projects: %w", err)
|
||||
|
||||
+229
-146
@@ -2,7 +2,6 @@ package docker
|
||||
|
||||
import (
|
||||
"archive/tar"
|
||||
"bytes"
|
||||
"compress/gzip"
|
||||
"errors"
|
||||
"fmt"
|
||||
@@ -14,27 +13,30 @@ import (
|
||||
v1 "github.com/google/go-containerregistry/pkg/v1"
|
||||
"github.com/google/go-containerregistry/pkg/v1/remote"
|
||||
"github.com/google/go-containerregistry/pkg/v1/tarball"
|
||||
"golang.org/x/sync/errgroup"
|
||||
"google.golang.org/protobuf/proto"
|
||||
"google.golang.org/protobuf/types/known/anypb"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/context"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/source_metadatapb"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
|
||||
)
|
||||
|
||||
const SourceType = sourcespb.SourceType_SOURCE_TYPE_DOCKER
|
||||
|
||||
type Source struct {
|
||||
name string
|
||||
sourceId int64
|
||||
jobId int64
|
||||
verify bool
|
||||
conn sourcespb.Docker
|
||||
name string
|
||||
sourceId sources.SourceID
|
||||
jobId sources.JobID
|
||||
verify bool
|
||||
concurrency int
|
||||
conn sourcespb.Docker
|
||||
sources.Progress
|
||||
sources.CommonSourceUnitUnmarshaller
|
||||
}
|
||||
|
||||
var FilesizeLimitBytes int64 = 50 * 1024 * 1024 // 50MB
|
||||
|
||||
// Ensure the Source satisfies the interfaces at compile time.
|
||||
var _ sources.Source = (*Source)(nil)
|
||||
var _ sources.SourceUnitUnmarshaller = (*Source)(nil)
|
||||
@@ -42,23 +44,24 @@ var _ sources.SourceUnitUnmarshaller = (*Source)(nil)
|
||||
// Type returns the type of source.
|
||||
// It is used for matching source types in configuration and job input.
|
||||
func (s *Source) Type() sourcespb.SourceType {
|
||||
return sourcespb.SourceType_SOURCE_TYPE_DOCKER
|
||||
return SourceType
|
||||
}
|
||||
|
||||
func (s *Source) SourceID() int64 {
|
||||
func (s *Source) SourceID() sources.SourceID {
|
||||
return s.sourceId
|
||||
}
|
||||
|
||||
func (s *Source) JobID() int64 {
|
||||
func (s *Source) JobID() sources.JobID {
|
||||
return s.jobId
|
||||
}
|
||||
|
||||
// Init initializes the source.
|
||||
func (s *Source) Init(_ context.Context, name string, jobId, sourceId int64, verify bool, connection *anypb.Any, concurrency int) error {
|
||||
func (s *Source) Init(_ context.Context, name string, jobId sources.JobID, sourceId sources.SourceID, verify bool, connection *anypb.Any, concurrency int) error {
|
||||
s.name = name
|
||||
s.sourceId = sourceId
|
||||
s.jobId = jobId
|
||||
s.verify = verify
|
||||
s.concurrency = concurrency
|
||||
|
||||
// Reset metrics for this source at initialization time.
|
||||
dockerImagesScanned.WithLabelValues(s.name).Set(0)
|
||||
@@ -71,164 +74,207 @@ func (s *Source) Init(_ context.Context, name string, jobId, sourceId int64, ver
|
||||
return nil
|
||||
}
|
||||
|
||||
type imageInfo struct {
|
||||
image v1.Image
|
||||
base string
|
||||
tag string
|
||||
}
|
||||
|
||||
type layerInfo struct {
|
||||
digest v1.Hash
|
||||
base string
|
||||
tag string
|
||||
}
|
||||
|
||||
// Chunks emits data over a channel that is decoded and scanned for secrets.
|
||||
func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) error {
|
||||
remoteOpts, err := s.remoteOpts()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk, _ ...sources.ChunkingTarget) error {
|
||||
ctx = context.WithValues(ctx, "source_type", s.Type(), "source_name", s.name)
|
||||
|
||||
logger := ctx.Logger().WithValues("source_type", s.Type(), "source_name", s.name)
|
||||
workers := new(errgroup.Group)
|
||||
workers.SetLimit(s.concurrency)
|
||||
|
||||
scanErrs := sources.NewScanErrors()
|
||||
for _, image := range s.conn.GetImages() {
|
||||
var img v1.Image
|
||||
var err error
|
||||
var base, tag string
|
||||
var hasDigest bool
|
||||
var imageName name.Reference
|
||||
image := image
|
||||
workers.Go(func() error {
|
||||
if common.IsDone(ctx) {
|
||||
return nil
|
||||
}
|
||||
|
||||
if strings.HasPrefix(image, "file://") {
|
||||
image = strings.TrimPrefix(image, "file://")
|
||||
base = image
|
||||
img, err = tarball.ImageFromPath(image, nil)
|
||||
imgInfo, err := s.processImage(ctx, image)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
} else {
|
||||
base, tag, hasDigest = baseAndTagFromImage(image)
|
||||
|
||||
if hasDigest {
|
||||
imageName, err = name.NewDigest(image)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
} else {
|
||||
imageName, err = name.NewTag(image)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
scanErrs.Add(err)
|
||||
return nil
|
||||
}
|
||||
|
||||
img, err = remote.Image(imageName, remoteOpts...)
|
||||
ctx = context.WithValues(ctx, "image", imgInfo.base, "tag", imgInfo.tag)
|
||||
ctx.Logger().V(2).Info("scanning image")
|
||||
|
||||
layers, err := imgInfo.image.Layers()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
logger = logger.WithValues("image", base, "tag", tag)
|
||||
logger.V(2).Info("scanning image")
|
||||
|
||||
layers, err := img.Layers()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
for _, layer := range layers {
|
||||
|
||||
digest, err := layer.Digest()
|
||||
if err != nil {
|
||||
return err
|
||||
scanErrs.Add(err)
|
||||
return nil
|
||||
}
|
||||
|
||||
logger = logger.WithValues("layer", digest.String())
|
||||
logger.V(2).Info("scanning layer")
|
||||
|
||||
rc, err := layer.Compressed()
|
||||
if err != nil {
|
||||
return err
|
||||
for _, layer := range layers {
|
||||
if err := s.processLayer(ctx, layer, imgInfo, chunksChan); err != nil {
|
||||
scanErrs.Add(err)
|
||||
return nil
|
||||
}
|
||||
dockerLayersScanned.WithLabelValues(s.name).Inc()
|
||||
}
|
||||
|
||||
defer rc.Close()
|
||||
dockerImagesScanned.WithLabelValues(s.name).Inc()
|
||||
|
||||
gzipReader, err := gzip.NewReader(rc)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
defer gzipReader.Close()
|
||||
|
||||
tarReader := tar.NewReader(gzipReader)
|
||||
|
||||
for {
|
||||
header, err := tarReader.Next()
|
||||
if errors.Is(err, io.EOF) {
|
||||
break // End of archive
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Skip files larger than FilesizeLimitBytes
|
||||
if header.Size > FilesizeLimitBytes {
|
||||
continue
|
||||
}
|
||||
|
||||
file := bytes.NewBuffer(make([]byte, 0, header.Size))
|
||||
|
||||
_, err = io.Copy(file, tarReader)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
chunk := &sources.Chunk{
|
||||
SourceType: s.Type(),
|
||||
SourceName: s.name,
|
||||
SourceID: s.SourceID(),
|
||||
JobID: s.JobID(),
|
||||
Data: file.Bytes(),
|
||||
SourceMetadata: &source_metadatapb.MetaData{
|
||||
Data: &source_metadatapb.MetaData_Docker{
|
||||
Docker: &source_metadatapb.Docker{
|
||||
File: "/" + header.Name, // Prepend /, because the file is relative to the root of the image
|
||||
Image: base,
|
||||
Tag: tag,
|
||||
Layer: digest.String(),
|
||||
},
|
||||
},
|
||||
},
|
||||
Verify: s.verify,
|
||||
}
|
||||
|
||||
select {
|
||||
case chunksChan <- chunk:
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
}
|
||||
}
|
||||
|
||||
dockerLayersScanned.WithLabelValues(s.name).Inc()
|
||||
}
|
||||
|
||||
dockerImagesScanned.WithLabelValues(s.name).Inc()
|
||||
return nil
|
||||
})
|
||||
}
|
||||
_ = workers.Wait()
|
||||
if scanErrs.Count() > 0 {
|
||||
ctx.Logger().V(2).Info("scan errors", "errors", scanErrs.String())
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func baseAndTagFromImage(image string) (base, tag string, hasDigest bool) {
|
||||
regRepoDelimiter := "/"
|
||||
tagDelim := ":"
|
||||
digestDelim := "@"
|
||||
// processImage processes an individual image and prepares it for further processing.
|
||||
func (s *Source) processImage(ctx context.Context, image string) (imageInfo, error) {
|
||||
var (
|
||||
imgInfo imageInfo
|
||||
hasDigest bool
|
||||
imageName name.Reference
|
||||
)
|
||||
|
||||
// Split on digest first, if present.
|
||||
parts := strings.SplitN(image, digestDelim, 2)
|
||||
if len(parts) > 1 {
|
||||
base = parts[0]
|
||||
tag = parts[1]
|
||||
hasDigest = true
|
||||
return
|
||||
remoteOpts, err := s.remoteOpts()
|
||||
if err != nil {
|
||||
return imgInfo, err
|
||||
}
|
||||
parts = strings.Split(image, tagDelim)
|
||||
// Verify that we aren't confusing a tag for a hostname w/ port for the purposes of weak validation.
|
||||
if len(parts) > 1 && !strings.Contains(parts[len(parts)-1], regRepoDelimiter) {
|
||||
base = strings.Join(parts[:len(parts)-1], tagDelim)
|
||||
tag = parts[len(parts)-1]
|
||||
|
||||
const filePrefix = "file://"
|
||||
if strings.HasPrefix(image, filePrefix) {
|
||||
image = strings.TrimPrefix(image, filePrefix)
|
||||
imgInfo.base = image
|
||||
imgInfo.image, err = tarball.ImageFromPath(image, nil)
|
||||
if err != nil {
|
||||
return imgInfo, err
|
||||
}
|
||||
} else {
|
||||
base = image
|
||||
tag = "latest"
|
||||
imgInfo.base, imgInfo.tag, hasDigest = baseAndTagFromImage(image)
|
||||
|
||||
if hasDigest {
|
||||
imageName, err = name.NewDigest(image)
|
||||
} else {
|
||||
imageName, err = name.NewTag(image)
|
||||
}
|
||||
if err != nil {
|
||||
return imgInfo, err
|
||||
}
|
||||
|
||||
imgInfo.image, err = remote.Image(imageName, remoteOpts...)
|
||||
if err != nil {
|
||||
return imgInfo, err
|
||||
}
|
||||
}
|
||||
|
||||
return
|
||||
ctx.Logger().WithValues("image", imgInfo.base, "tag", imgInfo.tag).V(2).Info("scanning image")
|
||||
|
||||
return imgInfo, nil
|
||||
}
|
||||
|
||||
// processLayer processes an individual layer of an image.
|
||||
func (s *Source) processLayer(ctx context.Context, layer v1.Layer, imgInfo imageInfo, chunksChan chan *sources.Chunk) error {
|
||||
layerInfo := layerInfo{
|
||||
base: imgInfo.base,
|
||||
tag: imgInfo.tag,
|
||||
}
|
||||
|
||||
var err error
|
||||
layerInfo.digest, err = layer.Digest()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
ctx.Logger().WithValues("layer", layerInfo.digest.String()).V(2).Info("scanning layer")
|
||||
|
||||
rc, err := layer.Compressed()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer rc.Close()
|
||||
|
||||
gzipReader, err := gzip.NewReader(rc)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer gzipReader.Close()
|
||||
|
||||
tarReader := tar.NewReader(gzipReader)
|
||||
for {
|
||||
header, err := tarReader.Next()
|
||||
if errors.Is(err, io.EOF) {
|
||||
break
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
info := chunkProcessingInfo{size: header.Size, name: header.Name, reader: tarReader, layer: layerInfo}
|
||||
if err := s.processChunk(ctx, info, chunksChan); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
type chunkProcessingInfo struct {
|
||||
size int64
|
||||
name string
|
||||
reader io.Reader
|
||||
layer layerInfo
|
||||
}
|
||||
|
||||
// processChunk processes an individual chunk of a layer.
|
||||
func (s *Source) processChunk(ctx context.Context, info chunkProcessingInfo, chunksChan chan *sources.Chunk) error {
|
||||
const filesizeLimitBytes int64 = 50 * 1024 * 1024 // 50MB
|
||||
if info.size > filesizeLimitBytes {
|
||||
ctx.Logger().V(4).Info("skipping large file", "file", info.name, "size", info.size)
|
||||
return nil
|
||||
}
|
||||
|
||||
chunkReader := sources.NewChunkReader()
|
||||
chunkResChan := chunkReader(ctx, info.reader)
|
||||
|
||||
for data := range chunkResChan {
|
||||
if err := data.Error(); err != nil {
|
||||
ctx.Logger().Error(err, "error reading chunk.")
|
||||
continue
|
||||
}
|
||||
|
||||
chunk := &sources.Chunk{
|
||||
SourceType: s.Type(),
|
||||
SourceName: s.name,
|
||||
SourceID: s.SourceID(),
|
||||
SourceMetadata: &source_metadatapb.MetaData{
|
||||
Data: &source_metadatapb.MetaData_Docker{
|
||||
Docker: &source_metadatapb.Docker{
|
||||
File: "/" + info.name,
|
||||
Image: info.layer.base,
|
||||
Tag: info.layer.tag,
|
||||
Layer: info.layer.digest.String(),
|
||||
},
|
||||
},
|
||||
},
|
||||
Verify: s.verify,
|
||||
}
|
||||
chunk.Data = data.Bytes()
|
||||
|
||||
if err := common.CancellableWrite(ctx, chunksChan, chunk); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Source) remoteOpts() ([]remote.Option, error) {
|
||||
@@ -256,3 +302,40 @@ func (s *Source) remoteOpts() ([]remote.Option, error) {
|
||||
return nil, fmt.Errorf("unknown credential type: %T", s.conn.Credential)
|
||||
}
|
||||
}
|
||||
|
||||
func baseAndTagFromImage(image string) (base, tag string, hasDigest bool) {
|
||||
if base, tag, hasDigest = extractDigest(image); hasDigest {
|
||||
return base, tag, true
|
||||
}
|
||||
|
||||
base, tag = extractTagOrUseDefault(image)
|
||||
return base, tag, false
|
||||
}
|
||||
|
||||
// extractDigest tries to split the image string on the digest delimiter.
|
||||
// If successful, it means the image has a digest.
|
||||
func extractDigest(image string) (base, tag string, hasDigest bool) {
|
||||
const digestDelim = "@"
|
||||
|
||||
if parts := strings.SplitN(image, digestDelim, 2); len(parts) > 1 {
|
||||
return parts[0], parts[1], true
|
||||
}
|
||||
return "", "", false
|
||||
}
|
||||
|
||||
// extractTagOrUseDefault extracts the tag from the image string.
|
||||
// If no tag is found, it defaults to "latest".
|
||||
func extractTagOrUseDefault(image string) (base, tag string) {
|
||||
const (
|
||||
tagDelim = ":"
|
||||
regRepoDelim = "/"
|
||||
)
|
||||
|
||||
parts := strings.Split(image, tagDelim)
|
||||
|
||||
// Check if the last part is not a hostname with a port (for weak validation)
|
||||
if len(parts) > 1 && !strings.Contains(parts[len(parts)-1], regRepoDelim) {
|
||||
return strings.Join(parts[:len(parts)-1], tagDelim), parts[len(parts)-1]
|
||||
}
|
||||
return image, "latest"
|
||||
}
|
||||
|
||||
@@ -22,10 +22,12 @@ import (
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
|
||||
)
|
||||
|
||||
const SourceType = sourcespb.SourceType_SOURCE_TYPE_FILESYSTEM
|
||||
|
||||
type Source struct {
|
||||
name string
|
||||
sourceId int64
|
||||
jobId int64
|
||||
sourceId sources.SourceID
|
||||
jobId sources.JobID
|
||||
verify bool
|
||||
paths []string
|
||||
log logr.Logger
|
||||
@@ -43,19 +45,19 @@ var _ sources.SourceUnitChunker = (*Source)(nil)
|
||||
// Type returns the type of source.
|
||||
// It is used for matching source types in configuration and job input.
|
||||
func (s *Source) Type() sourcespb.SourceType {
|
||||
return sourcespb.SourceType_SOURCE_TYPE_FILESYSTEM
|
||||
return SourceType
|
||||
}
|
||||
|
||||
func (s *Source) SourceID() int64 {
|
||||
func (s *Source) SourceID() sources.SourceID {
|
||||
return s.sourceId
|
||||
}
|
||||
|
||||
func (s *Source) JobID() int64 {
|
||||
func (s *Source) JobID() sources.JobID {
|
||||
return s.jobId
|
||||
}
|
||||
|
||||
// Init returns an initialized Filesystem source.
|
||||
func (s *Source) Init(aCtx context.Context, name string, jobId, sourceId int64, verify bool, connection *anypb.Any, _ int) error {
|
||||
func (s *Source) Init(aCtx context.Context, name string, jobId sources.JobID, sourceId sources.SourceID, verify bool, connection *anypb.Any, _ int) error {
|
||||
s.log = aCtx.Logger()
|
||||
|
||||
s.name = name
|
||||
@@ -77,7 +79,7 @@ func (s *Source) WithFilter(filter *common.Filter) {
|
||||
}
|
||||
|
||||
// Chunks emits chunks of bytes over a channel.
|
||||
func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) error {
|
||||
func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk, _ ...sources.ChunkingTarget) error {
|
||||
for i, path := range s.paths {
|
||||
logger := ctx.Logger().WithValues("path", path)
|
||||
if common.IsDone(ctx) {
|
||||
|
||||
+12
-8
@@ -28,7 +28,11 @@ import (
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
|
||||
)
|
||||
|
||||
const defaultCachePersistIncrement = 2500
|
||||
const (
|
||||
SourceType = sourcespb.SourceType_SOURCE_TYPE_GCS
|
||||
|
||||
defaultCachePersistIncrement = 2500
|
||||
)
|
||||
|
||||
// Ensure the Source satisfies the interfaces at compile time.
|
||||
var _ sources.Source = (*Source)(nil)
|
||||
@@ -37,16 +41,16 @@ var _ sources.SourceUnitUnmarshaller = (*Source)(nil)
|
||||
// Type returns the type of source.
|
||||
// It is used for matching source types in configuration and job input.
|
||||
func (s *Source) Type() sourcespb.SourceType {
|
||||
return sourcespb.SourceType_SOURCE_TYPE_GCS
|
||||
return SourceType
|
||||
}
|
||||
|
||||
// SourceID number for GCS Source.
|
||||
func (s *Source) SourceID() int64 {
|
||||
func (s *Source) SourceID() sources.SourceID {
|
||||
return s.sourceId
|
||||
}
|
||||
|
||||
// JobID number for GCS Source.
|
||||
func (s *Source) JobID() int64 {
|
||||
func (s *Source) JobID() sources.JobID {
|
||||
return s.jobId
|
||||
}
|
||||
|
||||
@@ -58,8 +62,8 @@ type objectManager interface {
|
||||
// Source represents a GCS source.
|
||||
type Source struct {
|
||||
name string
|
||||
jobId int64
|
||||
sourceId int64
|
||||
jobId sources.JobID
|
||||
sourceId sources.SourceID
|
||||
concurrency int
|
||||
verify bool
|
||||
|
||||
@@ -107,7 +111,7 @@ func (c *persistableCache) shouldPersist() (bool, string) {
|
||||
}
|
||||
|
||||
// Init returns an initialized GCS source.
|
||||
func (s *Source) Init(aCtx context.Context, name string, id int64, sourceID int64, verify bool, connection *anypb.Any, concurrency int) error {
|
||||
func (s *Source) Init(aCtx context.Context, name string, id sources.JobID, sourceID sources.SourceID, verify bool, connection *anypb.Any, concurrency int) error {
|
||||
s.log = aCtx.Logger()
|
||||
|
||||
s.name = name
|
||||
@@ -248,7 +252,7 @@ func (s *Source) enumerate(ctx context.Context) error {
|
||||
}
|
||||
|
||||
// Chunks emits chunks of bytes over a channel.
|
||||
func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) error {
|
||||
func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk, _ ...sources.ChunkingTarget) error {
|
||||
persistableCache := s.setupCache(ctx)
|
||||
|
||||
objectCh, err := s.gcsManager.ListObjects(ctx)
|
||||
|
||||
+38
-10
@@ -34,10 +34,12 @@ import (
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
|
||||
)
|
||||
|
||||
const SourceType = sourcespb.SourceType_SOURCE_TYPE_GIT
|
||||
|
||||
type Source struct {
|
||||
name string
|
||||
sourceId int64
|
||||
jobId int64
|
||||
sourceId sources.SourceID
|
||||
jobId sources.JobID
|
||||
verify bool
|
||||
git *Git
|
||||
sources.Progress
|
||||
@@ -51,8 +53,8 @@ type Source struct {
|
||||
type Git struct {
|
||||
sourceType sourcespb.SourceType
|
||||
sourceName string
|
||||
sourceID int64
|
||||
jobID int64
|
||||
sourceID sources.SourceID
|
||||
jobID sources.JobID
|
||||
sourceMetadataFunc func(file, email, commit, timestamp, repository string, line int64) *source_metadatapb.MetaData
|
||||
verify bool
|
||||
metrics metrics
|
||||
@@ -63,7 +65,7 @@ type metrics struct {
|
||||
commitsScanned uint64
|
||||
}
|
||||
|
||||
func NewGit(sourceType sourcespb.SourceType, jobID, sourceID int64, sourceName string, verify bool, concurrency int,
|
||||
func NewGit(sourceType sourcespb.SourceType, jobID sources.JobID, sourceID sources.SourceID, sourceName string, verify bool, concurrency int,
|
||||
sourceMetadataFunc func(file, email, commit, timestamp, repository string, line int64) *source_metadatapb.MetaData,
|
||||
) *Git {
|
||||
return &Git{
|
||||
@@ -84,14 +86,14 @@ var _ sources.SourceUnitUnmarshaller = (*Source)(nil)
|
||||
// Type returns the type of source.
|
||||
// It is used for matching source types in configuration and job input.
|
||||
func (s *Source) Type() sourcespb.SourceType {
|
||||
return sourcespb.SourceType_SOURCE_TYPE_GIT
|
||||
return SourceType
|
||||
}
|
||||
|
||||
func (s *Source) SourceID() int64 {
|
||||
func (s *Source) SourceID() sources.SourceID {
|
||||
return s.sourceId
|
||||
}
|
||||
|
||||
func (s *Source) JobID() int64 {
|
||||
func (s *Source) JobID() sources.JobID {
|
||||
return s.jobId
|
||||
}
|
||||
|
||||
@@ -109,7 +111,7 @@ func (s *Source) WithPreserveTempDirs(preserve bool) {
|
||||
}
|
||||
|
||||
// Init returns an initialized GitHub source.
|
||||
func (s *Source) Init(aCtx context.Context, name string, jobId, sourceId int64, verify bool, connection *anypb.Any, concurrency int) error {
|
||||
func (s *Source) Init(aCtx context.Context, name string, jobId sources.JobID, sourceId sources.SourceID, verify bool, connection *anypb.Any, concurrency int) error {
|
||||
s.name = name
|
||||
s.sourceId = sourceId
|
||||
s.jobId = jobId
|
||||
@@ -153,7 +155,7 @@ func (s *Source) Init(aCtx context.Context, name string, jobId, sourceId int64,
|
||||
}
|
||||
|
||||
// Chunks emits chunks of bytes over a channel.
|
||||
func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) error {
|
||||
func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk, _ ...sources.ChunkingTarget) error {
|
||||
if err := s.scanRepos(ctx, chunksChan); err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -370,6 +372,32 @@ func CloneRepo(ctx context.Context, userInfo *url.Userinfo, gitUrl string, args
|
||||
return clonePath, repo, nil
|
||||
}
|
||||
|
||||
// PingRepoUsingToken executes git ls-remote on a repo and returns any error that occurs. It can be used to validate
|
||||
// that a repo actually exists and is reachable.
|
||||
//
|
||||
// Pinging using other authentication methods is only unimplemented because there's been no pressing need for it yet.
|
||||
func PingRepoUsingToken(ctx context.Context, token, gitUrl, user string) error {
|
||||
if err := GitCmdCheck(); err != nil {
|
||||
return err
|
||||
}
|
||||
lsUrl, err := GitURLParse(gitUrl)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if lsUrl.User == nil {
|
||||
lsUrl.User = url.UserPassword(user, token)
|
||||
}
|
||||
|
||||
// We don't actually care about any refs on the remote, we just care whether can can list them at all. So we query
|
||||
// only for a ref that we know won't exist to minimize the search time on the remote. (By default, ls-remote exits
|
||||
// with 0 even if it doesn't find any matching refs.)
|
||||
fakeRef := "TRUFFLEHOG_CHECK_GIT_REMOTE_URL_REACHABILITY"
|
||||
gitArgs := []string{"ls-remote", lsUrl.String(), "--quiet", fakeRef}
|
||||
cmd := exec.Command("git", gitArgs...)
|
||||
_, err = cmd.CombinedOutput()
|
||||
return err
|
||||
}
|
||||
|
||||
// CloneRepoUsingToken clones a repo using a provided token.
|
||||
func CloneRepoUsingToken(ctx context.Context, token, gitUrl, user string, args ...string) (string, *git.Repository, error) {
|
||||
userInfo := url.UserPassword(user, token)
|
||||
|
||||
@@ -39,6 +39,8 @@ import (
|
||||
)
|
||||
|
||||
const (
|
||||
SourceType = sourcespb.SourceType_SOURCE_TYPE_GITHUB
|
||||
|
||||
unauthGithubOrgRateLimt = 30
|
||||
defaultPagination = 100
|
||||
membersAppPagination = 500
|
||||
@@ -51,8 +53,8 @@ type Source struct {
|
||||
githubUser string
|
||||
githubToken string
|
||||
|
||||
sourceID int64
|
||||
jobID int64
|
||||
sourceID sources.SourceID
|
||||
jobID sources.JobID
|
||||
verify bool
|
||||
repos []string
|
||||
members []string
|
||||
@@ -104,14 +106,14 @@ var endsWithGithub = regexp.MustCompile(`github\.com/?$`)
|
||||
// Type returns the type of source.
|
||||
// It is used for matching source types in configuration and job input.
|
||||
func (s *Source) Type() sourcespb.SourceType {
|
||||
return sourcespb.SourceType_SOURCE_TYPE_GITHUB
|
||||
return SourceType
|
||||
}
|
||||
|
||||
func (s *Source) SourceID() int64 {
|
||||
func (s *Source) SourceID() sources.SourceID {
|
||||
return s.sourceID
|
||||
}
|
||||
|
||||
func (s *Source) JobID() int64 {
|
||||
func (s *Source) JobID() sources.JobID {
|
||||
return s.jobID
|
||||
}
|
||||
|
||||
@@ -200,7 +202,7 @@ func (c *filteredRepoCache) includeRepo(s string) bool {
|
||||
}
|
||||
|
||||
// Init returns an initialized GitHub source.
|
||||
func (s *Source) Init(aCtx context.Context, name string, jobID, sourceID int64, verify bool, connection *anypb.Any, concurrency int) error {
|
||||
func (s *Source) Init(aCtx context.Context, name string, jobID sources.JobID, sourceID sources.SourceID, verify bool, connection *anypb.Any, concurrency int) error {
|
||||
s.log = aCtx.Logger()
|
||||
|
||||
s.name = name
|
||||
@@ -413,7 +415,7 @@ func (s *Source) visibilityOf(ctx context.Context, repoURL string) (visibility s
|
||||
}
|
||||
|
||||
// Chunks emits chunks of bytes over a channel.
|
||||
func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) error {
|
||||
func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk, _ ...sources.ChunkingTarget) error {
|
||||
apiEndpoint := s.conn.Endpoint
|
||||
if len(apiEndpoint) == 0 || endsWithGithub.MatchString(apiEndpoint) {
|
||||
apiEndpoint = "https://api.github.com"
|
||||
|
||||
@@ -190,13 +190,13 @@ func TestAddMembersByApp(t *testing.T) {
|
||||
gock.New("https://api.github.com").
|
||||
Get("/app/installations").
|
||||
Reply(200).
|
||||
JSON([]map[string]interface{}{
|
||||
JSON([]map[string]any{
|
||||
{"account": map[string]string{"login": "super-secret-org", "type": "Organization"}},
|
||||
})
|
||||
gock.New("https://api.github.com").
|
||||
Get("/orgs/super-secret-org/members").
|
||||
Reply(200).
|
||||
JSON([]map[string]interface{}{
|
||||
JSON([]map[string]any{
|
||||
{"login": "ssm1"},
|
||||
{"login": "ssm2"},
|
||||
{"login": "ssm3"},
|
||||
@@ -221,7 +221,7 @@ func TestAddReposByApp(t *testing.T) {
|
||||
gock.New("https://api.github.com").
|
||||
Get("/installation/repositories").
|
||||
Reply(200).
|
||||
JSON(map[string]interface{}{
|
||||
JSON(map[string]any{
|
||||
"repositories": []map[string]string{
|
||||
{"clone_url": "https://github/ssr1.git", "full_name": "ssr1"},
|
||||
{"clone_url": "https://github/ssr2.git", "full_name": "ssr2"},
|
||||
@@ -247,7 +247,7 @@ func TestAddOrgsByUser(t *testing.T) {
|
||||
gock.New("https://api.github.com").
|
||||
Get("/user/orgs").
|
||||
Reply(200).
|
||||
JSON([]map[string]interface{}{
|
||||
JSON([]map[string]any{
|
||||
{"login": "sso2"},
|
||||
})
|
||||
|
||||
|
||||
+139
-54
@@ -29,10 +29,12 @@ import (
|
||||
"google.golang.org/protobuf/types/known/anypb"
|
||||
)
|
||||
|
||||
const SourceType = sourcespb.SourceType_SOURCE_TYPE_GITLAB
|
||||
|
||||
type Source struct {
|
||||
name string
|
||||
sourceId int64
|
||||
jobId int64
|
||||
sourceId sources.SourceID
|
||||
jobId sources.JobID
|
||||
verify bool
|
||||
authMethod string
|
||||
user string
|
||||
@@ -53,23 +55,24 @@ type Source struct {
|
||||
// Ensure the Source satisfies the interfaces at compile time.
|
||||
var _ sources.Source = (*Source)(nil)
|
||||
var _ sources.SourceUnitUnmarshaller = (*Source)(nil)
|
||||
var _ sources.Validator = (*Source)(nil)
|
||||
|
||||
// Type returns the type of source.
|
||||
// It is used for matching source types in configuration and job input.
|
||||
func (s *Source) Type() sourcespb.SourceType {
|
||||
return sourcespb.SourceType_SOURCE_TYPE_GITLAB
|
||||
return SourceType
|
||||
}
|
||||
|
||||
func (s *Source) SourceID() int64 {
|
||||
func (s *Source) SourceID() sources.SourceID {
|
||||
return s.sourceId
|
||||
}
|
||||
|
||||
func (s *Source) JobID() int64 {
|
||||
func (s *Source) JobID() sources.JobID {
|
||||
return s.jobId
|
||||
}
|
||||
|
||||
// Init returns an initialized Gitlab source.
|
||||
func (s *Source) Init(_ context.Context, name string, jobId, sourceId int64, verify bool, connection *anypb.Any, concurrency int) error {
|
||||
func (s *Source) Init(_ context.Context, name string, jobId sources.JobID, sourceId sources.SourceID, verify bool, connection *anypb.Any, concurrency int) error {
|
||||
s.name = name
|
||||
s.sourceId = sourceId
|
||||
s.jobId = jobId
|
||||
@@ -139,16 +142,16 @@ func (s *Source) Init(_ context.Context, name string, jobId, sourceId int64, ver
|
||||
}
|
||||
|
||||
// Chunks emits chunks of bytes over a channel.
|
||||
func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) error {
|
||||
func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk, _ ...sources.ChunkingTarget) error {
|
||||
// Start client.
|
||||
apiClient, err := s.newClient()
|
||||
if err != nil {
|
||||
return errors.New(err)
|
||||
}
|
||||
// Get repo within target.
|
||||
repos, errs := s.getRepos()
|
||||
repos, errs := normalizeRepos(s.repos)
|
||||
for _, repoErr := range errs {
|
||||
ctx.Logger().Info("error getting repo", "error", repoErr)
|
||||
ctx.Logger().Info("error normalizing repo", "error", repoErr)
|
||||
}
|
||||
|
||||
// End early if we had errors getting specified repos but none were validated.
|
||||
@@ -158,26 +161,14 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) err
|
||||
|
||||
// Get all repos if not specified.
|
||||
if len(repos) == 0 {
|
||||
projects, err := s.getAllProjects(ctx, apiClient)
|
||||
if err != nil {
|
||||
return fmt.Errorf("error getting all projects: %v", err)
|
||||
}
|
||||
// Turn projects into URLs for Git cloner.
|
||||
for _, prj := range projects {
|
||||
if s.ignoreRepo(ctx, prj.PathWithNamespace) {
|
||||
continue
|
||||
}
|
||||
|
||||
// Ensure the urls are valid before adding them to the repo list.
|
||||
_, err := url.Parse(prj.HTTPURLToRepo)
|
||||
if err != nil {
|
||||
fmt.Printf("could not parse url given by project: %s", prj.HTTPURLToRepo)
|
||||
}
|
||||
repos = append(repos, prj.HTTPURLToRepo)
|
||||
}
|
||||
if len(repos) == 0 {
|
||||
return errors.Errorf("unable to discover any repos")
|
||||
ignoreRepo := buildIgnorer(s.ignoreRepos, func(err error, pattern string) {
|
||||
ctx.Logger().Error(err, "could not compile ignore repo glob", "glob", pattern)
|
||||
})
|
||||
gitlabRepos, err2, done := s.getReposFromGitlab(ctx, apiClient, ignoreRepo)
|
||||
if done {
|
||||
return err2
|
||||
}
|
||||
repos = gitlabRepos
|
||||
}
|
||||
|
||||
s.repos = repos
|
||||
@@ -187,6 +178,69 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) err
|
||||
return s.scanRepos(ctx, chunksChan)
|
||||
}
|
||||
|
||||
func (s *Source) Validate(ctx context.Context) []error {
|
||||
// The client is only used to query Gitlab for a repo list - it's not used to actually clone anything. Thus, we
|
||||
// don't use it if there is a list of explicitly configured repos. However, constructing it validates that the
|
||||
// configured authentication method is sensible, so we'll do it here.
|
||||
apiClient, err := s.newClient()
|
||||
if err != nil {
|
||||
return []error{err}
|
||||
}
|
||||
|
||||
_, _, err = apiClient.Users.CurrentUser()
|
||||
if err != nil {
|
||||
msg := fmt.Sprintf("gitlab authentication failed using method %v", s.authMethod)
|
||||
return []error{errors.WrapPrefix(err, msg, 0)}
|
||||
}
|
||||
|
||||
explicitlyConfiguredRepos, errs := normalizeRepos(s.repos)
|
||||
|
||||
if len(explicitlyConfiguredRepos) > 0 {
|
||||
user := s.user
|
||||
if user == "" {
|
||||
user = "placeholder"
|
||||
}
|
||||
|
||||
// We only check reachability for explicitly configured repositories. The purpose of source validation is to
|
||||
// help users validate their configuration files, and Gitlab telling us about repositories that it won't let us
|
||||
// access isn't a local configuration issue.
|
||||
for _, r := range explicitlyConfiguredRepos {
|
||||
if err := git.PingRepoUsingToken(ctx, s.token, r, user); err != nil {
|
||||
msg := fmt.Sprintf("could not reach git repository %q", r)
|
||||
err = errors.WrapPrefix(err, msg, 0)
|
||||
errs = append(errs, err)
|
||||
}
|
||||
}
|
||||
|
||||
if len(s.ignoreRepos) > 0 {
|
||||
errs = append(errs, fmt.Errorf("both repositories and ignore patterns were explicitly configured; ignore patterns will not be used"))
|
||||
}
|
||||
}
|
||||
|
||||
if len(explicitlyConfiguredRepos) > 0 || len(errs) > 0 {
|
||||
return errs
|
||||
}
|
||||
|
||||
ignoreProject := buildIgnorer(s.ignoreRepos, func(err error, pattern string) {
|
||||
msg := fmt.Sprintf("could not compile ignore repo pattern %q", pattern)
|
||||
errs = append(errs, errors.WrapPrefix(err, msg, 0))
|
||||
})
|
||||
|
||||
projects, err := s.getAllProjects(ctx, apiClient)
|
||||
if err != nil {
|
||||
errs = append(errs, err)
|
||||
return errs
|
||||
}
|
||||
|
||||
for _, p := range projects {
|
||||
if !ignoreProject(p.PathWithNamespace) {
|
||||
return errs
|
||||
}
|
||||
}
|
||||
|
||||
return append(errs, fmt.Errorf("ignore patterns excluded all projects"))
|
||||
}
|
||||
|
||||
func (s *Source) newClient() (*gitlab.Client, error) {
|
||||
// Initialize a new api instance.
|
||||
switch s.authMethod {
|
||||
@@ -322,23 +376,30 @@ func (s *Source) getAllProjects(ctx context.Context, apiClient *gitlab.Client) (
|
||||
return projectList, nil
|
||||
}
|
||||
|
||||
func (s *Source) getRepos() ([]string, []error) {
|
||||
if len(s.repos) == 0 {
|
||||
return nil, nil
|
||||
func (s *Source) getReposFromGitlab(ctx context.Context, apiClient *gitlab.Client, ignoreRepo func(repo string) bool) ([]string, error, bool) {
|
||||
projects, err := s.getAllProjects(ctx, apiClient)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("error getting all projects: %v", err), true
|
||||
}
|
||||
|
||||
var validRepos []string
|
||||
var errs []error
|
||||
for _, prj := range s.repos {
|
||||
repo, err := giturl.NormalizeGitlabRepo(prj)
|
||||
if err != nil {
|
||||
errs = append(errs, errors.WrapPrefix(err, fmt.Sprintf("unable to normalize gitlab repo url %s", prj), 0))
|
||||
// Turn projects into URLs for Git cloner.
|
||||
var repos []string
|
||||
for _, prj := range projects {
|
||||
if ignoreRepo(prj.PathWithNamespace) {
|
||||
continue
|
||||
}
|
||||
|
||||
validRepos = append(validRepos, repo)
|
||||
// Ensure the urls are valid before adding them to the repo list.
|
||||
_, err := url.Parse(prj.HTTPURLToRepo)
|
||||
if err != nil {
|
||||
fmt.Printf("could not parse url given by project: %s", prj.HTTPURLToRepo)
|
||||
}
|
||||
repos = append(repos, prj.HTTPURLToRepo)
|
||||
}
|
||||
return validRepos, errs
|
||||
if len(repos) == 0 {
|
||||
return nil, errors.Errorf("unable to discover any repos"), true
|
||||
}
|
||||
return repos, nil, false
|
||||
}
|
||||
|
||||
func (s *Source) scanRepos(ctx context.Context, chunksChan chan *sources.Chunk) error {
|
||||
@@ -411,21 +472,6 @@ func (s *Source) scanRepos(ctx context.Context, chunksChan chan *sources.Chunk)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Source) ignoreRepo(ctx context.Context, r string) bool {
|
||||
for _, ignore := range s.ignoreRepos {
|
||||
g, err := glob.Compile(ignore)
|
||||
if err != nil {
|
||||
ctx.Logger().Error(err, "could not compile ignore repo glob", "glob", ignore)
|
||||
continue
|
||||
}
|
||||
if g.Match(r) {
|
||||
ctx.Logger().V(2).Info("Ignoring repo", "repo", r)
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// setProgressCompleteWithRepo calls the s.SetProgressComplete after safely setting up the encoded resume info string.
|
||||
func (s *Source) setProgressCompleteWithRepo(index int, offset int, repoURL string) {
|
||||
s.resumeInfoMutex.Lock()
|
||||
@@ -445,3 +491,42 @@ func (s *Source) setProgressCompleteWithRepo(index int, offset int, repoURL stri
|
||||
func (s *Source) WithScanOptions(scanOptions *git.ScanOptions) {
|
||||
s.scanOptions = scanOptions
|
||||
}
|
||||
|
||||
func buildIgnorer(patterns []string, onCompileErr func(err error, pattern string)) func(repo string) bool {
|
||||
var globs []glob.Glob
|
||||
|
||||
for _, pattern := range patterns {
|
||||
g, err := glob.Compile(pattern)
|
||||
if err != nil {
|
||||
onCompileErr(err, pattern)
|
||||
continue
|
||||
}
|
||||
globs = append(globs, g)
|
||||
}
|
||||
|
||||
f := func(repo string) bool {
|
||||
for _, g := range globs {
|
||||
if g.Match(repo) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
return f
|
||||
}
|
||||
|
||||
func normalizeRepos(repos []string) ([]string, []error) {
|
||||
var validRepos []string
|
||||
var errs []error
|
||||
for _, prj := range repos {
|
||||
repo, err := giturl.NormalizeGitlabRepo(prj)
|
||||
if err != nil {
|
||||
errs = append(errs, errors.WrapPrefix(err, fmt.Sprintf("unable to normalize gitlab repo url %s", prj), 0))
|
||||
continue
|
||||
}
|
||||
|
||||
validRepos = append(validRepos, repo)
|
||||
}
|
||||
return validRepos, errs
|
||||
}
|
||||
|
||||
@@ -177,6 +177,150 @@ func TestSource_Scan(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestSource_Validate(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
|
||||
secret, err := common.GetTestSecret(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(fmt.Errorf("failed to access secret: %v", err))
|
||||
}
|
||||
token := secret.MustGetField("GITLAB_TOKEN")
|
||||
tokenWrongScope := secret.MustGetField("GITLAB_TOKEN_WRONG_SCOPE")
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
connection *sourcespb.GitLab
|
||||
wantErrCount int
|
||||
wantErrs []string
|
||||
}{
|
||||
{
|
||||
name: "basic auth did not authenticate",
|
||||
connection: &sourcespb.GitLab{
|
||||
Credential: &sourcespb.GitLab_BasicAuth{
|
||||
BasicAuth: &credentialspb.BasicAuth{
|
||||
Username: "bad-user",
|
||||
Password: "bad-password",
|
||||
},
|
||||
},
|
||||
},
|
||||
wantErrCount: 1,
|
||||
},
|
||||
{
|
||||
name: "token did not authenticate",
|
||||
connection: &sourcespb.GitLab{
|
||||
Credential: &sourcespb.GitLab_Token{
|
||||
Token: "bad-token",
|
||||
},
|
||||
},
|
||||
wantErrCount: 1,
|
||||
},
|
||||
{
|
||||
name: "bad repo urls",
|
||||
connection: &sourcespb.GitLab{
|
||||
Credential: &sourcespb.GitLab_Token{
|
||||
Token: token,
|
||||
},
|
||||
Repositories: []string{
|
||||
"https://gitlab.com/testermctestface/testy", // valid
|
||||
"https://gitlab.com/testermctestface/testy/", // trailing slash
|
||||
"ssh:git@gitlab.com/testermctestface/testy", // bad protocol
|
||||
"https://gitlab.com", // no path
|
||||
"https://gitlab.com/", // no org name
|
||||
"https://gitlab.com//testy", // no org name
|
||||
"https://gitlab.com/testermctestface/", // no repo name
|
||||
},
|
||||
},
|
||||
wantErrCount: 6,
|
||||
},
|
||||
{
|
||||
name: "token does not have permission to list projects",
|
||||
connection: &sourcespb.GitLab{
|
||||
Credential: &sourcespb.GitLab_Token{
|
||||
Token: tokenWrongScope,
|
||||
},
|
||||
},
|
||||
wantErrCount: 1,
|
||||
},
|
||||
{
|
||||
name: "repositories and ignore globs both configured",
|
||||
connection: &sourcespb.GitLab{
|
||||
Credential: &sourcespb.GitLab_Token{
|
||||
Token: token,
|
||||
},
|
||||
Repositories: []string{
|
||||
"https://gitlab.com/testermctestface/testy", // valid
|
||||
},
|
||||
IgnoreRepos: []string{
|
||||
"tes1188/*-gitlab",
|
||||
"[", // glob doesn't compile, but this won't be checked
|
||||
},
|
||||
},
|
||||
wantErrCount: 1,
|
||||
},
|
||||
{
|
||||
name: "could not compile ignore glob(s)",
|
||||
connection: &sourcespb.GitLab{
|
||||
Credential: &sourcespb.GitLab_Token{
|
||||
Token: token,
|
||||
},
|
||||
IgnoreRepos: []string{
|
||||
"tes1188/*-gitlab",
|
||||
"[", // glob doesn't compile
|
||||
"[a-]", // glob doesn't compile
|
||||
},
|
||||
},
|
||||
wantErrCount: 2,
|
||||
},
|
||||
{
|
||||
name: "repositories do not exist or are not accessible",
|
||||
connection: &sourcespb.GitLab{
|
||||
Credential: &sourcespb.GitLab_Token{
|
||||
Token: token,
|
||||
},
|
||||
Repositories: []string{
|
||||
"https://gitlab.com/testermctestface/testy",
|
||||
"https://gitlab.com/testermctestface/doesn't-exist",
|
||||
"https://gitlab.com/testermctestface/also-doesn't-exist",
|
||||
},
|
||||
},
|
||||
wantErrCount: 2,
|
||||
},
|
||||
{
|
||||
name: "ignore globs exclude all repos",
|
||||
connection: &sourcespb.GitLab{
|
||||
Credential: &sourcespb.GitLab_Token{
|
||||
Token: token,
|
||||
},
|
||||
IgnoreRepos: []string{
|
||||
"*",
|
||||
},
|
||||
},
|
||||
wantErrCount: 1,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
s := Source{}
|
||||
|
||||
conn, err := anypb.New(tt.connection)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
err = s.Init(ctx, tt.name, 0, 0, false, conn, 1)
|
||||
if err != nil {
|
||||
t.Fatalf("Source.Init() error: %v", err)
|
||||
}
|
||||
|
||||
errs := s.Validate(ctx)
|
||||
|
||||
assert.Equal(t, tt.wantErrCount, len(errs))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func Test_setProgressCompleteWithRepo_resumeInfo(t *testing.T) {
|
||||
tests := []struct {
|
||||
startingResumeInfoSlice []string
|
||||
|
||||
@@ -43,8 +43,8 @@ type JobProgressHook interface {
|
||||
// If the job supports it, the reference can also be used to cancel running via
|
||||
// CancelRun.
|
||||
type JobProgressRef struct {
|
||||
JobID int64
|
||||
SourceID int64
|
||||
JobID JobID
|
||||
SourceID SourceID
|
||||
SourceName string
|
||||
jobProgress *JobProgress
|
||||
}
|
||||
@@ -101,8 +101,8 @@ func (f ChunkError) Unwrap() error { return f.err }
|
||||
// JobProgress aggregates information about a run of a Source.
|
||||
type JobProgress struct {
|
||||
// Unique identifiers for this job.
|
||||
JobID int64
|
||||
SourceID int64
|
||||
JobID JobID
|
||||
SourceID SourceID
|
||||
SourceName string
|
||||
// Tracks whether the job is finished or not.
|
||||
ctx context.Context
|
||||
@@ -155,7 +155,7 @@ func WithCancel(cancel context.CancelCauseFunc) func(*JobProgress) {
|
||||
}
|
||||
|
||||
// NewJobProgress creates a new job report for the given source and job ID.
|
||||
func NewJobProgress(jobID, sourceID int64, sourceName string, opts ...func(*JobProgress)) *JobProgress {
|
||||
func NewJobProgress(jobID JobID, sourceID SourceID, sourceName string, opts ...func(*JobProgress)) *JobProgress {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
jp := &JobProgress{
|
||||
JobID: jobID,
|
||||
|
||||
@@ -38,8 +38,8 @@ func TestJobProgressFatalErrors(t *testing.T) {
|
||||
func TestJobProgressRef(t *testing.T) {
|
||||
jp := NewJobProgress(123, 456, "source name")
|
||||
ref := jp.Ref()
|
||||
assert.Equal(t, int64(123), ref.JobID)
|
||||
assert.Equal(t, int64(456), ref.SourceID)
|
||||
assert.Equal(t, JobID(123), ref.JobID)
|
||||
assert.Equal(t, SourceID(456), ref.SourceID)
|
||||
|
||||
// Test Done() blocks until Finish() is called.
|
||||
select {
|
||||
|
||||
@@ -41,7 +41,7 @@ func (m *MockJobProgressHook) End(arg0 JobProgressRef, arg1 time.Time) {
|
||||
}
|
||||
|
||||
// End indicates an expected call of End.
|
||||
func (mr *MockJobProgressHookMockRecorder) End(arg0, arg1 interface{}) *gomock.Call {
|
||||
func (mr *MockJobProgressHookMockRecorder) End(arg0, arg1 any) *gomock.Call {
|
||||
mr.mock.ctrl.T.Helper()
|
||||
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "End", reflect.TypeOf((*MockJobProgressHook)(nil).End), arg0, arg1)
|
||||
}
|
||||
@@ -53,7 +53,7 @@ func (m *MockJobProgressHook) EndEnumerating(arg0 JobProgressRef, arg1 time.Time
|
||||
}
|
||||
|
||||
// EndEnumerating indicates an expected call of EndEnumerating.
|
||||
func (mr *MockJobProgressHookMockRecorder) EndEnumerating(arg0, arg1 interface{}) *gomock.Call {
|
||||
func (mr *MockJobProgressHookMockRecorder) EndEnumerating(arg0, arg1 any) *gomock.Call {
|
||||
mr.mock.ctrl.T.Helper()
|
||||
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "EndEnumerating", reflect.TypeOf((*MockJobProgressHook)(nil).EndEnumerating), arg0, arg1)
|
||||
}
|
||||
@@ -65,7 +65,7 @@ func (m *MockJobProgressHook) EndUnitChunking(arg0 JobProgressRef, arg1 SourceUn
|
||||
}
|
||||
|
||||
// EndUnitChunking indicates an expected call of EndUnitChunking.
|
||||
func (mr *MockJobProgressHookMockRecorder) EndUnitChunking(arg0, arg1, arg2 interface{}) *gomock.Call {
|
||||
func (mr *MockJobProgressHookMockRecorder) EndUnitChunking(arg0, arg1, arg2 any) *gomock.Call {
|
||||
mr.mock.ctrl.T.Helper()
|
||||
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "EndUnitChunking", reflect.TypeOf((*MockJobProgressHook)(nil).EndUnitChunking), arg0, arg1, arg2)
|
||||
}
|
||||
@@ -77,7 +77,7 @@ func (m *MockJobProgressHook) Finish(arg0 JobProgressRef) {
|
||||
}
|
||||
|
||||
// Finish indicates an expected call of Finish.
|
||||
func (mr *MockJobProgressHookMockRecorder) Finish(arg0 interface{}) *gomock.Call {
|
||||
func (mr *MockJobProgressHookMockRecorder) Finish(arg0 any) *gomock.Call {
|
||||
mr.mock.ctrl.T.Helper()
|
||||
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Finish", reflect.TypeOf((*MockJobProgressHook)(nil).Finish), arg0)
|
||||
}
|
||||
@@ -89,7 +89,7 @@ func (m *MockJobProgressHook) ReportChunk(arg0 JobProgressRef, arg1 SourceUnit,
|
||||
}
|
||||
|
||||
// ReportChunk indicates an expected call of ReportChunk.
|
||||
func (mr *MockJobProgressHookMockRecorder) ReportChunk(arg0, arg1, arg2 interface{}) *gomock.Call {
|
||||
func (mr *MockJobProgressHookMockRecorder) ReportChunk(arg0, arg1, arg2 any) *gomock.Call {
|
||||
mr.mock.ctrl.T.Helper()
|
||||
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ReportChunk", reflect.TypeOf((*MockJobProgressHook)(nil).ReportChunk), arg0, arg1, arg2)
|
||||
}
|
||||
@@ -101,7 +101,7 @@ func (m *MockJobProgressHook) ReportError(arg0 JobProgressRef, arg1 error) {
|
||||
}
|
||||
|
||||
// ReportError indicates an expected call of ReportError.
|
||||
func (mr *MockJobProgressHookMockRecorder) ReportError(arg0, arg1 interface{}) *gomock.Call {
|
||||
func (mr *MockJobProgressHookMockRecorder) ReportError(arg0, arg1 any) *gomock.Call {
|
||||
mr.mock.ctrl.T.Helper()
|
||||
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ReportError", reflect.TypeOf((*MockJobProgressHook)(nil).ReportError), arg0, arg1)
|
||||
}
|
||||
@@ -113,7 +113,7 @@ func (m *MockJobProgressHook) ReportUnit(arg0 JobProgressRef, arg1 SourceUnit) {
|
||||
}
|
||||
|
||||
// ReportUnit indicates an expected call of ReportUnit.
|
||||
func (mr *MockJobProgressHookMockRecorder) ReportUnit(arg0, arg1 interface{}) *gomock.Call {
|
||||
func (mr *MockJobProgressHookMockRecorder) ReportUnit(arg0, arg1 any) *gomock.Call {
|
||||
mr.mock.ctrl.T.Helper()
|
||||
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ReportUnit", reflect.TypeOf((*MockJobProgressHook)(nil).ReportUnit), arg0, arg1)
|
||||
}
|
||||
@@ -125,7 +125,7 @@ func (m *MockJobProgressHook) Start(arg0 JobProgressRef, arg1 time.Time) {
|
||||
}
|
||||
|
||||
// Start indicates an expected call of Start.
|
||||
func (mr *MockJobProgressHookMockRecorder) Start(arg0, arg1 interface{}) *gomock.Call {
|
||||
func (mr *MockJobProgressHookMockRecorder) Start(arg0, arg1 any) *gomock.Call {
|
||||
mr.mock.ctrl.T.Helper()
|
||||
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Start", reflect.TypeOf((*MockJobProgressHook)(nil).Start), arg0, arg1)
|
||||
}
|
||||
@@ -137,7 +137,7 @@ func (m *MockJobProgressHook) StartEnumerating(arg0 JobProgressRef, arg1 time.Ti
|
||||
}
|
||||
|
||||
// StartEnumerating indicates an expected call of StartEnumerating.
|
||||
func (mr *MockJobProgressHookMockRecorder) StartEnumerating(arg0, arg1 interface{}) *gomock.Call {
|
||||
func (mr *MockJobProgressHookMockRecorder) StartEnumerating(arg0, arg1 any) *gomock.Call {
|
||||
mr.mock.ctrl.T.Helper()
|
||||
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "StartEnumerating", reflect.TypeOf((*MockJobProgressHook)(nil).StartEnumerating), arg0, arg1)
|
||||
}
|
||||
@@ -149,7 +149,7 @@ func (m *MockJobProgressHook) StartUnitChunking(arg0 JobProgressRef, arg1 Source
|
||||
}
|
||||
|
||||
// StartUnitChunking indicates an expected call of StartUnitChunking.
|
||||
func (mr *MockJobProgressHookMockRecorder) StartUnitChunking(arg0, arg1, arg2 interface{}) *gomock.Call {
|
||||
func (mr *MockJobProgressHookMockRecorder) StartUnitChunking(arg0, arg1, arg2 any) *gomock.Call {
|
||||
mr.mock.ctrl.T.Helper()
|
||||
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "StartUnitChunking", reflect.TypeOf((*MockJobProgressHook)(nil).StartUnitChunking), arg0, arg1, arg2)
|
||||
}
|
||||
|
||||
+131
-49
@@ -31,14 +31,17 @@ import (
|
||||
)
|
||||
|
||||
const (
|
||||
SourceType = sourcespb.SourceType_SOURCE_TYPE_S3
|
||||
|
||||
defaultAWSRegion = "us-east-1"
|
||||
defaultMaxObjectSize = 250 * 1024 * 1024 // 250 MiB
|
||||
maxObjectSizeLimit = 250 * 1024 * 1024 // 250 MiB
|
||||
)
|
||||
|
||||
type Source struct {
|
||||
name string
|
||||
sourceId int64
|
||||
jobId int64
|
||||
sourceId sources.SourceID
|
||||
jobId sources.JobID
|
||||
verify bool
|
||||
concurrency int
|
||||
log logr.Logger
|
||||
@@ -53,22 +56,23 @@ type Source struct {
|
||||
// Ensure the Source satisfies the interfaces at compile time
|
||||
var _ sources.Source = (*Source)(nil)
|
||||
var _ sources.SourceUnitUnmarshaller = (*Source)(nil)
|
||||
var _ sources.Validator = (*Source)(nil)
|
||||
|
||||
// Type returns the type of source
|
||||
func (s *Source) Type() sourcespb.SourceType {
|
||||
return sourcespb.SourceType_SOURCE_TYPE_S3
|
||||
return SourceType
|
||||
}
|
||||
|
||||
func (s *Source) SourceID() int64 {
|
||||
func (s *Source) SourceID() sources.SourceID {
|
||||
return s.sourceId
|
||||
}
|
||||
|
||||
func (s *Source) JobID() int64 {
|
||||
func (s *Source) JobID() sources.JobID {
|
||||
return s.jobId
|
||||
}
|
||||
|
||||
// Init returns an initialized AWS source
|
||||
func (s *Source) Init(aCtx context.Context, name string, jobId, sourceId int64, verify bool, connection *anypb.Any, concurrency int) error {
|
||||
func (s *Source) Init(aCtx context.Context, name string, jobId sources.JobID, sourceId sources.SourceID, verify bool, connection *anypb.Any, concurrency int) error {
|
||||
s.log = context.WithValues(aCtx, "source", s.Type(), "name", name).Logger()
|
||||
|
||||
s.name = name
|
||||
@@ -93,6 +97,23 @@ func (s *Source) Init(aCtx context.Context, name string, jobId, sourceId int64,
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Source) Validate(ctx context.Context) []error {
|
||||
var errors []error
|
||||
visitor := func(c context.Context, defaultRegionClient *s3.S3, roleArn string, buckets []string) {
|
||||
roleErrs := s.validateBucketAccess(c, defaultRegionClient, roleArn, buckets)
|
||||
if len(roleErrs) > 0 {
|
||||
errors = append(errors, roleErrs...)
|
||||
}
|
||||
}
|
||||
|
||||
err := s.visitRoles(ctx, visitor)
|
||||
if err != nil {
|
||||
errors = append(errors, err)
|
||||
}
|
||||
|
||||
return errors
|
||||
}
|
||||
|
||||
// setMaxObjectSize sets the maximum size of objects that will be scanned. If
|
||||
// not set, set to a negative number, or set larger than the
|
||||
// maxObjectSizeLimit, the defaultMaxObjectSizeLimit will be used.
|
||||
@@ -153,7 +174,7 @@ func (s *Source) getBucketsToScan(client *s3.S3) ([]string, error) {
|
||||
|
||||
res, err := client.ListBuckets(&s3.ListBucketsInput{})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("could not list s3 buckets: %w", err)
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var bucketsToScan []string
|
||||
@@ -163,32 +184,28 @@ func (s *Source) getBucketsToScan(client *s3.S3) ([]string, error) {
|
||||
return bucketsToScan, nil
|
||||
}
|
||||
|
||||
func (s *Source) scanBuckets(ctx context.Context, client *s3.S3, role string, bucketsToScan []string, chunksChan chan *sources.Chunk) error {
|
||||
const defaultAWSRegion = "us-east-1"
|
||||
func (s *Source) scanBuckets(ctx context.Context, client *s3.S3, role string, bucketsToScan []string, chunksChan chan *sources.Chunk) {
|
||||
objectCount := uint64(0)
|
||||
|
||||
logger := s.log
|
||||
if role != "" {
|
||||
logger = logger.WithValues("roleArn", role)
|
||||
}
|
||||
|
||||
for i, bucket := range bucketsToScan {
|
||||
logger := logger.WithValues("bucket", bucket)
|
||||
|
||||
if common.IsDone(ctx) {
|
||||
return nil
|
||||
return
|
||||
}
|
||||
|
||||
s.SetProgressComplete(i, len(bucketsToScan), fmt.Sprintf("Bucket: %s", bucket), "")
|
||||
s.log.Info("Scanning bucket", "bucket", bucket)
|
||||
region, err := s3manager.GetBucketRegionWithClient(ctx, client, bucket)
|
||||
if err != nil {
|
||||
s.log.Error(err, "could not get s3 region for bucket", "bucket: ", bucket)
|
||||
continue
|
||||
}
|
||||
logger.V(3).Info("Scanning bucket")
|
||||
|
||||
var regionalClient *s3.S3
|
||||
if region != defaultAWSRegion {
|
||||
regionalClient, err = s.newClient(region, role)
|
||||
if err != nil {
|
||||
s.log.Error(err, "could not make regional s3 client")
|
||||
continue
|
||||
}
|
||||
} else {
|
||||
regionalClient = client
|
||||
regionalClient, err := s.getRegionalClientForBucket(ctx, client, role, bucket)
|
||||
if err != nil {
|
||||
logger.Error(err, "could not get regional client for bucket")
|
||||
continue
|
||||
}
|
||||
|
||||
errorCount := sync.Map{}
|
||||
@@ -201,40 +218,46 @@ func (s *Source) scanBuckets(ctx context.Context, client *s3.S3, role string, bu
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
s.log.Error(err, "could not list objects in s3 bucket", "bucket: ", bucket)
|
||||
continue
|
||||
if role == "" {
|
||||
logger.Error(err, "could not list objects in bucket")
|
||||
} else {
|
||||
// Our documentation blesses specifying a role to assume without specifying buckets to scan, which will
|
||||
// often cause this to happen a lot (because in that case the scanner tries to scan every bucket in the
|
||||
// account, but the role probably doesn't have access to all of them). This makes it expected behavior
|
||||
// and therefore not an error.
|
||||
logger.V(3).Info("could not list objects in bucket",
|
||||
"err", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
s.SetProgressComplete(len(bucketsToScan), len(bucketsToScan), fmt.Sprintf("Completed scanning source %s. %d objects scanned.", s.name, objectCount), "")
|
||||
return nil
|
||||
}
|
||||
|
||||
// Chunks emits chunks of bytes over a channel.
|
||||
func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) error {
|
||||
const defaultAWSRegion = "us-east-1"
|
||||
|
||||
roles := s.conn.Roles
|
||||
if len(roles) == 0 {
|
||||
roles = []string{""}
|
||||
func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk, _ ...sources.ChunkingTarget) error {
|
||||
visitor := func(c context.Context, defaultRegionClient *s3.S3, roleArn string, buckets []string) {
|
||||
s.scanBuckets(c, defaultRegionClient, roleArn, buckets, chunksChan)
|
||||
}
|
||||
|
||||
for _, role := range roles {
|
||||
client, err := s.newClient(defaultAWSRegion, role)
|
||||
if err != nil {
|
||||
return errors.WrapPrefix(err, "could not create s3 client", 0)
|
||||
}
|
||||
return s.visitRoles(ctx, visitor)
|
||||
}
|
||||
|
||||
bucketsToScan, err := s.getBucketsToScan(client)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := s.scanBuckets(ctx, client, role, bucketsToScan, chunksChan); err != nil {
|
||||
return err
|
||||
}
|
||||
func (s *Source) getRegionalClientForBucket(ctx context.Context, defaultRegionClient *s3.S3, role, bucket string) (*s3.S3, error) {
|
||||
region, err := s3manager.GetBucketRegionWithClient(ctx, defaultRegionClient, bucket)
|
||||
if err != nil {
|
||||
return nil, errors.WrapPrefix(err, "could not get s3 region for bucket", 0)
|
||||
}
|
||||
|
||||
return nil
|
||||
if region == defaultAWSRegion {
|
||||
return defaultRegionClient, nil
|
||||
}
|
||||
|
||||
regionalClient, err := s.newClient(region, role)
|
||||
if err != nil {
|
||||
return nil, errors.WrapPrefix(err, "could not create regional s3 client", 0)
|
||||
}
|
||||
|
||||
return regionalClient, nil
|
||||
}
|
||||
|
||||
// pageChunker emits chunks onto the given channel from a page
|
||||
@@ -257,7 +280,7 @@ func (s *Source) pageChunker(ctx context.Context, client *s3.S3, chunksChan chan
|
||||
|
||||
// ignore large files
|
||||
if *obj.Size > s.maxObjectSize {
|
||||
s.log.V(3).Info("Skipping %d byte file (over maxObjectSize limit)", "object", *obj.Key)
|
||||
s.log.V(5).Info("Skipping %d byte file (over maxObjectSize limit)", "object", *obj.Key)
|
||||
continue
|
||||
}
|
||||
|
||||
@@ -396,6 +419,65 @@ func (s *Source) pageChunker(ctx context.Context, client *s3.S3, chunksChan chan
|
||||
_ = s.jobPool.Wait()
|
||||
}
|
||||
|
||||
func (s *Source) validateBucketAccess(ctx context.Context, client *s3.S3, roleArn string, buckets []string) []error {
|
||||
shouldHaveAccessToAllBuckets := roleArn == ""
|
||||
wasAbleToListAnyBucket := false
|
||||
var errors []error
|
||||
|
||||
for _, bucket := range buckets {
|
||||
if common.IsDone(ctx) {
|
||||
return append(errors, ctx.Err())
|
||||
}
|
||||
|
||||
regionalClient, err := s.getRegionalClientForBucket(ctx, client, roleArn, bucket)
|
||||
if err != nil {
|
||||
errors = append(errors, fmt.Errorf("could not get regional client for bucket %q: %w", bucket, err))
|
||||
continue
|
||||
}
|
||||
|
||||
_, err = regionalClient.ListObjectsV2(&s3.ListObjectsV2Input{Bucket: &bucket})
|
||||
|
||||
if err == nil {
|
||||
wasAbleToListAnyBucket = true
|
||||
} else if shouldHaveAccessToAllBuckets {
|
||||
errors = append(errors, fmt.Errorf("could not list objects in bucket %q: %w", bucket, err))
|
||||
}
|
||||
}
|
||||
|
||||
if !wasAbleToListAnyBucket {
|
||||
if roleArn == "" {
|
||||
errors = append(errors, fmt.Errorf("could not list objects in any bucket"))
|
||||
} else {
|
||||
errors = append(errors, fmt.Errorf("role %q could not list objects in any bucket", roleArn))
|
||||
}
|
||||
}
|
||||
|
||||
return errors
|
||||
}
|
||||
|
||||
func (s *Source) visitRoles(ctx context.Context, f func(c context.Context, defaultRegionClient *s3.S3, roleArn string, buckets []string)) error {
|
||||
roles := s.conn.Roles
|
||||
if len(roles) == 0 {
|
||||
roles = []string{""}
|
||||
}
|
||||
|
||||
for _, role := range roles {
|
||||
client, err := s.newClient(defaultAWSRegion, role)
|
||||
if err != nil {
|
||||
return errors.WrapPrefix(err, "could not create s3 client", 0)
|
||||
}
|
||||
|
||||
bucketsToScan, err := s.getBucketsToScan(client)
|
||||
if err != nil {
|
||||
return fmt.Errorf("role %q could not list any s3 buckets for scanning: %w", role, err)
|
||||
}
|
||||
|
||||
f(ctx, client, role, bucketsToScan)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// S3 links currently have the general format of:
|
||||
// https://[bucket].s3[.region unless us-east-1].amazonaws.com/[key]
|
||||
func makeS3Link(bucket, region, key string) string {
|
||||
|
||||
@@ -4,10 +4,14 @@
|
||||
package s3
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/credentialspb"
|
||||
"google.golang.org/protobuf/types/known/anypb"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/context"
|
||||
@@ -45,3 +49,127 @@ func TestSource_ChunksCount(t *testing.T) {
|
||||
}
|
||||
assert.Greater(t, got, wantChunkCount)
|
||||
}
|
||||
|
||||
func TestSource_Validate(t *testing.T) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), time.Second*15)
|
||||
defer cancel()
|
||||
|
||||
secret, err := common.GetTestSecret(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(fmt.Errorf("failed to access secret: %v", err))
|
||||
}
|
||||
|
||||
s3key := secret.MustGetField("AWS_S3_KEY")
|
||||
s3secret := secret.MustGetField("AWS_S3_SECRET")
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
roles []string
|
||||
buckets []string
|
||||
wantErrCount int
|
||||
}{
|
||||
{
|
||||
name: "buckets without roles, can access all buckets",
|
||||
buckets: []string{
|
||||
"truffletestbucket-s3-tests",
|
||||
},
|
||||
wantErrCount: 0,
|
||||
},
|
||||
{
|
||||
name: "buckets without roles, one error per inaccessible bucket",
|
||||
buckets: []string{
|
||||
"truffletestbucket-s3-tests",
|
||||
"truffletestbucket-s3-role-assumption",
|
||||
"truffletestbucket-no-access",
|
||||
},
|
||||
wantErrCount: 2,
|
||||
},
|
||||
{
|
||||
name: "roles without buckets, all can access at least one account bucket",
|
||||
roles: []string{
|
||||
"arn:aws:iam::619888638459:role/s3-test-assume-role",
|
||||
},
|
||||
wantErrCount: 0,
|
||||
},
|
||||
{
|
||||
name: "roles without buckets, one error per role that cannot access any account buckets",
|
||||
roles: []string{
|
||||
"arn:aws:iam::619888638459:role/s3-test-assume-role",
|
||||
"arn:aws:iam::619888638459:role/test-no-access",
|
||||
},
|
||||
wantErrCount: 1,
|
||||
},
|
||||
{
|
||||
name: "role and buckets, can access at least one bucket",
|
||||
roles: []string{
|
||||
"arn:aws:iam::619888638459:role/s3-test-assume-role",
|
||||
},
|
||||
buckets: []string{
|
||||
"truffletestbucket-s3-role-assumption",
|
||||
"truffletestbucket-no-access",
|
||||
},
|
||||
wantErrCount: 0,
|
||||
},
|
||||
{
|
||||
name: "roles and buckets, one error per role that cannot access at least one bucket",
|
||||
roles: []string{
|
||||
"arn:aws:iam::619888638459:role/s3-test-assume-role",
|
||||
"arn:aws:iam::619888638459:role/test-no-access",
|
||||
},
|
||||
buckets: []string{
|
||||
"truffletestbucket-s3-role-assumption",
|
||||
"truffletestbucket-no-access",
|
||||
},
|
||||
wantErrCount: 1,
|
||||
},
|
||||
{
|
||||
name: "role and buckets, a bucket doesn't even exist",
|
||||
roles: []string{
|
||||
"arn:aws:iam::619888638459:role/s3-test-assume-role",
|
||||
},
|
||||
buckets: []string{
|
||||
"truffletestbucket-s3-role-assumption",
|
||||
"not-a-real-bucket-asljdhmglasjgvklhsdaljfh", // need a bucket name that nobody is likely to ever create
|
||||
},
|
||||
wantErrCount: 1,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), time.Second*15)
|
||||
var cancelOnce sync.Once
|
||||
defer cancelOnce.Do(cancel)
|
||||
|
||||
// These are used by the tests that assume roles
|
||||
t.Setenv("AWS_ACCESS_KEY_ID", s3key)
|
||||
t.Setenv("AWS_SECRET_ACCESS_KEY", s3secret)
|
||||
|
||||
s := &Source{}
|
||||
|
||||
conn, err := anypb.New(&sourcespb.S3{
|
||||
// These are used by the tests that don't assume roles
|
||||
Credential: &sourcespb.S3_AccessKey{
|
||||
AccessKey: &credentialspb.KeySecret{
|
||||
Key: s3key,
|
||||
Secret: s3secret,
|
||||
},
|
||||
},
|
||||
Buckets: tt.buckets,
|
||||
Roles: tt.roles,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
err = s.Init(ctx, tt.name, 0, 0, false, conn, 0)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
errs := s.Validate(ctx)
|
||||
|
||||
assert.Equal(t, tt.wantErrCount, len(errs))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
+80
-170
@@ -2,10 +2,12 @@ package sources
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"runtime"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"github.com/marusama/semaphore/v2"
|
||||
"golang.org/x/sync/errgroup"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
@@ -13,33 +15,12 @@ import (
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
|
||||
)
|
||||
|
||||
// handle uniquely identifies a Source given to the manager to manage. If the
|
||||
// SourceManager is connected to the API, it will be equivalent to the unique
|
||||
// source ID, otherwise it behaves as a counter.
|
||||
type handle int64
|
||||
|
||||
// SourceInitFunc is a function that takes a source and job ID and returns an
|
||||
// initialized Source.
|
||||
type SourceInitFunc func(ctx context.Context, jobID, sourceID int64) (Source, error)
|
||||
|
||||
// sourceInfo is an aggregate struct to store source information provided on
|
||||
// initialization.
|
||||
type sourceInfo struct {
|
||||
initFunc SourceInitFunc
|
||||
sourceID int64
|
||||
name string
|
||||
}
|
||||
|
||||
type SourceManager struct {
|
||||
api apiClient
|
||||
hooks []JobProgressHook
|
||||
// Map of handle to source initializer.
|
||||
handles map[handle]sourceInfo
|
||||
handlesLock sync.Mutex
|
||||
// Pool limiting the amount of concurrent sources running.
|
||||
pool errgroup.Group
|
||||
poolLimit int
|
||||
currentRunningCount int32
|
||||
sem semaphore.Semaphore
|
||||
wg sync.WaitGroup
|
||||
// Max number of units to scan concurrently per source.
|
||||
concurrentUnits int
|
||||
// Run the sources using source unit enumeration / chunking if available.
|
||||
@@ -47,15 +28,16 @@ type SourceManager struct {
|
||||
// Downstream chunks channel to be scanned.
|
||||
outputChunks chan *Chunk
|
||||
// Set when Wait() returns.
|
||||
done bool
|
||||
firstErr chan error
|
||||
waitErr error
|
||||
done bool
|
||||
}
|
||||
|
||||
// apiClient is an interface for optionally communicating with an external API.
|
||||
type apiClient interface {
|
||||
// RegisterSource lets the API know we have a source and it returns a unique source ID for it.
|
||||
RegisterSource(ctx context.Context, name string, kind sourcespb.SourceType) (int64, error)
|
||||
// GetJobID queries the API for an existing job or new job ID.
|
||||
GetJobID(ctx context.Context, id int64) (int64, error)
|
||||
// GetIDs informs the API of the source that's about to run and returns
|
||||
// two identifiers used during source initialization.
|
||||
GetIDs(ctx context.Context, name string, kind sourcespb.SourceType) (SourceID, JobID, error)
|
||||
}
|
||||
|
||||
// WithAPI adds an API client to the manager for tracking jobs and progress. If
|
||||
@@ -73,8 +55,7 @@ func WithReportHook(hook JobProgressHook) func(*SourceManager) {
|
||||
// WithConcurrentSources limits the concurrent number of sources a manager can run.
|
||||
func WithConcurrentSources(concurrency int) func(*SourceManager) {
|
||||
return func(mgr *SourceManager) {
|
||||
mgr.pool.SetLimit(concurrency)
|
||||
mgr.poolLimit = concurrency
|
||||
mgr.sem.SetLimit(concurrency)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -100,8 +81,9 @@ func NewManager(opts ...func(*SourceManager)) *SourceManager {
|
||||
mgr := SourceManager{
|
||||
// Default to the headless API. Can be overwritten by the WithAPI option.
|
||||
api: &headlessAPI{},
|
||||
handles: make(map[handle]sourceInfo),
|
||||
sem: semaphore.New(runtime.NumCPU()),
|
||||
outputChunks: make(chan *Chunk),
|
||||
firstErr: make(chan error, 1),
|
||||
}
|
||||
for _, opt := range opts {
|
||||
opt(&mgr)
|
||||
@@ -109,129 +91,53 @@ func NewManager(opts ...func(*SourceManager)) *SourceManager {
|
||||
return &mgr
|
||||
}
|
||||
|
||||
// Enroll informs the SourceManager to track and manage a Source.
|
||||
func (s *SourceManager) Enroll(ctx context.Context, name string, kind sourcespb.SourceType, f SourceInitFunc) (handle, error) {
|
||||
if s.done {
|
||||
return 0, fmt.Errorf("manager is done")
|
||||
}
|
||||
id, err := s.api.RegisterSource(ctx, name, kind)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
handleID := handle(id)
|
||||
s.handlesLock.Lock()
|
||||
defer s.handlesLock.Unlock()
|
||||
if _, ok := s.handles[handleID]; ok {
|
||||
// TODO: smartly handle this?
|
||||
return 0, fmt.Errorf("handle ID '%d' already in use", handleID)
|
||||
}
|
||||
s.handles[handleID] = sourceInfo{
|
||||
initFunc: f,
|
||||
sourceID: int64(handleID),
|
||||
name: name,
|
||||
}
|
||||
return handleID, nil
|
||||
func (s *SourceManager) GetIDs(ctx context.Context, sourceName string, kind sourcespb.SourceType) (SourceID, JobID, error) {
|
||||
return s.api.GetIDs(ctx, sourceName, kind)
|
||||
}
|
||||
|
||||
// Run blocks until a resource is available to run the source, then
|
||||
// synchronously runs it. The first fatal error, if any, will be returned.
|
||||
func (s *SourceManager) Run(ctx context.Context, handle handle) (JobProgressRef, error) {
|
||||
progress, err := s.ScheduleRun(ctx, handle)
|
||||
if err != nil {
|
||||
return progress, err
|
||||
}
|
||||
<-progress.Done()
|
||||
return progress, progress.Snapshot().FatalError()
|
||||
}
|
||||
|
||||
// RunOnce blocks until a resource is available to run the source, then
|
||||
// synchronously runs it. The first fatal error, if any, will be returned.
|
||||
func (s *SourceManager) RunOnce(ctx context.Context, name string, source Source) (JobProgressRef, error) {
|
||||
progress, err := s.ScheduleRunOnce(ctx, name, source)
|
||||
if err != nil {
|
||||
return progress, err
|
||||
}
|
||||
<-progress.Done()
|
||||
return progress, progress.Snapshot().FatalError()
|
||||
}
|
||||
|
||||
// ScheduleRun blocks until a resource is available to run the source, then
|
||||
// asynchronously runs it. Error information is stored and accessible via the
|
||||
// JobProgressRef as it becomes available.
|
||||
func (s *SourceManager) ScheduleRun(ctx context.Context, handle handle) (JobProgressRef, error) {
|
||||
return s.asyncRun(ctx, handle)
|
||||
func (s *SourceManager) Run(ctx context.Context, sourceName string, source Source) (JobProgressRef, error) {
|
||||
return s.asyncRun(ctx, sourceName, source)
|
||||
}
|
||||
|
||||
// ScheduleRunOnce blocks until a resource is available to run the source, then
|
||||
// asynchronously runs it. Error information is stored and accessible via the
|
||||
// JobProgressRef as it becomes available.
|
||||
func (s *SourceManager) ScheduleRunOnce(ctx context.Context, name string, source Source) (JobProgressRef, error) {
|
||||
return s.asyncRunOnce(ctx, name, source)
|
||||
}
|
||||
|
||||
// asyncRun is a helper method to asynchronously run the Source. It calls out
|
||||
// to the API to get a job ID for this run, creates a JobProgress object, then
|
||||
// waits for an available goroutine to asynchronously run it.
|
||||
func (s *SourceManager) asyncRun(ctx context.Context, handle handle) (JobProgressRef, error) {
|
||||
// asyncRun is a helper method to asynchronously run the Source.
|
||||
func (s *SourceManager) asyncRun(ctx context.Context, sourceName string, source Source) (JobProgressRef, error) {
|
||||
sourceID, jobID := source.SourceID(), source.JobID()
|
||||
// Do preflight checks before waiting on the pool.
|
||||
if err := s.preflightChecks(ctx); err != nil {
|
||||
return JobProgressRef{}, err
|
||||
}
|
||||
// Get the name.
|
||||
sourceInfo, ok := s.getSourceInfo(handle)
|
||||
if !ok {
|
||||
return JobProgressRef{SourceID: int64(handle)}, fmt.Errorf("unrecognized handle")
|
||||
}
|
||||
sourceName := sourceInfo.name
|
||||
// Get a Job ID.
|
||||
ctx = context.WithValue(ctx, "source_id", int64(handle))
|
||||
jobID, err := s.api.GetJobID(ctx, int64(handle))
|
||||
if err != nil {
|
||||
return JobProgressRef{SourceID: int64(handle), SourceName: sourceName}, err
|
||||
return JobProgressRef{
|
||||
SourceName: sourceName,
|
||||
SourceID: sourceID,
|
||||
JobID: jobID,
|
||||
}, err
|
||||
}
|
||||
// Create a JobProgress object for tracking progress.
|
||||
ctx, cancel := context.WithCancelCause(ctx)
|
||||
progress := NewJobProgress(jobID, int64(handle), sourceName, WithHooks(s.hooks...), WithCancel(cancel))
|
||||
s.pool.Go(func() error {
|
||||
atomic.AddInt32(&s.currentRunningCount, 1)
|
||||
defer atomic.AddInt32(&s.currentRunningCount, -1)
|
||||
progress := NewJobProgress(jobID, sourceID, sourceName, WithHooks(s.hooks...), WithCancel(cancel))
|
||||
if err := s.sem.Acquire(ctx, 1); err != nil {
|
||||
// Context cancelled.
|
||||
progress.ReportError(Fatal{err})
|
||||
return progress.Ref(), Fatal{err}
|
||||
}
|
||||
s.wg.Add(1)
|
||||
go func() {
|
||||
defer s.sem.Release(1)
|
||||
defer s.wg.Done()
|
||||
ctx := context.WithValues(ctx,
|
||||
"job_id", jobID,
|
||||
"source_manager_worker_id", common.RandomID(5),
|
||||
)
|
||||
defer common.Recover(ctx)
|
||||
defer cancel(nil)
|
||||
return s.run(ctx, sourceInfo, jobID, progress)
|
||||
})
|
||||
return progress.Ref(), nil
|
||||
}
|
||||
|
||||
// asyncRunOnce is a helper method to asynchronously run an already initialized
|
||||
// Source once. Unlike asyncRun, this method doesn't call out to the API for a
|
||||
// job ID since it's already available in the Source.
|
||||
func (s *SourceManager) asyncRunOnce(ctx context.Context, sourceName string, source Source) (JobProgressRef, error) {
|
||||
if err := s.preflightChecks(ctx); err != nil {
|
||||
return JobProgressRef{}, err
|
||||
}
|
||||
ctx = context.WithValue(ctx, "source_id", source.SourceID())
|
||||
jobID := source.JobID()
|
||||
// Create a JobProgress object for tracking progress.
|
||||
ctx, cancel := context.WithCancelCause(ctx)
|
||||
progress := NewJobProgress(jobID, source.SourceID(), sourceName, WithHooks(s.hooks...), WithCancel(cancel))
|
||||
s.pool.Go(func() error {
|
||||
ctx := context.WithValues(ctx,
|
||||
"job_id", jobID,
|
||||
"source_manager_worker_id", common.RandomID(5),
|
||||
)
|
||||
defer common.Recover(ctx)
|
||||
defer cancel(nil)
|
||||
sourceInfo := sourceInfo{
|
||||
initFunc: func(context.Context, int64, int64) (Source, error) { return source, nil },
|
||||
sourceID: source.SourceID(),
|
||||
name: sourceName,
|
||||
if err := s.run(ctx, source, progress); err != nil {
|
||||
select {
|
||||
case s.firstErr <- err:
|
||||
default:
|
||||
}
|
||||
}
|
||||
return s.run(ctx, sourceInfo, jobID, progress)
|
||||
})
|
||||
}()
|
||||
return progress.Ref(), nil
|
||||
}
|
||||
|
||||
@@ -248,13 +154,19 @@ func (s *SourceManager) Chunks() <-chan *Chunk {
|
||||
func (s *SourceManager) Wait() error {
|
||||
// Check if the manager has been Waited.
|
||||
if s.done {
|
||||
return s.pool.Wait()
|
||||
return s.waitErr
|
||||
}
|
||||
defer close(s.outputChunks)
|
||||
defer func() { s.done = true }()
|
||||
s.done = true
|
||||
|
||||
// Return the first error returned by run.
|
||||
return s.pool.Wait()
|
||||
s.wg.Wait()
|
||||
select {
|
||||
case s.waitErr = <-s.firstErr:
|
||||
default:
|
||||
}
|
||||
close(s.outputChunks)
|
||||
close(s.firstErr)
|
||||
return s.waitErr
|
||||
}
|
||||
|
||||
// ScanChunk injects a chunk into the output stream of chunks to be scanned.
|
||||
@@ -265,16 +177,33 @@ func (s *SourceManager) ScanChunk(chunk *Chunk) {
|
||||
}
|
||||
|
||||
// AvailableCapacity returns the number of concurrent jobs the manager can
|
||||
// accommodate at this time. If there is no limit, -1 is returned.
|
||||
// accommodate at this time.
|
||||
func (s *SourceManager) AvailableCapacity() int {
|
||||
if s.poolLimit == 0 {
|
||||
return -1
|
||||
}
|
||||
runCount := atomic.LoadInt32(&s.currentRunningCount)
|
||||
return s.poolLimit - int(runCount)
|
||||
return s.sem.GetLimit() - s.sem.GetCount()
|
||||
}
|
||||
|
||||
// preflightChecks is a helper method to check the Manager or the context isn't done.
|
||||
// MaxConcurrentSources returns the maximum configured limit of concurrent
|
||||
// sources the manager will run.
|
||||
func (s *SourceManager) MaxConcurrentSources() int {
|
||||
return s.sem.GetLimit()
|
||||
}
|
||||
|
||||
// ConcurrentSources returns the current number of concurrently running
|
||||
// sources.
|
||||
func (s *SourceManager) ConcurrentSources() int {
|
||||
return s.sem.GetCount()
|
||||
}
|
||||
|
||||
// SetMaxConcurrentSources sets the maximum number of concurrently running
|
||||
// sources. If the count is lower than the already existing number of
|
||||
// concurrently running sources, no sources will be scheduled to run until the
|
||||
// existing sources complete.
|
||||
func (s *SourceManager) SetMaxConcurrentSources(maxRunCount int) {
|
||||
s.sem.SetLimit(maxRunCount)
|
||||
}
|
||||
|
||||
// preflightChecks is a helper method to check the Manager or the context isn't
|
||||
// done.
|
||||
func (s *SourceManager) preflightChecks(ctx context.Context) error {
|
||||
// Check if the manager has been Waited.
|
||||
if s.done {
|
||||
@@ -286,7 +215,7 @@ func (s *SourceManager) preflightChecks(ctx context.Context) error {
|
||||
// run is a helper method to sychronously run the source. It does not check for
|
||||
// acquired resources. An error is returned if there was a fatal error during
|
||||
// the run. This information is also recorded in the JobProgress.
|
||||
func (s *SourceManager) run(ctx context.Context, sourceInfo sourceInfo, jobID int64, report *JobProgress) error {
|
||||
func (s *SourceManager) run(ctx context.Context, source Source, report *JobProgress) error {
|
||||
defer report.Finish()
|
||||
report.Start(time.Now())
|
||||
defer func() { report.End(time.Now()) }()
|
||||
@@ -297,16 +226,10 @@ func (s *SourceManager) run(ctx context.Context, sourceInfo sourceInfo, jobID in
|
||||
}
|
||||
}()
|
||||
|
||||
// Initialize the source.
|
||||
source, err := sourceInfo.initFunc(ctx, jobID, sourceInfo.sourceID)
|
||||
if err != nil {
|
||||
report.ReportError(Fatal{err})
|
||||
return Fatal{err}
|
||||
}
|
||||
report.TrackProgress(source.GetProgress())
|
||||
ctx = context.WithValues(ctx,
|
||||
"source_type", source.Type().String(),
|
||||
"source_name", sourceInfo.name,
|
||||
"source_name", report.SourceName,
|
||||
)
|
||||
// Check for the preferred method of tracking source units.
|
||||
if enumChunker, ok := source.(SourceUnitEnumChunker); ok && s.useSourceUnits {
|
||||
@@ -422,28 +345,15 @@ func (s *SourceManager) runWithUnits(ctx context.Context, source SourceUnitEnumC
|
||||
}
|
||||
}
|
||||
|
||||
// getSourceInfo is a helper method for safe concurrent access to the
|
||||
// map[handle]SourceInitFunc map.
|
||||
func (s *SourceManager) getSourceInfo(handle handle) (sourceInfo, bool) {
|
||||
s.handlesLock.Lock()
|
||||
defer s.handlesLock.Unlock()
|
||||
f, ok := s.handles[handle]
|
||||
return f, ok
|
||||
}
|
||||
|
||||
// headlessAPI implements the apiClient interface locally.
|
||||
type headlessAPI struct {
|
||||
// Counters for assigning handle and job IDs.
|
||||
// Counters for assigning source and job IDs.
|
||||
sourceIDCounter int64
|
||||
jobIDCounter int64
|
||||
}
|
||||
|
||||
func (api *headlessAPI) RegisterSource(ctx context.Context, name string, kind sourcespb.SourceType) (int64, error) {
|
||||
return atomic.AddInt64(&api.sourceIDCounter, 1), nil
|
||||
}
|
||||
|
||||
func (api *headlessAPI) GetJobID(ctx context.Context, id int64) (int64, error) {
|
||||
return atomic.AddInt64(&api.jobIDCounter, 1), nil
|
||||
func (api *headlessAPI) GetIDs(context.Context, string, sourcespb.SourceType) (SourceID, JobID, error) {
|
||||
return SourceID(atomic.AddInt64(&api.sourceIDCounter, 1)), JobID(atomic.AddInt64(&api.jobIDCounter, 1)), nil
|
||||
}
|
||||
|
||||
// mgrUnitReporter implements the UnitReporter interface.
|
||||
|
||||
@@ -6,25 +6,24 @@ import (
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"google.golang.org/protobuf/types/known/anypb"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/context"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
|
||||
"google.golang.org/protobuf/types/known/anypb"
|
||||
)
|
||||
|
||||
// DummySource implements Source and is used for testing a SourceManager.
|
||||
type DummySource struct {
|
||||
sourceID int64
|
||||
jobID int64
|
||||
initCalled bool
|
||||
sourceID SourceID
|
||||
jobID JobID
|
||||
chunker
|
||||
}
|
||||
|
||||
func (d *DummySource) Type() sourcespb.SourceType { return 1337 }
|
||||
func (d *DummySource) SourceID() int64 { return d.sourceID }
|
||||
func (d *DummySource) JobID() int64 { return d.jobID }
|
||||
func (d *DummySource) Init(_ context.Context, _ string, jobID, sourceID int64, _ bool, _ *anypb.Any, _ int) error {
|
||||
d.initCalled = true
|
||||
func (d *DummySource) SourceID() SourceID { return d.sourceID }
|
||||
func (d *DummySource) JobID() JobID { return d.jobID }
|
||||
func (d *DummySource) Init(_ context.Context, _ string, jobID JobID, sourceID SourceID, _ bool, _ *anypb.Any, _ int) error {
|
||||
d.sourceID = sourceID
|
||||
d.jobID = jobID
|
||||
return nil
|
||||
@@ -33,7 +32,7 @@ func (d *DummySource) GetProgress() *Progress { return nil }
|
||||
|
||||
// Interface to easily test different chunking methods.
|
||||
type chunker interface {
|
||||
Chunks(context.Context, chan *Chunk) error
|
||||
Chunks(context.Context, chan *Chunk, ...ChunkingTarget) error
|
||||
ChunkUnit(ctx context.Context, unit SourceUnit, reporter ChunkReporter) error
|
||||
Enumerate(ctx context.Context, reporter UnitReporter) error
|
||||
}
|
||||
@@ -44,7 +43,7 @@ type counterChunker struct {
|
||||
count int
|
||||
}
|
||||
|
||||
func (c *counterChunker) Chunks(ctx context.Context, ch chan *Chunk) error {
|
||||
func (c *counterChunker) Chunks(ctx context.Context, ch chan *Chunk, _ ...ChunkingTarget) error {
|
||||
for i := 0; i < c.count; i++ {
|
||||
select {
|
||||
case ch <- &Chunk{Data: []byte{c.chunkCounter}}:
|
||||
@@ -77,20 +76,17 @@ func (c *counterChunker) ChunkUnit(ctx context.Context, unit SourceUnit, reporte
|
||||
// Chunk method that always returns an error.
|
||||
type errorChunker struct{ error }
|
||||
|
||||
func (c errorChunker) Chunks(context.Context, chan *Chunk) error { return c }
|
||||
func (c errorChunker) Enumerate(context.Context, UnitReporter) error { return c }
|
||||
func (c errorChunker) ChunkUnit(context.Context, SourceUnit, ChunkReporter) error { return c }
|
||||
func (c errorChunker) Chunks(context.Context, chan *Chunk, ...ChunkingTarget) error { return c }
|
||||
func (c errorChunker) Enumerate(context.Context, UnitReporter) error { return c }
|
||||
func (c errorChunker) ChunkUnit(context.Context, SourceUnit, ChunkReporter) error { return c }
|
||||
|
||||
// enrollDummy is a helper function to enroll a DummySource with a SourceManager.
|
||||
func enrollDummy(mgr *SourceManager, chunkMethod chunker) (handle, error) {
|
||||
return mgr.Enroll(context.Background(), "dummy", 1337,
|
||||
func(ctx context.Context, jobID, sourceID int64) (Source, error) {
|
||||
source := &DummySource{chunker: chunkMethod}
|
||||
if err := source.Init(ctx, "dummy", jobID, sourceID, true, nil, 42); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return source, nil
|
||||
})
|
||||
// buildDummy is a helper function to enroll a DummySource with a SourceManager.
|
||||
func buildDummy(chunkMethod chunker) (Source, error) {
|
||||
source := &DummySource{chunker: chunkMethod}
|
||||
if err := source.Init(context.Background(), "dummy", 123, 456, true, nil, 42); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return source, nil
|
||||
}
|
||||
|
||||
// tryRead is a helper function that will try to read from a channel and return
|
||||
@@ -106,11 +102,13 @@ func tryRead(ch <-chan *Chunk) (*Chunk, error) {
|
||||
|
||||
func TestSourceManagerRun(t *testing.T) {
|
||||
mgr := NewManager(WithBufferedOutput(8))
|
||||
handle, err := enrollDummy(mgr, &counterChunker{count: 1})
|
||||
source, err := buildDummy(&counterChunker{count: 1})
|
||||
assert.NoError(t, err)
|
||||
for i := 0; i < 3; i++ {
|
||||
_, err = mgr.Run(context.Background(), handle)
|
||||
ref, err := mgr.Run(context.Background(), "dummy", source)
|
||||
<-ref.Done()
|
||||
assert.NoError(t, err)
|
||||
assert.NoError(t, ref.Snapshot().FatalError())
|
||||
chunk, err := tryRead(mgr.Chunks())
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, []byte{byte(i)}, chunk.Data)
|
||||
@@ -122,35 +120,31 @@ func TestSourceManagerRun(t *testing.T) {
|
||||
|
||||
func TestSourceManagerWait(t *testing.T) {
|
||||
mgr := NewManager()
|
||||
handle, err := enrollDummy(mgr, &counterChunker{count: 1})
|
||||
source, err := buildDummy(&counterChunker{count: 1})
|
||||
assert.NoError(t, err)
|
||||
// Asynchronously run the source.
|
||||
_, err = mgr.ScheduleRun(context.Background(), handle)
|
||||
_, err = mgr.Run(context.Background(), "dummy", source)
|
||||
assert.NoError(t, err)
|
||||
// Read the 1 chunk we're expecting so Waiting completes.
|
||||
<-mgr.Chunks()
|
||||
// Wait for all resources to complete.
|
||||
assert.NoError(t, mgr.Wait())
|
||||
// Enroll and run should return an error now.
|
||||
_, err = enrollDummy(mgr, &counterChunker{count: 1})
|
||||
assert.Error(t, err)
|
||||
_, err = mgr.ScheduleRun(context.Background(), handle)
|
||||
// Run should return an error now.
|
||||
_, err = buildDummy(&counterChunker{count: 1})
|
||||
assert.NoError(t, err)
|
||||
_, err = mgr.Run(context.Background(), "dummy", source)
|
||||
assert.Error(t, err)
|
||||
}
|
||||
|
||||
func TestSourceManagerError(t *testing.T) {
|
||||
mgr := NewManager()
|
||||
handle, err := enrollDummy(mgr, errorChunker{fmt.Errorf("oops")})
|
||||
source, err := buildDummy(errorChunker{fmt.Errorf("oops")})
|
||||
assert.NoError(t, err)
|
||||
// A synchronous run should fail.
|
||||
_, err = mgr.Run(context.Background(), handle)
|
||||
assert.Error(t, err)
|
||||
// Scheduling a run should not fail, but the error should surface in
|
||||
// Wait().
|
||||
ref, err := mgr.ScheduleRun(context.Background(), handle)
|
||||
ref, err := mgr.Run(context.Background(), "dummy", source)
|
||||
assert.NoError(t, err)
|
||||
assert.Error(t, mgr.Wait())
|
||||
<-ref.Done()
|
||||
assert.Error(t, ref.Snapshot().FatalError())
|
||||
assert.Error(t, mgr.Wait())
|
||||
}
|
||||
|
||||
func TestSourceManagerReport(t *testing.T) {
|
||||
@@ -160,11 +154,11 @@ func TestSourceManagerReport(t *testing.T) {
|
||||
{WithBufferedOutput(8), WithSourceUnits(), WithConcurrentUnits(1)},
|
||||
} {
|
||||
mgr := NewManager(opts...)
|
||||
handle, err := enrollDummy(mgr, &counterChunker{count: 4})
|
||||
source, err := buildDummy(&counterChunker{count: 4})
|
||||
assert.NoError(t, err)
|
||||
// Synchronously run the source.
|
||||
ref, err := mgr.Run(context.Background(), handle)
|
||||
ref, err := mgr.Run(context.Background(), "dummy", source)
|
||||
assert.NoError(t, err)
|
||||
<-ref.Done()
|
||||
assert.Equal(t, 0, len(ref.Snapshot().Errors))
|
||||
assert.Equal(t, uint64(4), ref.Snapshot().TotalChunks)
|
||||
}
|
||||
@@ -178,7 +172,7 @@ type unitChunk struct {
|
||||
|
||||
type unitChunker struct{ steps []unitChunk }
|
||||
|
||||
func (c *unitChunker) Chunks(ctx context.Context, ch chan *Chunk) error {
|
||||
func (c *unitChunker) Chunks(ctx context.Context, ch chan *Chunk, _ ...ChunkingTarget) error {
|
||||
for _, step := range c.steps {
|
||||
if step.err != "" {
|
||||
continue
|
||||
@@ -222,10 +216,11 @@ func TestSourceManagerNonFatalError(t *testing.T) {
|
||||
{unit: "three", err: "not again"},
|
||||
}
|
||||
mgr := NewManager(WithBufferedOutput(8), WithSourceUnits())
|
||||
handle, err := enrollDummy(mgr, &unitChunker{input})
|
||||
source, err := buildDummy(&unitChunker{input})
|
||||
assert.NoError(t, err)
|
||||
ref, err := mgr.Run(context.Background(), handle)
|
||||
ref, err := mgr.Run(context.Background(), "dummy", source)
|
||||
assert.NoError(t, err)
|
||||
<-ref.Done()
|
||||
report := ref.Snapshot()
|
||||
assert.Equal(t, len(input), int(report.TotalUnits))
|
||||
assert.Equal(t, len(input), int(report.FinishedUnits))
|
||||
@@ -236,11 +231,11 @@ func TestSourceManagerNonFatalError(t *testing.T) {
|
||||
|
||||
func TestSourceManagerContextCancelled(t *testing.T) {
|
||||
mgr := NewManager(WithBufferedOutput(8))
|
||||
handle, err := enrollDummy(mgr, &counterChunker{count: 100})
|
||||
source, err := buildDummy(&counterChunker{count: 100})
|
||||
assert.NoError(t, err)
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
ref, err := mgr.ScheduleRun(ctx, handle)
|
||||
ref, err := mgr.Run(ctx, "dummy", source)
|
||||
assert.NoError(t, err)
|
||||
|
||||
cancel()
|
||||
@@ -250,60 +245,33 @@ func TestSourceManagerContextCancelled(t *testing.T) {
|
||||
}
|
||||
|
||||
type DummyAPI struct {
|
||||
registerSource func(context.Context, string, sourcespb.SourceType) (int64, error)
|
||||
getJobID func(context.Context, int64) (int64, error)
|
||||
registerSource func(context.Context, string, sourcespb.SourceType) (SourceID, error)
|
||||
getJobID func(context.Context, SourceID) (JobID, error)
|
||||
}
|
||||
|
||||
func (api DummyAPI) RegisterSource(ctx context.Context, name string, kind sourcespb.SourceType) (int64, error) {
|
||||
func (api DummyAPI) RegisterSource(ctx context.Context, name string, kind sourcespb.SourceType) (SourceID, error) {
|
||||
return api.registerSource(ctx, name, kind)
|
||||
}
|
||||
|
||||
func (api DummyAPI) GetJobID(ctx context.Context, id int64) (int64, error) {
|
||||
func (api DummyAPI) GetJobID(ctx context.Context, id SourceID) (JobID, error) {
|
||||
return api.getJobID(ctx, id)
|
||||
}
|
||||
|
||||
func TestSourceManagerJobAndSourceIDs(t *testing.T) {
|
||||
mgr := NewManager(WithAPI(DummyAPI{
|
||||
registerSource: func(context.Context, string, sourcespb.SourceType) (int64, error) {
|
||||
return 1337, nil
|
||||
},
|
||||
getJobID: func(context.Context, int64) (int64, error) {
|
||||
return 9001, nil
|
||||
},
|
||||
}))
|
||||
var (
|
||||
initializedJobID int64
|
||||
initializedSourceID int64
|
||||
)
|
||||
handle, err := mgr.Enroll(context.Background(), "dummy", 1337,
|
||||
func(ctx context.Context, jobID, sourceID int64) (Source, error) {
|
||||
initializedJobID = jobID
|
||||
initializedSourceID = sourceID
|
||||
return nil, fmt.Errorf("ignore")
|
||||
})
|
||||
assert.NoError(t, err)
|
||||
|
||||
ref, _ := mgr.Run(context.Background(), handle)
|
||||
assert.Equal(t, int64(1337), initializedSourceID)
|
||||
assert.Equal(t, int64(1337), ref.SourceID)
|
||||
assert.Equal(t, int64(9001), initializedJobID)
|
||||
assert.Equal(t, int64(9001), ref.JobID)
|
||||
assert.Equal(t, "dummy", ref.SourceName)
|
||||
}
|
||||
|
||||
// Chunk method that has a custom callback for the Chunks method.
|
||||
type callbackChunker struct {
|
||||
cb func(context.Context, chan *Chunk) error
|
||||
}
|
||||
|
||||
func (c callbackChunker) Chunks(ctx context.Context, ch chan *Chunk) error { return c.cb(ctx, ch) }
|
||||
func (c callbackChunker) Chunks(ctx context.Context, ch chan *Chunk, _ ...ChunkingTarget) error {
|
||||
return c.cb(ctx, ch)
|
||||
}
|
||||
func (c callbackChunker) Enumerate(context.Context, UnitReporter) error { return nil }
|
||||
func (c callbackChunker) ChunkUnit(context.Context, SourceUnit, ChunkReporter) error { return nil }
|
||||
|
||||
func TestSourceManagerCancelRun(t *testing.T) {
|
||||
mgr := NewManager(WithBufferedOutput(8))
|
||||
var returnedErr error
|
||||
handle, err := enrollDummy(mgr, callbackChunker{func(ctx context.Context, _ chan *Chunk) error {
|
||||
source, err := buildDummy(callbackChunker{func(ctx context.Context, _ chan *Chunk) error {
|
||||
// The context passed to Chunks should get cancelled when ref.CancelRun() is called.
|
||||
<-ctx.Done()
|
||||
returnedErr = fmt.Errorf("oh no: %w", ctx.Err())
|
||||
@@ -311,7 +279,7 @@ func TestSourceManagerCancelRun(t *testing.T) {
|
||||
}})
|
||||
assert.NoError(t, err)
|
||||
|
||||
ref, err := mgr.ScheduleRun(context.Background(), handle)
|
||||
ref, err := mgr.Run(context.Background(), "dummy", source)
|
||||
assert.NoError(t, err)
|
||||
|
||||
cancelErr := fmt.Errorf("abort! abort!")
|
||||
@@ -325,7 +293,7 @@ func TestSourceManagerCancelRun(t *testing.T) {
|
||||
func TestSourceManagerAvailableCapacity(t *testing.T) {
|
||||
mgr := NewManager(WithConcurrentSources(1337))
|
||||
start, end := make(chan struct{}), make(chan struct{})
|
||||
handle, err := enrollDummy(mgr, callbackChunker{func(context.Context, chan *Chunk) error {
|
||||
source, err := buildDummy(callbackChunker{func(context.Context, chan *Chunk) error {
|
||||
start <- struct{}{} // Send start signal.
|
||||
<-end // Wait for end signal.
|
||||
return nil
|
||||
@@ -333,7 +301,7 @@ func TestSourceManagerAvailableCapacity(t *testing.T) {
|
||||
assert.NoError(t, err)
|
||||
|
||||
assert.Equal(t, 1337, mgr.AvailableCapacity())
|
||||
ref, err := mgr.ScheduleRun(context.Background(), handle)
|
||||
ref, err := mgr.Run(context.Background(), "dummy", source)
|
||||
assert.NoError(t, err)
|
||||
|
||||
<-start // Wait for start signal.
|
||||
@@ -342,21 +310,3 @@ func TestSourceManagerAvailableCapacity(t *testing.T) {
|
||||
<-ref.Done() // Wait for the job to finish.
|
||||
assert.Equal(t, 1337, mgr.AvailableCapacity())
|
||||
}
|
||||
|
||||
func TestSourceManagerRunOnce(t *testing.T) {
|
||||
mgr := NewManager()
|
||||
var chunksCalled bool
|
||||
source := &DummySource{sourceID: 123, jobID: 456, chunker: callbackChunker{
|
||||
func(context.Context, chan *Chunk) error {
|
||||
chunksCalled = true
|
||||
return nil
|
||||
},
|
||||
}}
|
||||
ref, err := mgr.RunOnce(context.Background(), "test", source)
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, source.jobID, ref.JobID)
|
||||
assert.Equal(t, source.sourceID, ref.SourceID)
|
||||
assert.Equal(t, "test", ref.SourceName)
|
||||
assert.True(t, chunksCalled)
|
||||
assert.False(t, source.initCalled)
|
||||
}
|
||||
|
||||
+46
-7
@@ -11,14 +11,19 @@ import (
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
|
||||
)
|
||||
|
||||
type (
|
||||
SourceID int64
|
||||
JobID int64
|
||||
)
|
||||
|
||||
// Chunk contains data to be decoded and scanned along with context on where it came from.
|
||||
type Chunk struct {
|
||||
// SourceName is the name of the Source that produced the chunk.
|
||||
SourceName string
|
||||
// SourceID is the ID of the source that the Chunk originated from.
|
||||
SourceID int64
|
||||
SourceID SourceID
|
||||
// JobID is the ID of the job that the Chunk originated from.
|
||||
JobID int64
|
||||
JobID JobID
|
||||
// SourceType is the type of Source that produced the chunk.
|
||||
SourceType sourcespb.SourceType
|
||||
// SourceMetadata holds the context of where the Chunk was found.
|
||||
@@ -30,18 +35,32 @@ type Chunk struct {
|
||||
Verify bool
|
||||
}
|
||||
|
||||
// ChunkingTarget specifies criteria for a targeted chunking process.
|
||||
// Instead of collecting data indiscriminately, this struct allows the caller
|
||||
// to specify particular subsets of data they're interested in. This becomes
|
||||
// especially useful when one needs to verify or recheck specific data points
|
||||
// without processing the entire dataset.
|
||||
type ChunkingTarget struct {
|
||||
// QueryCriteria represents specific parameters or conditions to target the chunking process.
|
||||
QueryCriteria source_metadatapb.MetaData
|
||||
}
|
||||
|
||||
// Source defines the interface required to implement a source chunker.
|
||||
type Source interface {
|
||||
// Type returns the source type, used for matching against configuration and jobs.
|
||||
Type() sourcespb.SourceType
|
||||
// SourceID returns the initialized source ID used for tracking relationships in the DB.
|
||||
SourceID() int64
|
||||
SourceID() SourceID
|
||||
// JobID returns the initialized job ID used for tracking relationships in the DB.
|
||||
JobID() int64
|
||||
JobID() JobID
|
||||
// Init initializes the source.
|
||||
Init(aCtx context.Context, name string, jobId, sourceId int64, verify bool, connection *anypb.Any, concurrency int) error
|
||||
// Chunks emits data over a channel that is decoded and scanned for secrets.
|
||||
Chunks(ctx context.Context, chunksChan chan *Chunk) error
|
||||
Init(aCtx context.Context, name string, jobId JobID, sourceId SourceID, verify bool, connection *anypb.Any, concurrency int) error
|
||||
// Chunks emits data over a channel which is then decoded and scanned for secrets.
|
||||
// By default, data is obtained indiscriminately. However, by providing one or more
|
||||
// ChunkingTarget parameters, the caller can direct the function to retrieve
|
||||
// specific chunks of data. This targeted approach allows for efficient and
|
||||
// intentional data processing, beneficial when verifying or rechecking specific data points.
|
||||
Chunks(ctx context.Context, chunksChan chan *Chunk, targets ...ChunkingTarget) error
|
||||
// GetProgress is the completion progress (percentage) for Scanned Source.
|
||||
GetProgress() *Progress
|
||||
}
|
||||
@@ -258,6 +277,9 @@ type Validator interface {
|
||||
// scope should be the len(scopedItems)
|
||||
// message is the public facing user information about the current progress
|
||||
// encodedResumeInfo is an optional string representing any information necessary to resume the job if interrupted
|
||||
//
|
||||
// NOTE: SetProgressOngoing should be used when source does not yet know how many items it is scanning (scope)
|
||||
// and does not want to display a percentage complete
|
||||
func (p *Progress) SetProgressComplete(i, scope int, message, encodedResumeInfo string) {
|
||||
p.mut.Lock()
|
||||
defer p.mut.Unlock()
|
||||
@@ -276,6 +298,23 @@ func (p *Progress) SetProgressComplete(i, scope int, message, encodedResumeInfo
|
||||
p.PercentComplete = int64((float64(i) / float64(scope)) * 100)
|
||||
}
|
||||
|
||||
// SetProgressOngoing sets information about the current running job based on
|
||||
// the highest level objects in the source.
|
||||
// message is the public facing user information about the current progress
|
||||
// encodedResumeInfo is an optional string representing any information necessary to resume the job if interrupted
|
||||
//
|
||||
// NOTE: This method should be used over SetProgressComplete when the source does
|
||||
// not yet know how many items it is scanning and does not want to display a percentage complete.
|
||||
func (p *Progress) SetProgressOngoing(message string, encodedResumeInfo string) {
|
||||
p.mut.Lock()
|
||||
defer p.mut.Unlock()
|
||||
|
||||
p.Message = message
|
||||
p.EncodedResumeInfo = encodedResumeInfo
|
||||
// Explicitly set SectionsRemaining to 0 so the frontend does not display a percent.
|
||||
p.SectionsRemaining = 0
|
||||
}
|
||||
|
||||
// GetProgress gets job completion percentage for metrics reporting.
|
||||
func (p *Progress) GetProgress() *Progress {
|
||||
p.mut.Lock()
|
||||
|
||||
@@ -23,12 +23,16 @@ import (
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
|
||||
)
|
||||
|
||||
const nilString = ""
|
||||
const (
|
||||
SourceType = sourcespb.SourceType_SOURCE_TYPE_SYSLOG
|
||||
|
||||
nilString = ""
|
||||
)
|
||||
|
||||
type Source struct {
|
||||
name string
|
||||
sourceId int64
|
||||
jobId int64
|
||||
sourceId sources.SourceID
|
||||
jobId sources.JobID
|
||||
verify bool
|
||||
syslog *Syslog
|
||||
sources.Progress
|
||||
@@ -38,14 +42,14 @@ type Source struct {
|
||||
type Syslog struct {
|
||||
sourceType sourcespb.SourceType
|
||||
sourceName string
|
||||
sourceID int64
|
||||
jobID int64
|
||||
sourceID sources.SourceID
|
||||
jobID sources.JobID
|
||||
sourceMetadataFunc func(hostname, appname, procid, timestamp, facility, client string) *source_metadatapb.MetaData
|
||||
verify bool
|
||||
concurrency *semaphore.Weighted
|
||||
}
|
||||
|
||||
func NewSyslog(sourceType sourcespb.SourceType, jobID, sourceID int64, sourceName string, verify bool, concurrency int,
|
||||
func NewSyslog(sourceType sourcespb.SourceType, jobID sources.JobID, sourceID sources.SourceID, sourceName string, verify bool, concurrency int,
|
||||
sourceMetadataFunc func(hostname, appname, procid, timestamp, facility, client string) *source_metadatapb.MetaData,
|
||||
) *Syslog {
|
||||
return &Syslog{
|
||||
@@ -103,14 +107,14 @@ var _ sources.Source = (*Source)(nil)
|
||||
// Type returns the type of source.
|
||||
// It is used for matching source types in configuration and job input.
|
||||
func (s *Source) Type() sourcespb.SourceType {
|
||||
return sourcespb.SourceType_SOURCE_TYPE_SYSLOG
|
||||
return SourceType
|
||||
}
|
||||
|
||||
func (s *Source) SourceID() int64 {
|
||||
func (s *Source) SourceID() sources.SourceID {
|
||||
return s.sourceId
|
||||
}
|
||||
|
||||
func (s *Source) JobID() int64 {
|
||||
func (s *Source) JobID() sources.JobID {
|
||||
return s.jobId
|
||||
}
|
||||
|
||||
@@ -119,7 +123,7 @@ func (s *Source) InjectConnection(conn *sourcespb.Syslog) {
|
||||
}
|
||||
|
||||
// Init returns an initialized Syslog source.
|
||||
func (s *Source) Init(_ context.Context, name string, jobId, sourceId int64, verify bool, connection *anypb.Any, concurrency int) error {
|
||||
func (s *Source) Init(_ context.Context, name string, jobId sources.JobID, sourceId sources.SourceID, verify bool, connection *anypb.Any, concurrency int) error {
|
||||
|
||||
s.name = name
|
||||
s.sourceId = sourceId
|
||||
@@ -182,7 +186,7 @@ func (s *Source) verifyConnectionConfig() error {
|
||||
}
|
||||
|
||||
// Chunks emits chunks of bytes over a channel.
|
||||
func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) error {
|
||||
func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk, _ ...sources.ChunkingTarget) error {
|
||||
switch {
|
||||
case s.conn.TlsCert != nilString || s.conn.TlsKey != nilString:
|
||||
cert, err := tls.X509KeyPair([]byte(s.conn.TlsCert), []byte(s.conn.TlsKey))
|
||||
|
||||
@@ -936,6 +936,8 @@ enum DetectorType {
|
||||
Salesforce = 925;
|
||||
HuggingFace = 926;
|
||||
Snowflake = 927;
|
||||
Sourcegraph = 928;
|
||||
Tailscale = 929;
|
||||
}
|
||||
|
||||
message Result {
|
||||
|
||||
Reference in New Issue
Block a user