Compare commits
5 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 Bill Rich
|
0a3679671a | ||
|
Bill Rich
|
3b49183caf | ||
|
Bill Rich
|
809b35a843 | ||
|
Bill Rich
|
6dd13a0b7c | ||
|
Bill Rich
|
bd1539d22a |
@@ -6,7 +6,11 @@ labels: bug, needs triage
|
||||
assignees: trufflesecurity/product-eng
|
||||
---
|
||||
|
||||
Please review the [Community Note](https://github.com/trufflesecurity/trufflehog/blob/main/.github/community_note.md) before submitting
|
||||
### Community Note
|
||||
|
||||
* Please vote on this issue by adding a 👍 [reaction](https://blog.github.com/2016-03-10-add-reactions-to-pull-requests-issues-and-comments/) to the original issue to help the community and maintainers prioritize this request
|
||||
* Please do not leave "+1" or other comments that do not add relevant new information or questions, they generate extra noise for issue followers and do not help prioritize the request
|
||||
* If you are interested in working on this issue or have submitted a pull request, please leave a comment
|
||||
|
||||
### TruffleHog Version
|
||||
<!--- Please run `trufflehog --version` to show the version. If you are not running the latest version, please upgrade because your issue may have already been fixed. --->
|
||||
|
||||
@@ -6,27 +6,33 @@ labels: enhancement, needs triage
|
||||
assignees: trufflesecurity/product-eng
|
||||
---
|
||||
|
||||
Please review the [Community Note](https://github.com/trufflesecurity/trufflehog/blob/main/.github/community_note.md) before submitting
|
||||
### Community Note
|
||||
|
||||
* Please vote on this issue by adding a 👍 [reaction](https://blog.github.com/2016-03-10-add-reactions-to-pull-requests-issues-and-comments/) to the original issue to help the community and maintainers prioritize this request
|
||||
* Please do not leave "+1" or other comments that do not add relevant new information or questions, they generate extra noise for issue followers and do not help prioritize the request
|
||||
* If you are interested in working on this issue or have submitted a pull request, please leave a comment
|
||||
|
||||
### Description
|
||||
|
||||
## Description
|
||||
<!--- Please leave a helpful description of the feature request here. --->
|
||||
|
||||
### Preferred Solution
|
||||
<!--- A clear and concise description of what you want to happen. What
|
||||
information may be required and what would be the preferred way to provide it?
|
||||
## Problem to be Addressed
|
||||
<!--- A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] --->
|
||||
|
||||
## Description of the Preferred Solution
|
||||
<!--- A clear and concise description of what you want to happen. What
|
||||
information may be required and what would be the preferred way to provide it?
|
||||
What should the output include? --->
|
||||
|
||||
### Additional Context
|
||||
## Additional Context
|
||||
<!--- Add any other context or screenshots about the feature request here. --->
|
||||
|
||||
#### References
|
||||
### References
|
||||
|
||||
<!---
|
||||
Information about referencing Github Issues: https://help.github.com/articles/basic-writing-and-formatting-syntax/#referencing-issues-and-pull-requests
|
||||
|
||||
Are there any other GitHub issues (open or closed) or pull requests that should be linked here? Vendor blog posts or documentation? For example:
|
||||
|
||||
* #0000
|
||||
--->
|
||||
|
||||
|
||||
* #0000
|
||||
|
||||
@@ -2,11 +2,3 @@
|
||||
Please create an issue to collect feedback prior to feature additions. Please also reference that issue in any PRs.
|
||||
If possible try to keep PRs scoped to one feature, and add tests for new features.
|
||||
-->
|
||||
|
||||
### Description:
|
||||
Explain the purpose of the PR.
|
||||
|
||||
### Checklist:
|
||||
* [ ] Tests passing (`make test-community`)?
|
||||
* [ ] Lint passing (`make lint` this requires [golangci-lint](https://golangci-lint.run/usage/install/#local-installation))?
|
||||
|
||||
|
||||
@@ -1,7 +0,0 @@
|
||||
## Community Note
|
||||
|
||||
Please vote on this issue by adding a 👍 reaction to the original issue to help the community and maintainers prioritize this request.
|
||||
|
||||
Please do not leave "+1" or other comments that do not add relevant new information or questions, they generate extra noise for issue followers and do not help prioritize the request.
|
||||
|
||||
If you are interested in working on this issue or have submitted a pull request, please leave a comment.
|
||||
@@ -0,0 +1,16 @@
|
||||
# To get started with Dependabot version updates, you'll need to specify which
|
||||
# package ecosystems to update and where the package manifests are located.
|
||||
# Please see the documentation for all configuration options:
|
||||
# https://help.github.com/github/administering-a-repository/configuration-options-for-dependency-updates
|
||||
|
||||
version: 2
|
||||
updates:
|
||||
- package-ecosystem: "gomod" # See documentation for possible values
|
||||
directory: "/" # Location of package manifests
|
||||
schedule:
|
||||
interval: "weekly"
|
||||
- package-ecosystem: "github-actions"
|
||||
directory: "/"
|
||||
schedule:
|
||||
# Check for updates to GitHub Actions every weekday
|
||||
interval: "weekly"
|
||||
@@ -1,8 +0,0 @@
|
||||
{
|
||||
"$schema": "https://docs.renovatebot.com/renovate-schema.json",
|
||||
"extends": [
|
||||
"config:base"
|
||||
],
|
||||
"prConcurrentLimit": 3,
|
||||
"prHourlyLimit": 2
|
||||
}
|
||||
@@ -1,19 +0,0 @@
|
||||
# Testing
|
||||
|
||||
Most testing is handled automatically by our GitHub Actions workflows.
|
||||
|
||||
## Local GitHub Action Testing
|
||||
|
||||
In some cases you may wish to submit changes to the Trufflehog GitHub Action. Unfortunately GitHub does not provide a 1st-party testing environment for testing actions outside of GitHub Actions.
|
||||
|
||||
Fortunately [nektos/act](https://github.com/nektos/act) enables local testing of GitHub Actions.
|
||||
|
||||
### Instructions
|
||||
|
||||
1. Please follow [the installation instructions](http://https://github.com/nektos/act#installation) for your OS.
|
||||
2. The first run of `act` will ask you to specify an image. `Medium` should suffice.
|
||||
3. You'll need to configure a personal-access-token(PAT) with: `repo:status`, `repo_deployment`, and `public_repo` permissions.
|
||||
4. Set an environment variable named `GITHUB_TOKEN` with the PAT from the previous step as the value: `$ export GITHUB_TOKEN=<your_PAT>`
|
||||
5. Run the following command from the repository root: `act pull_request -j test -W .github/workflows/secrets.yml -s GITHUB_TOKEN --defaultbranch main`
|
||||
6. If the job was successful, you should expect to see output from the scanner showing several detected secrets.
|
||||
7. If you want to omit the context of a pull request event and just test that the action starts successfully, run: `act -j test -W .github/workflows/secrets.yml -s GITHUB_TOKEN --defaultbranch main`
|
||||
@@ -38,7 +38,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v4
|
||||
uses: actions/checkout@v3
|
||||
|
||||
# Initializes the CodeQL tools for scanning.
|
||||
- name: Initialize CodeQL
|
||||
|
||||
@@ -1,46 +0,0 @@
|
||||
name: detector test aggregation
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
schedule:
|
||||
- cron: "0 8 * * *"
|
||||
|
||||
jobs:
|
||||
test-detectors:
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
actions: "read"
|
||||
contents: "read"
|
||||
id-token: "write"
|
||||
steps:
|
||||
- name: Install Go
|
||||
uses: actions/setup-go@v4
|
||||
with:
|
||||
go-version: "1.21"
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
- id: "auth"
|
||||
uses: "google-github-actions/auth@v2"
|
||||
with:
|
||||
workload_identity_provider: "projects/811013774421/locations/global/workloadIdentityPools/github-pool/providers/github-provider"
|
||||
service_account: "github-ci-external@trufflehog-testing.iam.gserviceaccount.com"
|
||||
- name: Test
|
||||
run: make test-integration
|
||||
- name: Set up gotestsum
|
||||
run: |
|
||||
go install gotest.tools/gotestsum@latest
|
||||
mkdir -p tmp/test-results
|
||||
- name: Test
|
||||
run: |
|
||||
CGO_ENABLED=1 gotestsum --junitfile tmp/test-results/test.xml --raw-command -- go test -json -tags=detectors -timeout=15m $(go list ./... | grep pkg/detectors)
|
||||
- name: Upload test results to BuildPulse for flaky test detection
|
||||
if: ${{ !cancelled() }} # Run this step even when the tests fail. Skip if the workflow is cancelled.
|
||||
uses: buildpulse/buildpulse-action@main
|
||||
with:
|
||||
account: 79229934
|
||||
repository: 694446374
|
||||
path: |
|
||||
tmp/test-results/*.xml
|
||||
key: ${{ secrets.BUILDPULSE_DETECTORS_ACCESS_KEY_ID }}
|
||||
secret: ${{ secrets.BUILDPULSE_DETECTORS_SECRET_ACCESS_KEY }}
|
||||
tags: detectors
|
||||
@@ -4,6 +4,8 @@ on:
|
||||
push:
|
||||
tags:
|
||||
- v*
|
||||
branches:
|
||||
- main
|
||||
pull_request:
|
||||
|
||||
permissions:
|
||||
@@ -15,10 +17,10 @@ jobs:
|
||||
name: lint
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/setup-go@v4
|
||||
- uses: actions/setup-go@v3
|
||||
with:
|
||||
go-version: '1.21'
|
||||
- uses: actions/checkout@v4
|
||||
go-version: '1.18'
|
||||
- uses: actions/checkout@v3
|
||||
- name: golangci-lint
|
||||
uses: golangci/golangci-lint-action@v3
|
||||
with:
|
||||
@@ -28,10 +30,13 @@ jobs:
|
||||
# working-directory: somedir
|
||||
|
||||
# Optional: golangci-lint command line arguments.
|
||||
args: --enable bodyclose --enable exportloopref --timeout 10m
|
||||
args: --enable bodyclose --timeout 10m
|
||||
|
||||
# Optional: show only new issues if it's a pull request. The default value is `false`.
|
||||
only-new-issues: true
|
||||
|
||||
# Optional: if set to true then the action don't cache or restore ~/go/pkg.
|
||||
# skip-pkg-cache: true
|
||||
|
||||
# Optional: if set to true then the action don't cache or restore ~/.cache/go-build.
|
||||
# skip-build-cache: true
|
||||
# skip-build-cache: true
|
||||
@@ -1,92 +0,0 @@
|
||||
name: Performance Test
|
||||
|
||||
on: [pull_request]
|
||||
|
||||
jobs:
|
||||
speed:
|
||||
# skip if PR is from a fork.
|
||||
# TODO: this could probabaly be refactored a bit so that it runs on forks
|
||||
if: ${{ ! github.event.pull_request.head.repo.fork }}
|
||||
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Install Go
|
||||
uses: actions/setup-go@v4
|
||||
with:
|
||||
go-version: '1.21'
|
||||
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 0
|
||||
ref: ${{ github.head_ref }}
|
||||
|
||||
- name: Run Head
|
||||
run: |
|
||||
go build -o current .
|
||||
repo_tmp=$(mktemp -d)
|
||||
git clone https://github.com/trufflesecurity/trufflehog.git $repo_tmp
|
||||
|
||||
user_time_sum=0
|
||||
|
||||
for i in {1..5}
|
||||
do
|
||||
tmpfile=$(mktemp)
|
||||
/usr/bin/time -o $tmpfile $GITHUB_WORKSPACE/current git "file://$repo_tmp" --no-verification --no-update > out.txt
|
||||
cat $tmpfile
|
||||
time_output=$(cat $tmpfile)
|
||||
rm $tmpfile
|
||||
user_time=$(echo $time_output | awk '{print $1}' | sed 's/user//')
|
||||
|
||||
# Add the user time to the sum
|
||||
user_time_sum=$(echo "$user_time_sum + $user_time" | bc)
|
||||
done
|
||||
|
||||
average_user_time=$(echo "scale=3; $user_time_sum / 5" | bc)
|
||||
echo HEAD_TIME=$average_user_time >> $GITHUB_ENV
|
||||
|
||||
- name: Figure out previous tag
|
||||
run: |
|
||||
git fetch --tags
|
||||
git tag -l --sort=-v:refname | head -n 1 > previous_tag.txt
|
||||
echo PREVIOUS_TAG=$(cat previous_tag.txt) >> $GITHUB_ENV
|
||||
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 0
|
||||
ref: ${{ env.PREVIOUS_TAG }}
|
||||
|
||||
- name: Run Previous
|
||||
run: |
|
||||
go build -o previous .
|
||||
repo_tmp=$(mktemp -d)
|
||||
git clone https://github.com/trufflesecurity/trufflehog.git $repo_tmp
|
||||
|
||||
user_time_sum=0
|
||||
|
||||
for i in {1..5}
|
||||
do
|
||||
tmpfile=$(mktemp)
|
||||
/usr/bin/time -o $tmpfile $GITHUB_WORKSPACE/previous git "file://$repo_tmp" --no-verification --no-update > out.txt
|
||||
cat $tmpfile
|
||||
time_output=$(cat $tmpfile)
|
||||
rm $tmpfile
|
||||
user_time=$(echo $time_output | awk '{print $1}' | sed 's/user//')
|
||||
|
||||
# Add the user time to the sum
|
||||
user_time_sum=$(echo "$user_time_sum + $user_time" | bc)
|
||||
done
|
||||
|
||||
average_user_time=$(echo "scale=3; $user_time_sum / 5" | bc)
|
||||
echo PREVIOUS_TIME=$average_user_time >> $GITHUB_ENV
|
||||
|
||||
- name: Compare Results
|
||||
run: |
|
||||
echo "head ($GITHUB_SHA) avg time (n=5): $HEAD_TIME"
|
||||
echo "$PREVIOUS_TAG avg time (n=5): $PREVIOUS_TIME"
|
||||
if [ $(echo "$HEAD_TIME > $PREVIOUS_TIME * 1.1" | bc) -eq 1 ]
|
||||
then
|
||||
echo "HEAD run time is at least 10% slower than PREVIOUS run time"
|
||||
exit 1
|
||||
fi
|
||||
@@ -1,14 +1,20 @@
|
||||
name: Release
|
||||
|
||||
# on:
|
||||
# push:
|
||||
# tags:
|
||||
# - 'v*'
|
||||
|
||||
on:
|
||||
push:
|
||||
tags:
|
||||
- v*
|
||||
branches:
|
||||
- main
|
||||
|
||||
permissions:
|
||||
contents: write
|
||||
packages: write
|
||||
id-token: write
|
||||
|
||||
jobs:
|
||||
Release:
|
||||
@@ -17,30 +23,28 @@ jobs:
|
||||
DOCKER_CLI_EXPERIMENTAL: "enabled"
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
uses: actions/checkout@v3
|
||||
with:
|
||||
fetch-depth: 0
|
||||
- name: Set up QEMU
|
||||
uses: docker/setup-qemu-action@v3
|
||||
uses: docker/setup-qemu-action@v2
|
||||
- name: Docker Login to DockerHub
|
||||
uses: docker/login-action@v3
|
||||
uses: docker/login-action@v2
|
||||
with:
|
||||
username: ${{ secrets.DOCKERHUB_USERNAME }}
|
||||
password: ${{ secrets.DOCKERHUB_TOKEN }}
|
||||
- name: Docker Login to GitHub Container Registry
|
||||
uses: docker/login-action@v3
|
||||
uses: docker/login-action@v2
|
||||
with:
|
||||
registry: ghcr.io
|
||||
username: ${{ github.repository_owner }}
|
||||
password: ${{ secrets.GITHUB_TOKEN }}
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v4
|
||||
uses: actions/setup-go@v3
|
||||
with:
|
||||
go-version: '1.21'
|
||||
- name: Cosign install
|
||||
uses: sigstore/cosign-installer@1fc5bd396d372bee37d608f955b336615edf79c8 # v3.2.0
|
||||
go-version: '1.18'
|
||||
- name: Run GoReleaser
|
||||
uses: goreleaser/goreleaser-action@v5
|
||||
uses: goreleaser/goreleaser-action@v3
|
||||
with:
|
||||
distribution: goreleaser
|
||||
version: latest
|
||||
|
||||
@@ -7,18 +7,17 @@ on:
|
||||
- v*
|
||||
branches:
|
||||
- main
|
||||
pull_request:
|
||||
|
||||
jobs:
|
||||
test:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Install Go
|
||||
uses: actions/setup-go@v4
|
||||
uses: actions/setup-go@v3
|
||||
with:
|
||||
go-version: '1.21'
|
||||
go-version: '1.18'
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
uses: actions/checkout@v3
|
||||
with:
|
||||
fetch-depth: 0
|
||||
ref: ${{ github.head_ref }}
|
||||
@@ -28,5 +27,4 @@ jobs:
|
||||
with:
|
||||
path: ./
|
||||
base: ${{ github.event.repository.default_branch }}
|
||||
head: HEAD
|
||||
extra_args: --only-verified
|
||||
head: HEAD
|
||||
@@ -1,21 +0,0 @@
|
||||
|
||||
name: Smoke
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
|
||||
jobs:
|
||||
smoke:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Install Go
|
||||
uses: actions/setup-go@v4
|
||||
with:
|
||||
go-version: '1.21'
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
- name: Smoke
|
||||
run: |
|
||||
set -e
|
||||
go run . git https://github.com/dustin-decker/secretsandstuff.git
|
||||
go run . github --repo https://github.com/dustin-decker/secretsandstuff.git
|
||||
@@ -16,9 +16,9 @@ jobs:
|
||||
name: Run Snifftest
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/setup-go@v4
|
||||
- uses: actions/setup-go@v3
|
||||
with:
|
||||
go-version: '1.21'
|
||||
- uses: actions/checkout@v4
|
||||
go-version: '1.18'
|
||||
- uses: actions/checkout@v3
|
||||
- name: Run Snifftest
|
||||
run: make snifftest
|
||||
run: make snifftest
|
||||
+34
-51
@@ -1,3 +1,4 @@
|
||||
|
||||
name: Test
|
||||
|
||||
on:
|
||||
@@ -10,60 +11,42 @@ on:
|
||||
|
||||
jobs:
|
||||
test:
|
||||
if: ${{ github.repository == 'trufflesecurity/trufflehog' && !github.event.pull_request.head.repo.fork }}
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
actions: "read"
|
||||
contents: "read"
|
||||
id-token: "write"
|
||||
contents: 'read'
|
||||
id-token: 'write'
|
||||
steps:
|
||||
- name: Install Go
|
||||
uses: actions/setup-go@v4
|
||||
with:
|
||||
go-version: "1.21"
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
- id: "auth"
|
||||
uses: "google-github-actions/auth@v2"
|
||||
with:
|
||||
workload_identity_provider: "projects/811013774421/locations/global/workloadIdentityPools/github-pool/providers/github-provider"
|
||||
service_account: "github-ci-external@trufflehog-testing.iam.gserviceaccount.com"
|
||||
- name: Set up gotestsum
|
||||
run: |
|
||||
go install gotest.tools/gotestsum@latest
|
||||
mkdir -p tmp/test-results
|
||||
- name: Test
|
||||
run: |
|
||||
CGO_ENABLED=1 gotestsum --junitfile tmp/test-results/test.xml --raw-command -- go test -json -tags=sources $(go list ./... | grep -v /vendor/ | grep -v pkg/detectors)
|
||||
if: ${{ success() || failure() }} # always run this step, even if there were previous errors
|
||||
- name: Upload test results to BuildPulse for flaky test detection
|
||||
if: ${{ !cancelled() }} # Run this step even when the tests fail. Skip if the workflow is cancelled.
|
||||
uses: buildpulse/buildpulse-action@main
|
||||
with:
|
||||
account: 79229934
|
||||
repository: 77726177
|
||||
path: |
|
||||
tmp/test-results/*.xml
|
||||
key: ${{ secrets.BUILDPULSE_ACCESS_KEY_ID }}
|
||||
secret: ${{ secrets.BUILDPULSE_SECRET_ACCESS_KEY }}
|
||||
tags: integration
|
||||
- name: Annotate test results
|
||||
uses: mikepenz/action-junit-report@v4
|
||||
if: success() || failure() # always run even if the previous step fails
|
||||
with:
|
||||
report_paths: "tmp/test-results/*.xml"
|
||||
test-community:
|
||||
if: ${{ github.event.pull_request.head.repo.fork }}
|
||||
- name: Install Go
|
||||
uses: actions/setup-go@v3
|
||||
with:
|
||||
go-version: '1.18'
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v3
|
||||
- id: 'auth'
|
||||
uses: 'google-github-actions/auth@v0'
|
||||
with:
|
||||
workload_identity_provider: 'projects/811013774421/locations/global/workloadIdentityPools/github-pool/providers/github-provider'
|
||||
service_account: 'github-ci-external@trufflehog-testing.iam.gserviceaccount.com'
|
||||
- name: Test
|
||||
run: make test
|
||||
test-detectors:
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
actions: "read"
|
||||
contents: "read"
|
||||
contents: 'read'
|
||||
id-token: 'write'
|
||||
steps:
|
||||
- name: Install Go
|
||||
uses: actions/setup-go@v4
|
||||
with:
|
||||
go-version: "1.21"
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
- name: Test
|
||||
run: make test-community
|
||||
- name: Install Go
|
||||
uses: actions/setup-go@v3
|
||||
with:
|
||||
go-version: '1.18'
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v3
|
||||
- id: 'auth'
|
||||
uses: 'google-github-actions/auth@v0'
|
||||
with:
|
||||
workload_identity_provider: 'projects/811013774421/locations/global/workloadIdentityPools/github-pool/providers/github-provider'
|
||||
service_account: 'github-ci-external@trufflehog-testing.iam.gserviceaccount.com'
|
||||
- name: Test
|
||||
run: make test-detectors
|
||||
continue-on-error: true
|
||||
|
||||
+1
-4
@@ -1,7 +1,4 @@
|
||||
.idea
|
||||
dist
|
||||
.env
|
||||
*.test
|
||||
|
||||
# binary
|
||||
trufflehog
|
||||
*.test
|
||||
@@ -96,15 +96,3 @@ brews:
|
||||
homepage: "https://github.com/trufflesecurity/trufflehog"
|
||||
install: |
|
||||
bin.install "trufflehog"
|
||||
signs:
|
||||
- cmd: cosign
|
||||
signature: "${artifact}.sig"
|
||||
certificate: "${artifact}.pem"
|
||||
args:
|
||||
- "sign-blob"
|
||||
- "--oidc-issuer=https://token.actions.githubusercontent.com"
|
||||
- "--output-certificate=${certificate}"
|
||||
- "--output-signature=${signature}"
|
||||
- "${artifact}"
|
||||
- "--yes"
|
||||
artifacts: checksum
|
||||
@@ -1,10 +0,0 @@
|
||||
repos:
|
||||
- repo: https://github.com/rhysd/actionlint
|
||||
rev: v1.6.24
|
||||
hooks:
|
||||
- id: actionlint
|
||||
|
||||
- repo: https://github.com/mpalmer/action-validator
|
||||
rev: v0.5.1
|
||||
hooks:
|
||||
- id: action-validator
|
||||
@@ -1,6 +0,0 @@
|
||||
- id: trufflehog
|
||||
name: TruffleHog
|
||||
description: Detect secrets in your data with TruffleHog.
|
||||
entry: trufflehog git file://. --since-commit HEAD --only-verified --fail
|
||||
language: golang
|
||||
pass_filenames: false
|
||||
+4
-3
@@ -8,9 +8,10 @@ RUN --mount=type=cache,target=/go/pkg/mod \
|
||||
--mount=type=cache,target=/root/.cache/go-build \
|
||||
GOOS=${TARGETOS} GOARCH=${TARGETARCH} go build -o trufflehog .
|
||||
|
||||
FROM alpine:3.18
|
||||
RUN apk add --no-cache bash git openssh-client ca-certificates rpm2cpio binutils cpio \
|
||||
&& rm -rf /var/cache/apk/* && update-ca-certificates
|
||||
FROM alpine:3.15
|
||||
RUN apk add --no-cache git ca-certificates \
|
||||
&& rm -rf /var/cache/apk/* && \
|
||||
update-ca-certificates
|
||||
COPY --from=builder /build/trufflehog /usr/bin/trufflehog
|
||||
COPY entrypoint.sh /etc/entrypoint.sh
|
||||
RUN chmod +x /etc/entrypoint.sh
|
||||
|
||||
@@ -1,7 +1,8 @@
|
||||
FROM alpine:3.18
|
||||
FROM alpine:3.15
|
||||
|
||||
RUN apk add --no-cache bash git openssh-client ca-certificates rpm2cpio binutils cpio \
|
||||
&& rm -rf /var/cache/apk/* && update-ca-certificates
|
||||
RUN apk add --no-cache git ca-certificates \
|
||||
&& rm -rf /var/cache/apk/* && \
|
||||
update-ca-certificates
|
||||
WORKDIR /usr/bin/
|
||||
COPY trufflehog .
|
||||
COPY entrypoint.sh /etc/entrypoint.sh
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
PROTOS_IMAGE ?= trufflesecurity/protos:1.21-0
|
||||
PROTOS_IMAGE ?= trufflesecurity/protos:1.18-0
|
||||
|
||||
.PHONY: check
|
||||
.PHONY: lint
|
||||
@@ -22,7 +22,7 @@ check:
|
||||
go vet $(shell go list ./... | grep -v /vendor/)
|
||||
|
||||
lint:
|
||||
golangci-lint run --enable bodyclose --enable exportloopref --out-format=colored-line-number --timeout 10m
|
||||
golangci-lint run --out-format=colored-line-number --timeout 10m
|
||||
|
||||
test-failing:
|
||||
CGO_ENABLED=0 go test -timeout=5m $(shell go list ./... | grep -v /vendor/) | grep FAIL
|
||||
@@ -30,18 +30,12 @@ test-failing:
|
||||
test:
|
||||
CGO_ENABLED=0 go test -timeout=5m $(shell go list ./... | grep -v /vendor/ | grep -v pkg/detectors)
|
||||
|
||||
test-integration:
|
||||
CGO_ENABLED=0 go test -timeout=5m -tags=integration $(shell go list ./... | grep -v /vendor/ | grep -v pkg/detectors)
|
||||
|
||||
test-race:
|
||||
CGO_ENABLED=1 go test -timeout=5m -race $(shell go list ./... | grep -v /vendor/ | grep -v pkg/detectors)
|
||||
|
||||
test-detectors:
|
||||
CGO_ENABLED=0 go test -tags=detectors -timeout=5m $(shell go list ./... | grep pkg/detectors)
|
||||
|
||||
test-community:
|
||||
CGO_ENABLED=0 go test -timeout=5m $(shell go list ./... | grep -v /vendor/ | grep -v pkg/detectors | grep -v pkg/sources)
|
||||
|
||||
bench:
|
||||
CGO_ENABLED=0 go test $(shell go list ./pkg/secrets/... | grep -v /vendor/) -benchmem -run=xxx -bench .
|
||||
|
||||
@@ -52,17 +46,17 @@ run-debug:
|
||||
CGO_ENABLED=0 go run . git file://. --json --debug
|
||||
|
||||
protos:
|
||||
docker run --rm -u "$(shell id -u)" -v "$(shell dirname $(realpath $(firstword $(MAKEFILE_LIST))))":/pwd "${PROTOS_IMAGE}" bash -c "cd /pwd; /pwd/scripts/gen_proto.sh"
|
||||
docker run -u "$(shell id -u)" -v "$(shell dirname $(realpath $(firstword $(MAKEFILE_LIST))))":/pwd "${PROTOS_IMAGE}" bash -c "cd /pwd; /pwd/scripts/gen_proto.sh"
|
||||
|
||||
protos-windows:
|
||||
docker run --rm -v "$(shell cygpath -w $(shell pwd))":/pwd "${PROTOS_IMAGE}" bash -c "cd /pwd; ./scripts/gen_proto.sh"
|
||||
docker run -v "$(shell cygpath -w $(shell pwd))":/pwd "${PROTOS_IMAGE}" bash -c "cd /pwd; ./scripts/gen_proto.sh"
|
||||
|
||||
release-protos-image:
|
||||
docker buildx build --push --platform=linux/amd64,linux/arm64 \
|
||||
-t trufflesecurity/protos:1.21-0 -f hack/Dockerfile.protos .
|
||||
-t trufflesecurity/protos:1.18-0 -f hack/Dockerfile.protos .
|
||||
|
||||
snifftest:
|
||||
./hack/snifftest/snifftest.sh
|
||||
|
||||
test-release:
|
||||
goreleaser release --rm-dist --skip-publish --snapshot
|
||||
goreleaser release --rm-dist --skip-publish --snapshot
|
||||
@@ -8,6 +8,7 @@
|
||||
|
||||
<div align="center">
|
||||
|
||||
[](https://github.com/trufflesecurity/trufflehog/actions)
|
||||
[](https://goreportcard.com/report/github.com/trufflesecurity/trufflehog/v3)
|
||||
[](/LICENSE)
|
||||
[](/pkg/detectors)
|
||||
@@ -16,235 +17,93 @@
|
||||
|
||||
---
|
||||
|
||||
# :mag_right: _Now Scanning_
|
||||
## Join The Slack
|
||||
Have questions? Feedback? Jump in slack and hang out with us
|
||||
|
||||
<div align="center">
|
||||
https://join.slack.com/t/trufflehog-community/shared_invite/zt-pw2qbi43-Aa86hkiimstfdKH9UCpPzQ
|
||||
|
||||
<img src="assets/scanning_logos.svg">
|
||||
|
||||
**...and more**
|
||||
|
||||
</div>
|
||||
|
||||
# :loudspeaker: Join Our Community
|
||||
|
||||
Have questions? Feedback? Jump in slack or discord and hang out with us
|
||||
|
||||
Join our [Slack Community](https://join.slack.com/t/trufflehog-community/shared_invite/zt-pw2qbi43-Aa86hkiimstfdKH9UCpPzQ)
|
||||
|
||||
Join the [Secret Scanning Discord](https://discord.gg/8Hzbrnkr7E)
|
||||
|
||||
# :tv: Demo
|
||||
## Demo
|
||||
|
||||
|
||||
|
||||
```bash
|
||||
docker run --rm -it -v "$PWD:/pwd" trufflesecurity/trufflehog:latest github --org=trufflesecurity
|
||||
docker run -it -v "$PWD:/pwd" trufflesecurity/trufflehog:latest github --org=trufflesecurity
|
||||
```
|
||||
|
||||
# :floppy_disk: Installation
|
||||
|
||||
Several options available for you:
|
||||
|
||||
```bash
|
||||
# MacOS users
|
||||
brew install trufflesecurity/trufflehog/trufflehog
|
||||
|
||||
# Docker
|
||||
docker run --rm -it -v "$PWD:/pwd" trufflesecurity/trufflehog:latest github --repo https://github.com/trufflesecurity/test_keys
|
||||
|
||||
# Docker for M1 and M2 Mac
|
||||
docker run --platform linux/arm64 --rm -it -v "$PWD:/pwd" trufflesecurity/trufflehog:latest github --repo https://github.com/trufflesecurity/test_keys
|
||||
|
||||
# Binary releases
|
||||
Download and unpack from https://github.com/trufflesecurity/trufflehog/releases
|
||||
|
||||
# Compile from source
|
||||
git clone https://github.com/trufflesecurity/trufflehog.git
|
||||
cd trufflehog; go install
|
||||
|
||||
# Using installation script
|
||||
curl -sSfL https://raw.githubusercontent.com/trufflesecurity/trufflehog/main/scripts/install.sh | sh -s -- -b /usr/local/bin
|
||||
# Using installation script to install a specific version
|
||||
curl -sSfL https://raw.githubusercontent.com/trufflesecurity/trufflehog/main/scripts/install.sh | sh -s -- -b /usr/local/bin <ReleaseTag like v3.56.0>
|
||||
```
|
||||
|
||||
# :closed_lock_with_key: Verifying the artifacts
|
||||
|
||||
Checksums are applied to all artifacts, and the resulting checksum file is signed using cosign.
|
||||
|
||||
You need the following tool to verify signature:
|
||||
|
||||
- [Cosign](https://docs.sigstore.dev/cosign/installation/)
|
||||
|
||||
Verification steps are as follow:
|
||||
|
||||
1. Download the artifact files you want, and the following files from the [releases](https://github.com/trufflesecurity/trufflehog/releases) page.
|
||||
|
||||
- trufflehog\_{version}\_checksums.txt
|
||||
- trufflehog\_{version}\_checksums.txt.pem
|
||||
- trufflehog\_{version}\_checksums.txt.sig
|
||||
|
||||
2. Verify the signature:
|
||||
|
||||
```shell
|
||||
cosign verify-blob <path to trufflehog_{version}_checksums.txt> \
|
||||
--certificate <path to trufflehog_{version}_checksums.txt.pem> \
|
||||
--signature <path to trufflehog_{version}_checksums.txt.sig> \
|
||||
--certificate-identity-regexp 'https://github\.com/trufflesecurity/trufflehog/\.github/workflows/.+' \
|
||||
--certificate-oidc-issuer "https://token.actions.githubusercontent.com"
|
||||
```
|
||||
|
||||
3. Once the signature is confirmed as valid, you can proceed to validate that the SHA256 sums align with the downloaded artifact:
|
||||
|
||||
```shell
|
||||
sha256sum --ignore-missing -c trufflehog_{version}_checksums.txt
|
||||
```
|
||||
|
||||
Replace `{version}` with the downloaded files version
|
||||
|
||||
# :rocket: Quick Start
|
||||
|
||||
## 1: Scan a repo for only verified secrets
|
||||
|
||||
Command:
|
||||
|
||||
```bash
|
||||
trufflehog git https://github.com/trufflesecurity/test_keys --only-verified
|
||||
```
|
||||
|
||||
Expected output:
|
||||
|
||||
```
|
||||
🐷🔑🐷 TruffleHog. Unearth your secrets. 🐷🔑🐷
|
||||
|
||||
Found verified result 🐷🔑
|
||||
Detector Type: AWS
|
||||
Decoder Type: PLAIN
|
||||
Raw result: AKIAYVP4CIPPERUVIFXG
|
||||
Line: 4
|
||||
Commit: fbc14303ffbf8fb1c2c1914e8dda7d0121633aca
|
||||
File: keys
|
||||
Email: counter <counter@counters-MacBook-Air.local>
|
||||
Repository: https://github.com/trufflesecurity/test_keys
|
||||
Timestamp: 2022-06-16 10:17:40 -0700 PDT
|
||||
...
|
||||
```
|
||||
|
||||
## 2: Scan a GitHub Org for only verified secrets
|
||||
|
||||
```bash
|
||||
trufflehog github --org=trufflesecurity --only-verified
|
||||
```
|
||||
|
||||
## 3: Scan a GitHub Repo for only verified keys and get JSON output
|
||||
|
||||
Command:
|
||||
|
||||
```bash
|
||||
trufflehog git https://github.com/trufflesecurity/test_keys --only-verified --json
|
||||
```
|
||||
|
||||
Expected output:
|
||||
|
||||
```
|
||||
{"SourceMetadata":{"Data":{"Git":{"commit":"fbc14303ffbf8fb1c2c1914e8dda7d0121633aca","file":"keys","email":"counter \u003ccounter@counters-MacBook-Air.local\u003e","repository":"https://github.com/trufflesecurity/test_keys","timestamp":"2022-06-16 10:17:40 -0700 PDT","line":4}}},"SourceID":0,"SourceType":16,"SourceName":"trufflehog - git","DetectorType":2,"DetectorName":"AWS","DecoderName":"PLAIN","Verified":true,"Raw":"AKIAYVP4CIPPERUVIFXG","Redacted":"AKIAYVP4CIPPERUVIFXG","ExtraData":{"account":"595918472158","arn":"arn:aws:iam::595918472158:user/canarytokens.com@@mirux23ppyky6hx3l6vclmhnj","user_id":"AIDAYVP4CIPPJ5M54LRCY"},"StructuredData":null}
|
||||
...
|
||||
```
|
||||
|
||||
## 4: Scan a GitHub Repo + its Issues and Pull Requests.
|
||||
|
||||
```bash
|
||||
trufflehog github --repo=https://github.com/trufflesecurity/test_keys --issue-comments --pr-comments
|
||||
```
|
||||
|
||||
## 5: Scan an S3 bucket for verified keys
|
||||
|
||||
```bash
|
||||
trufflehog s3 --bucket=<bucket name> --only-verified
|
||||
```
|
||||
|
||||
## 6: Scan S3 buckets using IAM Roles
|
||||
|
||||
```bash
|
||||
trufflehog s3 --role-arn=<iam role arn>
|
||||
```
|
||||
|
||||
## 7: Scan a Github Repo using SSH authentication in docker
|
||||
|
||||
```bash
|
||||
docker run --rm -v "$HOME/.ssh:/root/.ssh:ro" trufflesecurity/trufflehog:latest git ssh://github.com/trufflesecurity/test_keys
|
||||
```
|
||||
|
||||
## 8: Scan individual files or directories
|
||||
|
||||
```bash
|
||||
trufflehog filesystem path/to/file1.txt path/to/file2.txt path/to/dir
|
||||
```
|
||||
|
||||
## 9: Scan GCS buckets for verified secrets.
|
||||
|
||||
```bash
|
||||
trufflehog gcs --project-id=<project-ID> --cloud-environment --only-verified
|
||||
```
|
||||
|
||||
## 10: Scan a Docker image for verified secrets.
|
||||
|
||||
Use the `--image` flag multiple times to scan multiple images.
|
||||
|
||||
```bash
|
||||
trufflehog docker --image trufflesecurity/secrets --only-verified
|
||||
```
|
||||
|
||||
## 11: Scan in CI
|
||||
|
||||
Set the `--since-commit` flag to your default branch that people merge into (ex: "main"). Set the `--branch` flag to your PR's branch name (ex: "feature-1"). Depending on the CI/CD platform you use, this value can be pulled in dynamically (ex: [CIRCLE_BRANCH in Circle CI](https://circleci.com/docs/variables/) and [TRAVIS_PULL_REQUEST_BRANCH in Travis CI](https://docs.travis-ci.com/user/environment-variables/)). If the repo is cloned and the target branch is already checked out during the CI/CD workflow, then `--branch HEAD` should be sufficient. The `--fail` flag will return an 183 error code if valid credentials are found.
|
||||
|
||||
```bash
|
||||
trufflehog git file://. --since-commit main --branch feature-1 --only-verified --fail
|
||||
```
|
||||
|
||||
# :question: FAQ
|
||||
|
||||
- All I see is `🐷🔑🐷 TruffleHog. Unearth your secrets. 🐷🔑🐷` and the program exits, what gives?
|
||||
- That means no secrets were detected
|
||||
- Why is the scan taking a long time when I scan a GitHub org
|
||||
- Unauthenticated GitHub scans have rate limits. To improve your rate limits, include the `--token` flag with a personal access token
|
||||
- It says a private key was verified, what does that mean?
|
||||
- Check out our Driftwood blog post to learn how to do this, in short we've confirmed the key can be used live for SSH or SSL [Blog post](https://trufflesecurity.com/blog/driftwood-know-if-private-keys-are-sensitive/)
|
||||
- Is there an easy way to ignore specific secrets?
|
||||
- If the scanned source [supports line numbers](https://github.com/trufflesecurity/trufflehog/blob/d6375ba92172fd830abb4247cca15e3176448c5d/pkg/engine/engine.go#L358-L365), then you can add a `trufflehog:ignore` comment on the line containing the secret to ignore that secrets.
|
||||
|
||||
# :newspaper: What's new in v3?
|
||||
# What's new in v3?
|
||||
|
||||
TruffleHog v3 is a complete rewrite in Go with many new powerful features.
|
||||
|
||||
- We've **added over 700 credential detectors that support active verification against their respective APIs**.
|
||||
- We've also added native **support for scanning GitHub, GitLab, Docker, filesystems, S3, GCS, Circle CI and Travis CI**.
|
||||
- We've also added native **support for scanning GitHub, GitLab, filesystems, and S3**.
|
||||
- **Instantly verify private keys** against millions of github users and **billions** of TLS certificates using our [Driftwood](https://trufflesecurity.com/blog/driftwood) technology.
|
||||
- Scan binaries, documents, and other file formats
|
||||
- Available as a GitHub Action and a pre-commit hook
|
||||
|
||||
|
||||
## What is credential verification?
|
||||
For every potential credential that is detected, we've painstakingly implemented programatic verification against the API that we think it belongs to. Verification eliminates false positives. For example, the [AWS credential detector](pkg/detectors/aws/aws.go) performs a `GetCallerIdentity` API call against the AWS API to verify if an AWS credential is active.
|
||||
|
||||
For every potential credential that is detected, we've painstakingly implemented programmatic verification against the API that we think it belongs to. Verification eliminates false positives. For example, the [AWS credential detector](pkg/detectors/aws/aws.go) performs a `GetCallerIdentity` API call against the AWS API to verify if an AWS credential is active.
|
||||
## Installation
|
||||
|
||||
# :memo: Usage
|
||||
Several options:
|
||||
|
||||
### 1. Go
|
||||
```
|
||||
git clone https://github.com/trufflesecurity/trufflehog.git
|
||||
|
||||
cd trufflehog; go install
|
||||
```
|
||||
|
||||
### 2. [Release binaries](https://github.com/trufflesecurity/trufflehog/releases)
|
||||
|
||||
### 3. Docker
|
||||
|
||||
|
||||
> Note: Apple M1 hardware users should run with `docker run --platform linux/arm64` for better performance.
|
||||
|
||||
#### **Most users**
|
||||
|
||||
```bash
|
||||
docker run -it -v "$PWD:/pwd" trufflesecurity/trufflehog:latest github --repo https://github.com/trufflesecurity/test_keys
|
||||
```
|
||||
|
||||
#### **Apple M1 users**
|
||||
|
||||
The `linux/arm64` image is better to run on the M1 than the amd64 image.
|
||||
Even better is running the native darwin binary avilable, but there is not container image for that.
|
||||
|
||||
```bash
|
||||
docker run --platform linux/arm64 -it -v "$PWD:/pwd" trufflesecurity/trufflehog:latest github --repo https://github.com/trufflesecurity/test_keys
|
||||
```
|
||||
|
||||
### 4. Pip (help wanted)
|
||||
|
||||
It's possible to distribute binaries in pip wheels.
|
||||
|
||||
Here is an example of a [project that does it](https://github.com/Yelp/dumb-init).
|
||||
|
||||
Help with setting up this packaging would be appreciated!
|
||||
|
||||
### 5. Brew
|
||||
|
||||
```bash
|
||||
brew tap trufflesecurity/trufflehog
|
||||
brew install trufflehog
|
||||
```
|
||||
|
||||
## Usage
|
||||
|
||||
TruffleHog has a sub-command for each source of data that you may want to scan:
|
||||
|
||||
- git
|
||||
- github
|
||||
- gitlab
|
||||
- docker
|
||||
- S3
|
||||
- filesystem (files and directories)
|
||||
- filesystem
|
||||
- syslog
|
||||
- circleci
|
||||
- travisci
|
||||
- GCS (Google Cloud Storage)
|
||||
- file and stdin (coming soon)
|
||||
|
||||
Each subcommand can have options that you can see with the `--help` flag provided to the sub command:
|
||||
Each subcommand can have options that you can see with the `-h` flag provided to the sub command:
|
||||
|
||||
```
|
||||
$ trufflehog git --help
|
||||
@@ -253,110 +112,73 @@ usage: TruffleHog git [<flags>] <uri>
|
||||
Find credentials in git repositories.
|
||||
|
||||
Flags:
|
||||
-h, --help Show context-sensitive help (also try --help-long and --help-man).
|
||||
--debug Run in debug mode.
|
||||
--trace Run in trace mode.
|
||||
--profile Enables profiling and sets a pprof and fgprof server on :18066.
|
||||
-j, --json Output in JSON format.
|
||||
--json-legacy Use the pre-v3.0 JSON format. Only works with git, gitlab, and github sources.
|
||||
--github-actions Output in GitHub Actions format.
|
||||
--concurrency=8 Number of concurrent workers.
|
||||
--no-verification Don't verify the results.
|
||||
--only-verified Only output verified results.
|
||||
--filter-unverified Only output first unverified result per chunk per detector if there are more than one results.
|
||||
--filter-entropy=FILTER-ENTROPY
|
||||
Filter unverified results with Shannon entropy. Start with 3.0.
|
||||
--config=CONFIG Path to configuration file.
|
||||
--print-avg-detector-time
|
||||
Print the average time spent on each detector.
|
||||
--no-update Don't check for updates.
|
||||
--fail Exit with code 183 if results are found.
|
||||
--verifier=VERIFIER ... Set custom verification endpoints.
|
||||
--archive-max-size=ARCHIVE-MAX-SIZE
|
||||
Maximum size of archive to scan. (Byte units eg. 512B, 2KB, 4MB)
|
||||
--archive-max-depth=ARCHIVE-MAX-DEPTH
|
||||
Maximum depth of archive to scan.
|
||||
--archive-timeout=ARCHIVE-TIMEOUT
|
||||
Maximum time to spend extracting an archive.
|
||||
--include-detectors="all" Comma separated list of detector types to include. Protobuf name or IDs may be used, as well as ranges.
|
||||
--exclude-detectors=EXCLUDE-DETECTORS
|
||||
Comma separated list of detector types to exclude. Protobuf name or IDs may be used, as well as ranges. IDs defined here take precedence over the include list.
|
||||
--version Show application version.
|
||||
--help Show context-sensitive help (also try --help-long and --help-man).
|
||||
--debug Run in debug mode
|
||||
--version Prints trufflehog version.
|
||||
-j, --json Output in JSON format.
|
||||
--json-legacy Use the pre-v3.0 JSON format. Only works with git, gitlab, and github sources.
|
||||
--concurrency=1 Number of concurrent workers.
|
||||
--no-verification Don't verify the results.
|
||||
--only-verified Only output verified results.
|
||||
--print-avg-detector-time Print the average time spent on each detector.
|
||||
--no-update Don't check for updates.
|
||||
-i, --include-paths=INCLUDE-PATHS
|
||||
Path to file with newline separated regexes for files to include in scan.
|
||||
-x, --exclude-paths=EXCLUDE-PATHS
|
||||
Path to file with newline separated regexes for files to exclude in scan.
|
||||
--exclude-globs=EXCLUDE-GLOBS
|
||||
Comma separated list of globs to exclude in scan. This option filters at the `git log` level, resulting in faster scans.
|
||||
--since-commit=SINCE-COMMIT
|
||||
Commit to start scan from.
|
||||
--branch=BRANCH Branch to scan.
|
||||
--max-depth=MAX-DEPTH Maximum depth of commits to scan.
|
||||
--bare Scan bare repository (e.g. useful while using in pre-receive hooks)
|
||||
--allow No-op flag for backwards compat.
|
||||
--entropy No-op flag for backwards compat.
|
||||
--regex No-op flag for backwards compat.
|
||||
|
||||
Args:
|
||||
<uri> Git repository URL. https://, file://, or ssh:// schema expected.
|
||||
<uri> Git repository URL. https:// or file:// schema expected.
|
||||
```
|
||||
|
||||
For example, to scan a `git` repository, start with
|
||||
For example, to scan a `git` repository, start with
|
||||
|
||||
```
|
||||
$ trufflehog git https://github.com/trufflesecurity/trufflehog.git
|
||||
```
|
||||
|
||||
## S3
|
||||
|
||||
The S3 source supports assuming IAM roles for scanning in addition to IAM users. This makes it easier for users to scan multiple AWS accounts without needing to rely on hardcoded credentials for each account.
|
||||
|
||||
The IAM identity that TruffleHog uses initially will need to have `AssumeRole` privileges as a principal in the [trust policy](https://aws.amazon.com/blogs/security/how-to-use-trust-policies-with-iam-roles/) of each IAM role to assume.
|
||||
|
||||
To scan a specific bucket using locally set credentials or instance metadata if on an EC2 instance:
|
||||
|
||||
```bash
|
||||
trufflehog s3 --bucket=<bucket-name>
|
||||
```
|
||||
|
||||
To scan a specific bucket using an assumed role:
|
||||
|
||||
```bash
|
||||
trufflehog s3 --bucket=<bucket-name> --role-arn=<iam-role-arn>
|
||||
```
|
||||
|
||||
Multiple roles can be passed as separate arguments. The following command will attempt to scan every bucket each role has permissions to list in the S3 API:
|
||||
|
||||
```bash
|
||||
trufflehog s3 --role-arn=<iam-role-arn-1> --role-arn=<iam-role-arn-2>
|
||||
```
|
||||
|
||||
Exit Codes:
|
||||
|
||||
- 0: No errors and no results were found.
|
||||
- 1: An error was encountered. Sources may not have completed scans.
|
||||
- 183: No errors were encountered, but results were found. Will only be returned if `--fail` flag is used.
|
||||
|
||||
## :octocat: TruffleHog Github Action
|
||||
#### Scanning an organization
|
||||
|
||||
Try scanning an entire GitHub organization with the following:
|
||||
|
||||
```bash
|
||||
docker run -it -v "$PWD:/pwd" trufflesecurity/trufflehog:latest github --org=trufflesecurity
|
||||
```
|
||||
|
||||
### TruffleHog OSS Github Action
|
||||
|
||||
```yaml
|
||||
- name: TruffleHog
|
||||
- name: TruffleHog OSS
|
||||
uses: trufflesecurity/trufflehog@main
|
||||
with:
|
||||
# Repository path
|
||||
path:
|
||||
path:
|
||||
# Start scanning from here (usually main branch).
|
||||
base:
|
||||
base:
|
||||
# Scan commits until here (usually dev branch).
|
||||
head: # optional
|
||||
# Extra args to be passed to the trufflehog cli.
|
||||
extra_args: --debug --only-verified
|
||||
extra_args: # optional
|
||||
```
|
||||
|
||||
The TruffleHog OSS Github Action can be used to scan a range of commits for leaked credentials. The action will fail if
|
||||
any results are found.
|
||||
|
||||
For example, to scan the contents of pull requests you could use the following workflow:
|
||||
|
||||
```yaml
|
||||
name: TruffleHog Secrets Scan
|
||||
name: Leaked Secrets Scan
|
||||
on: [pull_request]
|
||||
jobs:
|
||||
TruffleHog:
|
||||
@@ -367,155 +189,53 @@ jobs:
|
||||
with:
|
||||
fetch-depth: 0
|
||||
- name: TruffleHog OSS
|
||||
uses: trufflesecurity/trufflehog@main
|
||||
uses: trufflesecurity/trufflehog@v3.4.3
|
||||
with:
|
||||
path: ./
|
||||
base: ${{ github.event.repository.default_branch }}
|
||||
head: HEAD
|
||||
extra_args: --debug --only-verified
|
||||
extra_args: --debug
|
||||
```
|
||||
|
||||
## Pre-commit Hook
|
||||
### Precommit Hook
|
||||
|
||||
Trufflehog can be used in a pre-commit hook to prevent credentials from leaking before they ever leave your computer.
|
||||
Trufflehog can be used in a precommit hook to prevent credentials from leaking before they ever leave your computer.
|
||||
An example `.pre-commit-config.yaml` is provided (see [pre-commit.com](https://pre-commit.com/) for installation).
|
||||
|
||||
```yaml
|
||||
repos:
|
||||
- repo: local
|
||||
hooks:
|
||||
- id: trufflehog
|
||||
name: TruffleHog
|
||||
description: Detect secrets in your data.
|
||||
entry: bash -c 'trufflehog git file://. --since-commit HEAD --only-verified --fail'
|
||||
# For running trufflehog in docker, use the following entry instead:
|
||||
# entry: bash -c 'docker run --rm -v "$(pwd):/workdir" -i --rm trufflesecurity/trufflehog:latest git file:///workdir --since-commit HEAD --only-verified --fail'
|
||||
language: system
|
||||
stages: ["commit", "push"]
|
||||
- repo: local
|
||||
hooks:
|
||||
- id: trufflehog
|
||||
name: TruffleHog
|
||||
description: Detect secrets in your data.
|
||||
entry: bash -c 'trufflehog git file://. --only-verified --fail'
|
||||
# For running trufflehog in docker, use the following entry instead:
|
||||
# entry: bash -c 'docker run -v "$(pwd):/workdir" -i --rm trufflesecurity/trufflehog:latest git file:///workdir --only-verified --fail'
|
||||
language: system
|
||||
stages: ["commit", "push"]
|
||||
```
|
||||
|
||||
## Regex Detector (alpha)
|
||||
|
||||
Trufflehog supports detection and verification of custom regular expressions.
|
||||
For detection, at least one **regular expression** and **keyword** is required.
|
||||
A **keyword** is a fixed literal string identifier that appears in or around
|
||||
the regex to be detected. To allow maximum flexibility for verification, a
|
||||
webhook is used containing the regular expression matches.
|
||||
|
||||
Trufflehog will send a JSON POST request containing the regex matches to a
|
||||
configured webhook endpoint. If the endpoint responds with a `200 OK` response
|
||||
status code, the secret is considered verified.
|
||||
|
||||
**NB:** This feature is alpha and subject to change.
|
||||
|
||||
## Regex Detector Example
|
||||
|
||||
```yaml
|
||||
# config.yaml
|
||||
detectors:
|
||||
- name: hog detector
|
||||
keywords:
|
||||
- hog
|
||||
regex:
|
||||
adjective: hogs are (\S+)
|
||||
verify:
|
||||
- endpoint: http://localhost:8000/
|
||||
# unsafe must be set if the endpoint is HTTP
|
||||
unsafe: true
|
||||
headers:
|
||||
- "Authorization: super secret authorization header"
|
||||
```
|
||||
|
||||
```
|
||||
$ trufflehog filesystem /tmp --config config.yaml --only-verified
|
||||
🐷🔑🐷 TruffleHog. Unearth your secrets. 🐷🔑🐷
|
||||
|
||||
Found verified result 🐷🔑
|
||||
Detector Type: CustomRegex
|
||||
Decoder Type: PLAIN
|
||||
Raw result: hogs are cool
|
||||
File: /tmp/hog-facts.txt
|
||||
```
|
||||
|
||||
## Verification Server Example (Python)
|
||||
|
||||
Unless you run a verification server, secrets found by the custom regex
|
||||
detector will be unverified. Here is an example Python implementation of a
|
||||
verification server for the above `config.yaml` file.
|
||||
|
||||
```python
|
||||
import json
|
||||
from http.server import BaseHTTPRequestHandler, HTTPServer
|
||||
|
||||
AUTH_HEADER = 'super secret authorization header'
|
||||
|
||||
|
||||
class Verifier(BaseHTTPRequestHandler):
|
||||
def do_GET(self):
|
||||
self.send_response(405)
|
||||
self.end_headers()
|
||||
|
||||
def do_POST(self):
|
||||
try:
|
||||
if self.headers['Authorization'] != AUTH_HEADER:
|
||||
self.send_response(401)
|
||||
self.end_headers()
|
||||
return
|
||||
|
||||
# read the body
|
||||
length = int(self.headers['Content-Length'])
|
||||
request = json.loads(self.rfile.read(length))
|
||||
self.log_message("%s", request)
|
||||
|
||||
# check the match
|
||||
if request['hog detector']['adjective'][-1] == 'cool':
|
||||
self.send_response(200)
|
||||
self.end_headers()
|
||||
else:
|
||||
# any other response besides 200
|
||||
self.send_response(406)
|
||||
self.end_headers()
|
||||
except Exception:
|
||||
self.send_response(400)
|
||||
self.end_headers()
|
||||
|
||||
|
||||
with HTTPServer(('', 8000), Verifier) as server:
|
||||
try:
|
||||
server.serve_forever()
|
||||
except KeyboardInterrupt:
|
||||
pass
|
||||
```
|
||||
|
||||
# :heart: Contributors
|
||||
## Contributors
|
||||
|
||||
This project exists thanks to all the people who contribute. [[Contribute](CONTRIBUTING.md)].
|
||||
|
||||
|
||||
<a href="https://github.com/trufflesecurity/trufflehog/graphs/contributors">
|
||||
<img src="https://contrib.rocks/image?repo=trufflesecurity/trufflehog" />
|
||||
</a>
|
||||
|
||||
# :computer: Contributing
|
||||
|
||||
## Contributing
|
||||
|
||||
Contributions are very welcome! Please see our [contribution guidelines first](CONTRIBUTING.md).
|
||||
|
||||
We no longer accept contributions to TruffleHog v2, but that code is available in the `v2` branch.
|
||||
|
||||
## Adding new secret detectors
|
||||
### Adding new secret detectors
|
||||
|
||||
We have published some [documentation and tooling to get started on adding new secret detectors](hack/docs/Adding_Detectors_external.md). Let's improve detection together!
|
||||
|
||||
# Use as a library
|
||||
|
||||
Currently, trufflehog is in heavy development and no guarantees can be made on
|
||||
the stability of the public APIs at this time.
|
||||
|
||||
# License Change
|
||||
## License Change
|
||||
|
||||
Since v3.0, TruffleHog is released under a AGPL 3 license, included in [`LICENSE`](LICENSE). TruffleHog v3.0 uses none of the previous codebase, but care was taken to preserve backwards compatibility on the command line interface. The work previous to this release is still available licensed under GPL 2.0 in the history of this repository and the previous package releases and tags. A completed CLA is required for us to accept contributions going forward.
|
||||
|
||||
# :money_with_wings: Enterprise product
|
||||
|
||||
Are you interested in continuously monitoring your Git, Jira, Slack, Confluence, etc.. for credentials? We have an enterprise product that can help. Reach out here to learn more https://trufflesecurity.com/contact/
|
||||
|
||||
We take the revenue from the enterprise product to fund more awesome open source projects that the whole community can benefit from.
|
||||
|
||||
+1
-4
@@ -8,8 +8,7 @@ inputs:
|
||||
required: true
|
||||
base:
|
||||
description: Start scanning from here (usually main branch).
|
||||
required: false
|
||||
default: ''
|
||||
required: true
|
||||
head:
|
||||
description: Scan commits until here (usually dev branch).
|
||||
required: false
|
||||
@@ -31,6 +30,4 @@ runs:
|
||||
- --branch
|
||||
- ${{ inputs.head }}
|
||||
- --fail
|
||||
- --no-update
|
||||
- --github-actions
|
||||
- ${{ inputs.extra_args }}
|
||||
|
||||
File diff suppressed because one or more lines are too long
|
Before Width: | Height: | Size: 83 KiB |
+4
-5
@@ -1,6 +1,5 @@
|
||||
#!/usr/bin/env bash
|
||||
#!/usr/bin/env ash
|
||||
|
||||
# Parse the last argument into an array of extra_args.
|
||||
mapfile -t extra_args < <(bash -c "for arg in ${*: -1}; do echo \$arg; done")
|
||||
|
||||
/usr/bin/trufflehog "${@: 1: $#-1}" "${extra_args[@]}"
|
||||
# `$*` expands the `args` supplied in an `array` individually
|
||||
# or splits `args` in a string separated by whitespace.
|
||||
/usr/bin/trufflehog $*
|
||||
@@ -1,14 +0,0 @@
|
||||
# Examples
|
||||
This folder contains various examples like custom detectors, scripts, etc. Feel free to contribute!
|
||||
|
||||
### Generic Detector
|
||||
An often requested feature for TruffleHog is a generic detector. By default, we do not support generic detection as it would result in lots of false positives. However, if you want to attempt detect generic secrets you can use a custom detector.
|
||||
|
||||
#### Try it out:
|
||||
```
|
||||
wget https://raw.githubusercontent.com/trufflesecurity/trufflehog/main/examples/generic.yml
|
||||
trufflehog filesystem --config=$PWD/generic.yml $PWD
|
||||
|
||||
# to filter so that _only_ generic credentials are logged:
|
||||
trufflehog filesystem --config=$PWD/generic.yml --json --no-verification $PWD | awk '/generic-api-key/{print $0}'
|
||||
```
|
||||
@@ -1,15 +0,0 @@
|
||||
detectors:
|
||||
- name: generic-api-key
|
||||
keywords:
|
||||
- key
|
||||
- api
|
||||
- token
|
||||
- secret
|
||||
- client
|
||||
- passwd
|
||||
- password
|
||||
- auth
|
||||
- access
|
||||
regex:
|
||||
# borrowing the gitleaks generic-api-key regex
|
||||
generic-api-key: "(?i)(?:key|api|token|secret|client|passwd|password|auth|access)(?:[0-9a-z\\-_\\t .]{0,20})(?:[\\s|']|[\\s|\"]){0,3}(?:=|>|:{1,3}=|\\|\\|:|<=|=>|:|\\?=)(?:'|\"|\\s|=|\\x60){0,5}([0-9a-z\\-_.=]{10,150})(?:['|\"|\\n|\\r|\\s|\\x60|;]|$)"
|
||||
@@ -1,105 +1,64 @@
|
||||
module github.com/trufflesecurity/trufflehog/v3
|
||||
|
||||
go 1.21
|
||||
go 1.18
|
||||
|
||||
replace github.com/jpillora/overseer => github.com/trufflesecurity/overseer v1.2.7
|
||||
replace github.com/jpillora/overseer => github.com/trufflesecurity/overseer v1.1.7-custom5
|
||||
|
||||
replace github.com/zricethezav/gitleaks/v8 => github.com/trufflesecurity/gitleaks/v8 v8.6.1-custom10
|
||||
|
||||
replace github.com/gitleaks/go-gitdiff => github.com/trufflesecurity/go-gitdiff v0.7.6-zombies2
|
||||
|
||||
replace github.com/bill-rich/gitparser => /home/hrich/go/src/github.com/bill-rich/gitparse
|
||||
|
||||
require (
|
||||
cloud.google.com/go/secretmanager v1.11.4
|
||||
cloud.google.com/go/storage v1.35.1
|
||||
github.com/Azure/go-autorest/autorest/azure/auth v0.5.12
|
||||
github.com/AzureAD/microsoft-authentication-library-for-go v1.2.0
|
||||
github.com/BobuSumisu/aho-corasick v1.0.3
|
||||
github.com/TheZeroSlave/zapsentry v1.19.0
|
||||
github.com/alecthomas/kingpin/v2 v2.4.0
|
||||
github.com/aws/aws-sdk-go v1.48.12
|
||||
github.com/aymanbagabas/go-osc52 v1.2.2
|
||||
cloud.google.com/go/secretmanager v1.5.0
|
||||
github.com/Azure/go-autorest/autorest/azure/auth v0.5.11
|
||||
github.com/aws/aws-sdk-go v1.44.61
|
||||
github.com/bill-rich/disk-buffer-reader v0.1.2
|
||||
github.com/bill-rich/gitparser v0.0.0-00010101000000-000000000000
|
||||
github.com/bill-rich/go-syslog v0.0.0-20220413021637-49edb52a574c
|
||||
github.com/bitfinexcom/bitfinex-api-go v0.0.0-20210608095005-9e0b26f200fb
|
||||
github.com/bradleyfalzon/ghinstallation/v2 v2.8.0
|
||||
github.com/charmbracelet/bubbles v0.16.1
|
||||
github.com/charmbracelet/bubbletea v0.24.2
|
||||
github.com/charmbracelet/glamour v0.6.0
|
||||
github.com/charmbracelet/lipgloss v0.9.1
|
||||
github.com/coinbase/waas-client-library-go v1.0.8
|
||||
github.com/couchbase/gocb/v2 v2.7.0
|
||||
github.com/bradleyfalzon/ghinstallation/v2 v2.1.0
|
||||
github.com/crewjam/rfc5424 v0.1.0
|
||||
github.com/denisenkom/go-mssqldb v0.12.3
|
||||
github.com/envoyproxy/protoc-gen-validate v1.0.2
|
||||
github.com/fatih/color v1.16.0
|
||||
github.com/felixge/fgprof v0.9.3
|
||||
github.com/getsentry/sentry-go v0.25.0
|
||||
github.com/go-errors/errors v1.5.1
|
||||
github.com/go-git/go-git/v5 v5.10.1
|
||||
github.com/go-ldap/ldap/v3 v3.4.6
|
||||
github.com/go-logr/logr v1.3.0
|
||||
github.com/go-logr/zapr v1.3.0
|
||||
github.com/go-redis/redis v6.15.9+incompatible
|
||||
github.com/go-sql-driver/mysql v1.7.1
|
||||
github.com/gobwas/glob v0.2.3
|
||||
github.com/envoyproxy/protoc-gen-validate v0.6.7
|
||||
github.com/fatih/color v1.13.0
|
||||
github.com/felixge/fgprof v0.9.2
|
||||
github.com/go-errors/errors v1.4.2
|
||||
github.com/go-git/go-git/v5 v5.4.2
|
||||
github.com/golang-jwt/jwt v3.2.2+incompatible
|
||||
github.com/google/go-cmp v0.6.0
|
||||
github.com/google/go-containerregistry v0.17.0
|
||||
github.com/google/go-github/v42 v42.0.0
|
||||
github.com/google/uuid v1.4.0
|
||||
github.com/googleapis/gax-go/v2 v2.12.0
|
||||
github.com/gorilla/mux v1.8.0
|
||||
github.com/h2non/filetype v1.1.3
|
||||
github.com/hashicorp/go-retryablehttp v0.7.5
|
||||
github.com/hashicorp/golang-lru v0.6.0
|
||||
github.com/hashicorp/golang-lru/v2 v2.0.7
|
||||
github.com/jlaffaye/ftp v0.2.0
|
||||
github.com/joho/godotenv v1.5.1
|
||||
github.com/hashicorp/go-retryablehttp v0.7.1
|
||||
github.com/joho/godotenv v1.4.0
|
||||
github.com/jpillora/overseer v1.1.6
|
||||
github.com/kylelemons/godebug v1.1.0
|
||||
github.com/launchdarkly/go-server-sdk/v6 v6.1.1
|
||||
github.com/lib/pq v1.10.9
|
||||
github.com/lrstanley/bubblezone v0.0.0-20230911164824-e3824f1adde9
|
||||
github.com/marusama/semaphore/v2 v2.5.0
|
||||
github.com/mattn/go-isatty v0.0.20
|
||||
github.com/mattn/go-sqlite3 v1.14.18
|
||||
github.com/mholt/archiver/v4 v4.0.0-alpha.8
|
||||
github.com/mitchellh/go-ps v1.0.0
|
||||
github.com/muesli/reflow v0.3.0
|
||||
github.com/patrickmn/go-cache v2.1.0+incompatible
|
||||
github.com/mattn/go-colorable v0.1.12
|
||||
github.com/mholt/archiver/v4 v4.0.0-alpha.7
|
||||
github.com/paulbellamy/ratecounter v0.2.0
|
||||
github.com/pkg/errors v0.9.1
|
||||
github.com/prometheus/client_golang v1.17.0
|
||||
github.com/rabbitmq/amqp091-go v1.9.0
|
||||
github.com/sergi/go-diff v1.3.1
|
||||
github.com/shuheiktgw/go-travis v0.3.1
|
||||
github.com/snowflakedb/gosnowflake v1.6.25
|
||||
github.com/stretchr/testify v1.8.4
|
||||
github.com/razorpay/razorpay-go v0.0.0-20210728161131-0341409a6ab2
|
||||
github.com/rs/zerolog v1.27.0
|
||||
github.com/sergi/go-diff v1.2.0
|
||||
github.com/sirupsen/logrus v1.9.0
|
||||
github.com/stretchr/testify v1.8.0
|
||||
github.com/tailscale/depaware v0.0.0-20210622194025-720c4b409502
|
||||
github.com/trufflesecurity/disk-buffer-reader v0.2.1
|
||||
github.com/xanzy/go-gitlab v0.94.0
|
||||
go.mongodb.org/mongo-driver v1.12.1
|
||||
go.uber.org/mock v0.3.0
|
||||
go.uber.org/zap v1.26.0
|
||||
golang.org/x/crypto v0.16.0
|
||||
golang.org/x/exp v0.0.0-20231127185646-65229373498e
|
||||
golang.org/x/net v0.19.0
|
||||
golang.org/x/oauth2 v0.15.0
|
||||
golang.org/x/sync v0.5.0
|
||||
golang.org/x/text v0.14.0
|
||||
google.golang.org/api v0.152.0
|
||||
google.golang.org/protobuf v1.31.0
|
||||
github.com/xanzy/go-gitlab v0.69.0
|
||||
github.com/zricethezav/gitleaks/v8 v8.10.3
|
||||
golang.org/x/crypto v0.0.0-20211215165025-cf75a172585e
|
||||
golang.org/x/net v0.0.0-20220617184016-355a448f1bc9
|
||||
golang.org/x/oauth2 v0.0.0-20220608161450-d0670ef3b1eb
|
||||
golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f
|
||||
google.golang.org/genproto v0.0.0-20220617124728-180714bec0ad
|
||||
google.golang.org/protobuf v1.28.1
|
||||
gopkg.in/alecthomas/kingpin.v2 v2.2.6
|
||||
gopkg.in/h2non/gock.v1 v1.1.2
|
||||
pgregory.net/rapid v1.1.0
|
||||
sigs.k8s.io/yaml v1.4.0
|
||||
)
|
||||
|
||||
require (
|
||||
cloud.google.com/go v0.110.10 // indirect
|
||||
cloud.google.com/go/compute v1.23.3 // indirect
|
||||
cloud.google.com/go/compute/metadata v0.2.3 // indirect
|
||||
cloud.google.com/go/iam v1.1.5 // indirect
|
||||
cloud.google.com/go/longrunning v0.5.4 // indirect
|
||||
dario.cat/mergo v1.0.0 // indirect
|
||||
github.com/99designs/go-keychain v0.0.0-20191008050251-8e49817e8af4 // indirect
|
||||
github.com/99designs/keyring v1.2.2 // indirect
|
||||
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.4.0 // indirect
|
||||
github.com/Azure/azure-sdk-for-go/sdk/internal v1.1.2 // indirect
|
||||
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.0.0 // indirect
|
||||
cloud.google.com/go v0.102.0 // indirect
|
||||
cloud.google.com/go/compute v1.7.0 // indirect
|
||||
cloud.google.com/go/iam v0.3.0 // indirect
|
||||
github.com/Azure/go-autorest v14.2.0+incompatible // indirect
|
||||
github.com/Azure/go-autorest/autorest v0.11.24 // indirect
|
||||
github.com/Azure/go-autorest/autorest/adal v0.9.18 // indirect
|
||||
@@ -107,161 +66,59 @@ require (
|
||||
github.com/Azure/go-autorest/autorest/date v0.3.0 // indirect
|
||||
github.com/Azure/go-autorest/logger v0.2.1 // indirect
|
||||
github.com/Azure/go-autorest/tracing v0.6.0 // indirect
|
||||
github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 // indirect
|
||||
github.com/JohnCGriffin/overflow v0.0.0-20211019200055-46fa312c352c // indirect
|
||||
github.com/Microsoft/go-winio v0.6.1 // indirect
|
||||
github.com/ProtonMail/go-crypto v0.0.0-20230828082145-3c4c8a2d2371 // indirect
|
||||
github.com/alecthomas/chroma v0.10.0 // indirect
|
||||
github.com/Microsoft/go-winio v0.4.16 // indirect
|
||||
github.com/ProtonMail/go-crypto v0.0.0-20210428141323-04723f9f07d7 // indirect
|
||||
github.com/acomagu/bufpipe v1.0.3 // indirect
|
||||
github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 // indirect
|
||||
github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 // indirect
|
||||
github.com/andybalholm/brotli v1.0.6 // indirect
|
||||
github.com/apache/arrow/go/v12 v12.0.1 // indirect
|
||||
github.com/apache/thrift v0.16.0 // indirect
|
||||
github.com/atotto/clipboard v0.1.4 // indirect
|
||||
github.com/aws/aws-sdk-go-v2 v1.17.7 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.10 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.13.18 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.59 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.31 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.25 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/v4a v1.0.23 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.11 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.26 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.25 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.14.0 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.31.0 // indirect
|
||||
github.com/aws/smithy-go v1.13.5 // indirect
|
||||
github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect
|
||||
github.com/aymerick/douceur v0.2.0 // indirect
|
||||
github.com/beorn7/perks v1.0.1 // indirect
|
||||
github.com/bodgit/plumbing v1.3.0 // indirect
|
||||
github.com/bodgit/sevenzip v1.4.5 // indirect
|
||||
github.com/bodgit/windows v1.0.1 // indirect
|
||||
github.com/cespare/xxhash/v2 v2.2.0 // indirect
|
||||
github.com/cloudflare/circl v1.3.3 // indirect
|
||||
github.com/containerd/console v1.0.4-0.20230313162750-1ae8d489ac81 // indirect
|
||||
github.com/containerd/stargz-snapshotter/estargz v0.14.3 // indirect
|
||||
github.com/couchbase/gocbcore/v10 v10.3.0 // indirect
|
||||
github.com/couchbase/gocbcoreps v0.1.0 // indirect
|
||||
github.com/couchbase/goprotostellar v1.0.0 // indirect
|
||||
github.com/couchbaselabs/gocbconnstr/v2 v2.0.0-20230515165046-68b522a21131 // indirect
|
||||
github.com/cyphar/filepath-securejoin v0.2.4 // indirect
|
||||
github.com/danieljoos/wincred v1.1.2 // indirect
|
||||
github.com/andybalholm/brotli v1.0.4 // indirect
|
||||
github.com/davecgh/go-spew v1.1.1 // indirect
|
||||
github.com/dimchansky/utfbom v1.1.1 // indirect
|
||||
github.com/dlclark/regexp2 v1.4.0 // indirect
|
||||
github.com/docker/cli v24.0.0+incompatible // indirect
|
||||
github.com/docker/distribution v2.8.2+incompatible // indirect
|
||||
github.com/docker/docker v24.0.7+incompatible // indirect
|
||||
github.com/docker/docker-credential-helpers v0.7.0 // indirect
|
||||
github.com/dsnet/compress v0.0.1 // indirect
|
||||
github.com/dvsekhvalnov/jose2go v1.5.0 // indirect
|
||||
github.com/emirpasic/gods v1.18.1 // indirect
|
||||
github.com/form3tech-oss/jwt-go v3.2.5+incompatible // indirect
|
||||
github.com/gabriel-vasile/mimetype v1.4.2 // indirect
|
||||
github.com/go-asn1-ber/asn1-ber v1.5.5 // indirect
|
||||
github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect
|
||||
github.com/go-git/go-billy/v5 v5.5.0 // indirect
|
||||
github.com/emirpasic/gods v1.12.0 // indirect
|
||||
github.com/gitleaks/go-gitdiff v0.7.6 // indirect
|
||||
github.com/go-git/gcfg v1.5.0 // indirect
|
||||
github.com/go-git/go-billy/v5 v5.3.1 // indirect
|
||||
github.com/go-ole/go-ole v1.2.6 // indirect
|
||||
github.com/goccy/go-json v0.10.0 // indirect
|
||||
github.com/godbus/dbus v0.0.0-20190726142602-4481cbc300e2 // indirect
|
||||
github.com/golang-jwt/jwt/v4 v4.5.0 // indirect
|
||||
github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe // indirect
|
||||
github.com/golang-sql/sqlexp v0.1.0 // indirect
|
||||
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
|
||||
github.com/golang/protobuf v1.5.3 // indirect
|
||||
github.com/golang-jwt/jwt/v4 v4.4.1 // indirect
|
||||
github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e // indirect
|
||||
github.com/golang/protobuf v1.5.2 // indirect
|
||||
github.com/golang/snappy v0.0.4 // indirect
|
||||
github.com/google/flatbuffers v23.1.21+incompatible // indirect
|
||||
github.com/google/go-github/v56 v56.0.0 // indirect
|
||||
github.com/google/go-cmp v0.5.8 // indirect
|
||||
github.com/google/go-github/v45 v45.2.0 // indirect
|
||||
github.com/google/go-querystring v1.1.0 // indirect
|
||||
github.com/google/pprof v0.0.0-20211214055906-6f57359322fd // indirect
|
||||
github.com/google/s2a-go v0.1.7 // indirect
|
||||
github.com/googleapis/enterprise-certificate-proxy v0.3.2 // indirect
|
||||
github.com/gorilla/css v1.0.0 // indirect
|
||||
github.com/gregjones/httpcache v0.0.0-20171119193500-2bcd89a1743f // indirect
|
||||
github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 // indirect
|
||||
github.com/grpc-ecosystem/grpc-gateway/v2 v2.15.2 // indirect
|
||||
github.com/gsterjov/go-libsecret v0.0.0-20161001094733-a6f4afe4910c // indirect
|
||||
github.com/google/pprof v0.0.0-20220729232143-a41b82acbcb1 // indirect
|
||||
github.com/googleapis/enterprise-certificate-proxy v0.1.0 // indirect
|
||||
github.com/googleapis/gax-go/v2 v2.4.0 // indirect
|
||||
github.com/h2non/parth v0.0.0-20190131123155-b4df798d6542 // indirect
|
||||
github.com/hashicorp/errwrap v1.0.0 // indirect
|
||||
github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
|
||||
github.com/hashicorp/go-multierror v1.1.1 // indirect
|
||||
github.com/imdario/mergo v0.3.12 // indirect
|
||||
github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect
|
||||
github.com/jmespath/go-jmespath v0.4.0 // indirect
|
||||
github.com/josharian/intern v1.0.0 // indirect
|
||||
github.com/jpillora/s3 v1.1.4 // indirect
|
||||
github.com/kevinburke/ssh_config v1.2.0 // indirect
|
||||
github.com/klauspost/asmfmt v1.3.2 // indirect
|
||||
github.com/klauspost/compress v1.17.4 // indirect
|
||||
github.com/klauspost/cpuid/v2 v2.2.3 // indirect
|
||||
github.com/kevinburke/ssh_config v0.0.0-20201106050909-4977a11b4351 // indirect
|
||||
github.com/klauspost/compress v1.15.5 // indirect
|
||||
github.com/klauspost/pgzip v1.2.5 // indirect
|
||||
github.com/launchdarkly/ccache v1.1.0 // indirect
|
||||
github.com/launchdarkly/eventsource v1.6.2 // indirect
|
||||
github.com/launchdarkly/go-jsonstream/v3 v3.0.0 // indirect
|
||||
github.com/launchdarkly/go-sdk-common/v3 v3.0.1 // indirect
|
||||
github.com/launchdarkly/go-sdk-events/v2 v2.0.2 // indirect
|
||||
github.com/launchdarkly/go-semver v1.0.2 // indirect
|
||||
github.com/launchdarkly/go-server-sdk-evaluation/v2 v2.0.2 // indirect
|
||||
github.com/lucasb-eyer/go-colorful v1.2.0 // indirect
|
||||
github.com/mailru/easyjson v0.7.7 // indirect
|
||||
github.com/mattn/go-colorable v0.1.13 // indirect
|
||||
github.com/mattn/go-localereader v0.0.1 // indirect
|
||||
github.com/mattn/go-runewidth v0.0.15 // indirect
|
||||
github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
|
||||
github.com/microcosm-cc/bluemonday v1.0.23 // indirect
|
||||
github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8 // indirect
|
||||
github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3 // indirect
|
||||
github.com/mattn/go-isatty v0.0.14 // indirect
|
||||
github.com/mitchellh/go-homedir v1.1.0 // indirect
|
||||
github.com/montanaflynn/stats v0.7.0 // indirect
|
||||
github.com/mtibben/percent v0.2.1 // indirect
|
||||
github.com/muesli/ansi v0.0.0-20211031195517-c9f0611b6c70 // indirect
|
||||
github.com/muesli/cancelreader v0.2.2 // indirect
|
||||
github.com/muesli/termenv v0.15.2 // indirect
|
||||
github.com/nwaples/rardecode/v2 v2.0.0-beta.2 // indirect
|
||||
github.com/olekukonko/tablewriter v0.0.5 // indirect
|
||||
github.com/onsi/ginkgo v1.16.5 // indirect
|
||||
github.com/opencontainers/go-digest v1.0.0 // indirect
|
||||
github.com/opencontainers/image-spec v1.1.0-rc3 // indirect
|
||||
github.com/pierrec/lz4/v4 v4.1.19 // indirect
|
||||
github.com/pjbgf/sha1cd v0.3.0 // indirect
|
||||
github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 // indirect
|
||||
github.com/pierrec/lz4/v4 v4.1.14 // indirect
|
||||
github.com/pkg/diff v0.0.0-20200914180035-5b29258ca4f7 // indirect
|
||||
github.com/pmezard/go-difflib v1.0.0 // indirect
|
||||
github.com/prometheus/client_model v0.4.1-0.20230718164431-9a2bf3000d16 // indirect
|
||||
github.com/prometheus/common v0.44.0 // indirect
|
||||
github.com/prometheus/procfs v0.11.1 // indirect
|
||||
github.com/rivo/uniseg v0.4.2 // indirect
|
||||
github.com/sahilm/fuzzy v0.1.0 // indirect
|
||||
github.com/sirupsen/logrus v1.9.1 // indirect
|
||||
github.com/skeema/knownhosts v1.2.1 // indirect
|
||||
github.com/therootcompany/xz v1.0.1 // indirect
|
||||
github.com/ulikunitz/xz v0.5.11 // indirect
|
||||
github.com/vbatts/tar-split v0.11.3 // indirect
|
||||
github.com/xanzy/ssh-agent v0.3.3 // indirect
|
||||
github.com/xdg-go/pbkdf2 v1.0.0 // indirect
|
||||
github.com/xdg-go/scram v1.1.2 // indirect
|
||||
github.com/xdg-go/stringprep v1.0.4 // indirect
|
||||
github.com/xhit/go-str2duration/v2 v2.1.0 // indirect
|
||||
github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d // indirect
|
||||
github.com/yuin/goldmark v1.5.2 // indirect
|
||||
github.com/yuin/goldmark-emoji v1.0.1 // indirect
|
||||
github.com/ulikunitz/xz v0.5.10 // indirect
|
||||
github.com/xanzy/ssh-agent v0.3.0 // indirect
|
||||
github.com/yusufpapurcu/wmi v1.2.2 // indirect
|
||||
github.com/zeebo/xxh3 v1.0.2 // indirect
|
||||
go.einride.tech/aip v0.60.0 // indirect
|
||||
go.opencensus.io v0.24.0 // indirect
|
||||
go.uber.org/multierr v1.11.0 // indirect
|
||||
go4.org v0.0.0-20200411211856-f5505b9728dd // indirect
|
||||
golang.org/x/mod v0.14.0 // indirect
|
||||
golang.org/x/sys v0.15.0 // indirect
|
||||
golang.org/x/term v0.15.0 // indirect
|
||||
golang.org/x/time v0.5.0 // indirect
|
||||
golang.org/x/tools v0.16.0 // indirect
|
||||
golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect
|
||||
go.opencensus.io v0.23.0 // indirect
|
||||
golang.org/x/mod v0.5.0 // indirect
|
||||
golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab // indirect
|
||||
golang.org/x/text v0.3.7 // indirect
|
||||
golang.org/x/time v0.0.0-20220411224347-583f2d630306 // indirect
|
||||
golang.org/x/tools v0.1.7 // indirect
|
||||
golang.org/x/xerrors v0.0.0-20220609144429-65e65417b02f // indirect
|
||||
google.golang.org/api v0.85.0 // indirect
|
||||
google.golang.org/appengine v1.6.7 // indirect
|
||||
google.golang.org/genproto v0.0.0-20231106174013-bbf56f31fb17 // indirect
|
||||
google.golang.org/genproto/googleapis/api v0.0.0-20231106174013-bbf56f31fb17 // indirect
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20231120223509-83a465c0220f // indirect
|
||||
google.golang.org/grpc v1.59.0 // indirect
|
||||
gopkg.in/square/go-jose.v2 v2.6.0 // indirect
|
||||
google.golang.org/grpc v1.47.0 // indirect
|
||||
gopkg.in/warnings.v0 v0.1.2 // indirect
|
||||
gopkg.in/yaml.v3 v3.0.1 // indirect
|
||||
)
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
# trufflesecurity/protos:1.18-0
|
||||
|
||||
FROM golang:1.21-bullseye
|
||||
FROM golang:1.18-buster
|
||||
|
||||
ARG TARGETARCH
|
||||
ARG TARGETOS
|
||||
|
||||
@@ -1,10 +0,0 @@
|
||||
set terminal png size 800,600
|
||||
set output "hack/bench/versions.png"
|
||||
|
||||
set title "User Time vs. Version"
|
||||
set xlabel "Version"
|
||||
set ylabel "Average User Time (s)"
|
||||
|
||||
set xtics rotate by -45
|
||||
|
||||
plot "hack/bench/plot.txt" using 2:xtic(1) with linespoints linestyle 1 notitle
|
||||
@@ -1,15 +0,0 @@
|
||||
#!/bin/bash
|
||||
|
||||
if [ $# -ne 2 ]; then
|
||||
echo "Usage: $0 <repository to clone> <number_of_versions_back_to_test>"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Get the number of versions back to test from command line argument
|
||||
num_versions="$2"
|
||||
|
||||
test_repo="$1"
|
||||
|
||||
bash hack/bench/versions.sh $test_repo $num_versions | tee hack/bench/plot.txt
|
||||
|
||||
gnuplot hack/bench/plot.gp
|
||||
@@ -1,24 +0,0 @@
|
||||
v3.33.0: 1.402
|
||||
v3.32.2: 1.298
|
||||
v3.32.1: 1.332
|
||||
v3.32.0: 1.348
|
||||
v3.31.6: 2.470
|
||||
v3.31.5: 2.462
|
||||
v3.31.4: 2.460
|
||||
v3.31.3: 2.418
|
||||
v3.31.2: 1.384
|
||||
v3.31.1: 1.344
|
||||
v3.31.0: 1.354
|
||||
v3.30.0: 1.392
|
||||
v3.29.1: 1.382
|
||||
v3.29.0: 1.340
|
||||
v3.28.7: 1.380
|
||||
v3.28.6: 1.308
|
||||
v3.28.5: 2.596
|
||||
v3.28.4: 2.554
|
||||
v3.28.3: 2.582
|
||||
v3.28.1: 2.578
|
||||
v3.28.2: 2.566
|
||||
v3.28.0: 2.552
|
||||
v3.27.1: 2.574
|
||||
v3.26.0: 2.538
|
||||
Binary file not shown.
|
Before Width: | Height: | Size: 9.3 KiB |
@@ -1,80 +0,0 @@
|
||||
#!/bin/bash
|
||||
|
||||
if [ $# -ne 2 ]; then
|
||||
echo "Usage: $0 <repository to clone> <number_of_versions_back_to_test>"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Get the number of versions back to test from command line argument
|
||||
num_versions="$2"
|
||||
|
||||
test_repo="$1"
|
||||
|
||||
num_iterations=5
|
||||
|
||||
# Create a temporary folder to clone the repository
|
||||
repo_tmp=$(mktemp -d)
|
||||
# Set up a trap to remove the temporary folder on exit or failure
|
||||
trap "rm -rf $repo_tmp" EXIT
|
||||
# Clone the test repository to a temporary folder
|
||||
git clone --quiet "$test_repo" $repo_tmp
|
||||
|
||||
|
||||
# Get list of git tags, sorted from newest to oldest
|
||||
tags=$(git tag --sort=-creatordate)
|
||||
|
||||
# Counter to keep track of number of tags checked out
|
||||
count=0
|
||||
|
||||
|
||||
# Loop over tags and checkout each one in turn, up to the specified number of versions
|
||||
for tag in $tags
|
||||
do
|
||||
if [[ $count -eq $num_versions ]]; then
|
||||
break
|
||||
fi
|
||||
|
||||
# Skip RC tags
|
||||
if [[ $tag == *"rc"* ]]; then
|
||||
continue
|
||||
fi
|
||||
|
||||
# Skip alpha tags
|
||||
if [[ $tag == *"alpha"* ]]; then
|
||||
continue
|
||||
fi
|
||||
|
||||
# Use git checkout with the quiet flag to suppress output
|
||||
git checkout $tag --quiet
|
||||
|
||||
# Run make install with suppressed output
|
||||
make install > /dev/null
|
||||
|
||||
# Initialize the variable to store the sum of user times
|
||||
user_time_sum=0
|
||||
|
||||
# Run each iteration 5 times and calculate the average user time
|
||||
for i in {1..$num_iterations}
|
||||
do
|
||||
# Run trufflehog with suppressed output and capture user time with /usr/bin/time
|
||||
tmpfile=$(mktemp)
|
||||
/usr/bin/time -o $tmpfile trufflehog git "file://$repo_tmp" --no-verification --no-update >/dev/null 2>&1
|
||||
time_output=$(cat $tmpfile)
|
||||
rm $tmpfile
|
||||
|
||||
# Extract the user time from the output
|
||||
user_time=$(echo $time_output | awk '{print $3}')
|
||||
|
||||
# Add the user time to the sum
|
||||
user_time_sum=$(echo "$user_time_sum + $user_time" | bc)
|
||||
done
|
||||
|
||||
# Calculate the average user time
|
||||
average_user_time=$(echo "scale=3; $user_time_sum / $num_iterations" | bc)
|
||||
|
||||
# Print the average user time output for this iteration in the specified format
|
||||
echo "$tag: $average_user_time"
|
||||
|
||||
# Increment the counter
|
||||
count=$((count+1))
|
||||
done
|
||||
@@ -44,18 +44,6 @@ If you think that something should be included outside of these guidelines, plea
|
||||
- Go 1.17+
|
||||
- Make
|
||||
|
||||
### Adding New Token Formats to an Existing Scanner
|
||||
|
||||
In some instances, services will update their token format, requiring a new regex to properly detect secrets in addition to supporting the previous token format. Accomodating this can be done without adding a net-new detector. [We provide a `Versioner` interface](https://github.com/trufflesecurity/trufflehog/blob/e18cfd5e0af1469a9f05b8d5732bcc94c39da49c/pkg/detectors/detectors.go#L30) that can be implemented.
|
||||
|
||||
1. Create a copy of the package and append `_v2` to the package and file names. Ex: `<packagename>/` -> `<packagename>_v2`, `<packagename>.go` -> `<packagename>_v2.go`
|
||||
|
||||
Note: Be sure to update the tests to reference the new secret values in GSM, or the tests will fail.
|
||||
|
||||
2. Implement the `Versioner` interface. [GitHub example implementation.](/pkg/detectors/github_old/github_old.go#L22)
|
||||
|
||||
3. Proceed from step 3 of [Creating a new Secret Scanner](#creating-a-new-secret-scanner)
|
||||
|
||||
### Creating a new Secret Scanner
|
||||
|
||||
1. Identify the Secret Detector name from the [/proto/detectors.proto](/proto/detectors.proto) `DetectorType` enum.
|
||||
@@ -74,31 +62,15 @@ Note: Be sure to update the tests to reference the new secret values in GSM, or
|
||||
1. Add the test secret to GCP Secrets. See [managing test secrets](#managing-test-secrets)
|
||||
2. Update the pattern regex and keywords. Try iterating with [regex101.com](http://regex101.com/).
|
||||
3. Update the verifier code to use a non-destructive API call that can determine whether the secret is valid or not.
|
||||
* Make sure you understand [verification indeterminacy](#verification-indeterminacy).
|
||||
4. Update the tests with these test cases at minimum:
|
||||
1. Found and verified (using a credential loaded from GCP Secrets)
|
||||
2. Found and unverified (determinately, i.e. the secret is invalid)
|
||||
3. Found and unverified (indeterminately due to timeout)
|
||||
4. Found and unverified (indeterminately due to an unexpected API response)
|
||||
5. Not found
|
||||
6. Any false positive cases that you come across
|
||||
2. Found and unverified
|
||||
3. Not found
|
||||
4. Any false positive cases that you come across
|
||||
5. Create a merge request for review. CI tests must be passing.
|
||||
|
||||
## Addendum
|
||||
|
||||
### Verification indeterminacy
|
||||
|
||||
There are two types of reasons that secret verification can fail:
|
||||
* The candidate secret is not actually a valid secret.
|
||||
* Something went wrong in the process unrelated to the candidate secret, such as a transient network error or an unexpected API response.
|
||||
|
||||
In Trufflehog parlance, the first type of verification response is called _determinate_ and the second type is called _indeterminate_. Verification code should distinguish between the two by returning an error object in the result struct **only** for indeterminate failures. In general, a verifier should return an error (indicating an indeterminate failure) in all cases that haven't been explicitly identified as determinate failure states.
|
||||
|
||||
For example, consider a hypothetical authentication endpoint that returns `200 OK` for valid credentials and `403 Forbidden` for invalid credentials. The verifier for this endpoint could make an HTTP request and use the response status code to decide what to return:
|
||||
* A `200` response would indicate that verification succeeded. (Or maybe any `2xx` response.)
|
||||
* A `403` response would indicate that verification failed **determinately** and no error object should be returned.
|
||||
* Any other response would indicate that verification failed **indeterminately** and an error object should be returned.
|
||||
|
||||
### Managing Test Secrets
|
||||
|
||||
Do not embed test credentials in the test code. Instead, use GCP Secrets Manager.
|
||||
@@ -108,7 +80,7 @@ Do not embed test credentials in the test code. Instead, use GCP Secrets Manager
|
||||
Note: `/tmp/s` is a valid path on Linux. You will need to change that for Windows or OSX, otherwise you will see an error. On Windows you will also need to install [WSL](https://docs.microsoft.com/en-us/windows/wsl/install).
|
||||
|
||||
```bash
|
||||
gcloud secrets versions access --project trufflehog-testing --secret detectors5 latest > /tmp/s
|
||||
gcloud secrets versions access --project trufflehog-testing --secret detectors3 latest > /tmp/s
|
||||
```
|
||||
|
||||
2. Add the secret that you need for testing.
|
||||
@@ -125,9 +97,8 @@ Do not embed test credentials in the test code. Instead, use GCP Secrets Manager
|
||||
3. Update the secret version with your modification.
|
||||
|
||||
```bash
|
||||
gcloud secrets versions add --project trufflehog-testing detectors5 --data-file /tmp/s
|
||||
gcloud secrets versions add --project trufflehog-testing detectors3 --data-file /tmp/s
|
||||
```
|
||||
Note: We increment the detectors file name `detectors(n+1)` once the previous one exceeds the max size allowed by GSM (65kb).
|
||||
|
||||
4. Access the secret value as shown in the [example code](pkg/detectors/heroku/heroku_test.go).
|
||||
|
||||
@@ -152,7 +123,3 @@ Do not embed test credentials in the test code. Instead, use GCP Secrets Manager
|
||||
```bash
|
||||
make protos
|
||||
```
|
||||
### Testing a detector
|
||||
```bash
|
||||
go test ./pkg/detectors/<detector> -tags=detectors
|
||||
```
|
||||
|
||||
@@ -15,9 +15,9 @@ The purpose of Secret Detectors is to discover secrets with exceptionally high s
|
||||
+ [Sourcing Guidelines](#sourcing-guidelines)
|
||||
+ [Development Guidelines](#development-guidelines)
|
||||
+ [Development Dependencies](#development-dependencies)
|
||||
+ [Creating a new Secret Detector](#creating-a-new-secret-detector)
|
||||
+ [Testing the Detector](#testing-the-detector)
|
||||
+ [Creating a new Secret Scanner](#creating-a-new-secret-scanner)
|
||||
* [Addendum](#addendum)
|
||||
+ [Using a test secret file](#using-a-test-secret-file)
|
||||
+ [Adding Protos in Windows](#adding-protos-in-windows)
|
||||
|
||||
## Getting Started
|
||||
@@ -40,48 +40,34 @@ If you think that something should be included outside of these guidelines, plea
|
||||
- Go 1.17+
|
||||
- Make
|
||||
|
||||
### Adding New Token Formats to an Existing Scanner
|
||||
### Creating a new Secret Scanner
|
||||
|
||||
In some instances, services will update their token format, requiring a new regex to properly detect secrets in addition to supporting the previous token format. Accomodating this can be done without adding a net-new detector. [We provide a `Versioner` interface](https://github.com/trufflesecurity/trufflehog/blob/e18cfd5e0af1469a9f05b8d5732bcc94c39da49c/pkg/detectors/detectors.go#L30) that can be implemented.
|
||||
1. Identify the Secret Detector name from the [/proto/detectors.proto](/proto/detectors.proto) `DetectorType` enum. If necessary, run `make protos` when adding new ones.
|
||||
|
||||
1. Create a copy of the package and append `_v2` to the package and file names. Ex: `<packagename>/` -> `<packagename>_v2`, `<packagename>.go` -> `<packagename>_v2.go`
|
||||
|
||||
Note: Be sure to update the tests to reference the new secret values in GSM, or the tests will fail.
|
||||
|
||||
2. Implement the `Versioner` interface. [GitHub example implementation.](/pkg/detectors/github_old/github_old.go#L22)
|
||||
|
||||
3. Proceed from step 3 of [Creating a new Secret Scanner](#creating-a-new-secret-scanner)
|
||||
|
||||
### Creating a new Secret Detector
|
||||
|
||||
1. Add a new Secret Detector enum to the [`DetectorType` list here](/proto/detectors.proto).
|
||||
|
||||
2. Run `make protos` to update the `.pb` files.
|
||||
|
||||
3. Generate the Secret Detector
|
||||
2. Generate the Secret Detector
|
||||
|
||||
```bash
|
||||
go run hack/generate/generate.go detector <DetectorType enum name>
|
||||
example: go run hack/generate/generate.go detector SampleAPI
|
||||
```
|
||||
4. Add the Secret Detector to TruffleHog's Default Detectors
|
||||
|
||||
Add the secret scanner to the [`pkg/engine/defaults.go`](https://github.com/trufflesecurity/trufflehog/blob/main/pkg/engine/defaults.go) file like [`github.com/trufflesecurity/trufflehog/v3/pkg/detectors/<detector_name>`](https://github.com/trufflesecurity/trufflehog/blob/b71ea27a696bdf1c3141f637fda4ee4936c2f2d6/pkg/engine/defaults.go#L9) and
|
||||
[`<detector_name>.Scanner{}`](https://github.com/trufflesecurity/trufflehog/blob/b71ea27a696bdf1c3141f637fda4ee4936c2f2d6/pkg/engine/defaults.go#L1546)
|
||||
|
||||
5. Complete the Secret Detector.
|
||||
3. Complete the secret detector.
|
||||
|
||||
The previous step templated a boilerplate + some example code as a package in the `pkg/detectors` folder for you to work on.
|
||||
The Secret Detector can be completed with these general steps:
|
||||
The secret detector can be completed with these general steps:
|
||||
|
||||
1. Update the pattern regex and keywords. Try iterating with [regex101.com](http://regex101.com/).
|
||||
2. Update the verifier code to use a non-destructive API call that can determine whether the secret is valid or not.
|
||||
* Make sure you understand [verification indeterminacy](#verification-indeterminacy).
|
||||
3. Create a [test for the detector](#testing-the-detector)
|
||||
4. Create a pull request for review.
|
||||
1. Create a [test secrets file, and export the variable](#using-a-test-secret-file)
|
||||
2. Update the pattern regex and keywords. Try iterating with [regex101.com](http://regex101.com/).
|
||||
3. Update the verifier code to use a non-destructive API call that can determine whether the secret is valid or not.
|
||||
4. Update the tests with these test cases at minimum:
|
||||
1. Found and verified (using a credential loaded from GCP Secrets)
|
||||
2. Found and unverified
|
||||
3. Not found
|
||||
4. Any false positive cases that you come across
|
||||
5. Create a pull request for review.
|
||||
|
||||
### Testing the Detector
|
||||
To ensure the quality of your PR, make sure your tests are passing with verified credentials.
|
||||
## Addendum
|
||||
|
||||
### Using a test secret file
|
||||
|
||||
1. Create a file called `.env` with this env file format:
|
||||
|
||||
@@ -95,49 +81,8 @@ To ensure the quality of your PR, make sure your tests are passing with verified
|
||||
```bash
|
||||
export TEST_SECRET_FILE=".env"
|
||||
```
|
||||
The `.env` file should be in the new detector's directory like this:
|
||||
```
|
||||
├── tailscale
|
||||
│ ├── .env
|
||||
│ ├── tailscale.go
|
||||
│ └── tailscale_test.go
|
||||
```
|
||||
Now that a `.env` file is present, the test file can load secrets locally.
|
||||
|
||||
3. Next, update the tests as necessary. A test file has already been generated by the `go run hack/generate/generate.go` command from earlier. There are 5 cases that have been generated:
|
||||
1. Found and verified (using a credential loaded from the .env file)
|
||||
2. Found and unverified (determinately, i.e. the secret is invalid)
|
||||
3. Found and unverified (indeterminately due to timeout)
|
||||
4. Found and unverified (indeterminately due to an unexpected API response)
|
||||
5. Not found
|
||||
|
||||
Make any necessary updates to the tests. Note there might not be any changes required as the tests generated by the `go run hack/generate/generate.go` command are pretty good.
|
||||
[Here is an exemplary test file for a detector which covers all 5 test cases](https://github.com/trufflesecurity/trufflehog/blob/6f9065b0aae981133a7fa3431c17a5c6213be226/pkg/detectors/browserstack/browserstack_test.go).
|
||||
|
||||
4. Now run the tests and check to make sure they are passing ✔️!
|
||||
```bash
|
||||
go test ./pkg/detectors/<detector> -tags=detectors
|
||||
```
|
||||
|
||||
If the tests are passing, feel free to open a PR!
|
||||
|
||||
|
||||
|
||||
|
||||
## Addendum
|
||||
|
||||
### Verification indeterminacy
|
||||
|
||||
There are two types of reasons that secret verification can fail:
|
||||
* The candidate secret is not actually a valid secret.
|
||||
* Something went wrong in the process unrelated to the candidate secret, such as a transient network error or an unexpected API response.
|
||||
|
||||
In Trufflehog parlance, the first type of verification response is called _determinate_ and the second type is called _indeterminate_. Verification code should distinguish between the two by returning an error object in the result struct **only** for indeterminate failures. In general, a verifier should return an error (indicating an indeterminate failure) in all cases that haven't been explicitly identified as determinate failure states.
|
||||
|
||||
For example, consider a hypothetical authentication endpoint that returns `200 OK` for valid credentials and `403 Forbidden` for invalid credentials. The verifier for this endpoint could make an HTTP request and use the response status code to decide what to return:
|
||||
* A `200` response would indicate that verification succeeded. (Or maybe any `2xx` response.)
|
||||
* A `403` response would indicate that verification failed **determinately** and no error object should be returned.
|
||||
* Any other response would indicate that verification failed **indeterminately** and an error object should be returned.
|
||||
Now, the detector test should attempt to load the given env key from that file.
|
||||
|
||||
### Adding Protos in Windows
|
||||
|
||||
@@ -155,4 +100,3 @@ For example, consider a hypothetical authentication endpoint that returns `200 O
|
||||
```bash
|
||||
make protos
|
||||
```
|
||||
|
||||
|
||||
+10
-11
@@ -2,16 +2,15 @@ package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"log"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"text/template"
|
||||
|
||||
"github.com/alecthomas/kingpin/v2"
|
||||
"github.com/go-errors/errors"
|
||||
"golang.org/x/text/cases"
|
||||
"golang.org/x/text/language"
|
||||
"gopkg.in/alecthomas/kingpin.v2"
|
||||
)
|
||||
|
||||
var (
|
||||
@@ -26,7 +25,7 @@ func main() {
|
||||
log.SetPrefix("😲 [generate] ")
|
||||
|
||||
kingpin.MustParse(app.Parse(os.Args[1:]))
|
||||
nameTitle = cases.Title(language.AmericanEnglish).String(*name)
|
||||
nameTitle = strings.Title(*name)
|
||||
nameLower = strings.ToLower(*name)
|
||||
nameUpper = strings.ToUpper(*name)
|
||||
|
||||
@@ -34,14 +33,14 @@ func main() {
|
||||
case "detector":
|
||||
mustWriteTemplates([]templateJob{
|
||||
{
|
||||
TemplatePath: "pkg/detectors/alchemy/alchemy.go",
|
||||
TemplatePath: "pkg/detectors/heroku/heroku.go",
|
||||
WritePath: filepath.Join(folderPath(), nameLower+".go"),
|
||||
ReplaceString: []string{"alchemy"},
|
||||
ReplaceString: []string{"heroku"},
|
||||
},
|
||||
{
|
||||
TemplatePath: "pkg/detectors/alchemy/alchemy_test.go",
|
||||
TemplatePath: "pkg/detectors/heroku/heroku_test.go",
|
||||
WritePath: filepath.Join(folderPath(), nameLower+"_test.go"),
|
||||
ReplaceString: []string{"alchemy"},
|
||||
ReplaceString: []string{"heroku"},
|
||||
},
|
||||
})
|
||||
// case "source":
|
||||
@@ -67,7 +66,7 @@ type templateJob struct {
|
||||
}
|
||||
|
||||
func mustWriteTemplates(jobs []templateJob) {
|
||||
log.Printf("Generating %s %s\n", cases.Title(language.AmericanEnglish).String(*kind), nameTitle)
|
||||
log.Printf("Generating %s %s\n", strings.Title(*kind), nameTitle)
|
||||
|
||||
// Make the folder.
|
||||
log.Printf("Creating folder %s\n", folderPath())
|
||||
@@ -78,7 +77,7 @@ func mustWriteTemplates(jobs []templateJob) {
|
||||
|
||||
// Write the files from templates.
|
||||
for _, job := range jobs {
|
||||
tmplBytes, err := os.ReadFile(job.TemplatePath)
|
||||
tmplBytes, err := ioutil.ReadFile(job.TemplatePath)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
@@ -86,7 +85,7 @@ func mustWriteTemplates(jobs []templateJob) {
|
||||
|
||||
for _, rplString := range job.ReplaceString {
|
||||
tmplRaw = strings.ReplaceAll(tmplRaw, strings.ToLower(rplString), "<<.NameLower>>")
|
||||
tmplRaw = strings.ReplaceAll(tmplRaw, cases.Title(language.AmericanEnglish).String(rplString), "<<.NameTitle>>")
|
||||
tmplRaw = strings.ReplaceAll(tmplRaw, strings.Title(rplString), "<<.NameTitle>>")
|
||||
tmplRaw = strings.ReplaceAll(tmplRaw, strings.ToUpper(rplString), "<<.NameUpper>>")
|
||||
}
|
||||
|
||||
|
||||
+24
-43
@@ -1,6 +1,7 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"reflect"
|
||||
@@ -10,15 +11,14 @@ import (
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"github.com/alecthomas/kingpin/v2"
|
||||
"github.com/paulbellamy/ratecounter"
|
||||
log "github.com/sirupsen/logrus"
|
||||
"golang.org/x/sync/semaphore"
|
||||
"gopkg.in/alecthomas/kingpin.v2"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/context"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/decoders"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/engine"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/log"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/source_metadatapb"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
|
||||
@@ -42,20 +42,6 @@ var (
|
||||
)
|
||||
|
||||
func main() {
|
||||
// setup logger
|
||||
logger, flush := log.New("trufflehog", log.WithConsoleSink(os.Stderr))
|
||||
// make it the default logger for contexts
|
||||
context.SetDefaultLogger(logger)
|
||||
defer func() { _ = flush() }()
|
||||
logFatal := func(err error, message string, keyAndVals ...any) {
|
||||
logger.Error(err, message, keyAndVals...)
|
||||
if err != nil {
|
||||
os.Exit(1)
|
||||
return
|
||||
}
|
||||
os.Exit(0)
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), time.Hour*2)
|
||||
var cancelOnce sync.Once
|
||||
defer cancelOnce.Do(cancel)
|
||||
@@ -74,7 +60,7 @@ func main() {
|
||||
selectedScanners := map[string]detectors.Detector{}
|
||||
allScanners := getAllScanners()
|
||||
|
||||
allDecoders := decoders.DefaultDecoders()
|
||||
decoders := decoders.DefaultDecoders()
|
||||
|
||||
input := strings.ToLower(*scanCmdDetector)
|
||||
if input == "all" {
|
||||
@@ -82,19 +68,19 @@ func main() {
|
||||
} else {
|
||||
_, ok := allScanners[input]
|
||||
if !ok {
|
||||
logFatal(fmt.Errorf("invalid input"), "could not find scanner by that name")
|
||||
log.Fatal("could not find scanner by that name")
|
||||
}
|
||||
selectedScanners[input] = allScanners[input]
|
||||
}
|
||||
if len(selectedScanners) == 0 {
|
||||
logFatal(fmt.Errorf("invalid input"), "no detectors selected")
|
||||
log.Fatal("no detectors selected")
|
||||
}
|
||||
|
||||
for _, excluded := range *scanCmdExclude {
|
||||
delete(selectedScanners, excluded)
|
||||
}
|
||||
|
||||
logger.Info("loaded secret detectors", "count", len(selectedScanners)+3)
|
||||
log.Infof("loaded %d secret detectors", len(selectedScanners)+3)
|
||||
|
||||
var wgScanners sync.WaitGroup
|
||||
|
||||
@@ -106,7 +92,7 @@ func main() {
|
||||
time.Sleep(60 * time.Second)
|
||||
counter.Incr(int64(chunkCounter - prev))
|
||||
prev = chunkCounter
|
||||
logger.Info("chunk scan rate per second", "rate", counter.Rate()/60)
|
||||
log.Infof("chunk scan rate: %d/sec", counter.Rate()/60)
|
||||
}
|
||||
}()
|
||||
|
||||
@@ -121,7 +107,7 @@ func main() {
|
||||
|
||||
for chunk := range chunksChan {
|
||||
for name, scanner := range selectedScanners {
|
||||
for _, dec := range allDecoders {
|
||||
for _, dec := range decoders {
|
||||
decoded := dec.FromChunk(&sources.Chunk{Data: chunk.Data})
|
||||
if decoded != nil {
|
||||
foundKeyword := false
|
||||
@@ -136,7 +122,7 @@ func main() {
|
||||
|
||||
res, err := scanner.FromData(ctx, *scanVerify, decoded.Data)
|
||||
if err != nil {
|
||||
logFatal(err, "error scanning chunk")
|
||||
log.Fatal(err)
|
||||
}
|
||||
if len(res) > 0 {
|
||||
if resCounter[name] == nil {
|
||||
@@ -145,19 +131,16 @@ func main() {
|
||||
}
|
||||
atomic.AddUint64(resCounter[name], uint64(len(res)))
|
||||
if *scanThreshold != 0 && int(*resCounter[name]) > *scanThreshold {
|
||||
logger.Error(
|
||||
fmt.Errorf("exceeded result threshold"), "snifftest failed",
|
||||
"scanner", name, "threshold", *scanThreshold,
|
||||
)
|
||||
log.WithField("scanner", name).Errorf("exceeded result threshold of %d", *scanThreshold)
|
||||
failed = true
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
if *scanPrintRes {
|
||||
for _, r := range res {
|
||||
logger := logger.WithValues("secret", name, "meta", chunk.SourceMetadata, "result", string(r.Raw))
|
||||
logger := log.WithField("secret", name).WithField("meta", chunk.SourceMetadata).WithField("result", string(r.Raw))
|
||||
if *scanPrintChunkRes {
|
||||
logger = logger.WithValues("chunk", string(decoded.Data))
|
||||
logger = logger.WithField("chunk", string(decoded.Data))
|
||||
}
|
||||
logger.Info("result")
|
||||
}
|
||||
@@ -173,20 +156,18 @@ func main() {
|
||||
}
|
||||
|
||||
for _, repo := range strings.Split(*scanCmdRepo, ",") {
|
||||
if err := sem.Acquire(ctx, 1); err != nil {
|
||||
logFatal(err, "timed out waiting for semaphore")
|
||||
}
|
||||
sem.Acquire(ctx, 1)
|
||||
wgChunkers.Add(1)
|
||||
go func(r string) {
|
||||
defer sem.Release(1)
|
||||
defer wgChunkers.Done()
|
||||
logger.Info("cloning repo", "repo", r)
|
||||
path, repo, err := git.CloneRepoUsingUnauthenticated(ctx, r)
|
||||
log.Infof("cloning %s", r)
|
||||
path, repo, err := git.CloneRepoUsingUnauthenticated(r)
|
||||
if err != nil {
|
||||
logFatal(err, "error cloning repo", "repo", r)
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
logger.Info("cloned repo", "repo", r)
|
||||
log.Infof("cloned %s", r)
|
||||
|
||||
s := git.NewGit(sourcespb.SourceType_SOURCE_TYPE_GIT, 0, 0, "snifftest", false, runtime.NumCPU(),
|
||||
func(file, email, commit, timestamp, repository string, line int64) *source_metadatapb.MetaData {
|
||||
@@ -203,12 +184,12 @@ func main() {
|
||||
}
|
||||
})
|
||||
|
||||
logger.Info("scanning repo", "repo", r)
|
||||
err = s.ScanRepo(ctx, repo, path, git.NewScanOptions(), sources.ChanReporter{Ch: chunksChan})
|
||||
log.Infof("scanning %s", r)
|
||||
err = s.ScanRepo(ctx, repo, path, git.NewScanOptions(), chunksChan)
|
||||
if err != nil {
|
||||
logFatal(err, "error scanning repo")
|
||||
log.Fatal(err)
|
||||
}
|
||||
logger.Info("scanned repo", "repo", r)
|
||||
log.Infof("scanned %s", r)
|
||||
defer os.RemoveAll(path)
|
||||
}(repo)
|
||||
}
|
||||
@@ -220,9 +201,9 @@ func main() {
|
||||
|
||||
wgScanners.Wait()
|
||||
|
||||
logger.Info("completed snifftest", "chunks", chunkCounter)
|
||||
log.WithField("chunks", chunkCounter).Info("completed")
|
||||
for scanner, resultsCount := range resCounter {
|
||||
logger.Info(scanner, "results", *resultsCount)
|
||||
log.WithField("results", *resultsCount).Info(scanner)
|
||||
}
|
||||
|
||||
if failed {
|
||||
|
||||
@@ -1,7 +1,9 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"log"
|
||||
"net/http"
|
||||
_ "net/http/pprof"
|
||||
"os"
|
||||
@@ -9,123 +11,77 @@ import (
|
||||
"strconv"
|
||||
"strings"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"github.com/alecthomas/kingpin/v2"
|
||||
"github.com/felixge/fgprof"
|
||||
"github.com/go-logr/logr"
|
||||
"github.com/gorilla/mux"
|
||||
"github.com/jpillora/overseer"
|
||||
"github.com/mattn/go-isatty"
|
||||
"google.golang.org/protobuf/types/known/anypb"
|
||||
"github.com/sirupsen/logrus"
|
||||
"gopkg.in/alecthomas/kingpin.v2"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/cleantemp"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/config"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/context"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/decoders"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/engine"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/handlers"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/log"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/output"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/tui"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/updater"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/version"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/decoders"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/engine"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/output"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/sources/git"
|
||||
)
|
||||
|
||||
var (
|
||||
cli = kingpin.New("TruffleHog", "TruffleHog is a tool for finding credentials.")
|
||||
cmd string
|
||||
debug = cli.Flag("debug", "Run in debug mode.").Bool()
|
||||
trace = cli.Flag("trace", "Run in trace mode.").Bool()
|
||||
profile = cli.Flag("profile", "Enables profiling and sets a pprof and fgprof server on :18066.").Bool()
|
||||
localDev = cli.Flag("local-dev", "Hidden feature to disable overseer for local dev.").Hidden().Bool()
|
||||
jsonOut = cli.Flag("json", "Output in JSON format.").Short('j').Bool()
|
||||
jsonLegacy = cli.Flag("json-legacy", "Use the pre-v3.0 JSON format. Only works with git, gitlab, and github sources.").Bool()
|
||||
gitHubActionsFormat = cli.Flag("github-actions", "Output in GitHub Actions format.").Bool()
|
||||
concurrency = cli.Flag("concurrency", "Number of concurrent workers.").Default(strconv.Itoa(runtime.NumCPU())).Int()
|
||||
noVerification = cli.Flag("no-verification", "Don't verify the results.").Bool()
|
||||
onlyVerified = cli.Flag("only-verified", "Only output verified results.").Bool()
|
||||
filterUnverified = cli.Flag("filter-unverified", "Only output first unverified result per chunk per detector if there are more than one results.").Bool()
|
||||
filterEntropy = cli.Flag("filter-entropy", "Filter unverified results with Shannon entropy. Start with 3.0.").Float64()
|
||||
configFilename = cli.Flag("config", "Path to configuration file.").ExistingFile()
|
||||
cli = kingpin.New("TruffleHog", "TruffleHog is a tool for finding credentials.")
|
||||
cmd string
|
||||
debug = cli.Flag("debug", "Run in debug mode.").Bool()
|
||||
trace = cli.Flag("trace", "Run in trace mode.").Bool()
|
||||
jsonOut = cli.Flag("json", "Output in JSON format.").Short('j').Bool()
|
||||
jsonLegacy = cli.Flag("json-legacy", "Use the pre-v3.0 JSON format. Only works with git, gitlab, and github sources.").Bool()
|
||||
concurrency = cli.Flag("concurrency", "Number of concurrent workers.").Default(strconv.Itoa(runtime.NumCPU())).Int()
|
||||
noVerification = cli.Flag("no-verification", "Don't verify the results.").Bool()
|
||||
onlyVerified = cli.Flag("only-verified", "Only output verified results.").Bool()
|
||||
// rules = cli.Flag("rules", "Path to file with custom rules.").String()
|
||||
printAvgDetectorTime = cli.Flag("print-avg-detector-time", "Print the average time spent on each detector.").Bool()
|
||||
noUpdate = cli.Flag("no-update", "Don't check for updates.").Bool()
|
||||
fail = cli.Flag("fail", "Exit with code 183 if results are found.").Bool()
|
||||
verifiers = cli.Flag("verifier", "Set custom verification endpoints.").StringMap()
|
||||
archiveMaxSize = cli.Flag("archive-max-size", "Maximum size of archive to scan. (Byte units eg. 512B, 2KB, 4MB)").Bytes()
|
||||
archiveMaxDepth = cli.Flag("archive-max-depth", "Maximum depth of archive to scan.").Int()
|
||||
archiveTimeout = cli.Flag("archive-timeout", "Maximum time to spend extracting an archive.").Duration()
|
||||
includeDetectors = cli.Flag("include-detectors", "Comma separated list of detector types to include. Protobuf name or IDs may be used, as well as ranges.").Default("all").String()
|
||||
excludeDetectors = cli.Flag("exclude-detectors", "Comma separated list of detector types to exclude. Protobuf name or IDs may be used, as well as ranges. IDs defined here take precedence over the include list.").String()
|
||||
|
||||
gitScan = cli.Command("git", "Find credentials in git repositories.")
|
||||
gitScanURI = gitScan.Arg("uri", "Git repository URL. https://, file://, or ssh:// schema expected.").Required().String()
|
||||
gitScanURI = gitScan.Arg("uri", "Git repository URL. https:// or file:// schema expected.").Required().String()
|
||||
gitScanIncludePaths = gitScan.Flag("include-paths", "Path to file with newline separated regexes for files to include in scan.").Short('i').String()
|
||||
gitScanExcludePaths = gitScan.Flag("exclude-paths", "Path to file with newline separated regexes for files to exclude in scan.").Short('x').String()
|
||||
gitScanExcludeGlobs = gitScan.Flag("exclude-globs", "Comma separated list of globs to exclude in scan. This option filters at the `git log` level, resulting in faster scans.").String()
|
||||
gitScanSinceCommit = gitScan.Flag("since-commit", "Commit to start scan from.").String()
|
||||
gitScanBranch = gitScan.Flag("branch", "Branch to scan.").String()
|
||||
gitScanMaxDepth = gitScan.Flag("max-depth", "Maximum depth of commits to scan.").Int()
|
||||
gitScanBare = gitScan.Flag("bare", "Scan bare repository (e.g. useful while using in pre-receive hooks)").Bool()
|
||||
_ = gitScan.Flag("allow", "No-op flag for backwards compat.").Bool()
|
||||
_ = gitScan.Flag("entropy", "No-op flag for backwards compat.").Bool()
|
||||
_ = gitScan.Flag("regex", "No-op flag for backwards compat.").Bool()
|
||||
|
||||
githubScan = cli.Command("github", "Find credentials in GitHub repositories.")
|
||||
githubScanEndpoint = githubScan.Flag("endpoint", "GitHub endpoint.").Default("https://api.github.com").String()
|
||||
githubScanRepos = githubScan.Flag("repo", `GitHub repository to scan. You can repeat this flag. Example: "https://github.com/dustin-decker/secretsandstuff"`).Strings()
|
||||
githubScanOrgs = githubScan.Flag("org", `GitHub organization to scan. You can repeat this flag. Example: "trufflesecurity"`).Strings()
|
||||
githubScanToken = githubScan.Flag("token", "GitHub token. Can be provided with environment variable GITHUB_TOKEN.").Envar("GITHUB_TOKEN").String()
|
||||
githubIncludeForks = githubScan.Flag("include-forks", "Include forks in scan.").Bool()
|
||||
githubIncludeMembers = githubScan.Flag("include-members", "Include organization member repositories in scan.").Bool()
|
||||
githubIncludeRepos = githubScan.Flag("include-repos", `Repositories to include in an org scan. This can also be a glob pattern. You can repeat this flag. Must use Github repo full name. Example: "trufflesecurity/trufflehog", "trufflesecurity/t*"`).Strings()
|
||||
githubExcludeRepos = githubScan.Flag("exclude-repos", `Repositories to exclude in an org scan. This can also be a glob pattern. You can repeat this flag. Must use Github repo full name. Example: "trufflesecurity/driftwood", "trufflesecurity/d*"`).Strings()
|
||||
githubScanIncludePaths = githubScan.Flag("include-paths", "Path to file with newline separated regexes for files to include in scan.").Short('i').String()
|
||||
githubScanExcludePaths = githubScan.Flag("exclude-paths", "Path to file with newline separated regexes for files to exclude in scan.").Short('x').String()
|
||||
githubScanIssueComments = githubScan.Flag("issue-comments", "Include issue descriptions and comments in scan.").Bool()
|
||||
githubScanPRComments = githubScan.Flag("pr-comments", "Include pull request descriptions and comments in scan.").Bool()
|
||||
githubScanGistComments = githubScan.Flag("gist-comments", "Include gist comments in scan.").Bool()
|
||||
githubScan = cli.Command("github", "Find credentials in GitHub repositories.")
|
||||
githubScanEndpoint = githubScan.Flag("endpoint", "GitHub endpoint.").Default("https://api.github.com").String()
|
||||
githubScanRepos = githubScan.Flag("repo", `GitHub repository to scan. You can repeat this flag. Example: "https://github.com/dustin-decker/secretsandstuff"`).Strings()
|
||||
githubScanOrgs = githubScan.Flag("org", `GitHub organization to scan. You can repeat this flag. Example: "trufflesecurity"`).Strings()
|
||||
githubScanToken = githubScan.Flag("token", "GitHub token.").String()
|
||||
githubIncludeForks = githubScan.Flag("include-forks", "Include forks in scan.").Bool()
|
||||
githubIncludeMembers = githubScan.Flag("include-members", "Include organization member repositories in scan.").Bool()
|
||||
|
||||
gitlabScan = cli.Command("gitlab", "Find credentials in GitLab repositories.")
|
||||
// TODO: Add more GitLab options
|
||||
gitlabScanEndpoint = gitlabScan.Flag("endpoint", "GitLab endpoint.").Default("https://gitlab.com").String()
|
||||
gitlabScanRepos = gitlabScan.Flag("repo", "GitLab repo url. You can repeat this flag. Leave empty to scan all repos accessible with provided credential. Example: https://gitlab.com/org/repo.git").Strings()
|
||||
gitlabScanToken = gitlabScan.Flag("token", "GitLab token. Can be provided with environment variable GITLAB_TOKEN.").Envar("GITLAB_TOKEN").Required().String()
|
||||
gitlabScanIncludePaths = gitlabScan.Flag("include-paths", "Path to file with newline separated regexes for files to include in scan.").Short('i').String()
|
||||
gitlabScanExcludePaths = gitlabScan.Flag("exclude-paths", "Path to file with newline separated regexes for files to exclude in scan.").Short('x').String()
|
||||
gitlabScanEndpoint = gitlabScan.Flag("endpoint", "GitLab endpoint.").Default("https://gitlab.com").String()
|
||||
gitlabScanRepos = gitlabScan.Flag("repo", "GitLab repo url. You can repeat this flag. Leave empty to scan all repos accessible with provided credential. Example: https://gitlab.com/org/repo.git").Strings()
|
||||
gitlabScanToken = gitlabScan.Flag("token", "GitLab token.").Required().String()
|
||||
|
||||
filesystemScan = cli.Command("filesystem", "Find credentials in a filesystem.")
|
||||
filesystemPaths = filesystemScan.Arg("path", "Path to file or directory to scan.").Strings()
|
||||
// DEPRECATED: --directory is deprecated in favor of arguments.
|
||||
filesystemDirectories = filesystemScan.Flag("directory", "Path to directory to scan. You can repeat this flag.").Strings()
|
||||
filesystemScan = cli.Command("filesystem", "Find credentials in a filesystem.")
|
||||
filesystemDirectories = filesystemScan.Flag("directory", "Path to directory to scan. You can repeat this flag.").Required().Strings()
|
||||
// TODO: Add more filesystem scan options. Currently only supports scanning a list of directories.
|
||||
// filesystemScanRecursive = filesystemScan.Flag("recursive", "Scan recursively.").Short('r').Bool()
|
||||
filesystemScanIncludePaths = filesystemScan.Flag("include-paths", "Path to file with newline separated regexes for files to include in scan.").Short('i').String()
|
||||
filesystemScanExcludePaths = filesystemScan.Flag("exclude-paths", "Path to file with newline separated regexes for files to exclude in scan.").Short('x').String()
|
||||
// filesystemScanIncludePaths = filesystemScan.Flag("include-paths", "Path to file with newline separated regexes for files to include in scan.").Short('i').String()
|
||||
// filesystemScanExcludePaths = filesystemScan.Flag("exclude-paths", "Path to file with newline separated regexes for files to exclude in scan.").Short('x').String()
|
||||
|
||||
s3Scan = cli.Command("s3", "Find credentials in S3 buckets.")
|
||||
s3ScanKey = s3Scan.Flag("key", "S3 key used to authenticate. Can be provided with environment variable AWS_ACCESS_KEY_ID.").Envar("AWS_ACCESS_KEY_ID").String()
|
||||
s3ScanRoleArns = s3Scan.Flag("role-arn", "Specify the ARN of an IAM role to assume for scanning. You can repeat this flag.").Strings()
|
||||
s3ScanSecret = s3Scan.Flag("secret", "S3 secret used to authenticate. Can be provided with environment variable AWS_SECRET_ACCESS_KEY.").Envar("AWS_SECRET_ACCESS_KEY").String()
|
||||
s3ScanSessionToken = s3Scan.Flag("session-token", "S3 session token used to authenticate temporary credentials. Can be provided with environment variable AWS_SESSION_TOKEN.").Envar("AWS_SESSION_TOKEN").String()
|
||||
s3ScanCloudEnv = s3Scan.Flag("cloud-environment", "Use IAM credentials in cloud environment.").Bool()
|
||||
s3ScanBuckets = s3Scan.Flag("bucket", "Name of S3 bucket to scan. You can repeat this flag.").Strings()
|
||||
s3ScanMaxObjectSize = s3Scan.Flag("max-object-size", "Maximum size of objects to scan. Objects larger than this will be skipped. (Byte units eg. 512B, 2KB, 4MB)").Default("250MB").Bytes()
|
||||
|
||||
gcsScan = cli.Command("gcs", "Find credentials in GCS buckets.")
|
||||
gcsProjectID = gcsScan.Flag("project-id", "GCS project ID used to authenticate. Can NOT be used with unauth scan. Can be provided with environment variable GOOGLE_CLOUD_PROJECT.").Envar("GOOGLE_CLOUD_PROJECT").String()
|
||||
gcsCloudEnv = gcsScan.Flag("cloud-environment", "Use Application Default Credentials, IAM credentials to authenticate.").Bool()
|
||||
gcsServiceAccount = gcsScan.Flag("service-account", "Path to GCS service account JSON file.").ExistingFile()
|
||||
gcsWithoutAuth = gcsScan.Flag("without-auth", "Scan GCS buckets without authentication. This will only work for public buckets").Bool()
|
||||
gcsAPIKey = gcsScan.Flag("api-key", "GCS API key used to authenticate. Can be provided with environment variable GOOGLE_API_KEY.").Envar("GOOGLE_API_KEY").String()
|
||||
gcsIncludeBuckets = gcsScan.Flag("include-buckets", "Buckets to scan. Comma separated list of buckets. You can repeat this flag. Globs are supported").Short('I').Strings()
|
||||
gcsExcludeBuckets = gcsScan.Flag("exclude-buckets", "Buckets to exclude from scan. Comma separated list of buckets. Globs are supported").Short('X').Strings()
|
||||
gcsIncludeObjects = gcsScan.Flag("include-objects", "Objects to scan. Comma separated list of objects. you can repeat this flag. Globs are supported").Short('i').Strings()
|
||||
gcsExcludeObjects = gcsScan.Flag("exclude-objects", "Objects to exclude from scan. Comma separated list of objects. You can repeat this flag. Globs are supported").Short('x').Strings()
|
||||
gcsMaxObjectSize = gcsScan.Flag("max-object-size", "Maximum size of objects to scan. Objects larger than this will be skipped. (Byte units eg. 512B, 2KB, 4MB)").Default("10MB").Bytes()
|
||||
s3Scan = cli.Command("s3", "Find credentials in S3 buckets.")
|
||||
s3ScanKey = s3Scan.Flag("key", "S3 key used to authenticate.").String()
|
||||
s3ScanSecret = s3Scan.Flag("secret", "S3 secret used to authenticate.").String()
|
||||
s3ScanCloudEnv = s3Scan.Flag("cloud-environment", "Use IAM credentials in cloud environment.").Bool()
|
||||
s3ScanBuckets = s3Scan.Flag("bucket", "Name of S3 bucket to scan. You can repeat this flag.").Strings()
|
||||
|
||||
syslogScan = cli.Command("syslog", "Scan syslog")
|
||||
syslogAddress = syslogScan.Flag("address", "Address and port to listen on for syslog. Example: 127.0.0.1:514").String()
|
||||
@@ -133,15 +89,6 @@ var (
|
||||
syslogTLSCert = syslogScan.Flag("cert", "Path to TLS cert.").String()
|
||||
syslogTLSKey = syslogScan.Flag("key", "Path to TLS key.").String()
|
||||
syslogFormat = syslogScan.Flag("format", "Log format. Can be rfc3164 or rfc5424").String()
|
||||
|
||||
circleCiScan = cli.Command("circleci", "Scan CircleCI")
|
||||
circleCiScanToken = circleCiScan.Flag("token", "CircleCI token. Can also be provided with environment variable").Envar("CIRCLECI_TOKEN").Required().String()
|
||||
|
||||
dockerScan = cli.Command("docker", "Scan Docker Image")
|
||||
dockerScanImages = dockerScan.Flag("image", "Docker image to scan. Use the file:// prefix to point to a local tarball, otherwise a image registry is assumed.").Required().Strings()
|
||||
|
||||
travisCiScan = cli.Command("travisci", "Scan TravisCI")
|
||||
travisCiScanToken = travisCiScan.Flag("token", "TravisCI token. Can also be provided with environment variable").Envar("TRAVISCI_TOKEN").Required().String()
|
||||
)
|
||||
|
||||
func init() {
|
||||
@@ -154,49 +101,24 @@ func init() {
|
||||
}
|
||||
|
||||
cli.Version("trufflehog " + version.BuildVersion)
|
||||
|
||||
// Support -h for help
|
||||
cli.HelpFlag.Short('h')
|
||||
|
||||
if len(os.Args) <= 1 && isatty.IsTerminal(os.Stdout.Fd()) {
|
||||
args := tui.Run()
|
||||
if len(args) == 0 {
|
||||
os.Exit(0)
|
||||
}
|
||||
|
||||
// Overwrite the Args slice so overseer works properly.
|
||||
os.Args = os.Args[:1]
|
||||
os.Args = append(os.Args, args...)
|
||||
}
|
||||
|
||||
cmd = kingpin.MustParse(cli.Parse(os.Args[1:]))
|
||||
|
||||
if *jsonOut {
|
||||
logrus.SetFormatter(&logrus.JSONFormatter{})
|
||||
}
|
||||
switch {
|
||||
case *trace:
|
||||
log.SetLevel(5)
|
||||
logrus.SetLevel(logrus.TraceLevel)
|
||||
logrus.Debugf("running version %s", version.BuildVersion)
|
||||
case *debug:
|
||||
log.SetLevel(2)
|
||||
logrus.SetLevel(logrus.DebugLevel)
|
||||
logrus.Debugf("running version %s", version.BuildVersion)
|
||||
default:
|
||||
logrus.SetLevel(logrus.InfoLevel)
|
||||
}
|
||||
}
|
||||
|
||||
func main() {
|
||||
// setup logger
|
||||
logFormat := log.WithConsoleSink
|
||||
if *jsonOut {
|
||||
logFormat = log.WithJSONSink
|
||||
}
|
||||
logger, sync := log.New("trufflehog", logFormat(os.Stderr))
|
||||
// make it the default logger for contexts
|
||||
context.SetDefaultLogger(logger)
|
||||
|
||||
if *localDev {
|
||||
run(overseer.State{})
|
||||
os.Exit(0)
|
||||
}
|
||||
|
||||
defer func() { _ = sync() }()
|
||||
logFatal := logFatalFunc(logger)
|
||||
|
||||
updateCfg := overseer.Config{
|
||||
Program: run,
|
||||
Debug: *debug,
|
||||
@@ -214,20 +136,14 @@ func main() {
|
||||
|
||||
err := overseer.RunErr(updateCfg)
|
||||
if err != nil {
|
||||
logFatal(err, "error occurred with trufflehog updater 🐷")
|
||||
logrus.WithError(err).Fatal("error occured with trufflehog updater 🐷")
|
||||
}
|
||||
|
||||
ctx := context.Background()
|
||||
|
||||
go cleantemp.RunCleanupLoop(ctx)
|
||||
}
|
||||
|
||||
func run(state overseer.State) {
|
||||
ctx := context.Background()
|
||||
logger := ctx.Logger()
|
||||
logFatal := logFatalFunc(logger)
|
||||
|
||||
logger.V(2).Info(fmt.Sprintf("trufflehog %s", version.BuildVersion))
|
||||
if *debug {
|
||||
fmt.Println("trufflehog " + version.BuildVersion)
|
||||
}
|
||||
|
||||
if *githubScanToken != "" {
|
||||
// NOTE: this kludge is here to do an authenticated shallow commit
|
||||
@@ -240,394 +156,158 @@ func run(state overseer.State) {
|
||||
*concurrency = 1
|
||||
}
|
||||
|
||||
if *profile {
|
||||
if *debug {
|
||||
go func() {
|
||||
router := http.NewServeMux()
|
||||
router.Handle("/debug/pprof/", http.DefaultServeMux)
|
||||
router.Handle("/debug/fgprof", fgprof.Handler())
|
||||
logger.Info("starting pprof and fgprof server on :18066 /debug/pprof and /debug/fgprof")
|
||||
router := mux.NewRouter()
|
||||
router.PathPrefix("/debug/pprof").Handler(http.DefaultServeMux)
|
||||
router.PathPrefix("/debug/fgprof").Handler(fgprof.Handler())
|
||||
logrus.Info("starting pprof and fgprof server on :18066 /debug/pprof and /debug/fgprof")
|
||||
if err := http.ListenAndServe(":18066", router); err != nil {
|
||||
logger.Error(err, "error serving pprof and fgprof")
|
||||
logrus.Error(err)
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
conf := &config.Config{}
|
||||
if *configFilename != "" {
|
||||
var err error
|
||||
conf, err = config.Read(*configFilename)
|
||||
if err != nil {
|
||||
logFatal(err, "error parsing the provided configuration file")
|
||||
}
|
||||
ctx := context.TODO()
|
||||
e := engine.Start(ctx,
|
||||
engine.WithConcurrency(*concurrency),
|
||||
engine.WithDecoders(decoders.DefaultDecoders()...),
|
||||
engine.WithDetectors(!*noVerification, engine.DefaultDetectors()...),
|
||||
)
|
||||
|
||||
filter, err := common.FilterFromFiles(*gitScanIncludePaths, *gitScanExcludePaths)
|
||||
if err != nil {
|
||||
logrus.WithError(err).Fatal("could not create filter")
|
||||
}
|
||||
|
||||
if *archiveMaxSize != 0 {
|
||||
handlers.SetArchiveMaxSize(int(*archiveMaxSize))
|
||||
}
|
||||
if *archiveMaxDepth != 0 {
|
||||
handlers.SetArchiveMaxDepth(*archiveMaxDepth)
|
||||
}
|
||||
if *archiveTimeout != 0 {
|
||||
handlers.SetArchiveMaxTimeout(*archiveTimeout)
|
||||
}
|
||||
var repoPath string
|
||||
var remote bool
|
||||
switch cmd {
|
||||
case gitScan.FullCommand():
|
||||
repoPath, remote, err = git.PrepareRepoSinceCommit(*gitScanURI, *gitScanSinceCommit)
|
||||
if err != nil || repoPath == "" {
|
||||
logrus.WithError(err).Fatal("error preparing git repo for scanning")
|
||||
}
|
||||
if remote {
|
||||
defer os.RemoveAll(repoPath)
|
||||
}
|
||||
|
||||
// Build include and exclude detector sets for filtering on engine initialization.
|
||||
// Exit if there was an error to inform the user of the misconfiguration.
|
||||
var includeDetectorSet, excludeDetectorSet map[config.DetectorID]struct{}
|
||||
var detectorsWithCustomVerifierEndpoints map[config.DetectorID][]string
|
||||
{
|
||||
includeList, err := config.ParseDetectors(*includeDetectors)
|
||||
if err != nil {
|
||||
logFatal(err, "invalid include list detector configuration")
|
||||
config := sources.Config{
|
||||
RepoPath: repoPath,
|
||||
HeadRef: *gitScanBranch,
|
||||
BaseRef: *gitScanSinceCommit,
|
||||
MaxDepth: *gitScanMaxDepth,
|
||||
Filter: filter,
|
||||
}
|
||||
excludeList, err := config.ParseDetectors(*excludeDetectors)
|
||||
if err != nil {
|
||||
logFatal(err, "invalid exclude list detector configuration")
|
||||
}
|
||||
detectorsWithCustomVerifierEndpoints, err = config.ParseVerifierEndpoints(*verifiers)
|
||||
if err != nil {
|
||||
logFatal(err, "invalid verifier detector configuration")
|
||||
}
|
||||
includeDetectorSet = detectorTypeToSet(includeList)
|
||||
excludeDetectorSet = detectorTypeToSet(excludeList)
|
||||
}
|
||||
|
||||
// Verify that all the user-provided detectors support the optional
|
||||
// detector features.
|
||||
{
|
||||
if err, id := verifyDetectorsAreVersioner(includeDetectorSet); err != nil {
|
||||
logFatal(err, "invalid include list detector configuration", "detector", id)
|
||||
if err = e.ScanGit(ctx, config); err != nil {
|
||||
logrus.WithError(err).Fatal("Failed to scan Git.")
|
||||
}
|
||||
if err, id := verifyDetectorsAreVersioner(excludeDetectorSet); err != nil {
|
||||
logFatal(err, "invalid exclude list detector configuration", "detector", id)
|
||||
case githubScan.FullCommand():
|
||||
if len(*githubScanOrgs) == 0 && len(*githubScanRepos) == 0 {
|
||||
log.Fatal("You must specify at least one organization or repository.")
|
||||
}
|
||||
if err, id := verifyDetectorsAreVersioner(detectorsWithCustomVerifierEndpoints); err != nil {
|
||||
logFatal(err, "invalid verifier detector configuration", "detector", id)
|
||||
}
|
||||
// Extra check for endpoint customization.
|
||||
isEndpointCustomizer := engine.DefaultDetectorTypesImplementing[detectors.EndpointCustomizer]()
|
||||
for id := range detectorsWithCustomVerifierEndpoints {
|
||||
if _, ok := isEndpointCustomizer[id.ID]; !ok {
|
||||
logFatal(
|
||||
fmt.Errorf("endpoint provided but detector does not support endpoint customization"),
|
||||
"invalid custom verifier endpoint detector configuration",
|
||||
"detector", id,
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
includeFilter := func(d detectors.Detector) bool {
|
||||
_, ok := getWithDetectorID(d, includeDetectorSet)
|
||||
return ok
|
||||
}
|
||||
excludeFilter := func(d detectors.Detector) bool {
|
||||
_, ok := getWithDetectorID(d, excludeDetectorSet)
|
||||
return !ok
|
||||
}
|
||||
// Abuse filter to cause a side-effect.
|
||||
endpointCustomizer := func(d detectors.Detector) bool {
|
||||
urls, ok := getWithDetectorID(d, detectorsWithCustomVerifierEndpoints)
|
||||
if !ok {
|
||||
return true
|
||||
github := func(c sources.Config) {
|
||||
c.Endpoint = *githubScanEndpoint
|
||||
c.Repos = *githubScanRepos
|
||||
c.Orgs = *githubScanOrgs
|
||||
c.Token = *githubScanToken
|
||||
c.IncludeForks = *githubIncludeForks
|
||||
c.IncludeMembers = *githubIncludeMembers
|
||||
c.Concurrency = *concurrency
|
||||
}
|
||||
id := config.GetDetectorID(d)
|
||||
customizer, ok := d.(detectors.EndpointCustomizer)
|
||||
if !ok {
|
||||
// NOTE: We should never reach here due to validation above.
|
||||
logFatal(
|
||||
fmt.Errorf("failed to configure a detector endpoint"),
|
||||
"the provided detector does not support endpoint configuration",
|
||||
"detector", id,
|
||||
)
|
||||
}
|
||||
// TODO: Add flag to ignore the default endpoint.
|
||||
urls = append(urls, customizer.DefaultEndpoint())
|
||||
if err := customizer.SetEndpoints(urls...); err != nil {
|
||||
logFatal(err, "failed configuring custom endpoint for detector", "detector", id)
|
||||
}
|
||||
logger.Info("configured detector with verification urls",
|
||||
"detector", id, "urls", urls,
|
||||
)
|
||||
return true
|
||||
}
|
||||
|
||||
// Set how the engine will print its results.
|
||||
var printer engine.Printer
|
||||
switch {
|
||||
case *jsonLegacy:
|
||||
printer = new(output.LegacyJSONPrinter)
|
||||
case *jsonOut:
|
||||
printer = new(output.JSONPrinter)
|
||||
case *gitHubActionsFormat:
|
||||
printer = new(output.GitHubActionsPrinter)
|
||||
default:
|
||||
printer = new(output.PlainPrinter)
|
||||
if err = e.ScanGitHub(ctx, sources.NewConfig(github)); err != nil {
|
||||
logrus.WithError(err).Fatal("Failed to scan Github.")
|
||||
}
|
||||
case gitlabScan.FullCommand():
|
||||
gitlab := func(c sources.Config) {
|
||||
c.Endpoint = *gitlabScanEndpoint
|
||||
c.Token = *gitlabScanToken
|
||||
c.Repos = *gitlabScanRepos
|
||||
}
|
||||
|
||||
if err = e.ScanGitLab(ctx, sources.NewConfig(gitlab)); err != nil {
|
||||
logrus.WithError(err).Fatal("Failed to scan GitLab.")
|
||||
}
|
||||
case filesystemScan.FullCommand():
|
||||
fs := func(c sources.Config) {
|
||||
c.Directories = *filesystemDirectories
|
||||
}
|
||||
|
||||
if err = e.ScanFileSystem(ctx, sources.NewConfig(fs)); err != nil {
|
||||
logrus.WithError(err).Fatal("Failed to scan filesystem")
|
||||
}
|
||||
case s3Scan.FullCommand():
|
||||
s3 := func(c sources.Config) {
|
||||
c.Key = *s3ScanKey
|
||||
c.Secret = *s3ScanSecret
|
||||
c.Buckets = *s3ScanBuckets
|
||||
}
|
||||
|
||||
if err = e.ScanS3(ctx, sources.NewConfig(s3)); err != nil {
|
||||
logrus.WithError(err).Fatal("Failed to scan S3.")
|
||||
}
|
||||
case syslogScan.FullCommand():
|
||||
syslog := func(c sources.Config) {
|
||||
c.Address = *syslogAddress
|
||||
c.Protocol = *syslogProtocol
|
||||
c.CertPath = *syslogTLSCert
|
||||
c.KeyPath = *syslogTLSKey
|
||||
c.Format = *syslogFormat
|
||||
c.Concurrency = *concurrency
|
||||
}
|
||||
|
||||
if err = e.ScanSyslog(ctx, sources.NewConfig(syslog)); err != nil {
|
||||
logrus.WithError(err).Fatal("Failed to scan syslog.")
|
||||
}
|
||||
}
|
||||
// asynchronously wait for scanning to finish and cleanup
|
||||
go e.Finish()
|
||||
|
||||
if !*jsonLegacy && !*jsonOut {
|
||||
fmt.Fprintf(os.Stderr, "🐷🔑🐷 TruffleHog. Unearth your secrets. 🐷🔑🐷\n\n")
|
||||
}
|
||||
|
||||
e, err := engine.Start(ctx,
|
||||
engine.WithConcurrency(uint8(*concurrency)),
|
||||
engine.WithDecoders(decoders.DefaultDecoders()...),
|
||||
engine.WithDetectors(engine.DefaultDetectors()...),
|
||||
engine.WithDetectors(conf.Detectors...),
|
||||
engine.WithVerify(!*noVerification),
|
||||
engine.WithFilterDetectors(includeFilter),
|
||||
engine.WithFilterDetectors(excludeFilter),
|
||||
engine.WithFilterDetectors(endpointCustomizer),
|
||||
engine.WithFilterUnverified(*filterUnverified),
|
||||
engine.WithOnlyVerified(*onlyVerified),
|
||||
engine.WithPrintAvgDetectorTime(*printAvgDetectorTime),
|
||||
engine.WithPrinter(printer),
|
||||
engine.WithFilterEntropy(*filterEntropy),
|
||||
)
|
||||
if err != nil {
|
||||
logFatal(err, "error initializing engine")
|
||||
}
|
||||
// NOTE: this loop will terminate when the results channel is closed in
|
||||
// e.Finish()
|
||||
foundResults := false
|
||||
for r := range e.ResultsChan() {
|
||||
if *onlyVerified && !r.Verified {
|
||||
continue
|
||||
}
|
||||
foundResults = true
|
||||
|
||||
switch cmd {
|
||||
case gitScan.FullCommand():
|
||||
cfg := sources.GitConfig{
|
||||
URI: *gitScanURI,
|
||||
IncludePathsFile: *gitScanIncludePaths,
|
||||
ExcludePathsFile: *gitScanExcludePaths,
|
||||
HeadRef: *gitScanBranch,
|
||||
BaseRef: *gitScanSinceCommit,
|
||||
MaxDepth: *gitScanMaxDepth,
|
||||
Bare: *gitScanBare,
|
||||
ExcludeGlobs: *gitScanExcludeGlobs,
|
||||
}
|
||||
if err = e.ScanGit(ctx, cfg); err != nil {
|
||||
logFatal(err, "Failed to scan Git.")
|
||||
}
|
||||
case githubScan.FullCommand():
|
||||
filter, err := common.FilterFromFiles(*githubScanIncludePaths, *githubScanExcludePaths)
|
||||
if err != nil {
|
||||
logFatal(err, "could not create filter")
|
||||
}
|
||||
if len(*githubScanOrgs) == 0 && len(*githubScanRepos) == 0 {
|
||||
logFatal(fmt.Errorf("invalid config"), "You must specify at least one organization or repository.")
|
||||
}
|
||||
|
||||
cfg := sources.GithubConfig{
|
||||
Endpoint: *githubScanEndpoint,
|
||||
Token: *githubScanToken,
|
||||
IncludeForks: *githubIncludeForks,
|
||||
IncludeMembers: *githubIncludeMembers,
|
||||
Concurrency: *concurrency,
|
||||
ExcludeRepos: *githubExcludeRepos,
|
||||
IncludeRepos: *githubIncludeRepos,
|
||||
Repos: *githubScanRepos,
|
||||
Orgs: *githubScanOrgs,
|
||||
IncludeIssueComments: *githubScanIssueComments,
|
||||
IncludePullRequestComments: *githubScanPRComments,
|
||||
IncludeGistComments: *githubScanGistComments,
|
||||
Filter: filter,
|
||||
}
|
||||
if err := e.ScanGitHub(ctx, cfg); err != nil {
|
||||
logFatal(err, "Failed to scan Github.")
|
||||
}
|
||||
case gitlabScan.FullCommand():
|
||||
filter, err := common.FilterFromFiles(*gitlabScanIncludePaths, *gitlabScanExcludePaths)
|
||||
if err != nil {
|
||||
logFatal(err, "could not create filter")
|
||||
}
|
||||
|
||||
cfg := sources.GitlabConfig{
|
||||
Endpoint: *gitlabScanEndpoint,
|
||||
Token: *gitlabScanToken,
|
||||
Repos: *gitlabScanRepos,
|
||||
Filter: filter,
|
||||
}
|
||||
if err := e.ScanGitLab(ctx, cfg); err != nil {
|
||||
logFatal(err, "Failed to scan GitLab.")
|
||||
}
|
||||
case filesystemScan.FullCommand():
|
||||
filter, err := common.FilterFromFiles(*filesystemScanIncludePaths, *filesystemScanExcludePaths)
|
||||
if err != nil {
|
||||
logFatal(err, "could not create filter")
|
||||
}
|
||||
if len(*filesystemDirectories) > 0 {
|
||||
ctx.Logger().Info("--directory flag is deprecated, please pass directories as arguments")
|
||||
}
|
||||
paths := make([]string, 0, len(*filesystemPaths)+len(*filesystemDirectories))
|
||||
paths = append(paths, *filesystemPaths...)
|
||||
paths = append(paths, *filesystemDirectories...)
|
||||
cfg := sources.FilesystemConfig{
|
||||
Paths: paths,
|
||||
Filter: filter,
|
||||
}
|
||||
if err = e.ScanFileSystem(ctx, cfg); err != nil {
|
||||
logFatal(err, "Failed to scan filesystem")
|
||||
}
|
||||
case s3Scan.FullCommand():
|
||||
cfg := sources.S3Config{
|
||||
Key: *s3ScanKey,
|
||||
Secret: *s3ScanSecret,
|
||||
SessionToken: *s3ScanSessionToken,
|
||||
Buckets: *s3ScanBuckets,
|
||||
Roles: *s3ScanRoleArns,
|
||||
CloudCred: *s3ScanCloudEnv,
|
||||
MaxObjectSize: int64(*s3ScanMaxObjectSize),
|
||||
}
|
||||
if err := e.ScanS3(ctx, cfg); err != nil {
|
||||
logFatal(err, "Failed to scan S3.")
|
||||
}
|
||||
case syslogScan.FullCommand():
|
||||
cfg := sources.SyslogConfig{
|
||||
Address: *syslogAddress,
|
||||
Format: *syslogFormat,
|
||||
Protocol: *syslogProtocol,
|
||||
CertPath: *syslogTLSCert,
|
||||
KeyPath: *syslogTLSKey,
|
||||
Concurrency: *concurrency,
|
||||
}
|
||||
if err := e.ScanSyslog(ctx, cfg); err != nil {
|
||||
logFatal(err, "Failed to scan syslog.")
|
||||
}
|
||||
case circleCiScan.FullCommand():
|
||||
if err := e.ScanCircleCI(ctx, *circleCiScanToken); err != nil {
|
||||
logFatal(err, "Failed to scan CircleCI.")
|
||||
}
|
||||
case travisCiScan.FullCommand():
|
||||
if err := e.ScanTravisCI(ctx, *travisCiScanToken); err != nil {
|
||||
logFatal(err, "Failed to scan TravisCI.")
|
||||
}
|
||||
case gcsScan.FullCommand():
|
||||
cfg := sources.GCSConfig{
|
||||
ProjectID: *gcsProjectID,
|
||||
CloudCred: *gcsCloudEnv,
|
||||
ServiceAccount: *gcsServiceAccount,
|
||||
WithoutAuth: *gcsWithoutAuth,
|
||||
ApiKey: *gcsAPIKey,
|
||||
IncludeBuckets: commaSeparatedToSlice(*gcsIncludeBuckets),
|
||||
ExcludeBuckets: commaSeparatedToSlice(*gcsExcludeBuckets),
|
||||
IncludeObjects: commaSeparatedToSlice(*gcsIncludeObjects),
|
||||
ExcludeObjects: commaSeparatedToSlice(*gcsExcludeObjects),
|
||||
Concurrency: *concurrency,
|
||||
MaxObjectSize: int64(*gcsMaxObjectSize),
|
||||
}
|
||||
if err := e.ScanGCS(ctx, cfg); err != nil {
|
||||
logFatal(err, "Failed to scan GCS.")
|
||||
}
|
||||
case dockerScan.FullCommand():
|
||||
dockerConn := sourcespb.Docker{
|
||||
Images: *dockerScanImages,
|
||||
Credential: &sourcespb.Docker_DockerKeychain{
|
||||
DockerKeychain: true,
|
||||
},
|
||||
}
|
||||
anyConn, err := anypb.New(&dockerConn)
|
||||
if err != nil {
|
||||
logFatal(err, "Failed to marshal Docker connection")
|
||||
}
|
||||
if err := e.ScanDocker(ctx, anyConn); err != nil {
|
||||
logFatal(err, "Failed to scan Docker.")
|
||||
switch {
|
||||
case *jsonLegacy:
|
||||
output.PrintLegacyJSON(&r)
|
||||
case *jsonOut:
|
||||
output.PrintJSON(&r)
|
||||
default:
|
||||
output.PrintPlainOutput(&r)
|
||||
}
|
||||
}
|
||||
|
||||
// Wait for all workers to finish.
|
||||
if err = e.Finish(ctx); err != nil {
|
||||
logFatal(err, "engine failed to finish execution")
|
||||
}
|
||||
|
||||
metrics := e.GetMetrics()
|
||||
// Print results.
|
||||
logger.Info("finished scanning",
|
||||
"chunks", metrics.ChunksScanned,
|
||||
"bytes", metrics.BytesScanned,
|
||||
"verified_secrets", metrics.VerifiedSecretsFound,
|
||||
"unverified_secrets", metrics.UnverifiedSecretsFound,
|
||||
"scan_duration", metrics.ScanDuration.String(),
|
||||
)
|
||||
logrus.Debugf("scanned %d chunks", e.ChunksScanned())
|
||||
|
||||
if *printAvgDetectorTime {
|
||||
printAverageDetectorTime(e)
|
||||
}
|
||||
|
||||
if e.HasFoundResults() && *fail {
|
||||
logger.V(2).Info("exiting with code 183 because results were found")
|
||||
if foundResults && *fail {
|
||||
logrus.Debug("exiting with code 183 because results were found")
|
||||
os.Exit(183)
|
||||
}
|
||||
}
|
||||
|
||||
// logFatalFunc returns a log.Fatal style function. Calling the returned
|
||||
// function will terminate the program without cleanup.
|
||||
func logFatalFunc(logger logr.Logger) func(error, string, ...any) {
|
||||
return func(err error, message string, keyAndVals ...any) {
|
||||
logger.Error(err, message, keyAndVals...)
|
||||
if err != nil {
|
||||
os.Exit(1)
|
||||
return
|
||||
}
|
||||
os.Exit(0)
|
||||
}
|
||||
}
|
||||
|
||||
func commaSeparatedToSlice(s []string) []string {
|
||||
var result []string
|
||||
for _, items := range s {
|
||||
for _, item := range strings.Split(items, ",") {
|
||||
item = strings.TrimSpace(item)
|
||||
if item == "" {
|
||||
continue
|
||||
}
|
||||
result = append(result, item)
|
||||
}
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
func printAverageDetectorTime(e *engine.Engine) {
|
||||
fmt.Fprintln(os.Stderr, "Average detector time is the measurement of average time spent on each detector when results are returned.")
|
||||
for detectorName, duration := range e.GetDetectorsMetrics() {
|
||||
fmt.Fprintf(os.Stderr, "%s: %s\n", detectorName, duration)
|
||||
}
|
||||
}
|
||||
|
||||
// detectorTypeToSet is a helper function to convert a slice of detector IDs into a set.
|
||||
func detectorTypeToSet(detectors []config.DetectorID) map[config.DetectorID]struct{} {
|
||||
out := make(map[config.DetectorID]struct{}, len(detectors))
|
||||
for _, d := range detectors {
|
||||
out[d] = struct{}{}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// getWithDetectorID is a helper function to get a value from a map using a
|
||||
// detector's ID. This function behaves like a normal map lookup, with an extra
|
||||
// step of checking for the non-specific version of a detector.
|
||||
func getWithDetectorID[T any](d detectors.Detector, data map[config.DetectorID]T) (T, bool) {
|
||||
key := config.GetDetectorID(d)
|
||||
// Check if the specific ID is provided.
|
||||
if t, ok := data[key]; ok || key.Version == 0 {
|
||||
return t, ok
|
||||
}
|
||||
// Check if the generic type is provided without a version.
|
||||
// This means "all" versions of a type.
|
||||
key.Version = 0
|
||||
t, ok := data[key]
|
||||
return t, ok
|
||||
}
|
||||
|
||||
// verifyDetectorsAreVersioner checks all keys in a provided map to verify the
|
||||
// provided type is actually a Versioner.
|
||||
func verifyDetectorsAreVersioner[T any](data map[config.DetectorID]T) (error, config.DetectorID) {
|
||||
isVersioner := engine.DefaultDetectorTypesImplementing[detectors.Versioner]()
|
||||
for id := range data {
|
||||
if id.Version == 0 {
|
||||
// Version not provided.
|
||||
continue
|
||||
for detectorName, durations := range e.DetectorAvgTime() {
|
||||
var total time.Duration
|
||||
for _, d := range durations {
|
||||
total += d
|
||||
}
|
||||
if _, ok := isVersioner[id.ID]; ok {
|
||||
// Version provided for a Versioner detector.
|
||||
continue
|
||||
}
|
||||
// Version provided on a non-Versioner detector.
|
||||
return fmt.Errorf("version provided but detector does not have a version"), id
|
||||
avgDuration := total / time.Duration(len(durations))
|
||||
fmt.Fprintf(os.Stderr, "%s: %s\n", detectorName, avgDuration)
|
||||
}
|
||||
return nil, config.DetectorID{}
|
||||
}
|
||||
|
||||
Vendored
-24
@@ -1,24 +0,0 @@
|
||||
// Package cache provides an interface which can be implemented by different cache types.
|
||||
package cache
|
||||
|
||||
// Cache is used to store key/value pairs.
|
||||
type Cache interface {
|
||||
// Set stores the given key/value pair.
|
||||
Set(string, string)
|
||||
// Get returns the value for the given key and a boolean indicating if the key was found.
|
||||
Get(string) (string, bool)
|
||||
// Exists returns true if the given key exists in the cache.
|
||||
Exists(string) bool
|
||||
// Delete the given key from the cache.
|
||||
Delete(string)
|
||||
// Clear all key/value pairs from the cache.
|
||||
Clear()
|
||||
// Count the number of key/value pairs in the cache.
|
||||
Count() int
|
||||
// Keys returns all keys in the cache.
|
||||
Keys() []string
|
||||
// Values returns all values in the cache.
|
||||
Values() []string
|
||||
// Contents returns all keys in the cache encoded as a string.
|
||||
Contents() string
|
||||
}
|
||||
Vendored
-105
@@ -1,105 +0,0 @@
|
||||
package memory
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/patrickmn/go-cache"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/context"
|
||||
)
|
||||
|
||||
const (
|
||||
expirationInterval = 12 * time.Hour
|
||||
purgeInterval = 13 * time.Hour
|
||||
defaultExpiration = cache.DefaultExpiration
|
||||
)
|
||||
|
||||
// Cache is a wrapper around the go-cache library.
|
||||
type Cache struct {
|
||||
c *cache.Cache
|
||||
}
|
||||
|
||||
// New constructs a new in-memory cache.
|
||||
func New() *Cache {
|
||||
c := cache.New(expirationInterval, purgeInterval)
|
||||
return &Cache{c: c}
|
||||
}
|
||||
|
||||
// NewWithData constructs a new in-memory cache with existing data.
|
||||
func NewWithData(ctx context.Context, data []string) *Cache {
|
||||
ctx.Logger().V(3).Info("Loading cache", "num-items", len(data))
|
||||
|
||||
items := make(map[string]cache.Item, len(data))
|
||||
for _, d := range data {
|
||||
items[d] = cache.Item{Object: d, Expiration: int64(defaultExpiration)}
|
||||
}
|
||||
|
||||
c := cache.NewFrom(expirationInterval, purgeInterval, items)
|
||||
return &Cache{c: c}
|
||||
}
|
||||
|
||||
// Set adds a key-value pair to the cache.
|
||||
func (c *Cache) Set(key, value string) {
|
||||
c.c.Set(key, value, defaultExpiration)
|
||||
}
|
||||
|
||||
// Get returns the value for the given key.
|
||||
func (c *Cache) Get(key string) (string, bool) {
|
||||
res, ok := c.c.Get(key)
|
||||
if !ok {
|
||||
return "", ok
|
||||
}
|
||||
return res.(string), ok
|
||||
}
|
||||
|
||||
// Exists returns true if the given key exists in the cache.
|
||||
func (c *Cache) Exists(key string) bool {
|
||||
_, ok := c.c.Get(key)
|
||||
return ok
|
||||
}
|
||||
|
||||
// Delete removes the key-value pair from the cache.
|
||||
func (c *Cache) Delete(key string) {
|
||||
c.c.Delete(key)
|
||||
}
|
||||
|
||||
// Clear removes all key-value pairs from the cache.
|
||||
func (c *Cache) Clear() {
|
||||
c.c.Flush()
|
||||
}
|
||||
|
||||
// Count returns the number of key-value pairs in the cache.
|
||||
func (c *Cache) Count() int {
|
||||
return c.c.ItemCount()
|
||||
}
|
||||
|
||||
// Keys returns all keys in the cache.
|
||||
func (c *Cache) Keys() []string {
|
||||
items := c.c.Items()
|
||||
res := make([]string, 0, len(items))
|
||||
for k := range items {
|
||||
res = append(res, k)
|
||||
}
|
||||
return res
|
||||
}
|
||||
|
||||
// Values returns all values in the cache.
|
||||
func (c *Cache) Values() []string {
|
||||
items := c.c.Items()
|
||||
res := make([]string, 0, len(items))
|
||||
for _, v := range items {
|
||||
res = append(res, v.Object.(string))
|
||||
}
|
||||
return res
|
||||
}
|
||||
|
||||
// Contents returns a comma-separated string containing all keys in the cache.
|
||||
func (c *Cache) Contents() string {
|
||||
items := c.c.Items()
|
||||
res := make([]string, 0, len(items))
|
||||
for k := range items {
|
||||
res = append(res, k)
|
||||
}
|
||||
return strings.Join(res, ",")
|
||||
}
|
||||
Vendored
-169
@@ -1,169 +0,0 @@
|
||||
package memory
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/google/go-cmp/cmp"
|
||||
|
||||
logContext "github.com/trufflesecurity/trufflehog/v3/pkg/context"
|
||||
)
|
||||
|
||||
func TestCache(t *testing.T) {
|
||||
c := New()
|
||||
|
||||
// Test set and get.
|
||||
c.Set("key1", "key1")
|
||||
v, ok := c.Get("key1")
|
||||
if !ok || v != "key1" {
|
||||
t.Fatalf("Unexpected value for key1: %v, %v", v, ok)
|
||||
}
|
||||
|
||||
// Test exists.
|
||||
if !c.Exists("key1") {
|
||||
t.Fatalf("Expected key1 to exist")
|
||||
}
|
||||
|
||||
// Test the count.
|
||||
if c.Count() != 1 {
|
||||
t.Fatalf("Unexpected count: %d", c.Count())
|
||||
}
|
||||
|
||||
// Test delete.
|
||||
c.Delete("key1")
|
||||
v, ok = c.Get("key1")
|
||||
if ok || v != "" {
|
||||
t.Fatalf("Unexpected value for key1 after delete: %v, %v", v, ok)
|
||||
}
|
||||
|
||||
// Test clear.
|
||||
c.Set("key10", "key10")
|
||||
c.Clear()
|
||||
v, ok = c.Get("key10")
|
||||
if ok || v != "" {
|
||||
t.Fatalf("Unexpected value for key10 after clear: %v, %v", v, ok)
|
||||
}
|
||||
|
||||
// Test getting only the keys.
|
||||
keys := []string{"key1", "key2", "key3"}
|
||||
values := []string{"value1", "value2", "value3"}
|
||||
for i, k := range keys {
|
||||
c.Set(k, values[i])
|
||||
}
|
||||
k := c.Keys()
|
||||
sort.Strings(keys)
|
||||
sort.Strings(k)
|
||||
if !cmp.Equal(keys, k) {
|
||||
t.Fatalf("Unexpected keys: %v", k)
|
||||
}
|
||||
|
||||
// Test getting only the values.
|
||||
vals := c.Values()
|
||||
sort.Strings(vals)
|
||||
sort.Strings(values)
|
||||
if !cmp.Equal(values, vals) {
|
||||
t.Fatalf("Unexpected values: %v", vals)
|
||||
}
|
||||
|
||||
// Test contents.
|
||||
items := c.Contents()
|
||||
sort.Strings(keys)
|
||||
res := strings.Split(items, ",")
|
||||
sort.Strings(res)
|
||||
|
||||
if len(keys) != len(res) {
|
||||
t.Fatalf("Unexpected length of items: %d", len(res))
|
||||
}
|
||||
if !cmp.Equal(keys, res) {
|
||||
t.Fatalf("Unexpected items: %v", res)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCache_NewWithData(t *testing.T) {
|
||||
c := NewWithData(logContext.Background(), []string{"key1", "key2", "key3"})
|
||||
|
||||
// Test the count.
|
||||
if c.Count() != 3 {
|
||||
t.Fatalf("Unexpected count: %d", c.Count())
|
||||
}
|
||||
|
||||
// Test contents.
|
||||
keys := []string{"key1", "key2", "key3"}
|
||||
items := c.Contents()
|
||||
sort.Strings(keys)
|
||||
res := strings.Split(items, ",")
|
||||
sort.Strings(res)
|
||||
|
||||
if len(keys) != len(res) {
|
||||
t.Fatalf("Unexpected length of items: %d", len(res))
|
||||
}
|
||||
if !cmp.Equal(keys, res) {
|
||||
t.Fatalf("Unexpected items: %v", res)
|
||||
}
|
||||
}
|
||||
|
||||
func setupBenchmarks(b *testing.B) *Cache {
|
||||
b.Helper()
|
||||
|
||||
c := New()
|
||||
|
||||
for i := 0; i < 500_000; i++ {
|
||||
key := fmt.Sprintf("key%d", i)
|
||||
c.Set(key, key)
|
||||
}
|
||||
|
||||
return c
|
||||
}
|
||||
|
||||
func BenchmarkSet(b *testing.B) {
|
||||
c := New()
|
||||
|
||||
for i := 0; i < b.N; i++ {
|
||||
key := fmt.Sprintf("key%d", i)
|
||||
c.Set(key, key)
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkGet(b *testing.B) {
|
||||
c := setupBenchmarks(b)
|
||||
b.ResetTimer()
|
||||
|
||||
for i := 0; i < b.N; i++ {
|
||||
key := fmt.Sprintf("key%d", i)
|
||||
c.Get(key)
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkDelete(b *testing.B) {
|
||||
c := setupBenchmarks(b)
|
||||
b.ResetTimer()
|
||||
|
||||
for i := 0; i < b.N; i++ {
|
||||
key := fmt.Sprintf("key%d", i)
|
||||
c.Delete(key)
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkCount(b *testing.B) {
|
||||
c := setupBenchmarks(b)
|
||||
b.ResetTimer()
|
||||
|
||||
for i := 0; i < b.N; i++ {
|
||||
c.Count()
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkContents(b *testing.B) {
|
||||
c := setupBenchmarks(b)
|
||||
b.ResetTimer()
|
||||
|
||||
var s string
|
||||
|
||||
for i := 0; i < b.N; i++ {
|
||||
s = c.Contents()
|
||||
}
|
||||
|
||||
_ = s
|
||||
}
|
||||
@@ -1,127 +0,0 @@
|
||||
package cleantemp
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/mitchellh/go-ps"
|
||||
|
||||
logContext "github.com/trufflesecurity/trufflehog/v3/pkg/context"
|
||||
)
|
||||
|
||||
const (
|
||||
defaultExecPath = "trufflehog"
|
||||
defaultArtifactPrefixFormat = "%s-%d-"
|
||||
)
|
||||
|
||||
// MkdirTemp returns a temporary directory path formatted as:
|
||||
// trufflehog-<pid>-<randint>
|
||||
func MkdirTemp() (string, error) {
|
||||
pid := os.Getpid()
|
||||
tmpdir := fmt.Sprintf(defaultArtifactPrefixFormat, defaultExecPath, pid)
|
||||
dir, err := os.MkdirTemp(os.TempDir(), tmpdir)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return dir, nil
|
||||
}
|
||||
|
||||
// Unlike MkdirTemp, we only want to generate the filename string.
|
||||
// The tempfile creation in trufflehog we're interested in
|
||||
// is generally handled by "github.com/trufflesecurity/disk-buffer-reader"
|
||||
func MkFilename() string {
|
||||
pid := os.Getpid()
|
||||
filename := fmt.Sprintf(defaultArtifactPrefixFormat, defaultExecPath, pid)
|
||||
return filename
|
||||
}
|
||||
|
||||
// Only compile during startup.
|
||||
var trufflehogRE = regexp.MustCompile(`^trufflehog-\d+-\d+$`)
|
||||
|
||||
// CleanTempArtifacts deletes orphaned temp directories and files that do not contain running PID values.
|
||||
func CleanTempArtifacts(ctx logContext.Context) error {
|
||||
executablePath, err := os.Executable()
|
||||
if err != nil {
|
||||
executablePath = defaultExecPath
|
||||
}
|
||||
execName := filepath.Base(executablePath)
|
||||
|
||||
var pids []string
|
||||
procs, err := ps.Processes()
|
||||
if err != nil {
|
||||
return fmt.Errorf("error getting jobs PIDs: %w", err)
|
||||
}
|
||||
|
||||
for _, proc := range procs {
|
||||
if proc.Executable() == execName {
|
||||
pids = append(pids, strconv.Itoa(proc.Pid()))
|
||||
}
|
||||
}
|
||||
|
||||
tempDir := os.TempDir()
|
||||
artifacts, err := os.ReadDir(tempDir)
|
||||
if err != nil {
|
||||
return fmt.Errorf("error reading temp dir: %w", err)
|
||||
}
|
||||
|
||||
for _, artifact := range artifacts {
|
||||
if trufflehogRE.MatchString(artifact.Name()) {
|
||||
// Mark these artifacts initially as ones that should be deleted.
|
||||
shouldDelete := true
|
||||
// Check if the name matches any live PIDs.
|
||||
for _, pidval := range pids {
|
||||
if strings.Contains(artifact.Name(), fmt.Sprintf("-%s-", pidval)) {
|
||||
shouldDelete = false
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
if shouldDelete {
|
||||
artifactPath := filepath.Join(tempDir, artifact.Name())
|
||||
|
||||
var err error
|
||||
if artifact.IsDir() {
|
||||
err = os.RemoveAll(artifactPath)
|
||||
} else {
|
||||
err = os.Remove(artifactPath)
|
||||
}
|
||||
if err != nil {
|
||||
return fmt.Errorf("Error deleting temp artifact: %s", artifactPath)
|
||||
}
|
||||
|
||||
ctx.Logger().Info("Deleted orphaned temp artifact", "artifact", artifactPath)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// RunCleanupLoop runs a loop that cleans up orphaned directories every 15 seconds.
|
||||
func RunCleanupLoop(ctx logContext.Context) {
|
||||
err := CleanTempArtifacts(ctx)
|
||||
if err != nil {
|
||||
ctx.Logger().Error(err, "Error cleaning up orphaned directories ")
|
||||
}
|
||||
|
||||
const cleanupLoopInterval = 15 * time.Second
|
||||
ticker := time.NewTicker(cleanupLoopInterval)
|
||||
defer ticker.Stop()
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-ticker.C:
|
||||
if err := CleanTempArtifacts(ctx); err != nil {
|
||||
ctx.Logger().Error(err, "error cleaning up orphaned directories")
|
||||
}
|
||||
case <-ctx.Done():
|
||||
ctx.Logger().Info("Cleanup loop exiting due to context cancellation")
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,30 +0,0 @@
|
||||
package cleantemp
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"github.com/mitchellh/go-ps"
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
func TestExecName(t *testing.T) {
|
||||
executablePath, err := os.Executable()
|
||||
assert.Nil(t, err)
|
||||
execName := filepath.Base(executablePath)
|
||||
assert.Equal(t, "cleantemp.test", execName)
|
||||
|
||||
procs, err := ps.Processes()
|
||||
assert.Nil(t, err)
|
||||
assert.NotEmpty(t, procs)
|
||||
|
||||
found := false
|
||||
for _, proc := range procs {
|
||||
if proc.Executable() == execName {
|
||||
found = true
|
||||
}
|
||||
}
|
||||
|
||||
assert.True(t, found)
|
||||
}
|
||||
@@ -0,0 +1,36 @@
|
||||
package common
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"errors"
|
||||
"io"
|
||||
|
||||
log "github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
const (
|
||||
ChunkSize = 10 * 1024
|
||||
PeekSize = 3 * 1024
|
||||
)
|
||||
|
||||
func ChunkReader(r io.Reader) chan []byte {
|
||||
chunkChan := make(chan []byte)
|
||||
go func() {
|
||||
defer close(chunkChan)
|
||||
reader := bufio.NewReaderSize(bufio.NewReader(r), ChunkSize)
|
||||
for {
|
||||
chunk := make([]byte, ChunkSize)
|
||||
n, err := reader.Read(chunk)
|
||||
if err != nil && !errors.Is(err, io.EOF) {
|
||||
log.WithError(err).Error("Error chunking reader.")
|
||||
break
|
||||
}
|
||||
peekData, _ := reader.Peek(PeekSize)
|
||||
chunkChan <- append(chunk[:n], peekData...)
|
||||
if errors.Is(err, io.EOF) {
|
||||
break
|
||||
}
|
||||
}
|
||||
}()
|
||||
return chunkChan
|
||||
}
|
||||
@@ -0,0 +1,75 @@
|
||||
package common
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
"errors"
|
||||
"io"
|
||||
"net/http"
|
||||
"testing"
|
||||
|
||||
diskbufferreader "github.com/bill-rich/disk-buffer-reader"
|
||||
)
|
||||
|
||||
func TestChunker(t *testing.T) {
|
||||
resp, err := http.Get("https://raw.githubusercontent.com/bill-rich/bad-secrets/master/FifteenMB.gz")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
reReader, err := diskbufferreader.New(resp.Body)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer reReader.Close()
|
||||
|
||||
baseChunkCount := 0
|
||||
|
||||
// Count chunks from looping using chunk size.
|
||||
for {
|
||||
baseChunkCount++
|
||||
tmpChunk := make([]byte, ChunkSize)
|
||||
_, err := reReader.Read(tmpChunk)
|
||||
if err != nil {
|
||||
if errors.Is(err, io.EOF) {
|
||||
break
|
||||
}
|
||||
t.Error(err)
|
||||
}
|
||||
}
|
||||
_ = reReader.Reset()
|
||||
|
||||
// Get the first two chunks for comparing later.
|
||||
baseChunkOne := make([]byte, ChunkSize)
|
||||
baseChunkTwo := make([]byte, ChunkSize)
|
||||
|
||||
baseReader := bufio.NewReaderSize(reReader, ChunkSize)
|
||||
_, _ = baseReader.Read(baseChunkOne)
|
||||
peek, _ := baseReader.Peek(PeekSize)
|
||||
baseChunkOne = append(baseChunkOne, peek...)
|
||||
_, _ = baseReader.Read(baseChunkTwo)
|
||||
peek, _ = baseReader.Peek(PeekSize)
|
||||
baseChunkTwo = append(baseChunkTwo, peek...)
|
||||
|
||||
// Reset the reader to the beginning and use ChunkReader.
|
||||
_ = reReader.Reset()
|
||||
|
||||
testChunkCount := 0
|
||||
for chunk := range ChunkReader(reReader) {
|
||||
testChunkCount++
|
||||
switch testChunkCount {
|
||||
case 1:
|
||||
if !bytes.Equal(baseChunkOne, chunk) {
|
||||
t.Errorf("First chunk did not match expected. Got: %d bytes, expected: %d bytes", len(chunk), len(baseChunkOne))
|
||||
}
|
||||
case 2:
|
||||
if !bytes.Equal(baseChunkTwo, chunk) {
|
||||
t.Errorf("Second chunk did not match expected. Got: %d bytes, expected: %d bytes", len(chunk), len(baseChunkTwo))
|
||||
}
|
||||
}
|
||||
}
|
||||
if testChunkCount != baseChunkCount {
|
||||
t.Errorf("Wrong number of chunks received. Got %d, expected: %d.", testChunkCount, baseChunkCount)
|
||||
}
|
||||
|
||||
}
|
||||
@@ -10,20 +10,3 @@ func IsDone(ctx context.Context) bool {
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
// CancellableWrite blocks on writing the item to the channel but can be
|
||||
// cancelled by the context. If both the context is cancelled and the channel
|
||||
// write would succeed, either operation will be performed randomly.
|
||||
func CancellableWrite[T any](ctx context.Context, ch chan<- T, item T) error {
|
||||
select {
|
||||
case <-ctx.Done(): // priority to context cancellation
|
||||
return ctx.Err()
|
||||
default:
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
case ch <- item:
|
||||
return nil
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
+4
-15
@@ -6,7 +6,7 @@ import (
|
||||
"os"
|
||||
"regexp"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/context"
|
||||
log "github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
type Filter struct {
|
||||
@@ -20,8 +20,7 @@ type FilterRuleSet []regexp.Regexp
|
||||
func FilterEmpty() *Filter {
|
||||
filter, err := FilterFromFiles("", "")
|
||||
if err != nil {
|
||||
context.Background().Logger().Error(err, "could not create empty filter")
|
||||
os.Exit(1)
|
||||
log.WithError(err).Fatalf("could not create empty filter")
|
||||
}
|
||||
return filter
|
||||
}
|
||||
@@ -58,19 +57,15 @@ func FilterRulesFromFile(source string) (*FilterRuleSet, error) {
|
||||
}
|
||||
|
||||
commentPattern := regexp.MustCompile(`^\s*#`)
|
||||
emptyLinePattern := regexp.MustCompile(`^\s*$`)
|
||||
|
||||
file, err := os.Open(source)
|
||||
logger := context.Background().Logger().WithValues("file", source)
|
||||
if err != nil {
|
||||
logger.Error(err, "unable to open filter file", "file", source)
|
||||
os.Exit(1)
|
||||
log.WithError(err).Fatalf("unable to open filter file: %s", source)
|
||||
}
|
||||
defer func(file *os.File) {
|
||||
err := file.Close()
|
||||
if err != nil {
|
||||
logger.Error(err, "unable to close filter file")
|
||||
os.Exit(1)
|
||||
log.WithError(err).Fatalf("unable to close filter file: %s", source)
|
||||
}
|
||||
}(file)
|
||||
|
||||
@@ -80,9 +75,6 @@ func FilterRulesFromFile(source string) (*FilterRuleSet, error) {
|
||||
if commentPattern.MatchString(line) {
|
||||
continue
|
||||
}
|
||||
if emptyLinePattern.MatchString(line) {
|
||||
continue
|
||||
}
|
||||
pattern, err := regexp.Compile(line)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("can not compile regular expression: %s", line)
|
||||
@@ -94,9 +86,6 @@ func FilterRulesFromFile(source string) (*FilterRuleSet, error) {
|
||||
|
||||
// Pass returns true if the include FilterRuleSet matches the pattern and the exclude FilterRuleSet does not match.
|
||||
func (filter *Filter) Pass(object string) bool {
|
||||
if filter == nil {
|
||||
return true
|
||||
}
|
||||
excluded := filter.exclude.Matches(object)
|
||||
included := filter.include.Matches(object)
|
||||
return !excluded && included
|
||||
|
||||
@@ -129,13 +129,6 @@ func TestFilterFromFile(t *testing.T) {
|
||||
pattern: "test",
|
||||
pass: false,
|
||||
},
|
||||
"EmptyLinesAreIgnored": {
|
||||
includeFile: false,
|
||||
excludeFile: true,
|
||||
excludeFileContents: " \ntest.txt",
|
||||
pattern: "hello world.txt",
|
||||
pass: true,
|
||||
},
|
||||
}
|
||||
for name, test := range tests {
|
||||
var includeTestFile, excludeTestFile string
|
||||
|
||||
@@ -1,120 +0,0 @@
|
||||
package glob
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"github.com/gobwas/glob"
|
||||
)
|
||||
|
||||
// Filter is a generic filter for excluding and including globs (limited
|
||||
// regular expressions). Exclusion takes precedence if both include and exclude
|
||||
// lists are provided.
|
||||
type Filter struct {
|
||||
exclude []glob.Glob
|
||||
include []glob.Glob
|
||||
}
|
||||
|
||||
type globFilterOpt func(*Filter) error
|
||||
|
||||
// WithExcludeGlobs adds exclude globs to the filter.
|
||||
func WithExcludeGlobs(excludes ...string) globFilterOpt {
|
||||
return func(f *Filter) error {
|
||||
for _, exclude := range excludes {
|
||||
g, err := glob.Compile(exclude)
|
||||
if err != nil {
|
||||
return fmt.Errorf("invalid exclude glob %q: %w", exclude, err)
|
||||
}
|
||||
f.exclude = append(f.exclude, g)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
// WithIncludeGlobs adds include globs to the filter.
|
||||
func WithIncludeGlobs(includes ...string) globFilterOpt {
|
||||
return func(f *Filter) error {
|
||||
for _, include := range includes {
|
||||
g, err := glob.Compile(include)
|
||||
if err != nil {
|
||||
return fmt.Errorf("invalid include glob %q: %w", include, err)
|
||||
}
|
||||
f.include = append(f.include, g)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
// NewGlobFilter creates a new Filter with the provided options.
|
||||
func NewGlobFilter(opts ...globFilterOpt) (*Filter, error) {
|
||||
filter := &Filter{}
|
||||
for _, opt := range opts {
|
||||
if err := opt(filter); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
return filter, nil
|
||||
}
|
||||
|
||||
// ShouldInclude returns whether the object is in the include list or not in
|
||||
// the exclude list (exclude taking precedence).
|
||||
func (f *Filter) ShouldInclude(object string) bool {
|
||||
if f == nil {
|
||||
return true
|
||||
}
|
||||
exclude, include := len(f.exclude), len(f.include)
|
||||
if exclude == 0 && include == 0 {
|
||||
return true
|
||||
} else if exclude > 0 && include == 0 {
|
||||
return f.shouldIncludeFromExclude(object)
|
||||
} else if exclude == 0 && include > 0 {
|
||||
return f.shouldIncludeFromInclude(object)
|
||||
} else {
|
||||
if ok, err := f.shouldIncludeFromBoth(object); err == nil {
|
||||
return ok
|
||||
}
|
||||
// Ambiguous case.
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
// shouldIncludeFromExclude checks for explicitly excluded paths. This should
|
||||
// only be called when the include list is empty.
|
||||
func (f *Filter) shouldIncludeFromExclude(object string) bool {
|
||||
for _, g := range f.exclude {
|
||||
if g.Match(object) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// shouldIncludeFromInclude checks for explicitly included paths. This should
|
||||
// only be called when the exclude list is empty.
|
||||
func (f *Filter) shouldIncludeFromInclude(object string) bool {
|
||||
for _, g := range f.include {
|
||||
if g.Match(object) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// shouldIncludeFromBoth checks for either excluded or included paths. Exclusion
|
||||
// takes precedence. If neither list contains the object, true is returned.
|
||||
func (f *Filter) shouldIncludeFromBoth(object string) (bool, error) {
|
||||
// Exclude takes precedence. If we find the object in the exclude list,
|
||||
// we should not match.
|
||||
for _, g := range f.exclude {
|
||||
if g.Match(object) {
|
||||
return false, nil
|
||||
}
|
||||
}
|
||||
// If we find the object in the include list, we should match.
|
||||
for _, g := range f.include {
|
||||
if g.Match(object) {
|
||||
return true, nil
|
||||
}
|
||||
}
|
||||
// If we find it in neither, return an error to let the caller decide.
|
||||
return false, fmt.Errorf("ambiguous match")
|
||||
}
|
||||
@@ -1,138 +0,0 @@
|
||||
package glob
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"pgregory.net/rapid"
|
||||
)
|
||||
|
||||
type globTest struct {
|
||||
input string
|
||||
shouldInclude bool
|
||||
}
|
||||
|
||||
func testGlobs(t *testing.T, filter *Filter, tests ...globTest) {
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.input, func(t *testing.T) {
|
||||
// Invert because mentally it's easier to say whether an
|
||||
// input should be included.
|
||||
assert.Equal(t, tt.shouldInclude, filter.ShouldInclude(tt.input))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestGlobFilterExclude(t *testing.T) {
|
||||
filter, err := NewGlobFilter(WithExcludeGlobs("foo", "bar*"))
|
||||
assert.NoError(t, err)
|
||||
|
||||
testGlobs(t, filter,
|
||||
globTest{"foo", false},
|
||||
globTest{"bar", false},
|
||||
globTest{"bara", false},
|
||||
globTest{"barb", false},
|
||||
globTest{"barbosa", false},
|
||||
globTest{"foobar", true},
|
||||
globTest{"food", true},
|
||||
globTest{"anything else", true},
|
||||
)
|
||||
}
|
||||
|
||||
func TestGlobFilterInclude(t *testing.T) {
|
||||
filter, err := NewGlobFilter(WithIncludeGlobs("foo", "bar*"))
|
||||
assert.NoError(t, err)
|
||||
|
||||
testGlobs(t, filter,
|
||||
globTest{"foo", true},
|
||||
globTest{"bar", true},
|
||||
globTest{"bara", true},
|
||||
globTest{"barb", true},
|
||||
globTest{"barbosa", true},
|
||||
globTest{"foobar", false},
|
||||
globTest{"food", false},
|
||||
globTest{"anything else", false},
|
||||
)
|
||||
}
|
||||
|
||||
func TestGlobFilterEmpty(t *testing.T) {
|
||||
filter, err := NewGlobFilter()
|
||||
assert.NoError(t, err)
|
||||
|
||||
testGlobs(t, filter,
|
||||
globTest{"foo", true},
|
||||
globTest{"bar", true},
|
||||
globTest{"bara", true},
|
||||
globTest{"barb", true},
|
||||
globTest{"barbosa", true},
|
||||
globTest{"foobar", true},
|
||||
globTest{"food", true},
|
||||
globTest{"anything else", true},
|
||||
)
|
||||
}
|
||||
|
||||
func TestGlobFilterExcludeInclude(t *testing.T) {
|
||||
filter, err := NewGlobFilter(WithExcludeGlobs("/foo/bar/**"), WithIncludeGlobs("/foo/**"))
|
||||
assert.NoError(t, err)
|
||||
|
||||
testGlobs(t, filter,
|
||||
globTest{"/foo/a", true},
|
||||
globTest{"/foo/b", true},
|
||||
globTest{"/foo/c/d/e", true},
|
||||
globTest{"/foo/bar/a", false},
|
||||
globTest{"/foo/bar/b", false},
|
||||
globTest{"/foo/bar/c/d/e", false},
|
||||
globTest{"/any/other/path", false},
|
||||
)
|
||||
}
|
||||
|
||||
func TestGlobFilterExcludePrecedence(t *testing.T) {
|
||||
filter, err := NewGlobFilter(WithExcludeGlobs("foo"), WithIncludeGlobs("foo*"))
|
||||
assert.NoError(t, err)
|
||||
|
||||
testGlobs(t, filter,
|
||||
globTest{"foo", false},
|
||||
globTest{"foobar", true},
|
||||
)
|
||||
}
|
||||
|
||||
func TestGlobErrorContainsGlob(t *testing.T) {
|
||||
invalidGlob := "[this is invalid because it doesn't close the capture group"
|
||||
_, err := NewGlobFilter(WithExcludeGlobs(invalidGlob))
|
||||
assert.Error(t, err)
|
||||
assert.Contains(t, err.Error(), invalidGlob)
|
||||
}
|
||||
|
||||
// The filters in this test should be mutually exclusive because one includes
|
||||
// and the other excludes the same glob.
|
||||
func TestGlobInverse(t *testing.T) {
|
||||
for _, glob := range []string{
|
||||
"a",
|
||||
"a*",
|
||||
"a**",
|
||||
"*a",
|
||||
"**a",
|
||||
"*",
|
||||
} {
|
||||
include, err := NewGlobFilter(WithIncludeGlobs(glob))
|
||||
assert.NoError(t, err)
|
||||
exclude, err := NewGlobFilter(WithExcludeGlobs(glob))
|
||||
assert.NoError(t, err)
|
||||
rapid.Check(t, func(t *rapid.T) {
|
||||
input := rapid.String().Draw(t, "input")
|
||||
a, b := include.ShouldInclude(input), exclude.ShouldInclude(input)
|
||||
if a == b {
|
||||
t.Fatalf("Filter(Include(%q)) == Filter(Exclude(%q)) == %v for input %q", glob, glob, a, input)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestGlobDefaultFilters(t *testing.T) {
|
||||
for _, filter := range []*Filter{nil, {}} {
|
||||
rapid.Check(t, func(t *rapid.T) {
|
||||
if !filter.ShouldInclude(rapid.String().Draw(t, "input")) {
|
||||
t.Fatalf("filter %#v did not include input", filter)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
+3
-36
@@ -3,7 +3,6 @@ package common
|
||||
import (
|
||||
"crypto/tls"
|
||||
"crypto/x509"
|
||||
"io"
|
||||
"net"
|
||||
"net/http"
|
||||
"strings"
|
||||
@@ -76,14 +75,6 @@ func PinnedCertPool() *x509.CertPool {
|
||||
return trustedCerts
|
||||
}
|
||||
|
||||
type FakeTransport struct {
|
||||
CreateResponse func(req *http.Request) (*http.Response, error)
|
||||
}
|
||||
|
||||
func (t FakeTransport) RoundTrip(req *http.Request) (*http.Response, error) {
|
||||
return t.CreateResponse(req)
|
||||
}
|
||||
|
||||
type CustomTransport struct {
|
||||
T http.RoundTripper
|
||||
}
|
||||
@@ -100,21 +91,6 @@ func NewCustomTransport(T http.RoundTripper) *CustomTransport {
|
||||
return &CustomTransport{T}
|
||||
}
|
||||
|
||||
func ConstantResponseHttpClient(statusCode int, body string) *http.Client {
|
||||
return &http.Client{
|
||||
Timeout: DefaultResponseTimeout,
|
||||
Transport: FakeTransport{
|
||||
CreateResponse: func(req *http.Request) (*http.Response, error) {
|
||||
return &http.Response{
|
||||
Request: req,
|
||||
Body: io.NopCloser(strings.NewReader(body)),
|
||||
StatusCode: statusCode,
|
||||
}, nil
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func PinnedRetryableHttpClient() *http.Client {
|
||||
httpClient := retryablehttp.NewClient()
|
||||
httpClient.Logger = nil
|
||||
@@ -145,15 +121,6 @@ func RetryableHttpClient() *http.Client {
|
||||
return httpClient.StandardClient()
|
||||
}
|
||||
|
||||
func RetryableHttpClientTimeout(timeOutSeconds int64) *http.Client {
|
||||
httpClient := retryablehttp.NewClient()
|
||||
httpClient.RetryMax = 3
|
||||
httpClient.Logger = nil
|
||||
httpClient.HTTPClient.Timeout = time.Duration(timeOutSeconds) * time.Second
|
||||
httpClient.HTTPClient.Transport = NewCustomTransport(nil)
|
||||
return httpClient.StandardClient()
|
||||
}
|
||||
|
||||
const DefaultResponseTimeout = 5 * time.Second
|
||||
|
||||
var saneTransport = &http.Transport{
|
||||
@@ -175,10 +142,10 @@ func SaneHttpClient() *http.Client {
|
||||
return httpClient
|
||||
}
|
||||
|
||||
// SaneHttpClientTimeOut adds a custom timeout for some scanners
|
||||
func SaneHttpClientTimeOut(timeout time.Duration) *http.Client {
|
||||
//custom timeout for some scanners
|
||||
func SaneHttpClientTimeOut(timeOutSeconds int64) *http.Client {
|
||||
httpClient := &http.Client{}
|
||||
httpClient.Timeout = timeout
|
||||
httpClient.Timeout = time.Second * time.Duration(timeOutSeconds)
|
||||
httpClient.Transport = NewCustomTransport(nil)
|
||||
return httpClient
|
||||
}
|
||||
|
||||
@@ -1,8 +0,0 @@
|
||||
package common
|
||||
|
||||
const (
|
||||
// MetricsNamespace is the namespace for all metrics.
|
||||
MetricsNamespace = "trufflehog"
|
||||
// MetricsSubsystem is the subsystem for all metrics.
|
||||
MetricsSubsystem = "scanner"
|
||||
)
|
||||
+4
-34
@@ -2,9 +2,10 @@ package common
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
log "github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
const EmailPattern = `\b(?:[a-z0-9!#$%&'*+/=?^_\x60{|}~-]+(?:\.[a-z0-9!#$%&'*+/=?^_\x60{|}~-]+)*|"(?:[\x01-\x08\x0b\x0c\x0e-\x1f\x21\x23-\x5b\x5d-\x7f]|\\[\x01-\x09\x0b\x0c\x0e-\x7f])*")@(?:(?:[a-z0-9](?:[a-z0-9-]*[a-z0-9])?\.)+[a-z0-9](?:[a-z0-9-]*[a-z0-9])?|\[(?:(?:(2(5[0-5]|[0-4][0-9])|1[0-9][0-9]|[1-9]?[0-9]))\.){3}(?:(2(5[0-5]|[0-4][0-9])|1[0-9][0-9]|[1-9]?[0-9])|[a-z0-9-]*[a-z0-9]:(?:[\x01-\x08\x0b\x0c\x0e-\x1f\x21-\x5a\x53-\x7f]|\\[\x01-\x09\x0b\x0c\x0e-\x7f])+)\])\b`
|
||||
@@ -16,18 +17,14 @@ const RegexPattern = "0-9a-z"
|
||||
const AlphaNumPattern = "0-9a-zA-Z"
|
||||
const HexPattern = "0-9a-f"
|
||||
|
||||
type RegexState struct {
|
||||
compiledRegex *regexp.Regexp
|
||||
}
|
||||
|
||||
// Custom Regex functions
|
||||
//Custom Regex functions
|
||||
func BuildRegex(pattern string, specialChar string, length int) string {
|
||||
return fmt.Sprintf(`\b([%s%s]{%s})\b`, pattern, specialChar, strconv.Itoa(length))
|
||||
}
|
||||
|
||||
func BuildRegexJWT(firstRange, secondRange, thirdRange string) string {
|
||||
if RangeValidation(firstRange) || RangeValidation(secondRange) || RangeValidation(thirdRange) {
|
||||
panic("Min value should not be greater than or equal to max")
|
||||
log.Error("Min value should not be greater than or equal to max")
|
||||
}
|
||||
return fmt.Sprintf(`\b(ey[%s]{%s}.ey[%s-\/_]{%s}.[%s-\/_]{%s})\b`, AlphaNumPattern, firstRange, AlphaNumPattern, secondRange, AlphaNumPattern, thirdRange)
|
||||
}
|
||||
@@ -42,30 +39,3 @@ func RangeValidation(rangeInput string) bool {
|
||||
func ToUpperCase(input string) string {
|
||||
return strings.ToUpper(input)
|
||||
}
|
||||
|
||||
func (r RegexState) Matches(data []byte) []string {
|
||||
matches := r.compiledRegex.FindAllStringSubmatch(string(data), -1)
|
||||
|
||||
res := make([]string, 0, len(matches))
|
||||
|
||||
// trim off spaces and different quote types ('").
|
||||
for i := range matches {
|
||||
res = append(res, strings.Trim(matches[i][1], `"' )`))
|
||||
}
|
||||
|
||||
return res
|
||||
}
|
||||
|
||||
// UsernameRegexCheck constructs an username usernameRegex pattern from a given pattern of excluded characters.
|
||||
func UsernameRegexCheck(pattern string) RegexState {
|
||||
raw := fmt.Sprintf(`(?im)(?:user|usr)\S{0,40}?[:=\s]{1,3}[ '"=]{0,1}([^:%+v]{4,40})\b`, pattern)
|
||||
|
||||
return RegexState{regexp.MustCompile(raw)}
|
||||
}
|
||||
|
||||
// PasswordRegexCheck constructs an username usernameRegex pattern from a given pattern of excluded characters.
|
||||
func PasswordRegexCheck(pattern string) RegexState {
|
||||
raw := fmt.Sprintf(`(?im)(?:pass|password)\S{0,40}?[:=\s]{1,3}[ '"=]{0,1}([^:%+v]{4,40})`, pattern)
|
||||
|
||||
return RegexState{regexp.MustCompile(raw)}
|
||||
}
|
||||
|
||||
@@ -1,59 +0,0 @@
|
||||
package common
|
||||
|
||||
import (
|
||||
"github.com/stretchr/testify/assert"
|
||||
"regexp"
|
||||
"testing"
|
||||
)
|
||||
|
||||
const (
|
||||
usernamePattern = `?()/\+=\s\n`
|
||||
passwordPattern = `^<>;.*&|£\n\s`
|
||||
usernameRegex = `(?im)(?:user|usr)\S{0,40}?[:=\s]{1,3}[ '"=]{0,1}([^:?()/\+=\s\n]{4,40})\b`
|
||||
passwordRegex = `(?im)(?:pass|password)\S{0,40}?[:=\s]{1,3}[ '"=]{0,1}([^:^<>;.*&|£\n\s]{4,40})`
|
||||
)
|
||||
|
||||
func TestUsernameRegexCheck(t *testing.T) {
|
||||
usernameRegexPat := UsernameRegexCheck(usernamePattern)
|
||||
|
||||
expectedRegexPattern := regexp.MustCompile(usernameRegex)
|
||||
|
||||
if usernameRegexPat.compiledRegex.String() != expectedRegexPattern.String() {
|
||||
t.Errorf("\n got %v \n want %v", usernameRegexPat.compiledRegex, expectedRegexPattern)
|
||||
}
|
||||
|
||||
testString := `username = "johnsmith123"
|
||||
username='johnsmith123'
|
||||
username:="johnsmith123"
|
||||
username = johnsmith123
|
||||
username=johnsmith123`
|
||||
|
||||
expectedStr := []string{"johnsmith123", "johnsmith123", "johnsmith123", "johnsmith123", "johnsmith123"}
|
||||
|
||||
usernameRegexMatches := usernameRegexPat.Matches([]byte(testString))
|
||||
|
||||
assert.Exactly(t, usernameRegexMatches, expectedStr)
|
||||
|
||||
}
|
||||
|
||||
func TestPasswordRegexCheck(t *testing.T) {
|
||||
passwordRegexPat := PasswordRegexCheck(passwordPattern)
|
||||
|
||||
expectedRegexPattern := regexp.MustCompile(passwordRegex)
|
||||
assert.Equal(t, passwordRegexPat.compiledRegex, expectedRegexPattern)
|
||||
|
||||
testString := `password = "johnsmith123$!"
|
||||
password='johnsmith123$!'
|
||||
password:="johnsmith123$!"
|
||||
password = johnsmith123$!
|
||||
password=johnsmith123$!
|
||||
PasswordAuthenticator(username, "johnsmith123$!")`
|
||||
|
||||
expectedStr := []string{"johnsmith123$!", "johnsmith123$!", "johnsmith123$!", "johnsmith123$!", "johnsmith123$!",
|
||||
"johnsmith123$!"}
|
||||
|
||||
passwordRegexMatches := passwordRegexPat.Matches([]byte(testString))
|
||||
|
||||
assert.Exactly(t, passwordRegexMatches, expectedStr)
|
||||
|
||||
}
|
||||
@@ -1,46 +0,0 @@
|
||||
package common
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"runtime/debug"
|
||||
"time"
|
||||
|
||||
"github.com/getsentry/sentry-go"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/context"
|
||||
)
|
||||
|
||||
// Recover handles panics and reports to Sentry.
|
||||
func Recover(ctx context.Context) {
|
||||
if err := recover(); err != nil {
|
||||
panicStack := string(debug.Stack())
|
||||
if eventID := sentry.CurrentHub().Recover(err); eventID != nil {
|
||||
ctx.Logger().Info("panic captured", "event_id", *eventID)
|
||||
}
|
||||
ctx.Logger().Error(fmt.Errorf("panic"), panicStack,
|
||||
"recover", err,
|
||||
)
|
||||
if !sentry.Flush(time.Second * 5) {
|
||||
ctx.Logger().Info("sentry flush failed")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// RecoverWithExit handles panics and reports to Sentry before exiting.
|
||||
func RecoverWithExit(ctx context.Context) {
|
||||
if err := recover(); err != nil {
|
||||
panicStack := string(debug.Stack())
|
||||
if eventID := sentry.CurrentHub().Recover(err); eventID != nil {
|
||||
ctx.Logger().Info("panic captured", "event_id", *eventID)
|
||||
}
|
||||
ctx.Logger().Error(fmt.Errorf("panic"), "recovered from panic before exiting",
|
||||
"stack-trace", panicStack,
|
||||
"recover", err,
|
||||
)
|
||||
if !sentry.Flush(time.Second * 5) {
|
||||
ctx.Logger().Info("sentry flush failed")
|
||||
}
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
@@ -7,9 +7,9 @@ import (
|
||||
"time"
|
||||
|
||||
secretmanager "cloud.google.com/go/secretmanager/apiv1"
|
||||
"cloud.google.com/go/secretmanager/apiv1/secretmanagerpb"
|
||||
"github.com/joho/godotenv"
|
||||
"github.com/pkg/errors"
|
||||
secretmanagerpb "google.golang.org/genproto/googleapis/cloud/secretmanager/v1"
|
||||
)
|
||||
|
||||
type Secret struct{ kv map[string]string }
|
||||
|
||||
@@ -1,16 +1,5 @@
|
||||
package common
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"crypto/rand"
|
||||
"encoding/base32"
|
||||
"encoding/binary"
|
||||
"fmt"
|
||||
"io"
|
||||
"math/big"
|
||||
"strings"
|
||||
)
|
||||
|
||||
func AddStringSliceItem(item string, slice *[]string) {
|
||||
for _, i := range *slice {
|
||||
if i == item {
|
||||
@@ -28,53 +17,3 @@ func RemoveStringSliceItem(item string, slice *[]string) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func ResponseContainsSubstring(reader io.ReadCloser, target string) (bool, error) {
|
||||
scanner := bufio.NewScanner(reader)
|
||||
for scanner.Scan() {
|
||||
if strings.Contains(scanner.Text(), target) {
|
||||
return true, nil
|
||||
}
|
||||
}
|
||||
if err := scanner.Err(); err != nil {
|
||||
return false, err
|
||||
}
|
||||
return false, nil
|
||||
}
|
||||
|
||||
var letters = []rune("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789")
|
||||
|
||||
// RandomID returns a random string of the given length.
|
||||
func RandomID(length int) string {
|
||||
b := make([]rune, length)
|
||||
for i := range b {
|
||||
randInt, _ := rand.Int(rand.Reader, big.NewInt(int64(len(letters))))
|
||||
b[i] = letters[randInt.Int64()]
|
||||
}
|
||||
|
||||
return string(b)
|
||||
}
|
||||
|
||||
func GetAccountNumFromAWSID(AWSID string) (string, error) {
|
||||
// Function to get the account number from an AWS ID (no verification required)
|
||||
// Source: https://medium.com/@TalBeerySec/a-short-note-on-aws-key-id-f88cc4317489
|
||||
if len(AWSID) < 4 {
|
||||
return "", fmt.Errorf("AWSID is too short")
|
||||
}
|
||||
trimmedAWSID := AWSID[4:]
|
||||
decodedBytes, err := base32.StdEncoding.WithPadding(base32.NoPadding).DecodeString(strings.ToUpper(trimmedAWSID))
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
if len(decodedBytes) < 6 {
|
||||
return "", fmt.Errorf("Decoded AWSID is too short")
|
||||
}
|
||||
|
||||
data := make([]byte, 8)
|
||||
copy(data[2:], decodedBytes[0:6])
|
||||
z := binary.BigEndian.Uint64(data)
|
||||
const mask uint64 = 0x7fffffffff80
|
||||
accountNum := (z & mask) >> 7
|
||||
return fmt.Sprintf("%012d", accountNum), nil
|
||||
}
|
||||
|
||||
@@ -1,9 +1,7 @@
|
||||
package common
|
||||
|
||||
import (
|
||||
"io"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
@@ -76,60 +74,3 @@ func TestRemoveItem(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Test ParseResponseForKeywords with a reader that contains the keyword and a reader that doesn't.
|
||||
func TestParseResponseForKeywords(t *testing.T) {
|
||||
testCases := []struct {
|
||||
name string
|
||||
input string
|
||||
keyword string
|
||||
expected bool
|
||||
}{
|
||||
{
|
||||
name: "Should find keyword",
|
||||
input: "ey: abc",
|
||||
keyword: "ey",
|
||||
expected: true,
|
||||
},
|
||||
{
|
||||
name: "Should not find keyword",
|
||||
input: "fake response",
|
||||
keyword: "ey",
|
||||
expected: false,
|
||||
},
|
||||
{
|
||||
name: "Empty string",
|
||||
input: "",
|
||||
keyword: "ey",
|
||||
expected: false,
|
||||
},
|
||||
{
|
||||
name: "Keyword at end",
|
||||
input: "abc ey",
|
||||
keyword: "ey",
|
||||
expected: true,
|
||||
},
|
||||
{
|
||||
name: "Keyword at start",
|
||||
input: "ey abc",
|
||||
keyword: "ey",
|
||||
expected: true,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range testCases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
testReader := strings.NewReader(tc.input)
|
||||
testReadCloser := io.NopCloser(testReader)
|
||||
found, err := ResponseContainsSubstring(testReadCloser, tc.keyword)
|
||||
|
||||
if err != nil {
|
||||
t.Errorf("Error: %v", err)
|
||||
}
|
||||
|
||||
if found != tc.expected {
|
||||
t.Errorf("Expected %v, got %v", tc.expected, found)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
+12
-45
@@ -2,57 +2,24 @@ package common
|
||||
|
||||
import (
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"github.com/h2non/filetype"
|
||||
)
|
||||
|
||||
var (
|
||||
KB, MB, GB, TB, PB = 1e3, 1e6, 1e9, 1e12, 1e15
|
||||
IgnoredExtensions = []string{
|
||||
// multimedia/containers
|
||||
"mp4",
|
||||
"avi",
|
||||
"mpeg",
|
||||
"mpg",
|
||||
"mov",
|
||||
"wmv",
|
||||
"m4p",
|
||||
"swf",
|
||||
"mp2",
|
||||
"flv",
|
||||
"vob",
|
||||
"webm",
|
||||
"hdv",
|
||||
"3gp",
|
||||
"ogg",
|
||||
"mp3",
|
||||
"wav",
|
||||
"flac",
|
||||
"webp",
|
||||
|
||||
// images
|
||||
"png",
|
||||
"jpg",
|
||||
"jpeg",
|
||||
"gif",
|
||||
"tiff",
|
||||
|
||||
// binaries
|
||||
// These can theoretically contain secrets, but need decoding for users to make sense of them, and we don't have
|
||||
// any such decoders right now.
|
||||
"class",
|
||||
"dll",
|
||||
"xsb",
|
||||
"jdo",
|
||||
"jks",
|
||||
"ser",
|
||||
"idx",
|
||||
"hprof",
|
||||
}
|
||||
IGNORED_EXTENSIONS = []string{"pdf", "mp4", "avi", "mpeg", "mpg", "mov", "wmv", "m4p", "swf", "mp2", "flv", "vob", "webm", "hdv", "3gp", "ogg", "mp3", "wav", "flac", "tif", "tiff", "jpg", "jpeg", "png", "gif", "zip", "webp"}
|
||||
)
|
||||
|
||||
func SkipFile(filename string) bool {
|
||||
for _, ext := range IgnoredExtensions {
|
||||
if strings.TrimPrefix(filepath.Ext(filename), ".") == ext {
|
||||
func SkipFile(filename string, data []byte) bool {
|
||||
if filepath.Ext(filename) == "" {
|
||||
//no sepcified extension, check mimetype
|
||||
if filetype.IsArchive(data[:256]) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
for _, ext := range IGNORED_EXTENSIONS {
|
||||
if filepath.Ext(filename) == ext {
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,36 +0,0 @@
|
||||
package common
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestSkipFile(t *testing.T) {
|
||||
type testCase struct {
|
||||
file string
|
||||
want bool
|
||||
}
|
||||
|
||||
// Add a test case for each ignored extension.
|
||||
testCases := make([]testCase, 0, len(IgnoredExtensions)+1)
|
||||
for _, ext := range IgnoredExtensions {
|
||||
testCases = append(testCases, testCase{
|
||||
file: "test." + ext,
|
||||
want: true,
|
||||
})
|
||||
}
|
||||
|
||||
// Add a test case for a file that should not be skipped.
|
||||
testCases = append(testCases, testCase{file: "test.txt", want: false})
|
||||
|
||||
for _, tt := range testCases {
|
||||
t.Run(tt.file, func(t *testing.T) {
|
||||
if got := SkipFile(tt.file); got != tt.want {
|
||||
t.Errorf("SkipFile(%v) got %v, want %v", tt.file, got, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkSkipFile(b *testing.B) {
|
||||
for i := 0; i < b.N; i++ {
|
||||
SkipFile("test.mp4")
|
||||
}
|
||||
}
|
||||
@@ -1,45 +0,0 @@
|
||||
package config
|
||||
|
||||
import (
|
||||
"os"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/custom_detectors"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/custom_detectorspb"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/protoyaml"
|
||||
)
|
||||
|
||||
// Config holds user supplied configuration.
|
||||
type Config struct {
|
||||
Detectors []detectors.Detector
|
||||
}
|
||||
|
||||
// Read parses a given filename into a Config.
|
||||
func Read(filename string) (*Config, error) {
|
||||
input, err := os.ReadFile(filename)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return NewYAML(input)
|
||||
}
|
||||
|
||||
// NewYAML parses the given YAML data into a Config.
|
||||
func NewYAML(input []byte) (*Config, error) {
|
||||
// Parse the raw YAML into a structure.
|
||||
var messages custom_detectorspb.CustomDetectors
|
||||
if err := protoyaml.UnmarshalStrict(input, &messages); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// Convert the structured YAML into detectors.
|
||||
var d []detectors.Detector
|
||||
for _, detectorConfig := range messages.Detectors {
|
||||
detector, err := custom_detectors.NewWebhookCustomRegex(detectorConfig)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
d = append(d, detector)
|
||||
}
|
||||
return &Config{
|
||||
Detectors: d,
|
||||
}, nil
|
||||
}
|
||||
@@ -1,231 +0,0 @@
|
||||
package config
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"net/url"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
dpb "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
|
||||
)
|
||||
|
||||
var (
|
||||
specialGroups = map[string][]DetectorID{
|
||||
"all": allDetectors(),
|
||||
}
|
||||
|
||||
detectorTypeValue = make(map[string]dpb.DetectorType, len(dpb.DetectorType_value))
|
||||
validDetectors = make(map[dpb.DetectorType]struct{}, len(dpb.DetectorType_value))
|
||||
maxDetectorType dpb.DetectorType
|
||||
)
|
||||
|
||||
// Setup package local global variables.
|
||||
func init() {
|
||||
for k, v := range dpb.DetectorType_value {
|
||||
dt := dpb.DetectorType(v)
|
||||
detectorTypeValue[strings.ToLower(k)] = dt
|
||||
validDetectors[dt] = struct{}{}
|
||||
if dt > maxDetectorType {
|
||||
maxDetectorType = dt
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// DetectorID identifies a detector type and version. This struct is used as a
|
||||
// way for users to identify detectors, whether unique or not. A DetectorID
|
||||
// with Version = 0 indicates all possible versions of a detector.
|
||||
type DetectorID struct {
|
||||
ID dpb.DetectorType
|
||||
Version int
|
||||
}
|
||||
|
||||
// GetDetectorID extracts the DetectorID from a Detector.
|
||||
func GetDetectorID(d detectors.Detector) DetectorID {
|
||||
var version int
|
||||
if v, ok := d.(detectors.Versioner); ok {
|
||||
version = v.Version()
|
||||
}
|
||||
return DetectorID{
|
||||
ID: d.Type(),
|
||||
Version: version,
|
||||
}
|
||||
}
|
||||
|
||||
// ParseDetectors parses user supplied string into a list of detectors types.
|
||||
// "all" will return the list of all available detectors. The input is comma
|
||||
// separated and may use the case-insensitive detector name defined in the
|
||||
// protobuf, or the protobuf enum number. A range may be used as well in the
|
||||
// form "start-end". Order is preserved and duplicates are ignored.
|
||||
func ParseDetectors(input string) ([]DetectorID, error) {
|
||||
var output []DetectorID
|
||||
seenDetector := map[DetectorID]struct{}{}
|
||||
for _, item := range strings.Split(input, ",") {
|
||||
item = strings.TrimSpace(item)
|
||||
if item == "" {
|
||||
continue
|
||||
}
|
||||
allDetectors, ok := specialGroups[strings.ToLower(item)]
|
||||
if !ok {
|
||||
var err error
|
||||
allDetectors, err = asRange(item)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
for _, d := range allDetectors {
|
||||
if _, ok := seenDetector[d]; ok {
|
||||
continue
|
||||
}
|
||||
seenDetector[d] = struct{}{}
|
||||
output = append(output, d)
|
||||
}
|
||||
}
|
||||
return output, nil
|
||||
}
|
||||
|
||||
// ParseDetector parses a user supplied string into a single DetectorID. Input
|
||||
// is case-insensitive and either the detector name or ID may be used.
|
||||
func ParseDetector(input string) (DetectorID, error) {
|
||||
return asDetectorID(strings.TrimSpace(input))
|
||||
}
|
||||
|
||||
// ParseVerifierEndpoints parses a map of user supplied verifier URLs. The
|
||||
// input keys are detector IDs and the values are a comma separated list of
|
||||
// URLs. The URLs are validated as HTTPS endpoints.
|
||||
func ParseVerifierEndpoints(verifierURLs map[string]string) (map[DetectorID][]string, error) {
|
||||
verifiers := make(map[DetectorID][]string, len(verifierURLs))
|
||||
for detectorID, urls := range verifierURLs {
|
||||
key, err := ParseDetector(detectorID)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("invalid detector ID for verifier: %w", err)
|
||||
}
|
||||
verifierURLs := strings.Split(urls, ",")
|
||||
for i, rawEndpoint := range verifierURLs {
|
||||
rawEndpoint := strings.TrimSpace(rawEndpoint)
|
||||
verifierURLs[i] = rawEndpoint
|
||||
if endpoint, err := url.Parse(rawEndpoint); err != nil {
|
||||
return nil, fmt.Errorf("invalid verifier url %q: %w", rawEndpoint, err)
|
||||
} else if endpoint.Scheme != "https" {
|
||||
return nil, fmt.Errorf("verifier url must be https: %q", rawEndpoint)
|
||||
}
|
||||
}
|
||||
verifiers[key] = append(verifiers[key], verifierURLs...)
|
||||
}
|
||||
return verifiers, nil
|
||||
}
|
||||
|
||||
func (id DetectorID) String() string {
|
||||
name := dpb.DetectorType_name[int32(id.ID)]
|
||||
if name == "" {
|
||||
name = "<invalid ID>"
|
||||
}
|
||||
if id.Version == 0 {
|
||||
return name
|
||||
}
|
||||
return fmt.Sprintf("%s.v%d", name, id.Version)
|
||||
}
|
||||
|
||||
// allDetectors reutrns an ordered slice of all detector types.
|
||||
func allDetectors() []DetectorID {
|
||||
all := make([]DetectorID, 0, len(dpb.DetectorType_name))
|
||||
for id := range dpb.DetectorType_name {
|
||||
all = append(all, DetectorID{ID: dpb.DetectorType(id)})
|
||||
}
|
||||
sort.Slice(all, func(i, j int) bool { return all[i].ID < all[j].ID })
|
||||
return all
|
||||
}
|
||||
|
||||
// asRange converts a single input into a slice of detector types. If the input
|
||||
// is not in range format, a slice of length 1 is returned. Unbounded ranges
|
||||
// are allowed.
|
||||
func asRange(input string) ([]DetectorID, error) {
|
||||
// Check if it's a single detector type.
|
||||
dt, err := asDetectorID(input)
|
||||
if err == nil {
|
||||
return []DetectorID{dt}, nil
|
||||
}
|
||||
|
||||
// Check if it's a range; if not return the error from above.
|
||||
start, end, found := strings.Cut(input, "-")
|
||||
if !found {
|
||||
return nil, err
|
||||
}
|
||||
start, end = strings.TrimSpace(start), strings.TrimSpace(end)
|
||||
|
||||
// Convert the range start and end to a DetectorType.
|
||||
dtStart, err := asDetectorID(start)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
dtEnd, err := asDetectorID(end)
|
||||
// If end is empty it's an unbounded range.
|
||||
if err != nil && end != "" {
|
||||
return nil, err
|
||||
}
|
||||
if end == "" {
|
||||
dtEnd.ID = maxDetectorType
|
||||
}
|
||||
|
||||
// Ensure these ranges don't have versions.
|
||||
if dtEnd.Version != 0 || dtStart.Version != 0 {
|
||||
return nil, fmt.Errorf("versions within ranges are not supported: %s", input)
|
||||
}
|
||||
|
||||
step := dpb.DetectorType(1)
|
||||
if dtStart.ID > dtEnd.ID {
|
||||
step = -1
|
||||
}
|
||||
var output []DetectorID
|
||||
for dt := dtStart.ID; dt != dtEnd.ID; dt += step {
|
||||
if _, ok := validDetectors[dt]; !ok {
|
||||
continue
|
||||
}
|
||||
output = append(output, DetectorID{ID: dt})
|
||||
}
|
||||
return append(output, dtEnd), nil
|
||||
}
|
||||
|
||||
// asDetectorID converts the case-insensitive input into a DetectorID. Name or
|
||||
// ID may be used. Input is expected to be already trimmed of whitespace.
|
||||
func asDetectorID(input string) (DetectorID, error) {
|
||||
if input == "" {
|
||||
return DetectorID{}, fmt.Errorf("empty detector")
|
||||
}
|
||||
var detectorID DetectorID
|
||||
// Separate the version if there is one.
|
||||
if detector, version, hasVersion := strings.Cut(input, "."); hasVersion {
|
||||
parsedVersion, err := parseVersion(version)
|
||||
if err != nil {
|
||||
return DetectorID{}, fmt.Errorf("invalid version for input: %q error: %w", input, err)
|
||||
}
|
||||
detectorID.Version = parsedVersion
|
||||
// Because there was a version, the detector type input is the part before the '.'
|
||||
input = detector
|
||||
}
|
||||
|
||||
// Check if it's a named detector.
|
||||
if dt, ok := detectorTypeValue[strings.ToLower(input)]; ok {
|
||||
detectorID.ID = dt
|
||||
return detectorID, nil
|
||||
}
|
||||
// Check if it's a detector ID.
|
||||
if i, err := strconv.ParseInt(input, 10, 32); err == nil {
|
||||
dt := dpb.DetectorType(i)
|
||||
if _, ok := validDetectors[dt]; !ok {
|
||||
return DetectorID{}, fmt.Errorf("invalid detector ID: %s", input)
|
||||
}
|
||||
detectorID.ID = dt
|
||||
return detectorID, nil
|
||||
}
|
||||
return DetectorID{}, fmt.Errorf("unrecognized detector type: %s", input)
|
||||
}
|
||||
|
||||
func parseVersion(v string) (int, error) {
|
||||
if !strings.HasPrefix(strings.ToLower(v), "v") {
|
||||
return 0, fmt.Errorf("version must start with 'v'")
|
||||
}
|
||||
version := strings.TrimLeft(v, "vV")
|
||||
return strconv.Atoi(version)
|
||||
}
|
||||
@@ -1,47 +0,0 @@
|
||||
package config
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
dpb "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
|
||||
)
|
||||
|
||||
func TestDetectorParsing(t *testing.T) {
|
||||
tests := map[string]struct {
|
||||
input string
|
||||
expected []DetectorID
|
||||
}{
|
||||
"all": {"AlL", allDetectors()},
|
||||
"trailing range": {"0-", allDetectors()},
|
||||
"all after 1": {"1-", allDetectors()[1:]},
|
||||
"named and valid range": {"aWs,8-9", []DetectorID{{ID: dpb.DetectorType_AWS}, {ID: dpb.DetectorType_Github}, {ID: dpb.DetectorType_Gitlab}}},
|
||||
"duplicate order preserved": {"9, 8, 9", []DetectorID{{ID: 9}, {ID: 8}}},
|
||||
"named range": {"github - gitlab", []DetectorID{{ID: dpb.DetectorType_Github}, {ID: dpb.DetectorType_Gitlab}}},
|
||||
"range preserved": {"8-9, 7-10", []DetectorID{{ID: 8}, {ID: 9}, {ID: 7}, {ID: 10}}},
|
||||
"reverse range": {"9-8", []DetectorID{{ID: 9}, {ID: 8}}},
|
||||
"range preserved with all": {"10-,all", append(allDetectors()[10:], allDetectors()[:10]...)},
|
||||
"empty list item": {"8, ,9", []DetectorID{{ID: 8}, {ID: 9}}},
|
||||
"invalid end range": {"0-1337", nil},
|
||||
"invalid name": {"foo", nil},
|
||||
"negative": {"-1", nil},
|
||||
"github.v1": {"github.v1", []DetectorID{{ID: dpb.DetectorType_Github, Version: 1}}},
|
||||
"gitlab.v100": {"gitlab.v100", []DetectorID{{ID: dpb.DetectorType_Gitlab, Version: 100}}},
|
||||
"range with versions": {"github.v2 - gitlab.v1", nil},
|
||||
"invalid version no v": {"gitlab.2", nil},
|
||||
"invalid version no number": {"gitlab.github", nil},
|
||||
"capital V is fine": {"GiTlAb.V2", []DetectorID{{ID: dpb.DetectorType_Gitlab, Version: 2}}},
|
||||
"id number with version": {"8.v2", []DetectorID{{ID: 8, Version: 2}}},
|
||||
}
|
||||
|
||||
for name, tt := range tests {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
got, gotErr := ParseDetectors(tt.input)
|
||||
if tt.expected == nil {
|
||||
assert.Error(t, gotErr)
|
||||
return
|
||||
}
|
||||
assert.Equal(t, tt.expected, got)
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -1,178 +0,0 @@
|
||||
package context
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"time"
|
||||
|
||||
"github.com/go-logr/logr"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/log"
|
||||
)
|
||||
|
||||
var (
|
||||
// defaultLogger can be set via SetDefaultLogger.
|
||||
// It is initialized to write to stderr. To disable, you can call
|
||||
// SetDefaultLogger with logr.Discard().
|
||||
defaultLogger logr.Logger
|
||||
)
|
||||
|
||||
func init() {
|
||||
defaultLogger, _ = log.New("context", log.WithConsoleSink(os.Stderr))
|
||||
}
|
||||
|
||||
// Context wraps context.Context and includes an additional Logger() method.
|
||||
type Context interface {
|
||||
context.Context
|
||||
Logger() logr.Logger
|
||||
}
|
||||
|
||||
// CancelFunc and CancelCauseFunc are type aliases to allow use as if they are
|
||||
// the same types as the standard library variants.
|
||||
type CancelFunc = context.CancelFunc
|
||||
type CancelCauseFunc = context.CancelCauseFunc
|
||||
|
||||
// logCtx implements Context.
|
||||
type logCtx struct {
|
||||
// Embed context.Context to get all methods for free.
|
||||
context.Context
|
||||
log logr.Logger
|
||||
}
|
||||
|
||||
// Logger returns a structured logger.
|
||||
func (l logCtx) Logger() logr.Logger {
|
||||
return l.log
|
||||
}
|
||||
|
||||
// Background returns context.Background with a default logger.
|
||||
func Background() Context {
|
||||
return logCtx{
|
||||
log: defaultLogger,
|
||||
Context: context.Background(),
|
||||
}
|
||||
}
|
||||
|
||||
// TODO returns context.TODO with a default logger.
|
||||
func TODO() Context {
|
||||
return logCtx{
|
||||
log: defaultLogger,
|
||||
Context: context.TODO(),
|
||||
}
|
||||
}
|
||||
|
||||
// WithCancel returns context.WithCancel with the log object propagated.
|
||||
func WithCancel(parent Context) (Context, context.CancelFunc) {
|
||||
ctx, cancel := context.WithCancel(parent)
|
||||
lCtx := logCtx{
|
||||
log: parent.Logger(),
|
||||
Context: ctx,
|
||||
}
|
||||
return lCtx, cancel
|
||||
}
|
||||
|
||||
// WithCancelCause returns context.WithCancelCause with the log object propagated.
|
||||
func WithCancelCause(parent Context) (Context, context.CancelCauseFunc) {
|
||||
ctx, cancel := context.WithCancelCause(parent)
|
||||
lCtx := logCtx{
|
||||
log: parent.Logger(),
|
||||
Context: ctx,
|
||||
}
|
||||
return lCtx, cancel
|
||||
}
|
||||
|
||||
// WithDeadline returns context.WithDeadline with the log object propagated and
|
||||
// the deadline added to the structured log values.
|
||||
func WithDeadline(parent Context, d time.Time) (Context, context.CancelFunc) {
|
||||
ctx, cancel := context.WithDeadline(parent, d)
|
||||
lCtx := logCtx{
|
||||
log: parent.Logger().WithValues("deadline", d),
|
||||
Context: ctx,
|
||||
}
|
||||
return lCtx, cancel
|
||||
}
|
||||
|
||||
// WithDeadlineCause returns context.WithDeadlineCause with the log object
|
||||
// propagated and the deadline added to the structured log values.
|
||||
func WithDeadlineCause(parent Context, d time.Time, cause error) (Context, context.CancelFunc) {
|
||||
ctx, cancel := context.WithDeadlineCause(parent, d, cause)
|
||||
lCtx := logCtx{
|
||||
log: parent.Logger().WithValues("deadline", d),
|
||||
Context: ctx,
|
||||
}
|
||||
return lCtx, cancel
|
||||
}
|
||||
|
||||
// WithTimeout returns context.WithTimeout with the log object propagated and
|
||||
// the timeout added to the structured log values.
|
||||
func WithTimeout(parent Context, timeout time.Duration) (Context, context.CancelFunc) {
|
||||
ctx, cancel := context.WithTimeout(parent, timeout)
|
||||
lCtx := logCtx{
|
||||
log: parent.Logger().WithValues("timeout", timeout),
|
||||
Context: ctx,
|
||||
}
|
||||
return lCtx, cancel
|
||||
}
|
||||
|
||||
// WithTimeoutCause returns context.WithTimeoutCause with the log object
|
||||
// propagated and the timeout added to the structured log values.
|
||||
func WithTimeoutCause(parent Context, timeout time.Duration, cause error) (Context, context.CancelFunc) {
|
||||
ctx, cancel := context.WithTimeoutCause(parent, timeout, cause)
|
||||
lCtx := logCtx{
|
||||
log: parent.Logger().WithValues("timeout", timeout),
|
||||
Context: ctx,
|
||||
}
|
||||
return lCtx, cancel
|
||||
}
|
||||
|
||||
// Cause returns the context.Cause of the context.
|
||||
func Cause(ctx context.Context) error {
|
||||
return context.Cause(ctx)
|
||||
}
|
||||
|
||||
// WithValue returns context.WithValue with the log object propagated and
|
||||
// the value added to the structured log values (if the key is a string).
|
||||
func WithValue(parent Context, key, val any) Context {
|
||||
logger := parent.Logger()
|
||||
if k, ok := key.(string); ok {
|
||||
logger = logger.WithValues(k, val)
|
||||
}
|
||||
return logCtx{
|
||||
log: logger,
|
||||
Context: context.WithValue(parent, key, val),
|
||||
}
|
||||
}
|
||||
|
||||
// WithValues returns context.WithValue with the log object propagated and
|
||||
// the values added to the structured log values (if the key is a string).
|
||||
func WithValues(parent Context, keyAndVals ...any) Context {
|
||||
ctx := parent
|
||||
for i := 0; i < len(keyAndVals)-1; i += 2 {
|
||||
ctx = WithValue(ctx, keyAndVals[i], keyAndVals[i+1])
|
||||
}
|
||||
return ctx
|
||||
}
|
||||
|
||||
// WithLogger converts a context.Context into a Context by adding a logger.
|
||||
func WithLogger(parent context.Context, logger logr.Logger) Context {
|
||||
return logCtx{
|
||||
log: logger,
|
||||
Context: parent,
|
||||
}
|
||||
}
|
||||
|
||||
// AddLogger converts a context.Context into a Context. If the underlying type
|
||||
// is already a Context, that will be returned, otherwise a default logger will
|
||||
// be added.
|
||||
func AddLogger(parent context.Context) Context {
|
||||
if loggerCtx, ok := parent.(Context); ok {
|
||||
return loggerCtx
|
||||
}
|
||||
return WithLogger(parent, defaultLogger)
|
||||
}
|
||||
|
||||
// SetupDefaultLogger sets the package-level global default logger that will be
|
||||
// used for Background and TODO contexts. On startup, the default logger will
|
||||
// be configured to output logs to stderr. Use logr.Discard() to disable all
|
||||
// logs from Contexts.
|
||||
func SetDefaultLogger(l logr.Logger) {
|
||||
defaultLogger = l
|
||||
}
|
||||
@@ -1,187 +0,0 @@
|
||||
package context
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/go-logr/logr"
|
||||
"github.com/go-logr/zapr"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/log"
|
||||
"go.uber.org/zap"
|
||||
"go.uber.org/zap/zapcore"
|
||||
"go.uber.org/zap/zaptest"
|
||||
)
|
||||
|
||||
// testLogger is a helper function to create a logger with a closure callback.
|
||||
func testLogger(t *testing.T, f func(zapcore.Entry)) logr.Logger {
|
||||
return zapr.NewLogger(zaptest.NewLogger(t,
|
||||
zaptest.WrapOptions(zap.Hooks(func(e zapcore.Entry) error {
|
||||
f(e)
|
||||
return nil
|
||||
}))))
|
||||
}
|
||||
|
||||
// infoCounterContext is a helper function to create a Context that will count
|
||||
// the number of Info messages logged.
|
||||
func infoCounterContext(t *testing.T) (Context, *int) {
|
||||
var infoCount int
|
||||
logger := testLogger(t, func(e zapcore.Entry) {
|
||||
if e.Level == zap.InfoLevel {
|
||||
infoCount++
|
||||
}
|
||||
})
|
||||
return WithLogger(context.Background(), logger), &infoCount
|
||||
}
|
||||
|
||||
func TestWithCancel(t *testing.T) {
|
||||
parentCtx, infoCount := infoCounterContext(t)
|
||||
ctx, cancel := WithCancel(parentCtx)
|
||||
cancel()
|
||||
assert.Equal(t, 0, *infoCount)
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
ctx.Logger().Info("yay")
|
||||
case <-time.After(1 * time.Second):
|
||||
assert.Fail(t, "context should be done")
|
||||
}
|
||||
assert.Equal(t, 1, *infoCount)
|
||||
}
|
||||
|
||||
func TestWithTimeout(t *testing.T) {
|
||||
parentCtx, infoCount := infoCounterContext(t)
|
||||
ctx, cancel := WithTimeout(parentCtx, 10*time.Millisecond)
|
||||
defer cancel()
|
||||
|
||||
assert.Equal(t, 0, *infoCount)
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
ctx.Logger().Info("yay")
|
||||
case <-time.After(1 * time.Second):
|
||||
assert.Fail(t, "context should be done")
|
||||
}
|
||||
assert.Equal(t, 1, *infoCount)
|
||||
|
||||
ctx, cancel = WithTimeout(parentCtx, 1*time.Second)
|
||||
defer cancel()
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
assert.Fail(t, "context should not be done")
|
||||
case <-time.After(10 * time.Millisecond):
|
||||
ctx.Logger().Info("yay")
|
||||
}
|
||||
assert.Equal(t, 2, *infoCount)
|
||||
}
|
||||
|
||||
func TestWithLogger(t *testing.T) {
|
||||
var infoCount int
|
||||
logger := testLogger(t, func(e zapcore.Entry) {
|
||||
if e.Level == zap.InfoLevel {
|
||||
infoCount++
|
||||
}
|
||||
})
|
||||
|
||||
ctx := WithLogger(context.Background(), logger)
|
||||
assert.Equal(t, logger, ctx.Logger())
|
||||
|
||||
assert.Equal(t, 0, infoCount)
|
||||
ctx.Logger().Info("yay")
|
||||
assert.Equal(t, 1, infoCount)
|
||||
}
|
||||
|
||||
func TestAsContext(t *testing.T) {
|
||||
var gotValue any
|
||||
normalFuncThatTakesContext := func(ctx context.Context) {
|
||||
if logCtx, ok := ctx.(Context); ok {
|
||||
logCtx.Logger().Info("yay")
|
||||
}
|
||||
gotValue = ctx.Value("key")
|
||||
}
|
||||
parentCtx, infoCount := infoCounterContext(t)
|
||||
ctx := WithValue(parentCtx, "key", "value")
|
||||
|
||||
assert.Equal(t, 0, *infoCount)
|
||||
normalFuncThatTakesContext(ctx)
|
||||
assert.Equal(t, 1, *infoCount)
|
||||
assert.Equal(t, "value", gotValue)
|
||||
}
|
||||
|
||||
func TestWithValues(t *testing.T) {
|
||||
var buffer bytes.Buffer
|
||||
logger, sync := log.New("test",
|
||||
log.WithConsoleSink(&buffer),
|
||||
)
|
||||
defer func(prevLogger logr.Logger) {
|
||||
defaultLogger = prevLogger
|
||||
}(defaultLogger)
|
||||
SetDefaultLogger(logger)
|
||||
|
||||
{
|
||||
ctx1 := Background()
|
||||
ctx1.Logger().Info("only a", "a", 0)
|
||||
|
||||
ctx2 := WithValue(ctx1, "b", 1)
|
||||
ctx2.Logger().Info("only b")
|
||||
assert.Equal(t, 1, ctx2.Value("b"))
|
||||
|
||||
ctx3 := WithLogger(ctx2, ctx2.Logger().WithValues("c", 2, "d", 3))
|
||||
ctx3.Logger().Info("bcd")
|
||||
|
||||
ctx2.Logger().Info("only b again")
|
||||
|
||||
type customKey string
|
||||
ctx4 := WithValue(Background(), customKey("foo"), "bar")
|
||||
// foo:bar shouldn't be added to the logger because the key isn't a string
|
||||
ctx4.Logger().Info("foo")
|
||||
|
||||
ctx5 := WithValues(ctx2, "e", 4, "f", 5, 6, "six")
|
||||
ctx5.Logger().Info("bef")
|
||||
assert.Equal(t, "six", ctx5.Value(6))
|
||||
|
||||
ctx6 := WithValues(ctx2, "what does this do?")
|
||||
ctx6.Logger().Info("silently fail I suppose")
|
||||
}
|
||||
assert.Nil(t, sync())
|
||||
logs := strings.Split(strings.TrimSpace(buffer.String()), "\n")
|
||||
|
||||
assert.Equal(t, 7, len(logs))
|
||||
assert.Contains(t, logs[0], `{"a": 0}`)
|
||||
assert.Contains(t, logs[1], `{"b": 1}`)
|
||||
assert.Contains(t, logs[2], `{"b": 1, "c": 2, "d": 3}`)
|
||||
assert.Contains(t, logs[3], `{"b": 1}`)
|
||||
assert.NotContains(t, logs[4], `{"foo": "bar"}`)
|
||||
assert.Contains(t, logs[5], `{"b": 1, "e": 4, "f": 5}`)
|
||||
assert.Contains(t, logs[6], `silently fail`)
|
||||
assert.NotContains(t, logs[6], `what does this do?`)
|
||||
}
|
||||
|
||||
func TestDefaultLogger(t *testing.T) {
|
||||
var panicked bool
|
||||
defer func() {
|
||||
if r := recover(); r != nil {
|
||||
panicked = true
|
||||
}
|
||||
assert.False(t, panicked)
|
||||
}()
|
||||
ctx := Background()
|
||||
ctx.Logger().Info("this shouldn't panic")
|
||||
}
|
||||
|
||||
func TestRace(t *testing.T) {
|
||||
ctx, cancel := WithCancel(Background())
|
||||
go cancel()
|
||||
go func() { _ = ctx.Err() }()
|
||||
cancel()
|
||||
_ = ctx.Err()
|
||||
}
|
||||
|
||||
func TestCause(t *testing.T) {
|
||||
ctx, cancel := WithCancelCause(Background())
|
||||
err := fmt.Errorf("oh no")
|
||||
cancel(err)
|
||||
assert.Equal(t, err, Cause(ctx))
|
||||
}
|
||||
@@ -1,250 +0,0 @@
|
||||
package custom_detectors
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/custom_detectorspb"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
|
||||
"golang.org/x/sync/errgroup"
|
||||
)
|
||||
|
||||
// The maximum number of matches from one chunk. This const is used when
|
||||
// permutating each regex match to protect the scanner from doing too much work
|
||||
// for poorly defined regexps.
|
||||
const maxTotalMatches = 100
|
||||
|
||||
// CustomRegexWebhook is a CustomRegex with webhook validation that is
|
||||
// guaranteed to be valid (assuming the data is not changed after
|
||||
// initialization).
|
||||
type CustomRegexWebhook struct {
|
||||
*custom_detectorspb.CustomRegex
|
||||
}
|
||||
|
||||
// Ensure the Scanner satisfies the interface at compile time.
|
||||
var _ detectors.Detector = (*CustomRegexWebhook)(nil)
|
||||
|
||||
// NewWebhookCustomRegex initializes and validates a CustomRegexWebhook. An
|
||||
// unexported type is intentionally returned here to ensure the values have
|
||||
// been validated.
|
||||
func NewWebhookCustomRegex(pb *custom_detectorspb.CustomRegex) (*CustomRegexWebhook, error) {
|
||||
// TODO: Return all validation errors.
|
||||
if err := ValidateKeywords(pb.Keywords); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := ValidateRegex(pb.Regex); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
for _, verify := range pb.Verify {
|
||||
if err := ValidateVerifyEndpoint(verify.Endpoint, verify.Unsafe); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := ValidateVerifyHeaders(verify.Headers); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
// TODO: Copy only necessary data out of pb.
|
||||
return &CustomRegexWebhook{pb}, nil
|
||||
}
|
||||
|
||||
var httpClient = common.SaneHttpClient()
|
||||
|
||||
func (c *CustomRegexWebhook) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
|
||||
dataStr := string(data)
|
||||
regexMatches := make(map[string][][]string, len(c.GetRegex()))
|
||||
|
||||
// Find all submatches for each regex.
|
||||
for name, regex := range c.GetRegex() {
|
||||
regex, err := regexp.Compile(regex)
|
||||
if err != nil {
|
||||
// This will only happen if the regex is invalid.
|
||||
return nil, err
|
||||
}
|
||||
regexMatches[name] = regex.FindAllStringSubmatch(dataStr, -1)
|
||||
}
|
||||
|
||||
// Permutate each individual match.
|
||||
// {
|
||||
// "foo": [["match1"]]
|
||||
// "bar": [["match2"], ["match3"]]
|
||||
// }
|
||||
// becomes
|
||||
// [
|
||||
// {"foo": ["match1"], "bar": ["match2"]},
|
||||
// {"foo": ["match1"], "bar": ["match3"]},
|
||||
// ]
|
||||
matches := permutateMatches(regexMatches)
|
||||
|
||||
g := new(errgroup.Group)
|
||||
|
||||
// Create result object and test for verification.
|
||||
resultsCh := make(chan detectors.Result, maxTotalMatches)
|
||||
for _, match := range matches {
|
||||
match := match
|
||||
g.Go(func() error {
|
||||
return c.createResults(ctx, match, verify, resultsCh)
|
||||
})
|
||||
}
|
||||
|
||||
// Ignore any errors and collect as many of the results as we can.
|
||||
_ = g.Wait()
|
||||
close(resultsCh)
|
||||
|
||||
for result := range resultsCh {
|
||||
// NOTE: I don't believe this is being set anywhere else, hence the map assignment.
|
||||
result.ExtraData = map[string]string{
|
||||
"name": c.GetName(),
|
||||
}
|
||||
results = append(results, result)
|
||||
}
|
||||
|
||||
return results, nil
|
||||
}
|
||||
|
||||
func (c *CustomRegexWebhook) createResults(ctx context.Context, match map[string][]string, verify bool, results chan<- detectors.Result) error {
|
||||
if common.IsDone(ctx) {
|
||||
// TODO: Log we're possibly leaving out results.
|
||||
return ctx.Err()
|
||||
}
|
||||
var raw string
|
||||
for _, values := range match {
|
||||
// values[0] contains the entire regex match.
|
||||
raw += values[0]
|
||||
}
|
||||
result := detectors.Result{
|
||||
DetectorType: detectorspb.DetectorType_CustomRegex,
|
||||
DetectorName: c.GetName(),
|
||||
Raw: []byte(raw),
|
||||
}
|
||||
|
||||
if !verify {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
case results <- result:
|
||||
return nil
|
||||
}
|
||||
}
|
||||
// Verify via webhook.
|
||||
jsonBody, err := json.Marshal(map[string]map[string][]string{
|
||||
c.GetName(): match,
|
||||
})
|
||||
if err != nil {
|
||||
// This should never happen, but if it does, return nil to not
|
||||
// disrupt other verification.
|
||||
return nil
|
||||
}
|
||||
// Try each config until we successfully verify.
|
||||
for _, verifyConfig := range c.GetVerify() {
|
||||
if common.IsDone(ctx) {
|
||||
// TODO: Log we're possibly leaving out results.
|
||||
return ctx.Err()
|
||||
}
|
||||
req, err := http.NewRequestWithContext(ctx, "POST", verifyConfig.GetEndpoint(), bytes.NewReader(jsonBody))
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
for _, header := range verifyConfig.GetHeaders() {
|
||||
key, value, found := strings.Cut(header, ":")
|
||||
if !found {
|
||||
// Should be unreachable due to validation.
|
||||
continue
|
||||
}
|
||||
req.Header.Add(key, strings.TrimLeft(value, "\t\n\v\f\r "))
|
||||
}
|
||||
res, err := httpClient.Do(req)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
// TODO: Read response body.
|
||||
res.Body.Close()
|
||||
if res.StatusCode == http.StatusOK {
|
||||
result.Verified = true
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
case results <- result:
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
func (c *CustomRegexWebhook) Keywords() []string {
|
||||
return c.GetKeywords()
|
||||
}
|
||||
|
||||
// productIndices produces a permutation of indices for each length. Example:
|
||||
// productIndices(3, 2) -> [[0 0] [1 0] [2 0] [0 1] [1 1] [2 1]]. It returns
|
||||
// a slice of length no larger than maxTotalMatches.
|
||||
func productIndices(lengths ...int) [][]int {
|
||||
count := 1
|
||||
for _, l := range lengths {
|
||||
count *= l
|
||||
}
|
||||
if count == 0 {
|
||||
return nil
|
||||
}
|
||||
if count > maxTotalMatches {
|
||||
count = maxTotalMatches
|
||||
}
|
||||
|
||||
results := make([][]int, count)
|
||||
for i := 0; i < count; i++ {
|
||||
j := 1
|
||||
result := make([]int, 0, len(lengths))
|
||||
for _, l := range lengths {
|
||||
result = append(result, (i/j)%l)
|
||||
j *= l
|
||||
}
|
||||
results[i] = result
|
||||
}
|
||||
return results
|
||||
}
|
||||
|
||||
// permutateMatches converts the list of all regex matches into all possible
|
||||
// permutations selecting one from each named entry in the map. For example:
|
||||
// {"foo": [matchA, matchB], "bar": [matchC]} becomes
|
||||
//
|
||||
// [{"foo": matchA, "bar": matchC}, {"foo": matchB, "bar": matchC}]
|
||||
func permutateMatches(regexMatches map[string][][]string) []map[string][]string {
|
||||
// Get a consistent order for names and their matching lengths.
|
||||
// The lengths are used in calculating the permutation so order matters.
|
||||
names := make([]string, 0, len(regexMatches))
|
||||
lengths := make([]int, 0, len(regexMatches))
|
||||
for key, value := range regexMatches {
|
||||
names = append(names, key)
|
||||
lengths = append(lengths, len(value))
|
||||
}
|
||||
|
||||
// Permutate all the indices for each match. For example, if "foo" has
|
||||
// [matchA, matchB] and "bar" has [matchC], we will get indices [0 0] [1 0].
|
||||
permutationIndices := productIndices(lengths...)
|
||||
|
||||
// Build {"foo": matchA, "bar": matchC} and {"foo": matchB, "bar": matchC}
|
||||
// from the indices.
|
||||
var matches []map[string][]string
|
||||
for _, permutation := range permutationIndices {
|
||||
candidate := make(map[string][]string, len(permutationIndices))
|
||||
for i, name := range names {
|
||||
candidate[name] = regexMatches[name][permutation[i]]
|
||||
}
|
||||
matches = append(matches, candidate)
|
||||
}
|
||||
|
||||
return matches
|
||||
}
|
||||
|
||||
func (c *CustomRegexWebhook) Type() detectorspb.DetectorType {
|
||||
return detectorspb.DetectorType_CustomRegex
|
||||
}
|
||||
@@ -1,215 +0,0 @@
|
||||
package custom_detectors
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/custom_detectorspb"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/protoyaml"
|
||||
)
|
||||
|
||||
func TestCustomRegexTemplateParsing(t *testing.T) {
|
||||
testCustomRegexTemplateYaml := `name: Internal bi tool
|
||||
keywords:
|
||||
- secret_v1_
|
||||
- pat_v2_
|
||||
regex:
|
||||
id_pat_example: ([a-zA-Z0-9]{32})
|
||||
secret_pat_example: ([a-zA-Z0-9]{32})
|
||||
verify:
|
||||
- endpoint: http://localhost:8000/{id_pat_example}
|
||||
unsafe: true
|
||||
headers:
|
||||
- 'Authorization: Bearer {secret_pat_example.0}'
|
||||
successRanges:
|
||||
- 200-250
|
||||
- '288'`
|
||||
|
||||
var got custom_detectorspb.CustomRegex
|
||||
assert.NoError(t, protoyaml.UnmarshalStrict([]byte(testCustomRegexTemplateYaml), &got))
|
||||
assert.Equal(t, "Internal bi tool", got.Name)
|
||||
assert.Equal(t, []string{"secret_v1_", "pat_v2_"}, got.Keywords)
|
||||
assert.Equal(t, map[string]string{
|
||||
"id_pat_example": "([a-zA-Z0-9]{32})",
|
||||
"secret_pat_example": "([a-zA-Z0-9]{32})",
|
||||
}, got.Regex)
|
||||
assert.Equal(t, 1, len(got.Verify))
|
||||
assert.Equal(t, "http://localhost:8000/{id_pat_example}", got.Verify[0].Endpoint)
|
||||
assert.Equal(t, true, got.Verify[0].Unsafe)
|
||||
assert.Equal(t, []string{"Authorization: Bearer {secret_pat_example.0}"}, got.Verify[0].Headers)
|
||||
assert.Equal(t, []string{"200-250", "288"}, got.Verify[0].SuccessRanges)
|
||||
}
|
||||
|
||||
func TestCustomRegexWebhookParsing(t *testing.T) {
|
||||
testCustomRegexWebhookYaml := `name: Internal bi tool
|
||||
keywords:
|
||||
- secret_v1_
|
||||
- pat_v2_
|
||||
regex:
|
||||
id_pat_example: ([a-zA-Z0-9]{32})
|
||||
secret_pat_example: ([a-zA-Z0-9]{32})
|
||||
verify:
|
||||
- endpoint: http://localhost:8000/
|
||||
unsafe: true
|
||||
headers:
|
||||
- 'Authorization: Bearer token'`
|
||||
|
||||
var got custom_detectorspb.CustomRegex
|
||||
assert.NoError(t, protoyaml.UnmarshalStrict([]byte(testCustomRegexWebhookYaml), &got))
|
||||
assert.Equal(t, "Internal bi tool", got.Name)
|
||||
assert.Equal(t, []string{"secret_v1_", "pat_v2_"}, got.Keywords)
|
||||
assert.Equal(t, map[string]string{
|
||||
"id_pat_example": "([a-zA-Z0-9]{32})",
|
||||
"secret_pat_example": "([a-zA-Z0-9]{32})",
|
||||
}, got.Regex)
|
||||
assert.Equal(t, 1, len(got.Verify))
|
||||
assert.Equal(t, "http://localhost:8000/", got.Verify[0].Endpoint)
|
||||
assert.Equal(t, true, got.Verify[0].Unsafe)
|
||||
assert.Equal(t, []string{"Authorization: Bearer token"}, got.Verify[0].Headers)
|
||||
}
|
||||
|
||||
// TestCustomDetectorsParsing tests the full `detectors` configuration.
|
||||
func TestCustomDetectorsParsing(t *testing.T) {
|
||||
// TODO: Support both template and webhook.
|
||||
testYamlConfig := `detectors:
|
||||
- name: Internal bi tool
|
||||
keywords:
|
||||
- secret_v1_
|
||||
- pat_v2_
|
||||
regex:
|
||||
id_pat_example: ([a-zA-Z0-9]{32})
|
||||
secret_pat_example: ([a-zA-Z0-9]{32})
|
||||
verify:
|
||||
- endpoint: http://localhost:8000/
|
||||
unsafe: true
|
||||
headers:
|
||||
- 'Authorization: Bearer token'`
|
||||
|
||||
var messages custom_detectorspb.CustomDetectors
|
||||
assert.NoError(t, protoyaml.UnmarshalStrict([]byte(testYamlConfig), &messages))
|
||||
assert.Equal(t, 1, len(messages.Detectors))
|
||||
|
||||
got := messages.Detectors[0]
|
||||
assert.Equal(t, "Internal bi tool", got.Name)
|
||||
assert.Equal(t, []string{"secret_v1_", "pat_v2_"}, got.Keywords)
|
||||
assert.Equal(t, map[string]string{
|
||||
"id_pat_example": "([a-zA-Z0-9]{32})",
|
||||
"secret_pat_example": "([a-zA-Z0-9]{32})",
|
||||
}, got.Regex)
|
||||
assert.Equal(t, 1, len(got.Verify))
|
||||
assert.Equal(t, "http://localhost:8000/", got.Verify[0].Endpoint)
|
||||
assert.Equal(t, true, got.Verify[0].Unsafe)
|
||||
assert.Equal(t, []string{"Authorization: Bearer token"}, got.Verify[0].Headers)
|
||||
}
|
||||
|
||||
func TestFromData_InvalidRegEx(t *testing.T) {
|
||||
c := &CustomRegexWebhook{
|
||||
&custom_detectorspb.CustomRegex{
|
||||
Name: "Internal bi tool",
|
||||
Keywords: []string{"secret_v1_", "pat_v2_"},
|
||||
Regex: map[string]string{
|
||||
"test": "!!?(?:?)[a-zA-Z0-9]{32}", // invalid regex
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
_, err := c.FromData(context.Background(), false, []byte("test"))
|
||||
assert.Error(t, err)
|
||||
}
|
||||
|
||||
func TestProductIndices(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
input []int
|
||||
want [][]int
|
||||
}{
|
||||
{
|
||||
name: "zero",
|
||||
input: []int{3, 0},
|
||||
want: nil,
|
||||
},
|
||||
{
|
||||
name: "one input",
|
||||
input: []int{3},
|
||||
want: [][]int{{0}, {1}, {2}},
|
||||
},
|
||||
{
|
||||
name: "two inputs",
|
||||
input: []int{3, 2},
|
||||
want: [][]int{
|
||||
{0, 0}, {1, 0}, {2, 0},
|
||||
{0, 1}, {1, 1}, {2, 1},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "three inputs",
|
||||
input: []int{3, 2, 3},
|
||||
want: [][]int{
|
||||
{0, 0, 0}, {1, 0, 0}, {2, 0, 0},
|
||||
{0, 1, 0}, {1, 1, 0}, {2, 1, 0},
|
||||
{0, 0, 1}, {1, 0, 1}, {2, 0, 1},
|
||||
{0, 1, 1}, {1, 1, 1}, {2, 1, 1},
|
||||
{0, 0, 2}, {1, 0, 2}, {2, 0, 2},
|
||||
{0, 1, 2}, {1, 1, 2}, {2, 1, 2},
|
||||
},
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := productIndices(tt.input...)
|
||||
assert.Equal(t, tt.want, got)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestProductIndicesMax(t *testing.T) {
|
||||
got := productIndices(2, 3, 4, 5, 6)
|
||||
assert.GreaterOrEqual(t, 2*3*4*5*6, maxTotalMatches)
|
||||
assert.Equal(t, maxTotalMatches, len(got))
|
||||
}
|
||||
|
||||
func TestPermutateMatches(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
input map[string][][]string
|
||||
want []map[string][]string
|
||||
}{
|
||||
{
|
||||
name: "two matches",
|
||||
input: map[string][][]string{"foo": {{"matchA"}, {"matchB"}}, "bar": {{"matchC"}}},
|
||||
want: []map[string][]string{
|
||||
{"foo": {"matchA"}, "bar": {"matchC"}},
|
||||
{"foo": {"matchB"}, "bar": {"matchC"}},
|
||||
},
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := permutateMatches(tt.input)
|
||||
assert.Equal(t, tt.want, got)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestDetector(t *testing.T) {
|
||||
detector, err := NewWebhookCustomRegex(&custom_detectorspb.CustomRegex{
|
||||
Name: "test",
|
||||
// "password" is normally flagged as a false positive, but CustomRegex
|
||||
// should allow the user to decide and report it as a result.
|
||||
Keywords: []string{"password"},
|
||||
Regex: map[string]string{"regex": "password=.*"},
|
||||
})
|
||||
assert.NoError(t, err)
|
||||
results, err := detector.FromData(context.Background(), false, []byte(`password="123456"`))
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, 1, len(results))
|
||||
assert.Equal(t, results[0].Raw, []byte(`password="123456"`))
|
||||
}
|
||||
|
||||
func BenchmarkProductIndices(b *testing.B) {
|
||||
for i := 0; i < b.N; i++ {
|
||||
_ = productIndices(3, 2, 6)
|
||||
}
|
||||
}
|
||||
@@ -1,44 +0,0 @@
|
||||
package custom_detectors
|
||||
|
||||
import (
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// nameGroupRegex matches `{ name . group }` ignoring any whitespace.
|
||||
var nameGroupRegex = regexp.MustCompile(`{\s*([a-zA-Z0-9-_]+)\s*(\.\s*[0-9]*)?\s*}`)
|
||||
|
||||
// RegexVarString is a string with embedded {name.group} variables. A name may
|
||||
// only contain alphanumeric, hyphen, and underscore characters. Group is
|
||||
// optional but if provided it must be a non-negative integer. If the group is
|
||||
// omitted it defaults to 0.
|
||||
type RegexVarString struct {
|
||||
original string
|
||||
// map from name to group
|
||||
variables map[string]int
|
||||
}
|
||||
|
||||
func NewRegexVarString(original string) RegexVarString {
|
||||
variables := make(map[string]int)
|
||||
|
||||
matches := nameGroupRegex.FindAllStringSubmatch(original, -1)
|
||||
for _, match := range matches {
|
||||
name, group := match[1], 0
|
||||
// The second match will start with a period followed by any number
|
||||
// of whitespace.
|
||||
if len(match[2]) > 1 {
|
||||
g, err := strconv.Atoi(strings.TrimSpace(match[2][1:]))
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
group = g
|
||||
}
|
||||
variables[name] = group
|
||||
}
|
||||
|
||||
return RegexVarString{
|
||||
original: original,
|
||||
variables: variables,
|
||||
}
|
||||
}
|
||||
@@ -1,76 +0,0 @@
|
||||
package custom_detectors
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
func TestVarString(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
input string
|
||||
wantVars map[string]int
|
||||
}{
|
||||
{
|
||||
name: "empty",
|
||||
input: "{}",
|
||||
wantVars: map[string]int{},
|
||||
},
|
||||
{
|
||||
name: "no subgroup",
|
||||
input: "{hello}",
|
||||
wantVars: map[string]int{
|
||||
"hello": 0,
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "with subgroup",
|
||||
input: "{hello.123}",
|
||||
wantVars: map[string]int{
|
||||
"hello": 123,
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "subgroup with spaces",
|
||||
input: "{\thell0 . 123 }",
|
||||
wantVars: map[string]int{
|
||||
"hell0": 123,
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "multiple groups",
|
||||
input: "foo {bar} {bazz.buzz} {buzz.2}",
|
||||
wantVars: map[string]int{
|
||||
"bar": 0,
|
||||
"buzz": 2,
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "nested groups",
|
||||
input: "{foo {bar}}",
|
||||
wantVars: map[string]int{
|
||||
"bar": 0,
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "decimal without number",
|
||||
input: "{foo.}",
|
||||
wantVars: map[string]int{
|
||||
"foo": 0,
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "negative number",
|
||||
input: "{foo.-1}",
|
||||
wantVars: map[string]int{},
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := NewRegexVarString(tt.input)
|
||||
assert.Equal(t, tt.input, got.original)
|
||||
assert.Equal(t, tt.wantVars, got.variables)
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -1,109 +0,0 @@
|
||||
package custom_detectors
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
)
|
||||
|
||||
func ValidateKeywords(keywords []string) error {
|
||||
if len(keywords) == 0 {
|
||||
return fmt.Errorf("no keywords")
|
||||
}
|
||||
|
||||
for _, keyword := range keywords {
|
||||
if len(keyword) == 0 {
|
||||
return fmt.Errorf("empty keyword")
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func ValidateRegex(regex map[string]string) error {
|
||||
if len(regex) == 0 {
|
||||
return fmt.Errorf("no regex")
|
||||
}
|
||||
for name, reg := range regex {
|
||||
if _, err := regexp.Compile(reg); err != nil {
|
||||
return fmt.Errorf("regex '%s': %w", name, err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func ValidateVerifyEndpoint(endpoint string, unsafe bool) error {
|
||||
if len(endpoint) == 0 {
|
||||
return fmt.Errorf("no endpoint")
|
||||
}
|
||||
|
||||
if strings.HasPrefix(endpoint, "http://") && !unsafe {
|
||||
return fmt.Errorf("http endpoint must have unsafe=true")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func ValidateVerifyHeaders(headers []string) error {
|
||||
for _, header := range headers {
|
||||
if !strings.Contains(header, ":") {
|
||||
return fmt.Errorf("header %q must contain a colon", header)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func ValidateVerifyRanges(ranges []string) error {
|
||||
const httpLowerRange = 100
|
||||
const httpUpperRange = 599
|
||||
|
||||
for _, successRange := range ranges {
|
||||
if !strings.Contains(successRange, "-") {
|
||||
httpCode, err := strconv.Atoi(successRange)
|
||||
if err != nil {
|
||||
return fmt.Errorf("unable to convert http code to int %q", successRange)
|
||||
}
|
||||
|
||||
if httpCode < httpLowerRange || httpCode > httpUpperRange {
|
||||
return fmt.Errorf("invalid http status code %q", successRange)
|
||||
}
|
||||
|
||||
continue
|
||||
}
|
||||
|
||||
httpRange := strings.Split(successRange, "-")
|
||||
if len(httpRange) != 2 {
|
||||
return fmt.Errorf("invalid range format %q", successRange)
|
||||
}
|
||||
|
||||
lowerBound, err := strconv.Atoi(httpRange[0])
|
||||
if err != nil {
|
||||
return fmt.Errorf("unable to convert lower bound to int %q", successRange)
|
||||
}
|
||||
|
||||
upperBound, err := strconv.Atoi(httpRange[1])
|
||||
if err != nil {
|
||||
return fmt.Errorf("unable to convert upper bound to int %q", successRange)
|
||||
}
|
||||
|
||||
if lowerBound > upperBound {
|
||||
return fmt.Errorf("lower bound greater than upper bound on range %q", successRange)
|
||||
}
|
||||
|
||||
if lowerBound < httpLowerRange || upperBound > httpUpperRange {
|
||||
return fmt.Errorf("invalid http status code range %q", successRange)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func ValidateRegexVars(regex map[string]string, body ...string) error {
|
||||
for _, b := range body {
|
||||
matches := NewRegexVarString(b).variables
|
||||
for match := range matches {
|
||||
if _, ok := regex[match]; !ok {
|
||||
return fmt.Errorf("body %q contains an unknown variable", b)
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -1,234 +0,0 @@
|
||||
package custom_detectors
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestCustomDetectorsKeywordValidation(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
input []string
|
||||
wantErr bool
|
||||
}{
|
||||
{
|
||||
name: "Test empty list of keywords",
|
||||
input: []string{},
|
||||
wantErr: true,
|
||||
},
|
||||
{
|
||||
name: "Test empty keyword",
|
||||
input: []string{""},
|
||||
wantErr: true,
|
||||
},
|
||||
{
|
||||
name: "Test valid keywords",
|
||||
input: []string{"hello", "world"},
|
||||
wantErr: false,
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := ValidateKeywords(tt.input)
|
||||
|
||||
if (got != nil && !tt.wantErr) || (got == nil && tt.wantErr) {
|
||||
t.Errorf("ValidateKeywords() error = %v, wantErr %v", got, tt.wantErr)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestCustomDetectorsRegexValidation(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
input map[string]string
|
||||
wantErr bool
|
||||
}{
|
||||
{
|
||||
name: "Test list of keywords",
|
||||
input: map[string]string{
|
||||
"id_pat_example": "([a-zA-Z0-9]{32})",
|
||||
},
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "Test empty list of keywords",
|
||||
input: map[string]string{},
|
||||
wantErr: true,
|
||||
},
|
||||
{
|
||||
name: "Test invalid regex",
|
||||
input: map[string]string{
|
||||
"test": "!!?(?:?)[a-zA-Z0-9]{32}",
|
||||
},
|
||||
wantErr: true,
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := ValidateRegex(tt.input)
|
||||
|
||||
if (got != nil && !tt.wantErr) || (got == nil && tt.wantErr) {
|
||||
t.Errorf("ValidateRegex() error = %v, wantErr %v", got, tt.wantErr)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestCustomDetectorsVerifyEndpointValidation(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
endpoint string
|
||||
unsafe bool
|
||||
wantErr bool
|
||||
}{
|
||||
{
|
||||
name: "Test http endpoint with unsafe flag",
|
||||
endpoint: "http://localhost:8000/{id_pat_example}",
|
||||
unsafe: true,
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "Test http endpoint without unsafe flag",
|
||||
endpoint: "http://localhost:8000/{id_pat_example}",
|
||||
unsafe: false,
|
||||
wantErr: true,
|
||||
},
|
||||
{
|
||||
name: "Test https endpoint with unsafe flag",
|
||||
endpoint: "https://localhost:8000/{id_pat_example}",
|
||||
unsafe: true,
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "Test https endpoint without unsafe flag",
|
||||
endpoint: "https://localhost:8000/{id_pat_example}",
|
||||
unsafe: false,
|
||||
wantErr: false,
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := ValidateVerifyEndpoint(tt.endpoint, tt.unsafe)
|
||||
|
||||
if (got != nil && !tt.wantErr) || (got == nil && tt.wantErr) {
|
||||
t.Errorf("ValidateVerifyEndpoint() error = %v, wantErr %v", got, tt.wantErr)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestCustomDetectorsVerifyHeadersValidation(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
headers []string
|
||||
wantErr bool
|
||||
}{
|
||||
{
|
||||
name: "Test single header",
|
||||
headers: []string{"Authorization: Bearer {secret_pat_example.0}"},
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "Test invalid header",
|
||||
headers: []string{"Hello world"},
|
||||
wantErr: true,
|
||||
},
|
||||
{
|
||||
name: "Test ugly header",
|
||||
headers: []string{"Hello:::::::world::hi:"},
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "Test empty header",
|
||||
headers: []string{},
|
||||
wantErr: false,
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := ValidateVerifyHeaders(tt.headers)
|
||||
|
||||
if (got != nil && !tt.wantErr) || (got == nil && tt.wantErr) {
|
||||
t.Errorf("ValidateVerifyHeaders() error = %v, wantErr %v", got, tt.wantErr)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestCustomDetectorsVerifyRangeValidation(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
ranges []string
|
||||
wantErr bool
|
||||
}{
|
||||
{
|
||||
name: "Test multiple mixed ranges",
|
||||
ranges: []string{"200", "300-350"},
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "Test invalid non-number range",
|
||||
ranges: []string{"hi"},
|
||||
wantErr: true,
|
||||
},
|
||||
{
|
||||
name: "Test invalid lower to upper range",
|
||||
ranges: []string{"200-100"},
|
||||
wantErr: true,
|
||||
},
|
||||
{
|
||||
name: "Test invalid http range",
|
||||
ranges: []string{"400-1000"},
|
||||
wantErr: true,
|
||||
},
|
||||
{
|
||||
name: "Test multiple ranges with invalid inputs",
|
||||
ranges: []string{"322", "hello-world", "100-200"},
|
||||
wantErr: true,
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := ValidateVerifyRanges(tt.ranges)
|
||||
|
||||
if (got != nil && !tt.wantErr) || (got == nil && tt.wantErr) {
|
||||
t.Errorf("ValidateVerifyRanges() error = %v, wantErr %v", got, tt.wantErr)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestCustomDetectorsVerifyRegexVarsValidation(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
regex map[string]string
|
||||
body string
|
||||
wantErr bool
|
||||
}{
|
||||
{
|
||||
name: "Regex defined but not used in body",
|
||||
regex: map[string]string{"id": "[0-9]{1,10}", "id_pat_example": "([a-zA-Z0-9]{32})"},
|
||||
body: "hello world",
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "Regex defined and is used in body",
|
||||
regex: map[string]string{"id": "[0-9]{1,10}", "id_pat_example": "([a-zA-Z0-9]{32})"},
|
||||
body: "hello world {id}",
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "Regex var in body but not defined",
|
||||
regex: map[string]string{"id": "[0-9]{1,10}", "id_pat_example": "([a-zA-Z0-9]{32})"},
|
||||
body: "hello world {hello}",
|
||||
wantErr: true,
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := ValidateRegexVars(tt.regex, tt.body)
|
||||
|
||||
if (got != nil && !tt.wantErr) || (got == nil && tt.wantErr) {
|
||||
t.Errorf("ValidateRegexVars() error = %v, wantErr %v", got, tt.wantErr)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
+48
-99
@@ -3,127 +3,76 @@ package decoders
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/base64"
|
||||
"strings"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
|
||||
)
|
||||
|
||||
type (
|
||||
Base64 struct{}
|
||||
)
|
||||
type Base64 struct{}
|
||||
|
||||
var (
|
||||
b64Charset = []byte("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/-_=")
|
||||
b64EndChars = "+/-_="
|
||||
// Given characters are mostly ASCII, we can use a simple array to map.
|
||||
b64CharsetMapping [128]bool
|
||||
b64Charset = []byte("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/=")
|
||||
b64EndChars = "+/="
|
||||
)
|
||||
|
||||
func init() {
|
||||
// Build an array of all the characters in the base64 charset.
|
||||
for _, char := range b64Charset {
|
||||
b64CharsetMapping[char] = true
|
||||
func getSubstringsOfCharacterSet(data []byte, charset []byte, threshold int) []string {
|
||||
count := 0
|
||||
substrings := []string{}
|
||||
letters := strings.Builder{}
|
||||
if len(data) == 0 {
|
||||
return nil
|
||||
}
|
||||
for _, char := range string(data) {
|
||||
if bytes.ContainsRune(charset, char) {
|
||||
letters.WriteRune(char)
|
||||
count++
|
||||
} else {
|
||||
if count > threshold {
|
||||
substrings = appendB64Substring(letters, substrings)
|
||||
}
|
||||
letters.Reset()
|
||||
count = 0
|
||||
}
|
||||
}
|
||||
|
||||
if count > threshold && len(letters.String()) > 0 {
|
||||
substrings = appendB64Substring(letters, substrings)
|
||||
}
|
||||
|
||||
return substrings
|
||||
}
|
||||
|
||||
func (d *Base64) FromChunk(chunk *sources.Chunk) *DecodableChunk {
|
||||
decodableChunk := &DecodableChunk{Chunk: chunk, DecoderType: detectorspb.DecoderType_BASE64}
|
||||
encodedSubstrings := getSubstringsOfCharacterSet(chunk.Data, 20, b64CharsetMapping, b64EndChars)
|
||||
decodedSubstrings := make(map[string][]byte)
|
||||
func appendB64Substring(letters strings.Builder, substrings []string) []string {
|
||||
|
||||
substring := strings.TrimLeft(letters.String(), b64EndChars)
|
||||
// handle key=value
|
||||
if strings.Contains(strings.TrimRight(substring, b64EndChars), "=") {
|
||||
split := strings.SplitN(substring, "=", 2)
|
||||
substrings = append(substrings, split[len(split)-1])
|
||||
} else {
|
||||
substrings = append(substrings, substring)
|
||||
}
|
||||
return substrings
|
||||
}
|
||||
|
||||
func (d *Base64) FromChunk(chunk *sources.Chunk) *sources.Chunk {
|
||||
|
||||
encodedSubstrings := getSubstringsOfCharacterSet(chunk.Data, b64Charset, 20)
|
||||
decodedSubstrings := map[string][]byte{}
|
||||
|
||||
for _, str := range encodedSubstrings {
|
||||
dec, err := base64.StdEncoding.DecodeString(str)
|
||||
if err == nil {
|
||||
if len(dec) > 0 {
|
||||
decodedSubstrings[str] = dec
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
dec, err = base64.RawURLEncoding.DecodeString(str)
|
||||
if err == nil && len(dec) > 0 {
|
||||
decodedSubstrings[str] = dec
|
||||
}
|
||||
}
|
||||
|
||||
if len(decodedSubstrings) > 0 {
|
||||
var result bytes.Buffer
|
||||
result.Grow(len(chunk.Data))
|
||||
|
||||
start := 0
|
||||
for _, encoded := range encodedSubstrings {
|
||||
if decoded, ok := decodedSubstrings[encoded]; ok {
|
||||
end := bytes.Index(chunk.Data[start:], []byte(encoded))
|
||||
if end != -1 {
|
||||
result.Write(chunk.Data[start : start+end])
|
||||
result.Write(decoded)
|
||||
start += end + len(encoded)
|
||||
}
|
||||
}
|
||||
for substring, dec := range decodedSubstrings {
|
||||
chunk.Data = bytes.Replace(chunk.Data, []byte(substring), dec, 1)
|
||||
}
|
||||
result.Write(chunk.Data[start:])
|
||||
chunk.Data = result.Bytes()
|
||||
return decodableChunk
|
||||
return chunk
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func getSubstringsOfCharacterSet(data []byte, threshold int, charsetMapping [128]bool, endChars string) []string {
|
||||
if len(data) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
count := 0
|
||||
substringsCount := 0
|
||||
|
||||
// Determine the number of substrings that will be returned.
|
||||
// Pre-allocate the slice to avoid reallocations.
|
||||
for _, char := range data {
|
||||
if char < 128 && charsetMapping[char] {
|
||||
count++
|
||||
} else {
|
||||
if count > threshold {
|
||||
substringsCount++
|
||||
}
|
||||
count = 0
|
||||
}
|
||||
}
|
||||
if count > threshold {
|
||||
substringsCount++
|
||||
}
|
||||
|
||||
count = 0
|
||||
start := 0
|
||||
substrings := make([]string, 0, substringsCount)
|
||||
|
||||
for i, char := range data {
|
||||
if char < 128 && charsetMapping[char] {
|
||||
if count == 0 {
|
||||
start = i
|
||||
}
|
||||
count++
|
||||
} else {
|
||||
if count > threshold {
|
||||
substrings = appendB64Substring(data, start, count, substrings, endChars)
|
||||
}
|
||||
count = 0
|
||||
}
|
||||
}
|
||||
|
||||
if count > threshold {
|
||||
substrings = appendB64Substring(data, start, count, substrings, endChars)
|
||||
}
|
||||
|
||||
return substrings
|
||||
}
|
||||
|
||||
func appendB64Substring(data []byte, start, count int, substrings []string, endChars string) []string {
|
||||
substring := bytes.TrimLeft(data[start:start+count], endChars)
|
||||
if idx := bytes.IndexByte(bytes.TrimRight(substring, endChars), '='); idx != -1 {
|
||||
substrings = append(substrings, string(substring[idx+1:]))
|
||||
} else {
|
||||
substrings = append(substrings, string(substring))
|
||||
}
|
||||
return substrings
|
||||
}
|
||||
|
||||
@@ -11,9 +11,9 @@ import (
|
||||
|
||||
func TestBase64_FromChunk(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
chunk *sources.Chunk
|
||||
want *sources.Chunk
|
||||
name string
|
||||
}{
|
||||
{
|
||||
name: "only b64 chunk",
|
||||
@@ -69,60 +69,6 @@ func TestBase64_FromChunk(t *testing.T) {
|
||||
but only this encapsulated secret should be decoded.`),
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "b64-url-safe: only b64 chunk",
|
||||
chunk: &sources.Chunk{
|
||||
Data: []byte(`bG9uZ2VyLWVuY29kZWQtc2VjcmV0LXRlc3Q`),
|
||||
},
|
||||
want: &sources.Chunk{
|
||||
Data: []byte(`longer-encoded-secret-test`),
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "b64-url-safe: mixed content",
|
||||
chunk: &sources.Chunk{
|
||||
Data: []byte(`token: bG9uZ2VyLWVuY29kZWQtc2VjcmV0LXRlc3Q`),
|
||||
},
|
||||
want: &sources.Chunk{
|
||||
Data: []byte(`token: longer-encoded-secret-test`),
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "b64-url-safe: env var (looks like all b64 decodable but has `=` in the middle)",
|
||||
chunk: &sources.Chunk{
|
||||
Data: []byte(`some-encoded-secret=dGVzdHNlY3JldA`),
|
||||
},
|
||||
want: &sources.Chunk{
|
||||
Data: []byte(`some-encoded-secret=testsecret`),
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "b64-url-safe: has longer b64 inside",
|
||||
chunk: &sources.Chunk{
|
||||
Data: []byte(`some-encoded-secret="bG9uZ2VyLWVuY29kZWQtc2VjcmV0LXRlc3Q"`),
|
||||
},
|
||||
want: &sources.Chunk{
|
||||
Data: []byte(`some-encoded-secret="longer-encoded-secret-test"`),
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "b64-url-safe: hyphen url b64",
|
||||
chunk: &sources.Chunk{
|
||||
Data: []byte(`dHJ1ZmZsZWhvZz4-ZmluZHMtc2VjcmV0cw`),
|
||||
},
|
||||
want: &sources.Chunk{
|
||||
Data: []byte(`trufflehog>>finds-secrets`),
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "b64-url-safe: underscore url b64",
|
||||
chunk: &sources.Chunk{
|
||||
Data: []byte(`YjY0dXJsc2FmZS10ZXN0LXNlY3JldC11bmRlcnNjb3Jlcz8_`),
|
||||
},
|
||||
want: &sources.Chunk{
|
||||
Data: []byte(`b64urlsafe-test-secret-underscores??`),
|
||||
},
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
@@ -144,29 +90,13 @@ func TestBase64_FromChunk(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkFromChunkSmall(b *testing.B) {
|
||||
func BenchmarkFromChunk(benchmark *testing.B) {
|
||||
d := Base64{}
|
||||
data := detectors.MustGetBenchmarkData()["small"]
|
||||
|
||||
for n := 0; n < b.N; n++ {
|
||||
d.FromChunk(&sources.Chunk{Data: data})
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkFromChunkMedium(b *testing.B) {
|
||||
d := Base64{}
|
||||
data := detectors.MustGetBenchmarkData()["medium"]
|
||||
|
||||
for n := 0; n < b.N; n++ {
|
||||
d.FromChunk(&sources.Chunk{Data: data})
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkFromChunkLarge(b *testing.B) {
|
||||
d := Base64{}
|
||||
data := detectors.MustGetBenchmarkData()["big"]
|
||||
|
||||
for n := 0; n < b.N; n++ {
|
||||
d.FromChunk(&sources.Chunk{Data: data})
|
||||
for name, data := range detectors.MustGetBenchmarkData() {
|
||||
benchmark.Run(name, func(b *testing.B) {
|
||||
for n := 0; n < b.N; n++ {
|
||||
d.FromChunk(&sources.Chunk{Data: data})
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,28 +1,18 @@
|
||||
package decoders
|
||||
|
||||
import (
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
|
||||
)
|
||||
|
||||
func DefaultDecoders() []Decoder {
|
||||
return []Decoder{
|
||||
// UTF8 must be first for duplicate detection
|
||||
&UTF8{},
|
||||
&Plain{},
|
||||
&Base64{},
|
||||
&UTF16{},
|
||||
}
|
||||
}
|
||||
|
||||
// DecodableChunk is a chunk that includes the type of decoder used.
|
||||
// This allows us to avoid a type assertion on each decoder.
|
||||
type DecodableChunk struct {
|
||||
*sources.Chunk
|
||||
DecoderType detectorspb.DecoderType
|
||||
}
|
||||
|
||||
type Decoder interface {
|
||||
FromChunk(chunk *sources.Chunk) *DecodableChunk
|
||||
FromChunk(chunk *sources.Chunk) *sources.Chunk
|
||||
}
|
||||
|
||||
// Fuzz is an entrypoint for go-fuzz, which is an AFL-style fuzzing tool.
|
||||
|
||||
@@ -0,0 +1,14 @@
|
||||
package decoders
|
||||
|
||||
import (
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
|
||||
)
|
||||
|
||||
// Ensure the Decoder satisfies the interface at compile time
|
||||
var _ Decoder = (*Plain)(nil)
|
||||
|
||||
type Plain struct{}
|
||||
|
||||
func (d *Plain) FromChunk(chunk *sources.Chunk) *sources.Chunk {
|
||||
return chunk
|
||||
}
|
||||
@@ -1,48 +0,0 @@
|
||||
package decoders
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/binary"
|
||||
"unicode/utf8"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
|
||||
)
|
||||
|
||||
type UTF16 struct{}
|
||||
|
||||
func (d *UTF16) FromChunk(chunk *sources.Chunk) *DecodableChunk {
|
||||
if chunk == nil || len(chunk.Data) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
decodableChunk := &DecodableChunk{Chunk: chunk, DecoderType: detectorspb.DecoderType_UTF16}
|
||||
if utf16Data, err := utf16ToUTF8(chunk.Data); err == nil {
|
||||
if len(utf16Data) == 0 {
|
||||
return nil
|
||||
}
|
||||
chunk.Data = utf16Data
|
||||
return decodableChunk
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// utf16ToUTF8 converts a byte slice containing UTF-16 encoded data to a UTF-8 encoded byte slice.
|
||||
func utf16ToUTF8(b []byte) ([]byte, error) {
|
||||
var bufBE, bufLE bytes.Buffer
|
||||
for i := 0; i < len(b)-1; i += 2 {
|
||||
if r := rune(binary.BigEndian.Uint16(b[i:])); b[i] == 0 && utf8.ValidRune(r) {
|
||||
if isValidByte(byte(r)) {
|
||||
bufBE.WriteRune(r)
|
||||
}
|
||||
}
|
||||
if r := rune(binary.LittleEndian.Uint16(b[i:])); b[i+1] == 0 && utf8.ValidRune(r) {
|
||||
if isValidByte(byte(r)) {
|
||||
bufLE.WriteRune(r)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return append(bufLE.Bytes(), bufBE.Bytes()...), nil
|
||||
}
|
||||
Binary file not shown.
@@ -1,94 +0,0 @@
|
||||
package decoders
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"os"
|
||||
"testing"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
|
||||
)
|
||||
|
||||
func TestUTF16Decoder(t *testing.T) {
|
||||
testCases := []struct {
|
||||
name string
|
||||
input []byte
|
||||
expected []byte
|
||||
expectNil bool
|
||||
}{
|
||||
{
|
||||
name: "Valid UTF-16LE input",
|
||||
input: []byte{72, 0, 101, 0, 108, 0, 108, 0, 111, 0, 32, 0, 87, 0, 111, 0, 114, 0, 108, 0, 100, 0},
|
||||
expected: []byte("Hello World"),
|
||||
expectNil: false,
|
||||
},
|
||||
{
|
||||
name: "Valid UTF-16BE input",
|
||||
input: []byte{0, 72, 0, 101, 0, 108, 0, 108, 0, 111, 0, 32, 0, 87, 0, 111, 0, 114, 0, 108, 0, 100},
|
||||
expected: []byte("Hello World"),
|
||||
expectNil: false,
|
||||
},
|
||||
{
|
||||
name: "Invalid UTF-16 input (it's UTF-8)",
|
||||
input: []byte("Hello World!"),
|
||||
expected: nil,
|
||||
expectNil: true,
|
||||
},
|
||||
{
|
||||
name: "Invalid UTF-16 input (odd length)",
|
||||
input: []byte{72, 0, 101, 0, 108, 0, 108, 0, 111, 0, 32, 0, 87, 0, 111, 0, 114, 0, 108, 0, 0},
|
||||
expected: []byte("Hello Worl"),
|
||||
expectNil: false,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range testCases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
chunk := &sources.Chunk{Data: tc.input}
|
||||
decoder := &UTF16{}
|
||||
decodedChunk := decoder.FromChunk(chunk)
|
||||
|
||||
if tc.expectNil {
|
||||
if decodedChunk != nil {
|
||||
t.Errorf("Expected nil, got chunk with data: %v", decodedChunk.Data)
|
||||
}
|
||||
return
|
||||
}
|
||||
if decodedChunk == nil {
|
||||
t.Errorf("Expected chunk with data, got nil")
|
||||
return
|
||||
}
|
||||
if !bytes.Equal(decodedChunk.Data, tc.expected) {
|
||||
t.Errorf("Expected decoded data: %s, got: %s", tc.expected, decodedChunk.Data)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestDLL(t *testing.T) {
|
||||
data, err := os.ReadFile("utf16_test.dll")
|
||||
if err != nil {
|
||||
t.Errorf("Failed to read test data: %v", err)
|
||||
return
|
||||
}
|
||||
|
||||
chunk := &sources.Chunk{Data: data}
|
||||
decoder := &UTF16{}
|
||||
decodedChunk := decoder.FromChunk(chunk)
|
||||
if decodedChunk == nil {
|
||||
t.Errorf("Expected chunk with data, got nil")
|
||||
return
|
||||
}
|
||||
if !bytes.Contains(decodedChunk.Data, []byte("aws_secret_access_key")) {
|
||||
t.Errorf("Expected chunk to have aws_secret_access_key")
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkUtf16ToUtf8(b *testing.B) {
|
||||
// Example UTF-16LE encoded data
|
||||
data := []byte{72, 0, 101, 0, 108, 0, 108, 0, 111, 0, 32, 0, 87, 0, 111, 0, 114, 0, 108, 0, 100, 0}
|
||||
|
||||
for n := 0; n < b.N; n++ {
|
||||
_, _ = utf16ToUTF8(data)
|
||||
}
|
||||
}
|
||||
@@ -1,59 +0,0 @@
|
||||
package decoders
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"unicode/utf8"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
|
||||
)
|
||||
|
||||
type UTF8 struct{}
|
||||
|
||||
func (d *UTF8) FromChunk(chunk *sources.Chunk) *DecodableChunk {
|
||||
if chunk == nil || len(chunk.Data) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
decodableChunk := &DecodableChunk{Chunk: chunk, DecoderType: detectorspb.DecoderType_PLAIN}
|
||||
|
||||
if !utf8.Valid(chunk.Data) {
|
||||
chunk.Data = extractSubstrings(chunk.Data)
|
||||
return decodableChunk
|
||||
}
|
||||
|
||||
return decodableChunk
|
||||
}
|
||||
|
||||
// extractSubstrings performs similarly to the strings binutil,
|
||||
// extacting contigous portions of printable characters that we care
|
||||
// about from some bytes
|
||||
func extractSubstrings(b []byte) []byte {
|
||||
|
||||
field := make([]byte, len(b))
|
||||
fieldLen := 0
|
||||
buf := &bytes.Buffer{}
|
||||
for i, c := range b {
|
||||
if isValidByte(c) {
|
||||
field[fieldLen] = c
|
||||
fieldLen++
|
||||
} else {
|
||||
if fieldLen > 5 {
|
||||
buf.Write(field[:fieldLen])
|
||||
}
|
||||
fieldLen = 0
|
||||
}
|
||||
|
||||
if i == len(b)-1 && fieldLen > 5 {
|
||||
buf.Write(field[:fieldLen])
|
||||
}
|
||||
}
|
||||
|
||||
return buf.Bytes()
|
||||
}
|
||||
|
||||
func isValidByte(c byte) bool {
|
||||
// https://www.rapidtables.com/code/text/ascii-table.html
|
||||
// split on anything that is not ascii space through tilde
|
||||
return c > 31 && c < 127
|
||||
}
|
||||
@@ -1,88 +0,0 @@
|
||||
package decoders
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/kylelemons/godebug/pretty"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
|
||||
)
|
||||
|
||||
func TestUTF8_FromChunk(t *testing.T) {
|
||||
type args struct {
|
||||
chunk *sources.Chunk
|
||||
}
|
||||
tests := []struct {
|
||||
name string
|
||||
d *UTF8
|
||||
args args
|
||||
want *sources.Chunk
|
||||
wantErr bool
|
||||
}{
|
||||
{
|
||||
name: "successful UTF8 decode",
|
||||
d: &UTF8{},
|
||||
args: args{
|
||||
chunk: &sources.Chunk{Data: []byte("plain 'ol chunk that should decode successfully")},
|
||||
},
|
||||
want: &sources.Chunk{Data: []byte("plain 'ol chunk that should decode successfully")},
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "successful binary decode",
|
||||
d: &UTF8{},
|
||||
args: args{
|
||||
chunk: &sources.Chunk{Data: []byte("\xf0\x28\x8c\x28 not-entirely utf8 chunk that should decode successfully")},
|
||||
},
|
||||
want: &sources.Chunk{Data: []byte("( not-entirely utf8 chunk that should decode successfully")},
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "unsuccessful decode",
|
||||
d: &UTF8{},
|
||||
args: args{
|
||||
chunk: nil,
|
||||
},
|
||||
want: nil,
|
||||
wantErr: false,
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
d := &UTF8{}
|
||||
got := d.FromChunk(tt.args.chunk)
|
||||
if got != nil && tt.want != nil {
|
||||
if diff := pretty.Compare(string(got.Data), string(tt.want.Data)); diff != "" {
|
||||
t.Errorf("%s: Plain.FromChunk() diff: (-got +want)\n%s", tt.name, diff)
|
||||
}
|
||||
} else {
|
||||
if diff := pretty.Compare(got, tt.want); diff != "" {
|
||||
t.Errorf("%s: Plain.FromChunk() diff: (-got +want)\n%s", tt.name, diff)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
var testBytes = []byte(`some words with random spaces and
|
||||
|
||||
newlines with
|
||||
arbitrary length
|
||||
of
|
||||
|
||||
hey
|
||||
|
||||
the lines themselves.
|
||||
|
||||
and
|
||||
short
|
||||
words
|
||||
that
|
||||
go
|
||||
away.`)
|
||||
|
||||
func Benchmark_extractSubstrings(b *testing.B) {
|
||||
for i := 0; i < b.N; i++ {
|
||||
extractSubstrings(testBytes)
|
||||
}
|
||||
}
|
||||
@@ -2,6 +2,7 @@ package abbysale
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"strings"
|
||||
@@ -51,7 +52,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
req.Header.Add("x-api-key", resMatch)
|
||||
req.Header.Add("x-api-key", fmt.Sprintf("%s", resMatch))
|
||||
res, err := client.Do(req)
|
||||
if err == nil {
|
||||
defer res.Body.Close()
|
||||
@@ -69,9 +70,5 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
results = append(results, s1)
|
||||
}
|
||||
|
||||
return results, nil
|
||||
}
|
||||
|
||||
func (s Scanner) Type() detectorspb.DetectorType {
|
||||
return detectorspb.DetectorType_Abbysale
|
||||
return detectors.CleanResults(results), nil
|
||||
}
|
||||
|
||||
@@ -108,7 +108,6 @@ func BenchmarkFromData(benchmark *testing.B) {
|
||||
s := Scanner{}
|
||||
for name, data := range detectors.MustGetBenchmarkData() {
|
||||
benchmark.Run(name, func(b *testing.B) {
|
||||
b.ResetTimer()
|
||||
for n := 0; n < b.N; n++ {
|
||||
_, err := s.FromData(ctx, false, data)
|
||||
if err != nil {
|
||||
|
||||
@@ -70,9 +70,5 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
results = append(results, s1)
|
||||
}
|
||||
|
||||
return results, nil
|
||||
}
|
||||
|
||||
func (s Scanner) Type() detectorspb.DetectorType {
|
||||
return detectorspb.DetectorType_Abstract
|
||||
return detectors.CleanResults(results), nil
|
||||
}
|
||||
|
||||
@@ -108,7 +108,6 @@ func BenchmarkFromData(benchmark *testing.B) {
|
||||
s := Scanner{}
|
||||
for name, data := range detectors.MustGetBenchmarkData() {
|
||||
benchmark.Run(name, func(b *testing.B) {
|
||||
b.ResetTimer()
|
||||
for n := 0; n < b.N; n++ {
|
||||
_, err := s.FromData(ctx, false, data)
|
||||
if err != nil {
|
||||
|
||||
@@ -2,12 +2,14 @@ package abuseipdb
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io"
|
||||
"net/http"
|
||||
"io/ioutil"
|
||||
|
||||
// "log"
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"net/http"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
|
||||
@@ -56,11 +58,11 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
req.Header.Add("Key", resMatch)
|
||||
res, err := client.Do(req)
|
||||
if err == nil {
|
||||
bodyBytes, err := io.ReadAll(res.Body)
|
||||
bodyBytes, err := ioutil.ReadAll(res.Body)
|
||||
if err == nil {
|
||||
bodyString := string(bodyBytes)
|
||||
validResponse := strings.Contains(bodyString, `ipAddress`)
|
||||
// errCode := strings.Contains(bodyString, `AbuseIPDB APIv2 Server.`)
|
||||
//errCode := strings.Contains(bodyString, `AbuseIPDB APIv2 Server.`)
|
||||
|
||||
defer res.Body.Close()
|
||||
if res.StatusCode >= 200 && res.StatusCode < 300 {
|
||||
@@ -81,9 +83,5 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
results = append(results, s1)
|
||||
}
|
||||
|
||||
return results, nil
|
||||
}
|
||||
|
||||
func (s Scanner) Type() detectorspb.DetectorType {
|
||||
return detectorspb.DetectorType_AbuseIPDB
|
||||
return detectors.CleanResults(results), nil
|
||||
}
|
||||
|
||||
@@ -108,7 +108,6 @@ func BenchmarkFromData(benchmark *testing.B) {
|
||||
s := Scanner{}
|
||||
for name, data := range detectors.MustGetBenchmarkData() {
|
||||
benchmark.Run(name, func(b *testing.B) {
|
||||
b.ResetTimer()
|
||||
for n := 0; n < b.N; n++ {
|
||||
_, err := s.FromData(ctx, false, data)
|
||||
if err != nil {
|
||||
|
||||
@@ -68,9 +68,5 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
results = append(results, s1)
|
||||
}
|
||||
|
||||
return results, nil
|
||||
}
|
||||
|
||||
func (s Scanner) Type() detectorspb.DetectorType {
|
||||
return detectorspb.DetectorType_Accuweather
|
||||
return detectors.CleanResults(results), nil
|
||||
}
|
||||
|
||||
@@ -108,7 +108,6 @@ func BenchmarkFromData(benchmark *testing.B) {
|
||||
s := Scanner{}
|
||||
for name, data := range detectors.MustGetBenchmarkData() {
|
||||
benchmark.Run(name, func(b *testing.B) {
|
||||
b.ResetTimer()
|
||||
for n := 0; n < b.N; n++ {
|
||||
_, err := s.FromData(ctx, false, data)
|
||||
if err != nil {
|
||||
|
||||
@@ -68,9 +68,5 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
results = append(results, s1)
|
||||
}
|
||||
|
||||
return results, nil
|
||||
}
|
||||
|
||||
func (s Scanner) Type() detectorspb.DetectorType {
|
||||
return detectorspb.DetectorType_AdafruitIO
|
||||
return detectors.CleanResults(results), nil
|
||||
}
|
||||
|
||||
@@ -108,7 +108,6 @@ func BenchmarkFromData(benchmark *testing.B) {
|
||||
s := Scanner{}
|
||||
for name, data := range detectors.MustGetBenchmarkData() {
|
||||
benchmark.Run(name, func(b *testing.B) {
|
||||
b.ResetTimer()
|
||||
for n := 0; n < b.N; n++ {
|
||||
_, err := s.FromData(ctx, false, data)
|
||||
if err != nil {
|
||||
|
||||
@@ -80,9 +80,5 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
|
||||
}
|
||||
|
||||
return results, nil
|
||||
}
|
||||
|
||||
func (s Scanner) Type() detectorspb.DetectorType {
|
||||
return detectorspb.DetectorType_AdobeIO
|
||||
return detectors.CleanResults(results), nil
|
||||
}
|
||||
|
||||
@@ -109,7 +109,6 @@ func BenchmarkFromData(benchmark *testing.B) {
|
||||
s := Scanner{}
|
||||
for name, data := range detectors.MustGetBenchmarkData() {
|
||||
benchmark.Run(name, func(b *testing.B) {
|
||||
b.ResetTimer()
|
||||
for n := 0; n < b.N; n++ {
|
||||
_, err := s.FromData(ctx, false, data)
|
||||
if err != nil {
|
||||
|
||||
@@ -78,9 +78,5 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
}
|
||||
}
|
||||
|
||||
return results, nil
|
||||
}
|
||||
|
||||
func (s Scanner) Type() detectorspb.DetectorType {
|
||||
return detectorspb.DetectorType_Adzuna
|
||||
return detectors.CleanResults(results), nil
|
||||
}
|
||||
|
||||
@@ -109,7 +109,6 @@ func BenchmarkFromData(benchmark *testing.B) {
|
||||
s := Scanner{}
|
||||
for name, data := range detectors.MustGetBenchmarkData() {
|
||||
benchmark.Run(name, func(b *testing.B) {
|
||||
b.ResetTimer()
|
||||
for n := 0; n < b.N; n++ {
|
||||
_, err := s.FromData(ctx, false, data)
|
||||
if err != nil {
|
||||
|
||||
@@ -81,9 +81,5 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
|
||||
}
|
||||
|
||||
return results, nil
|
||||
}
|
||||
|
||||
func (s Scanner) Type() detectorspb.DetectorType {
|
||||
return detectorspb.DetectorType_Aeroworkflow
|
||||
return detectors.CleanResults(results), nil
|
||||
}
|
||||
|
||||
@@ -110,7 +110,6 @@ func BenchmarkFromData(benchmark *testing.B) {
|
||||
s := Scanner{}
|
||||
for name, data := range detectors.MustGetBenchmarkData() {
|
||||
benchmark.Run(name, func(b *testing.B) {
|
||||
b.ResetTimer()
|
||||
for n := 0; n < b.N; n++ {
|
||||
s.FromData(ctx, false, data)
|
||||
}
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user