Compare commits
1 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| def91bd138 |
@@ -0,0 +1,5 @@
|
||||
{
|
||||
"image": "trufflesecurity/protos:1.18-0",
|
||||
"features": {
|
||||
}
|
||||
}
|
||||
@@ -6,7 +6,11 @@ labels: bug, needs triage
|
||||
assignees: trufflesecurity/product-eng
|
||||
---
|
||||
|
||||
Please review the [Community Note](https://github.com/trufflesecurity/trufflehog/blob/main/.github/community_note.md) before submitting
|
||||
### Community Note
|
||||
|
||||
* Please vote on this issue by adding a 👍 [reaction](https://blog.github.com/2016-03-10-add-reactions-to-pull-requests-issues-and-comments/) to the original issue to help the community and maintainers prioritize this request
|
||||
* Please do not leave "+1" or other comments that do not add relevant new information or questions, they generate extra noise for issue followers and do not help prioritize the request
|
||||
* If you are interested in working on this issue or have submitted a pull request, please leave a comment
|
||||
|
||||
### TruffleHog Version
|
||||
<!--- Please run `trufflehog --version` to show the version. If you are not running the latest version, please upgrade because your issue may have already been fixed. --->
|
||||
|
||||
@@ -6,27 +6,33 @@ labels: enhancement, needs triage
|
||||
assignees: trufflesecurity/product-eng
|
||||
---
|
||||
|
||||
Please review the [Community Note](https://github.com/trufflesecurity/trufflehog/blob/main/.github/community_note.md) before submitting
|
||||
### Community Note
|
||||
|
||||
* Please vote on this issue by adding a 👍 [reaction](https://blog.github.com/2016-03-10-add-reactions-to-pull-requests-issues-and-comments/) to the original issue to help the community and maintainers prioritize this request
|
||||
* Please do not leave "+1" or other comments that do not add relevant new information or questions, they generate extra noise for issue followers and do not help prioritize the request
|
||||
* If you are interested in working on this issue or have submitted a pull request, please leave a comment
|
||||
|
||||
### Description
|
||||
|
||||
## Description
|
||||
<!--- Please leave a helpful description of the feature request here. --->
|
||||
|
||||
### Preferred Solution
|
||||
<!--- A clear and concise description of what you want to happen. What
|
||||
information may be required and what would be the preferred way to provide it?
|
||||
## Problem to be Addressed
|
||||
<!--- A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] --->
|
||||
|
||||
## Description of the Preferred Solution
|
||||
<!--- A clear and concise description of what you want to happen. What
|
||||
information may be required and what would be the preferred way to provide it?
|
||||
What should the output include? --->
|
||||
|
||||
### Additional Context
|
||||
## Additional Context
|
||||
<!--- Add any other context or screenshots about the feature request here. --->
|
||||
|
||||
#### References
|
||||
### References
|
||||
|
||||
<!---
|
||||
Information about referencing Github Issues: https://help.github.com/articles/basic-writing-and-formatting-syntax/#referencing-issues-and-pull-requests
|
||||
|
||||
Are there any other GitHub issues (open or closed) or pull requests that should be linked here? Vendor blog posts or documentation? For example:
|
||||
|
||||
* #0000
|
||||
--->
|
||||
|
||||
|
||||
* #0000
|
||||
|
||||
@@ -2,11 +2,3 @@
|
||||
Please create an issue to collect feedback prior to feature additions. Please also reference that issue in any PRs.
|
||||
If possible try to keep PRs scoped to one feature, and add tests for new features.
|
||||
-->
|
||||
|
||||
### Description:
|
||||
Explain the purpose of the PR.
|
||||
|
||||
### Checklist:
|
||||
* [ ] Tests passing (`make test-community`)?
|
||||
* [ ] Lint passing (`make lint` this requires [golangci-lint](https://golangci-lint.run/usage/install/#local-installation))?
|
||||
|
||||
|
||||
@@ -1,7 +0,0 @@
|
||||
## Community Note
|
||||
|
||||
Please vote on this issue by adding a 👍 reaction to the original issue to help the community and maintainers prioritize this request.
|
||||
|
||||
Please do not leave "+1" or other comments that do not add relevant new information or questions, they generate extra noise for issue followers and do not help prioritize the request.
|
||||
|
||||
If you are interested in working on this issue or have submitted a pull request, please leave a comment.
|
||||
@@ -0,0 +1,15 @@
|
||||
# To get started with Dependabot version updates, you'll need to specify which
|
||||
# package ecosystems to update and where the package manifests are located.
|
||||
# Please see the documentation for all configuration options:
|
||||
# https://help.github.com/github/administering-a-repository/configuration-options-for-dependency-updates
|
||||
|
||||
version: 2
|
||||
updates:
|
||||
- package-ecosystem: "gomod" # See documentation for possible values
|
||||
directory: "/" # Location of package manifests
|
||||
schedule:
|
||||
interval: "monthly"
|
||||
- package-ecosystem: "github-actions"
|
||||
directory: "/"
|
||||
schedule:
|
||||
interval: "monthly"
|
||||
@@ -1,8 +0,0 @@
|
||||
{
|
||||
"$schema": "https://docs.renovatebot.com/renovate-schema.json",
|
||||
"extends": [
|
||||
"config:base"
|
||||
],
|
||||
"prConcurrentLimit": 3,
|
||||
"prHourlyLimit": 2
|
||||
}
|
||||
@@ -38,7 +38,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v4
|
||||
uses: actions/checkout@v3
|
||||
|
||||
# Initializes the CodeQL tools for scanning.
|
||||
- name: Initialize CodeQL
|
||||
|
||||
@@ -1,47 +0,0 @@
|
||||
name: detector test aggregation
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
schedule:
|
||||
- cron: "0 8 * * *"
|
||||
|
||||
jobs:
|
||||
test-detectors:
|
||||
if: ${{ github.repository == 'trufflesecurity/trufflehog' }}
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
actions: "read"
|
||||
contents: "read"
|
||||
id-token: "write"
|
||||
steps:
|
||||
- name: Install Go
|
||||
uses: actions/setup-go@v4
|
||||
with:
|
||||
go-version: "1.21"
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
- id: "auth"
|
||||
uses: "google-github-actions/auth@v2"
|
||||
with:
|
||||
workload_identity_provider: "projects/811013774421/locations/global/workloadIdentityPools/github-pool/providers/github-provider"
|
||||
service_account: "github-ci-external@trufflehog-testing.iam.gserviceaccount.com"
|
||||
- name: Test
|
||||
run: make test-integration
|
||||
- name: Set up gotestsum
|
||||
run: |
|
||||
go install gotest.tools/gotestsum@latest
|
||||
mkdir -p tmp/test-results
|
||||
- name: Test
|
||||
run: |
|
||||
CGO_ENABLED=1 gotestsum --junitfile tmp/test-results/test.xml --raw-command -- go test -json -tags=detectors -timeout=15m $(go list ./... | grep pkg/detectors)
|
||||
- name: Upload test results to BuildPulse for flaky test detection
|
||||
if: ${{ !cancelled() }} # Run this step even when the tests fail. Skip if the workflow is cancelled.
|
||||
uses: buildpulse/buildpulse-action@main
|
||||
with:
|
||||
account: 79229934
|
||||
repository: 694446374
|
||||
path: |
|
||||
tmp/test-results/*.xml
|
||||
key: ${{ secrets.BUILDPULSE_DETECTORS_ACCESS_KEY_ID }}
|
||||
secret: ${{ secrets.BUILDPULSE_DETECTORS_SECRET_ACCESS_KEY }}
|
||||
tags: detectors
|
||||
@@ -17,8 +17,8 @@ jobs:
|
||||
steps:
|
||||
- uses: actions/setup-go@v4
|
||||
with:
|
||||
go-version: '1.21'
|
||||
- uses: actions/checkout@v4
|
||||
go-version: '1.20'
|
||||
- uses: actions/checkout@v3
|
||||
- name: golangci-lint
|
||||
uses: golangci/golangci-lint-action@v3
|
||||
with:
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
name: Performance Test
|
||||
name: Performance Test
|
||||
|
||||
on: [pull_request]
|
||||
|
||||
@@ -7,16 +7,16 @@ jobs:
|
||||
# skip if PR is from a fork.
|
||||
# TODO: this could probabaly be refactored a bit so that it runs on forks
|
||||
if: ${{ ! github.event.pull_request.head.repo.fork }}
|
||||
|
||||
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Install Go
|
||||
uses: actions/setup-go@v4
|
||||
with:
|
||||
go-version: '1.21'
|
||||
go-version: '1.20'
|
||||
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
uses: actions/checkout@v3
|
||||
with:
|
||||
fetch-depth: 0
|
||||
ref: ${{ github.head_ref }}
|
||||
@@ -52,7 +52,7 @@ jobs:
|
||||
echo PREVIOUS_TAG=$(cat previous_tag.txt) >> $GITHUB_ENV
|
||||
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
uses: actions/checkout@v3
|
||||
with:
|
||||
fetch-depth: 0
|
||||
ref: ${{ env.PREVIOUS_TAG }}
|
||||
|
||||
@@ -8,7 +8,6 @@ on:
|
||||
permissions:
|
||||
contents: write
|
||||
packages: write
|
||||
id-token: write
|
||||
|
||||
jobs:
|
||||
Release:
|
||||
@@ -17,18 +16,18 @@ jobs:
|
||||
DOCKER_CLI_EXPERIMENTAL: "enabled"
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
uses: actions/checkout@v3
|
||||
with:
|
||||
fetch-depth: 0
|
||||
- name: Set up QEMU
|
||||
uses: docker/setup-qemu-action@v3
|
||||
uses: docker/setup-qemu-action@v2
|
||||
- name: Docker Login to DockerHub
|
||||
uses: docker/login-action@v3
|
||||
uses: docker/login-action@v2
|
||||
with:
|
||||
username: ${{ secrets.DOCKERHUB_USERNAME }}
|
||||
password: ${{ secrets.DOCKERHUB_TOKEN }}
|
||||
- name: Docker Login to GitHub Container Registry
|
||||
uses: docker/login-action@v3
|
||||
uses: docker/login-action@v2
|
||||
with:
|
||||
registry: ghcr.io
|
||||
username: ${{ github.repository_owner }}
|
||||
@@ -36,11 +35,9 @@ jobs:
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v4
|
||||
with:
|
||||
go-version: '1.21'
|
||||
- name: Cosign install
|
||||
uses: sigstore/cosign-installer@9614fae9e5c5eddabb09f90a270fcb487c9f7149 # v3.3.0
|
||||
go-version: '1.20'
|
||||
- name: Run GoReleaser
|
||||
uses: goreleaser/goreleaser-action@v5
|
||||
uses: goreleaser/goreleaser-action@v4
|
||||
with:
|
||||
distribution: goreleaser
|
||||
version: latest
|
||||
|
||||
@@ -13,8 +13,12 @@ jobs:
|
||||
test:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Install Go
|
||||
uses: actions/setup-go@v4
|
||||
with:
|
||||
go-version: '1.20'
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
uses: actions/checkout@v3
|
||||
with:
|
||||
fetch-depth: 0
|
||||
ref: ${{ github.head_ref }}
|
||||
@@ -22,4 +26,7 @@ jobs:
|
||||
uses: ./
|
||||
id: dogfood
|
||||
with:
|
||||
path: ./
|
||||
base: ${{ github.event.repository.default_branch }}
|
||||
head: HEAD
|
||||
extra_args: --only-verified
|
||||
|
||||
@@ -11,9 +11,9 @@ jobs:
|
||||
- name: Install Go
|
||||
uses: actions/setup-go@v4
|
||||
with:
|
||||
go-version: '1.21'
|
||||
go-version: '1.20'
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
uses: actions/checkout@v3
|
||||
- name: Smoke
|
||||
run: |
|
||||
set -e
|
||||
|
||||
@@ -18,7 +18,7 @@ jobs:
|
||||
steps:
|
||||
- uses: actions/setup-go@v4
|
||||
with:
|
||||
go-version: '1.21'
|
||||
- uses: actions/checkout@v4
|
||||
go-version: '1.20'
|
||||
- uses: actions/checkout@v3
|
||||
- name: Run Snifftest
|
||||
run: make snifftest
|
||||
run: make snifftest
|
||||
+51
-50
@@ -1,3 +1,4 @@
|
||||
|
||||
name: Test
|
||||
|
||||
on:
|
||||
@@ -10,60 +11,60 @@ on:
|
||||
|
||||
jobs:
|
||||
test:
|
||||
if: ${{ github.repository == 'trufflesecurity/trufflehog' && !github.event.pull_request.head.repo.fork }}
|
||||
if: ${{ ! github.event.pull_request.head.repo.fork }}
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
actions: "read"
|
||||
contents: "read"
|
||||
id-token: "write"
|
||||
actions: 'read'
|
||||
contents: 'read'
|
||||
id-token: 'write'
|
||||
steps:
|
||||
- name: Install Go
|
||||
uses: actions/setup-go@v4
|
||||
with:
|
||||
go-version: "1.21"
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
- id: "auth"
|
||||
uses: "google-github-actions/auth@v2"
|
||||
with:
|
||||
workload_identity_provider: "projects/811013774421/locations/global/workloadIdentityPools/github-pool/providers/github-provider"
|
||||
service_account: "github-ci-external@trufflehog-testing.iam.gserviceaccount.com"
|
||||
- name: Set up gotestsum
|
||||
run: |
|
||||
go install gotest.tools/gotestsum@latest
|
||||
mkdir -p tmp/test-results
|
||||
- name: Test
|
||||
run: |
|
||||
CGO_ENABLED=1 gotestsum --junitfile tmp/test-results/test.xml --raw-command -- go test -json -tags=sources $(go list ./... | grep -v /vendor/ | grep -v pkg/detectors)
|
||||
if: ${{ success() || failure() }} # always run this step, even if there were previous errors
|
||||
- name: Upload test results to BuildPulse for flaky test detection
|
||||
if: ${{ !cancelled() }} # Run this step even when the tests fail. Skip if the workflow is cancelled.
|
||||
uses: buildpulse/buildpulse-action@main
|
||||
with:
|
||||
account: 79229934
|
||||
repository: 77726177
|
||||
path: |
|
||||
tmp/test-results/*.xml
|
||||
key: ${{ secrets.BUILDPULSE_ACCESS_KEY_ID }}
|
||||
secret: ${{ secrets.BUILDPULSE_SECRET_ACCESS_KEY }}
|
||||
tags: integration
|
||||
- name: Annotate test results
|
||||
uses: mikepenz/action-junit-report@v4
|
||||
if: success() || failure() # always run even if the previous step fails
|
||||
with:
|
||||
report_paths: "tmp/test-results/*.xml"
|
||||
test-community:
|
||||
- name: Install Go
|
||||
uses: actions/setup-go@v4
|
||||
with:
|
||||
go-version: '1.20'
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v3
|
||||
- id: 'auth'
|
||||
uses: 'google-github-actions/auth@v1'
|
||||
with:
|
||||
workload_identity_provider: 'projects/811013774421/locations/global/workloadIdentityPools/github-pool/providers/github-provider'
|
||||
service_account: 'github-ci-external@trufflehog-testing.iam.gserviceaccount.com'
|
||||
- name: Test
|
||||
run: make test-integration
|
||||
test-detectors:
|
||||
if: ${{ ! github.event.pull_request.head.repo.fork }}
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
actions: 'read'
|
||||
contents: 'read'
|
||||
id-token: 'write'
|
||||
steps:
|
||||
- name: Install Go
|
||||
uses: actions/setup-go@v4
|
||||
with:
|
||||
go-version: '1.20'
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v3
|
||||
- id: 'auth'
|
||||
uses: 'google-github-actions/auth@v1'
|
||||
with:
|
||||
workload_identity_provider: 'projects/811013774421/locations/global/workloadIdentityPools/github-pool/providers/github-provider'
|
||||
service_account: 'github-ci-external@trufflehog-testing.iam.gserviceaccount.com'
|
||||
- name: Test
|
||||
run: make test-detectors
|
||||
continue-on-error: true
|
||||
test-fork:
|
||||
if: ${{ github.event.pull_request.head.repo.fork }}
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
actions: "read"
|
||||
contents: "read"
|
||||
actions: 'read'
|
||||
contents: 'read'
|
||||
steps:
|
||||
- name: Install Go
|
||||
uses: actions/setup-go@v4
|
||||
with:
|
||||
go-version: "1.21"
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
- name: Test
|
||||
run: make test-community
|
||||
- name: Install Go
|
||||
uses: actions/setup-go@v4
|
||||
with:
|
||||
go-version: '1.20'
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v3
|
||||
- name: Test
|
||||
run: make test-forks
|
||||
|
||||
@@ -96,15 +96,3 @@ brews:
|
||||
homepage: "https://github.com/trufflesecurity/trufflehog"
|
||||
install: |
|
||||
bin.install "trufflehog"
|
||||
signs:
|
||||
- cmd: cosign
|
||||
signature: "${artifact}.sig"
|
||||
certificate: "${artifact}.pem"
|
||||
args:
|
||||
- "sign-blob"
|
||||
- "--oidc-issuer=https://token.actions.githubusercontent.com"
|
||||
- "--output-certificate=${certificate}"
|
||||
- "--output-signature=${signature}"
|
||||
- "${artifact}"
|
||||
- "--yes"
|
||||
artifacts: checksum
|
||||
+3
-3
@@ -8,9 +8,9 @@ RUN --mount=type=cache,target=/go/pkg/mod \
|
||||
--mount=type=cache,target=/root/.cache/go-build \
|
||||
GOOS=${TARGETOS} GOARCH=${TARGETARCH} go build -o trufflehog .
|
||||
|
||||
FROM alpine:3.19
|
||||
RUN apk add --no-cache bash git openssh-client ca-certificates rpm2cpio binutils cpio \
|
||||
&& rm -rf /var/cache/apk/* && update-ca-certificates
|
||||
FROM alpine:3.15
|
||||
RUN apk add --no-cache bash git openssh-client ca-certificates \
|
||||
&& update-ca-certificates
|
||||
COPY --from=builder /build/trufflehog /usr/bin/trufflehog
|
||||
COPY entrypoint.sh /etc/entrypoint.sh
|
||||
RUN chmod +x /etc/entrypoint.sh
|
||||
|
||||
@@ -1,7 +1,8 @@
|
||||
FROM alpine:3.19
|
||||
FROM alpine:3.15
|
||||
|
||||
RUN apk add --no-cache bash git openssh-client ca-certificates rpm2cpio binutils cpio \
|
||||
&& rm -rf /var/cache/apk/* && update-ca-certificates
|
||||
RUN apk add --no-cache bash git openssh-client ca-certificates \
|
||||
&& rm -rf /var/cache/apk/* && \
|
||||
update-ca-certificates
|
||||
WORKDIR /usr/bin/
|
||||
COPY trufflehog .
|
||||
COPY entrypoint.sh /etc/entrypoint.sh
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
PROTOS_IMAGE ?= trufflesecurity/protos:1.21-0
|
||||
PROTOS_IMAGE ?= trufflesecurity/protos:1.18-0
|
||||
|
||||
.PHONY: check
|
||||
.PHONY: lint
|
||||
@@ -39,7 +39,7 @@ test-race:
|
||||
test-detectors:
|
||||
CGO_ENABLED=0 go test -tags=detectors -timeout=5m $(shell go list ./... | grep pkg/detectors)
|
||||
|
||||
test-community:
|
||||
test-forks:
|
||||
CGO_ENABLED=0 go test -timeout=5m $(shell go list ./... | grep -v /vendor/ | grep -v pkg/detectors | grep -v pkg/sources)
|
||||
|
||||
bench:
|
||||
@@ -52,14 +52,14 @@ run-debug:
|
||||
CGO_ENABLED=0 go run . git file://. --json --debug
|
||||
|
||||
protos:
|
||||
docker run --rm -u "$(shell id -u)" -v "$(shell dirname $(realpath $(firstword $(MAKEFILE_LIST))))":/pwd "${PROTOS_IMAGE}" bash -c "cd /pwd; /pwd/scripts/gen_proto.sh"
|
||||
docker run -u "$(shell id -u)" -v "$(shell dirname $(realpath $(firstword $(MAKEFILE_LIST))))":/pwd "${PROTOS_IMAGE}" bash -c "cd /pwd; /pwd/scripts/gen_proto.sh"
|
||||
|
||||
protos-windows:
|
||||
docker run --rm -v "$(shell cygpath -w $(shell pwd))":/pwd "${PROTOS_IMAGE}" bash -c "cd /pwd; ./scripts/gen_proto.sh"
|
||||
docker run -v "$(shell cygpath -w $(shell pwd))":/pwd "${PROTOS_IMAGE}" bash -c "cd /pwd; ./scripts/gen_proto.sh"
|
||||
|
||||
release-protos-image:
|
||||
docker buildx build --push --platform=linux/amd64,linux/arm64 \
|
||||
-t trufflesecurity/protos:1.21-0 -f hack/Dockerfile.protos .
|
||||
-t trufflesecurity/protos:1.18-0 -f hack/Dockerfile.protos .
|
||||
|
||||
snifftest:
|
||||
./hack/snifftest/snifftest.sh
|
||||
|
||||
@@ -8,6 +8,7 @@
|
||||
|
||||
<div align="center">
|
||||
|
||||
[](https://github.com/trufflesecurity/trufflehog/actions)
|
||||
[](https://goreportcard.com/report/github.com/trufflesecurity/trufflehog/v3)
|
||||
[](/LICENSE)
|
||||
[](/pkg/detectors)
|
||||
@@ -32,7 +33,7 @@ Have questions? Feedback? Jump in slack or discord and hang out with us
|
||||
|
||||
Join our [Slack Community](https://join.slack.com/t/trufflehog-community/shared_invite/zt-pw2qbi43-Aa86hkiimstfdKH9UCpPzQ)
|
||||
|
||||
Join the [Secret Scanning Discord](https://discord.gg/8Hzbrnkr7E)
|
||||
Join the [Secret Scanning Discord](https://discord.gg/sydS6AHTUP)
|
||||
|
||||
# :tv: Demo
|
||||
|
||||
@@ -45,7 +46,6 @@ docker run --rm -it -v "$PWD:/pwd" trufflesecurity/trufflehog:latest github --or
|
||||
# :floppy_disk: Installation
|
||||
|
||||
Several options available for you:
|
||||
|
||||
```bash
|
||||
# MacOS users
|
||||
brew install trufflesecurity/trufflehog/trufflehog
|
||||
@@ -62,54 +62,8 @@ Download and unpack from https://github.com/trufflesecurity/trufflehog/releases
|
||||
# Compile from source
|
||||
git clone https://github.com/trufflesecurity/trufflehog.git
|
||||
cd trufflehog; go install
|
||||
|
||||
# Using installation script
|
||||
curl -sSfL https://raw.githubusercontent.com/trufflesecurity/trufflehog/main/scripts/install.sh | sh -s -- -b /usr/local/bin
|
||||
|
||||
# Using installation script, verify checksum signature (requires cosign to be installed)
|
||||
curl -sSfL https://raw.githubusercontent.com/trufflesecurity/trufflehog/main/scripts/install.sh | sh -s -- -v -b /usr/local/bin
|
||||
|
||||
# Using installation script to install a specific version
|
||||
curl -sSfL https://raw.githubusercontent.com/trufflesecurity/trufflehog/main/scripts/install.sh | sh -s -- -b /usr/local/bin <ReleaseTag like v3.56.0>
|
||||
```
|
||||
|
||||
# :closed_lock_with_key: Verifying the artifacts
|
||||
|
||||
Checksums are applied to all artifacts, and the resulting checksum file is signed using cosign.
|
||||
|
||||
You need the following tool to verify signature:
|
||||
|
||||
- [Cosign](https://docs.sigstore.dev/cosign/installation/)
|
||||
|
||||
Verification steps are as follow:
|
||||
|
||||
1. Download the artifact files you want, and the following files from the [releases](https://github.com/trufflesecurity/trufflehog/releases) page.
|
||||
|
||||
- trufflehog\_{version}\_checksums.txt
|
||||
- trufflehog\_{version}\_checksums.txt.pem
|
||||
- trufflehog\_{version}\_checksums.txt.sig
|
||||
|
||||
2. Verify the signature:
|
||||
|
||||
```shell
|
||||
cosign verify-blob <path to trufflehog_{version}_checksums.txt> \
|
||||
--certificate <path to trufflehog_{version}_checksums.txt.pem> \
|
||||
--signature <path to trufflehog_{version}_checksums.txt.sig> \
|
||||
--certificate-identity-regexp 'https://github\.com/trufflesecurity/trufflehog/\.github/workflows/.+' \
|
||||
--certificate-oidc-issuer "https://token.actions.githubusercontent.com"
|
||||
```
|
||||
|
||||
3. Once the signature is confirmed as valid, you can proceed to validate that the SHA256 sums align with the downloaded artifact:
|
||||
|
||||
```shell
|
||||
sha256sum --ignore-missing -c trufflehog_{version}_checksums.txt
|
||||
```
|
||||
|
||||
Replace `{version}` with the downloaded files version
|
||||
|
||||
Alternatively, if you are using installation script, pass `-v` option to perform signature verification.
|
||||
This required Cosign binary to be installed prior to running installation script.
|
||||
|
||||
# :rocket: Quick Start
|
||||
|
||||
## 1: Scan a repo for only verified secrets
|
||||
@@ -121,7 +75,6 @@ trufflehog git https://github.com/trufflesecurity/test_keys --only-verified
|
||||
```
|
||||
|
||||
Expected output:
|
||||
|
||||
```
|
||||
🐷🔑🐷 TruffleHog. Unearth your secrets. 🐷🔑🐷
|
||||
|
||||
@@ -153,49 +106,36 @@ trufflehog git https://github.com/trufflesecurity/test_keys --only-verified --js
|
||||
```
|
||||
|
||||
Expected output:
|
||||
|
||||
```
|
||||
{"SourceMetadata":{"Data":{"Git":{"commit":"fbc14303ffbf8fb1c2c1914e8dda7d0121633aca","file":"keys","email":"counter \u003ccounter@counters-MacBook-Air.local\u003e","repository":"https://github.com/trufflesecurity/test_keys","timestamp":"2022-06-16 10:17:40 -0700 PDT","line":4}}},"SourceID":0,"SourceType":16,"SourceName":"trufflehog - git","DetectorType":2,"DetectorName":"AWS","DecoderName":"PLAIN","Verified":true,"Raw":"AKIAYVP4CIPPERUVIFXG","Redacted":"AKIAYVP4CIPPERUVIFXG","ExtraData":{"account":"595918472158","arn":"arn:aws:iam::595918472158:user/canarytokens.com@@mirux23ppyky6hx3l6vclmhnj","user_id":"AIDAYVP4CIPPJ5M54LRCY"},"StructuredData":null}
|
||||
...
|
||||
```
|
||||
|
||||
## 4: Scan a GitHub Repo + its Issues and Pull Requests.
|
||||
|
||||
```bash
|
||||
trufflehog github --repo=https://github.com/trufflesecurity/test_keys --issue-comments --pr-comments
|
||||
```
|
||||
|
||||
## 5: Scan an S3 bucket for verified keys
|
||||
## 4: Scan an S3 bucket for verified keys
|
||||
|
||||
```bash
|
||||
trufflehog s3 --bucket=<bucket name> --only-verified
|
||||
```
|
||||
|
||||
## 6: Scan S3 buckets using IAM Roles
|
||||
|
||||
```bash
|
||||
trufflehog s3 --role-arn=<iam role arn>
|
||||
```
|
||||
|
||||
## 7: Scan a Github Repo using SSH authentication in docker
|
||||
## 5: Scan a Github Repo using SSH authentication in docker
|
||||
|
||||
```bash
|
||||
docker run --rm -v "$HOME/.ssh:/root/.ssh:ro" trufflesecurity/trufflehog:latest git ssh://github.com/trufflesecurity/test_keys
|
||||
```
|
||||
|
||||
## 8: Scan individual files or directories
|
||||
## 6: Scan individual files or directories
|
||||
|
||||
```bash
|
||||
trufflehog filesystem path/to/file1.txt path/to/file2.txt path/to/dir
|
||||
```
|
||||
|
||||
## 9: Scan GCS buckets for verified secrets.
|
||||
## 7: Scan GCS buckets for verified secrets.
|
||||
|
||||
```bash
|
||||
trufflehog gcs --project-id=<project-ID> --cloud-environment --only-verified
|
||||
```
|
||||
|
||||
## 10: Scan a Docker image for verified secrets.
|
||||
# 8: Scan a Docker image for verified secrets.
|
||||
|
||||
Use the `--image` flag multiple times to scan multiple images.
|
||||
|
||||
@@ -203,37 +143,30 @@ Use the `--image` flag multiple times to scan multiple images.
|
||||
trufflehog docker --image trufflesecurity/secrets --only-verified
|
||||
```
|
||||
|
||||
## 11: Scan in CI
|
||||
|
||||
Set the `--since-commit` flag to your default branch that people merge into (ex: "main"). Set the `--branch` flag to your PR's branch name (ex: "feature-1"). Depending on the CI/CD platform you use, this value can be pulled in dynamically (ex: [CIRCLE_BRANCH in Circle CI](https://circleci.com/docs/variables/) and [TRAVIS_PULL_REQUEST_BRANCH in Travis CI](https://docs.travis-ci.com/user/environment-variables/)). If the repo is cloned and the target branch is already checked out during the CI/CD workflow, then `--branch HEAD` should be sufficient. The `--fail` flag will return an 183 error code if valid credentials are found.
|
||||
|
||||
```bash
|
||||
trufflehog git file://. --since-commit main --branch feature-1 --only-verified --fail
|
||||
```
|
||||
|
||||
# :question: FAQ
|
||||
|
||||
- All I see is `🐷🔑🐷 TruffleHog. Unearth your secrets. 🐷🔑🐷` and the program exits, what gives?
|
||||
- That means no secrets were detected
|
||||
- Why is the scan taking a long time when I scan a GitHub org
|
||||
- Unauthenticated GitHub scans have rate limits. To improve your rate limits, include the `--token` flag with a personal access token
|
||||
- It says a private key was verified, what does that mean?
|
||||
- Check out our Driftwood blog post to learn how to do this, in short we've confirmed the key can be used live for SSH or SSL [Blog post](https://trufflesecurity.com/blog/driftwood-know-if-private-keys-are-sensitive/)
|
||||
- Is there an easy way to ignore specific secrets?
|
||||
- If the scanned source [supports line numbers](https://github.com/trufflesecurity/trufflehog/blob/d6375ba92172fd830abb4247cca15e3176448c5d/pkg/engine/engine.go#L358-L365), then you can add a `trufflehog:ignore` comment on the line containing the secret to ignore that secrets.
|
||||
+ All I see is `🐷🔑🐷 TruffleHog. Unearth your secrets. 🐷🔑🐷` and the program exits, what gives?
|
||||
+ That means no secrets were detected
|
||||
+ Why is the scan is taking a long time when I scan a GitHub org
|
||||
+ Unauthenticated GitHub scans have rate limits. To improve your rate limits, include the `--token` flag with a personal access token
|
||||
+ It says a private key was verified, what does that mean?
|
||||
+ Check out our Driftwood blog post to learn how to do this, in short we've confirmed the key can be used live for SSH or SSL [Blog post](https://trufflesecurity.com/blog/driftwood-know-if-private-keys-are-sensitive/)
|
||||
+ Is there an easy way to ignore specific secrets?
|
||||
+ If the scanned source [supports line numbers](https://github.com/trufflesecurity/trufflehog/blob/d6375ba92172fd830abb4247cca15e3176448c5d/pkg/engine/engine.go#L358-L365), then you can add a `trufflehog:ignore` comment on the line containing the secret to ignore that secrets.
|
||||
|
||||
|
||||
# :newspaper: What's new in v3?
|
||||
|
||||
TruffleHog v3 is a complete rewrite in Go with many new powerful features.
|
||||
|
||||
- We've **added over 700 credential detectors that support active verification against their respective APIs**.
|
||||
- We've also added native **support for scanning GitHub, GitLab, Docker, filesystems, S3, GCS, Circle CI and Travis CI**.
|
||||
- We've also added native **support for scanning GitHub, GitLab, filesystems, S3, GCS and Circle CI**.
|
||||
- **Instantly verify private keys** against millions of github users and **billions** of TLS certificates using our [Driftwood](https://trufflesecurity.com/blog/driftwood) technology.
|
||||
- Scan binaries, documents, and other file formats
|
||||
- Scan binaries and other file formats
|
||||
- Available as a GitHub Action and a pre-commit hook
|
||||
|
||||
## What is credential verification?
|
||||
|
||||
## What is credential verification?
|
||||
For every potential credential that is detected, we've painstakingly implemented programmatic verification against the API that we think it belongs to. Verification eliminates false positives. For example, the [AWS credential detector](pkg/detectors/aws/aws.go) performs a `GetCallerIdentity` API call against the AWS API to verify if an AWS credential is active.
|
||||
|
||||
# :memo: Usage
|
||||
@@ -248,8 +181,8 @@ TruffleHog has a sub-command for each source of data that you may want to scan:
|
||||
- filesystem (files and directories)
|
||||
- syslog
|
||||
- circleci
|
||||
- travisci
|
||||
- GCS (Google Cloud Storage)
|
||||
- stdin (coming soon)
|
||||
|
||||
Each subcommand can have options that you can see with the `--help` flag provided to the sub command:
|
||||
|
||||
@@ -260,144 +193,38 @@ usage: TruffleHog git [<flags>] <uri>
|
||||
Find credentials in git repositories.
|
||||
|
||||
Flags:
|
||||
-h, --help Show context-sensitive help (also try --help-long and --help-man).
|
||||
--debug Run in debug mode.
|
||||
--trace Run in trace mode.
|
||||
--profile Enables profiling and sets a pprof and fgprof server on :18066.
|
||||
-j, --json Output in JSON format.
|
||||
--json-legacy Use the pre-v3.0 JSON format. Only works with git, gitlab, and github sources.
|
||||
--github-actions Output in GitHub Actions format.
|
||||
--concurrency=8 Number of concurrent workers.
|
||||
--no-verification Don't verify the results.
|
||||
--only-verified Only output verified results.
|
||||
--filter-unverified Only output first unverified result per chunk per detector if there are more than one results.
|
||||
--filter-entropy=FILTER-ENTROPY
|
||||
Filter unverified results with Shannon entropy. Start with 3.0.
|
||||
--help Show context-sensitive help (also try --help-long and --help-man).
|
||||
--debug Run in debug mode.
|
||||
--trace Run in trace mode.
|
||||
--profile Enables profiling and sets a pprof and fgprof server on :18066.
|
||||
-j, --json Output in JSON format.
|
||||
--json-legacy Use the pre-v3.0 JSON format. Only works with git, gitlab, and github sources.
|
||||
--concurrency=10 Number of concurrent workers.
|
||||
--no-verification Don't verify the results.
|
||||
--only-verified Only output verified results.
|
||||
--filter-unverified Only output first unverified result per chunk per detector if there are more than one results.
|
||||
--config=CONFIG Path to configuration file.
|
||||
--print-avg-detector-time
|
||||
Print the average time spent on each detector.
|
||||
--no-update Don't check for updates.
|
||||
--fail Exit with code 183 if results are found.
|
||||
--verifier=VERIFIER ... Set custom verification endpoints.
|
||||
--archive-max-size=ARCHIVE-MAX-SIZE
|
||||
Maximum size of archive to scan. (Byte units eg. 512B, 2KB, 4MB)
|
||||
--archive-max-depth=ARCHIVE-MAX-DEPTH
|
||||
Maximum depth of archive to scan.
|
||||
--archive-timeout=ARCHIVE-TIMEOUT
|
||||
Maximum time to spend extracting an archive.
|
||||
--include-detectors="all" Comma separated list of detector types to include. Protobuf name or IDs may be used, as well as ranges.
|
||||
--exclude-detectors=EXCLUDE-DETECTORS
|
||||
Comma separated list of detector types to exclude. Protobuf name or IDs may be used, as well as ranges. IDs defined here take precedence over the include list.
|
||||
--version Show application version.
|
||||
-i, --include-paths=INCLUDE-PATHS
|
||||
Path to file with newline separated regexes for files to include in scan.
|
||||
-x, --exclude-paths=EXCLUDE-PATHS
|
||||
Path to file with newline separated regexes for files to exclude in scan.
|
||||
--exclude-globs=EXCLUDE-GLOBS
|
||||
Comma separated list of globs to exclude in scan. This option filters at the `git log` level, resulting in faster scans.
|
||||
--since-commit=SINCE-COMMIT
|
||||
Commit to start scan from.
|
||||
--branch=BRANCH Branch to scan.
|
||||
--max-depth=MAX-DEPTH Maximum depth of commits to scan.
|
||||
--bare Scan bare repository (e.g. useful while using in pre-receive hooks)
|
||||
--print-avg-detector-time Print the average time spent on each detector.
|
||||
--no-update Don't check for updates.
|
||||
--fail Exit with code 183 if results are found.
|
||||
--version Show application version.
|
||||
|
||||
Args:
|
||||
<uri> Git repository URL. https://, file://, or ssh:// schema expected.
|
||||
```
|
||||
|
||||
For example, to scan a `git` repository, start with
|
||||
For example, to scan a `git` repository, start with
|
||||
|
||||
```
|
||||
$ trufflehog git https://github.com/trufflesecurity/trufflehog.git
|
||||
```
|
||||
|
||||
## S3
|
||||
|
||||
The S3 source supports assuming IAM roles for scanning in addition to IAM users. This makes it easier for users to scan multiple AWS accounts without needing to rely on hardcoded credentials for each account.
|
||||
|
||||
The IAM identity that TruffleHog uses initially will need to have `AssumeRole` privileges as a principal in the [trust policy](https://aws.amazon.com/blogs/security/how-to-use-trust-policies-with-iam-roles/) of each IAM role to assume.
|
||||
|
||||
To scan a specific bucket using locally set credentials or instance metadata if on an EC2 instance:
|
||||
|
||||
```bash
|
||||
trufflehog s3 --bucket=<bucket-name>
|
||||
```
|
||||
|
||||
To scan a specific bucket using an assumed role:
|
||||
|
||||
```bash
|
||||
trufflehog s3 --bucket=<bucket-name> --role-arn=<iam-role-arn>
|
||||
```
|
||||
|
||||
Multiple roles can be passed as separate arguments. The following command will attempt to scan every bucket each role has permissions to list in the S3 API:
|
||||
|
||||
```bash
|
||||
trufflehog s3 --role-arn=<iam-role-arn-1> --role-arn=<iam-role-arn-2>
|
||||
```
|
||||
|
||||
Exit Codes:
|
||||
|
||||
- 0: No errors and no results were found.
|
||||
- 1: An error was encountered. Sources may not have completed scans.
|
||||
- 183: No errors were encountered, but results were found. Will only be returned if `--fail` flag is used.
|
||||
|
||||
## :octocat: TruffleHog Github Action
|
||||
|
||||
### General Usage
|
||||
|
||||
```
|
||||
on:
|
||||
push:
|
||||
branches:
|
||||
- main
|
||||
pull_request:
|
||||
|
||||
jobs:
|
||||
test:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 0
|
||||
- name: Secret Scanning
|
||||
uses: trufflesecurity/trufflehog@main
|
||||
with:
|
||||
extra_args: --only-verified
|
||||
```
|
||||
|
||||
In the example config above, we're scanning for live secrets in all PRs and Pushes to `main`. Only code changes in the referenced commits are scanned. If you'd like to scan an entire branch, please see the "Advanced Usage" section below.
|
||||
|
||||
|
||||
### Shallow Cloning
|
||||
|
||||
If you're incorporating TruffleHog into a standalone workflow and aren't running any other CI/CD tooling alongside TruffleHog, then we recommend using [Shallow Cloning](https://git-scm.com/docs/git-clone#Documentation/git-clone.txt---depthltdepthgt) to speed up your workflow. Here's an example for how to do it:
|
||||
|
||||
```
|
||||
...
|
||||
- shell: bash
|
||||
run: |
|
||||
if [ "${{ github.event_name }}" == "push" ]; then
|
||||
echo "depth=$(($(jq length <<< '${{ toJson(github.event.commits) }}') + 2))" >> $GITHUB_ENV
|
||||
echo "branch=${{ github.ref_name }}" >> $GITHUB_ENV
|
||||
fi
|
||||
if [ "${{ github.event_name }}" == "pull_request" ]; then
|
||||
echo "depth=$((${{ github.event.pull_request.commits }}+2))" >> $GITHUB_ENV
|
||||
echo "branch=${{ github.event.pull_request.head.ref }}" >> $GITHUB_ENV
|
||||
fi
|
||||
- uses: actions/checkout@v3
|
||||
with:
|
||||
ref: ${{env.branch}}
|
||||
fetch-depth: ${{env.depth}}
|
||||
- uses: trufflesecurity/trufflehog@main
|
||||
with:
|
||||
extra_args: --only-verified
|
||||
...
|
||||
```
|
||||
|
||||
Depending on the event type (push or PR), we calculate the number of commits present. Then we add 2, so that we can reference a base commit before our code changes. We pass that integer value to the `fetch-depth` flag in the checkout action in addition to the relevant branch. Now our checkout process should be much shorter.
|
||||
|
||||
### Advanced Usage
|
||||
# :octocat: TruffleHog Github Action
|
||||
|
||||
```yaml
|
||||
- name: TruffleHog
|
||||
@@ -413,44 +240,50 @@ Depending on the event type (push or PR), we calculate the number of commits pre
|
||||
extra_args: --debug --only-verified
|
||||
```
|
||||
|
||||
If you'd like to specify specific `base` and `head` refs, you can use the `base` argument (`--since-commit` flag in TruffleHog CLI) and the `head` argument (`--branch` flag in the TruffleHog CLI). We only recommend using these arguments for very specific use cases, where the default behavior does not work.
|
||||
The TruffleHog OSS Github Action can be used to scan a range of commits for leaked credentials. The action will fail if
|
||||
any results are found.
|
||||
|
||||
#### Advanced Usage: Scan entire branch
|
||||
```
|
||||
- name: scan-push
|
||||
For example, to scan the contents of pull requests you could use the following workflow:
|
||||
```yaml
|
||||
name: TruffleHog Secrets Scan
|
||||
on: [pull_request]
|
||||
jobs:
|
||||
TruffleHog:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v3
|
||||
with:
|
||||
fetch-depth: 0
|
||||
- name: TruffleHog OSS
|
||||
uses: trufflesecurity/trufflehog@main
|
||||
with:
|
||||
base: ""
|
||||
head: ${{ github.ref_name }}
|
||||
extra_args: --only-verified
|
||||
path: ./
|
||||
base: ${{ github.event.repository.default_branch }}
|
||||
head: HEAD
|
||||
extra_args: --debug --only-verified
|
||||
```
|
||||
|
||||
## Pre-commit Hook
|
||||
|
||||
Trufflehog can be used in a pre-commit hook to prevent credentials from leaking before they ever leave your computer.
|
||||
|
||||
**Key Usage Note:**
|
||||
|
||||
- **For optimal hook efficacy, execute `git add` followed by `git commit` separately.** This ensures Trufflehog analyzes all intended changes.
|
||||
- **Avoid using `git commit -am`, as it might bypass pre-commit hook execution for unstaged modifications.**
|
||||
# Precommit Hook
|
||||
|
||||
Trufflehog can be used in a precommit hook to prevent credentials from leaking before they ever leave your computer.
|
||||
An example `.pre-commit-config.yaml` is provided (see [pre-commit.com](https://pre-commit.com/) for installation).
|
||||
|
||||
```yaml
|
||||
repos:
|
||||
- repo: local
|
||||
hooks:
|
||||
- id: trufflehog
|
||||
name: TruffleHog
|
||||
description: Detect secrets in your data.
|
||||
entry: bash -c 'trufflehog git file://. --since-commit HEAD --only-verified --fail'
|
||||
# For running trufflehog in docker, use the following entry instead:
|
||||
# entry: bash -c 'docker run --rm -v "$(pwd):/workdir" -i --rm trufflesecurity/trufflehog:latest git file:///workdir --since-commit HEAD --only-verified --fail'
|
||||
language: system
|
||||
stages: ["commit", "push"]
|
||||
- repo: local
|
||||
hooks:
|
||||
- id: trufflehog
|
||||
name: TruffleHog
|
||||
description: Detect secrets in your data.
|
||||
entry: bash -c 'trufflehog git file://. --since-commit HEAD --only-verified --fail'
|
||||
# For running trufflehog in docker, use the following entry instead:
|
||||
# entry: bash -c 'docker run --rm -v "$(pwd):/workdir" -i --rm trufflesecurity/trufflehog:latest git file:///workdir --since-commit HEAD --only-verified --fail'
|
||||
language: system
|
||||
stages: ["commit", "push"]
|
||||
```
|
||||
|
||||
## Regex Detector (alpha)
|
||||
# Regex Detector (alpha)
|
||||
|
||||
Trufflehog supports detection and verification of custom regular expressions.
|
||||
For detection, at least one **regular expression** and **keyword** is required.
|
||||
@@ -469,17 +302,17 @@ status code, the secret is considered verified.
|
||||
```yaml
|
||||
# config.yaml
|
||||
detectors:
|
||||
- name: hog detector
|
||||
keywords:
|
||||
- hog
|
||||
regex:
|
||||
adjective: hogs are (\S+)
|
||||
verify:
|
||||
- endpoint: http://localhost:8000/
|
||||
# unsafe must be set if the endpoint is HTTP
|
||||
unsafe: true
|
||||
headers:
|
||||
- "Authorization: super secret authorization header"
|
||||
- name: hog detector
|
||||
keywords:
|
||||
- hog
|
||||
regex:
|
||||
adjective: hogs are (\S+)
|
||||
verify:
|
||||
- endpoint: http://localhost:8000/
|
||||
# unsafe must be set if the endpoint is HTTP
|
||||
unsafe: true
|
||||
headers:
|
||||
- 'Authorization: super secret authorization header'
|
||||
```
|
||||
|
||||
```
|
||||
@@ -547,29 +380,33 @@ with HTTPServer(('', 8000), Verifier) as server:
|
||||
|
||||
This project exists thanks to all the people who contribute. [[Contribute](CONTRIBUTING.md)].
|
||||
|
||||
|
||||
<a href="https://github.com/trufflesecurity/trufflehog/graphs/contributors">
|
||||
<img src="https://contrib.rocks/image?repo=trufflesecurity/trufflehog" />
|
||||
</a>
|
||||
|
||||
|
||||
# :computer: Contributing
|
||||
|
||||
Contributions are very welcome! Please see our [contribution guidelines first](CONTRIBUTING.md).
|
||||
|
||||
We no longer accept contributions to TruffleHog v2, but that code is available in the `v2` branch.
|
||||
|
||||
|
||||
## Adding new secret detectors
|
||||
|
||||
We have published some [documentation and tooling to get started on adding new secret detectors](hack/docs/Adding_Detectors_external.md). Let's improve detection together!
|
||||
|
||||
|
||||
# Use as a library
|
||||
|
||||
Currently, trufflehog is in heavy development and no guarantees can be made on
|
||||
the stability of the public APIs at this time.
|
||||
|
||||
# License Change
|
||||
|
||||
Since v3.0, TruffleHog is released under a AGPL 3 license, included in [`LICENSE`](LICENSE). TruffleHog v3.0 uses none of the previous codebase, but care was taken to preserve backwards compatibility on the command line interface. The work previous to this release is still available licensed under GPL 2.0 in the history of this repository and the previous package releases and tags. A completed CLA is required for us to accept contributions going forward.
|
||||
|
||||
|
||||
# :money_with_wings: Enterprise product
|
||||
|
||||
Are you interested in continuously monitoring your Git, Jira, Slack, Confluence, etc.. for credentials? We have an enterprise product that can help. Reach out here to learn more https://trufflesecurity.com/contact/
|
||||
|
||||
+15
-72
@@ -1,12 +1,11 @@
|
||||
name: 'TruffleHog OSS'
|
||||
description: 'Scan Github Actions with TruffleHog.'
|
||||
description: 'Scan Github Actions with TruffleHog'
|
||||
author: Truffle Security Co. <support@trufflesec.com>
|
||||
|
||||
inputs:
|
||||
path:
|
||||
description: Repository path
|
||||
required: false
|
||||
default: "./"
|
||||
required: true
|
||||
base:
|
||||
description: Start scanning from here (usually main branch).
|
||||
required: false
|
||||
@@ -21,73 +20,17 @@ inputs:
|
||||
branding:
|
||||
icon: "shield"
|
||||
color: "green"
|
||||
|
||||
runs:
|
||||
using: "composite"
|
||||
steps:
|
||||
- shell: bash
|
||||
env:
|
||||
REPO_PATH: ${{ inputs.path }}
|
||||
BASE: ${{ inputs.base }}
|
||||
HEAD: ${{ inputs.head }}
|
||||
ARGS: ${{ inputs.extra_args }}
|
||||
COMMITS: ${{ toJson(github.event.commits) }}
|
||||
run: |
|
||||
##########################################
|
||||
## ADVANCED USAGE ##
|
||||
## Scan by BASE & HEAD user inputs ##
|
||||
## If BASE == HEAD, exit with error ##
|
||||
##########################################
|
||||
if [ -n "$BASE" ] || [ -n "$HEAD" ]; then
|
||||
if [ -n "$BASE" ]; then
|
||||
base_commit=$(git rev-parse "$BASE" 2>/dev/null) || true
|
||||
else
|
||||
base_commit=""
|
||||
fi
|
||||
if [ -n "$HEAD" ]; then
|
||||
head_commit=$(git rev-parse "$HEAD" 2>/dev/null) || true
|
||||
else
|
||||
head_commit=""
|
||||
fi
|
||||
if [ $base_commit == $head_commit ] ; then
|
||||
echo "::error::BASE and HEAD commits are the same. TruffleHog won't scan anything. Please see documentation (https://github.com/trufflesecurity/trufflehog#octocat-trufflehog-github-action)."
|
||||
exit 1
|
||||
fi
|
||||
##########################################
|
||||
## Scan commits based on event type ##
|
||||
##########################################
|
||||
else
|
||||
if [ "${{ github.event_name }}" == "push" ]; then
|
||||
COMMIT_LENGTH=$(printenv COMMITS | jq length)
|
||||
if [ $COMMIT_LENGTH == "0" ]; then
|
||||
echo "No commits to scan"
|
||||
exit 0
|
||||
fi
|
||||
HEAD=${{ github.event.after }}
|
||||
if [ ${{ github.event.before }} == "0000000000000000000000000000000000000000" ]; then
|
||||
BASE=$(git rev-parse $HEAD~$COMMIT_LENGTH)
|
||||
else
|
||||
BASE=${{ github.event.before }}
|
||||
fi
|
||||
elif [ "${{ github.event_name }}" == "workflow_dispatch" ] || [ "${{ github.event_name }}" == "schedule" ]; then
|
||||
BASE=""
|
||||
HEAD=""
|
||||
elif [ "${{ github.event_name }}" == "pull_request" ]; then
|
||||
BASE=${{github.event.pull_request.base.sha}}
|
||||
HEAD=${{github.event.pull_request.head.sha}}
|
||||
fi
|
||||
fi
|
||||
##########################################
|
||||
## Run TruffleHog ##
|
||||
##########################################
|
||||
docker run --rm -v "$REPO_PATH":/tmp \
|
||||
ghcr.io/trufflesecurity/trufflehog:latest \
|
||||
git file:///tmp/ \
|
||||
--since-commit \
|
||||
${BASE:-''} \
|
||||
--branch \
|
||||
${HEAD:-''} \
|
||||
--fail \
|
||||
--no-update \
|
||||
--github-actions \
|
||||
${ARGS:-''}
|
||||
using: "docker"
|
||||
image: "docker://ghcr.io/trufflesecurity/trufflehog:latest"
|
||||
args:
|
||||
- git
|
||||
- file://${{ inputs.path }}
|
||||
- --since-commit
|
||||
- ${{ inputs.base }}
|
||||
- --branch
|
||||
- ${{ inputs.head }}
|
||||
- --fail
|
||||
- --no-update
|
||||
- --github-actions
|
||||
- ${{ inputs.extra_args }}
|
||||
|
||||
+25
-16
File diff suppressed because one or more lines are too long
|
Before Width: | Height: | Size: 83 KiB After Width: | Height: | Size: 11 KiB |
@@ -1,14 +0,0 @@
|
||||
# Examples
|
||||
This folder contains various examples like custom detectors, scripts, etc. Feel free to contribute!
|
||||
|
||||
### Generic Detector
|
||||
An often requested feature for TruffleHog is a generic detector. By default, we do not support generic detection as it would result in lots of false positives. However, if you want to attempt detect generic secrets you can use a custom detector.
|
||||
|
||||
#### Try it out:
|
||||
```
|
||||
wget https://raw.githubusercontent.com/trufflesecurity/trufflehog/main/examples/generic.yml
|
||||
trufflehog filesystem --config=$PWD/generic.yml $PWD
|
||||
|
||||
# to filter so that _only_ generic credentials are logged:
|
||||
trufflehog filesystem --config=$PWD/generic.yml --json --no-verification $PWD | awk '/generic-api-key/{print $0}'
|
||||
```
|
||||
@@ -1,15 +0,0 @@
|
||||
detectors:
|
||||
- name: generic-api-key
|
||||
keywords:
|
||||
- key
|
||||
- api
|
||||
- token
|
||||
- secret
|
||||
- client
|
||||
- passwd
|
||||
- password
|
||||
- auth
|
||||
- access
|
||||
regex:
|
||||
# borrowing the gitleaks generic-api-key regex
|
||||
generic-api-key: "(?i)(?:key|api|token|secret|client|passwd|password|auth|access)(?:[0-9a-z\\-_\\t .]{0,20})(?:[\\s|']|[\\s|\"]){0,3}(?:=|>|:{1,3}=|\\|\\|:|<=|=>|:|\\?=)(?:'|\"|\\s|=|\\x60){0,5}([0-9a-z\\-_.=]{10,150})(?:['|\"|\\n|\\r|\\s|\\x60|;]|$)"
|
||||
@@ -1,106 +1,82 @@
|
||||
module github.com/trufflesecurity/trufflehog/v3
|
||||
|
||||
go 1.21
|
||||
go 1.20
|
||||
|
||||
replace github.com/jpillora/overseer => github.com/trufflesecurity/overseer v1.2.7
|
||||
replace github.com/jpillora/overseer => github.com/trufflesecurity/overseer v1.1.7-custom5
|
||||
|
||||
// go wants to pull in the latest but after v10.2.5 has a breaking API change
|
||||
replace github.com/couchbase/gocbcore/v10 => github.com/couchbase/gocbcore/v10 v10.2.4
|
||||
|
||||
require (
|
||||
cloud.google.com/go/secretmanager v1.11.4
|
||||
cloud.google.com/go/storage v1.36.0
|
||||
github.com/Azure/go-autorest/autorest/azure/auth v0.5.12
|
||||
github.com/AzureAD/microsoft-authentication-library-for-go v1.2.1
|
||||
cloud.google.com/go/secretmanager v1.11.1
|
||||
cloud.google.com/go/storage v1.31.0
|
||||
github.com/Azure/go-autorest/autorest/azure/auth v0.5.11
|
||||
github.com/BobuSumisu/aho-corasick v1.0.3
|
||||
github.com/TheZeroSlave/zapsentry v1.19.0
|
||||
github.com/alecthomas/kingpin/v2 v2.4.0
|
||||
github.com/aws/aws-sdk-go v1.49.19
|
||||
github.com/aymanbagabas/go-osc52 v1.2.2
|
||||
github.com/TheZeroSlave/zapsentry v1.17.0
|
||||
github.com/aws/aws-sdk-go v1.44.83
|
||||
github.com/bill-rich/disk-buffer-reader v0.1.7
|
||||
github.com/bill-rich/go-syslog v0.0.0-20220413021637-49edb52a574c
|
||||
github.com/bitfinexcom/bitfinex-api-go v0.0.0-20210608095005-9e0b26f200fb
|
||||
github.com/bradleyfalzon/ghinstallation/v2 v2.9.0
|
||||
github.com/charmbracelet/bubbles v0.16.1
|
||||
github.com/charmbracelet/bubbletea v0.24.2
|
||||
github.com/charmbracelet/glamour v0.6.0
|
||||
github.com/charmbracelet/lipgloss v0.9.1
|
||||
github.com/coinbase/waas-client-library-go v1.0.8
|
||||
github.com/couchbase/gocb/v2 v2.7.0
|
||||
github.com/bradleyfalzon/ghinstallation/v2 v2.6.0
|
||||
github.com/couchbase/gocb/v2 v2.6.3
|
||||
github.com/crewjam/rfc5424 v0.1.0
|
||||
github.com/denisenkom/go-mssqldb v0.12.3
|
||||
github.com/envoyproxy/protoc-gen-validate v1.0.2
|
||||
github.com/fatih/color v1.16.0
|
||||
github.com/fatih/color v1.15.0
|
||||
github.com/felixge/fgprof v0.9.3
|
||||
github.com/gabriel-vasile/mimetype v1.4.3
|
||||
github.com/getsentry/sentry-go v0.25.0
|
||||
github.com/go-errors/errors v1.5.1
|
||||
github.com/go-git/go-git/v5 v5.11.0
|
||||
github.com/go-ldap/ldap/v3 v3.4.6
|
||||
github.com/go-logr/logr v1.3.0
|
||||
github.com/go-logr/zapr v1.3.0
|
||||
github.com/getsentry/sentry-go v0.22.0
|
||||
github.com/go-errors/errors v1.4.2
|
||||
github.com/go-git/go-git/v5 v5.8.0
|
||||
github.com/go-ldap/ldap/v3 v3.4.5
|
||||
github.com/go-logr/logr v1.2.4
|
||||
github.com/go-logr/zapr v1.2.4
|
||||
github.com/go-redis/redis v6.15.9+incompatible
|
||||
github.com/go-sql-driver/mysql v1.7.1
|
||||
github.com/gobwas/glob v0.2.3
|
||||
github.com/golang-jwt/jwt v3.2.2+incompatible
|
||||
github.com/google/go-cmp v0.6.0
|
||||
github.com/google/go-containerregistry v0.17.0
|
||||
github.com/golang/mock v1.4.3
|
||||
github.com/google/go-cmp v0.5.9
|
||||
github.com/google/go-containerregistry v0.15.2
|
||||
github.com/google/go-github/v42 v42.0.0
|
||||
github.com/google/uuid v1.4.0
|
||||
github.com/googleapis/gax-go/v2 v2.12.0
|
||||
github.com/h2non/filetype v1.1.3
|
||||
github.com/hashicorp/go-retryablehttp v0.7.5
|
||||
github.com/hashicorp/golang-lru v0.6.0
|
||||
github.com/hashicorp/golang-lru/v2 v2.0.7
|
||||
github.com/hashicorp/go-retryablehttp v0.7.4
|
||||
github.com/hashicorp/golang-lru v0.5.1
|
||||
github.com/jlaffaye/ftp v0.2.0
|
||||
github.com/joho/godotenv v1.5.1
|
||||
github.com/jpillora/overseer v1.1.6
|
||||
github.com/kylelemons/godebug v1.1.0
|
||||
github.com/launchdarkly/go-server-sdk/v6 v6.1.1
|
||||
github.com/lib/pq v1.10.9
|
||||
github.com/lrstanley/bubblezone v0.0.0-20230911164824-e3824f1adde9
|
||||
github.com/marusama/semaphore/v2 v2.5.0
|
||||
github.com/mattn/go-isatty v0.0.20
|
||||
github.com/mattn/go-sqlite3 v1.14.19
|
||||
github.com/mattn/go-sqlite3 v1.14.17
|
||||
github.com/mholt/archiver/v4 v4.0.0-alpha.8
|
||||
github.com/mitchellh/go-ps v1.0.0
|
||||
github.com/muesli/reflow v0.3.0
|
||||
github.com/patrickmn/go-cache v2.1.0+incompatible
|
||||
github.com/paulbellamy/ratecounter v0.2.0
|
||||
github.com/pkg/errors v0.9.1
|
||||
github.com/prometheus/client_golang v1.17.0
|
||||
github.com/rabbitmq/amqp091-go v1.9.0
|
||||
github.com/prometheus/client_golang v1.16.0
|
||||
github.com/rabbitmq/amqp091-go v1.8.1
|
||||
github.com/sergi/go-diff v1.3.1
|
||||
github.com/shuheiktgw/go-travis v0.3.1
|
||||
github.com/snowflakedb/gosnowflake v1.6.25
|
||||
github.com/stretchr/testify v1.8.4
|
||||
github.com/tailscale/depaware v0.0.0-20210622194025-720c4b409502
|
||||
github.com/trufflesecurity/disk-buffer-reader v0.2.1
|
||||
github.com/xanzy/go-gitlab v0.94.0
|
||||
go.mongodb.org/mongo-driver v1.12.1
|
||||
go.uber.org/mock v0.3.0
|
||||
go.uber.org/zap v1.26.0
|
||||
golang.org/x/crypto v0.17.0
|
||||
golang.org/x/exp v0.0.0-20240110193028-0dcbfd608b1e
|
||||
golang.org/x/net v0.19.0
|
||||
golang.org/x/oauth2 v0.15.0
|
||||
golang.org/x/sync v0.5.0
|
||||
golang.org/x/text v0.14.0
|
||||
google.golang.org/api v0.152.0
|
||||
github.com/xanzy/go-gitlab v0.88.0
|
||||
go.mongodb.org/mongo-driver v1.12.0
|
||||
go.uber.org/mock v0.2.0
|
||||
go.uber.org/zap v1.24.0
|
||||
golang.org/x/crypto v0.11.0
|
||||
golang.org/x/exp v0.0.0-20221018205818-5c77f4b2bbd7
|
||||
golang.org/x/oauth2 v0.10.0
|
||||
golang.org/x/sync v0.3.0
|
||||
golang.org/x/text v0.11.0
|
||||
google.golang.org/api v0.131.0
|
||||
google.golang.org/protobuf v1.31.0
|
||||
gopkg.in/alecthomas/kingpin.v2 v2.2.6
|
||||
gopkg.in/h2non/gock.v1 v1.1.2
|
||||
pgregory.net/rapid v1.1.0
|
||||
sigs.k8s.io/yaml v1.4.0
|
||||
sigs.k8s.io/yaml v1.3.0
|
||||
)
|
||||
|
||||
require (
|
||||
cloud.google.com/go v0.110.10 // indirect
|
||||
cloud.google.com/go/compute v1.23.3 // indirect
|
||||
cloud.google.com/go v0.110.2 // indirect
|
||||
cloud.google.com/go/compute v1.20.1 // indirect
|
||||
cloud.google.com/go/compute/metadata v0.2.3 // indirect
|
||||
cloud.google.com/go/iam v1.1.5 // indirect
|
||||
cloud.google.com/go/longrunning v0.5.4 // indirect
|
||||
dario.cat/mergo v1.0.0 // indirect
|
||||
github.com/99designs/go-keychain v0.0.0-20191008050251-8e49817e8af4 // indirect
|
||||
github.com/99designs/keyring v1.2.2 // indirect
|
||||
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.4.0 // indirect
|
||||
github.com/Azure/azure-sdk-for-go/sdk/internal v1.1.2 // indirect
|
||||
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.0.0 // indirect
|
||||
cloud.google.com/go/iam v1.1.0 // indirect
|
||||
github.com/Azure/go-autorest v14.2.0+incompatible // indirect
|
||||
github.com/Azure/go-autorest/autorest v0.11.24 // indirect
|
||||
github.com/Azure/go-autorest/autorest/adal v0.9.18 // indirect
|
||||
@@ -109,160 +85,103 @@ require (
|
||||
github.com/Azure/go-autorest/logger v0.2.1 // indirect
|
||||
github.com/Azure/go-autorest/tracing v0.6.0 // indirect
|
||||
github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 // indirect
|
||||
github.com/JohnCGriffin/overflow v0.0.0-20211019200055-46fa312c352c // indirect
|
||||
github.com/Microsoft/go-winio v0.6.1 // indirect
|
||||
github.com/ProtonMail/go-crypto v0.0.0-20230828082145-3c4c8a2d2371 // indirect
|
||||
github.com/alecthomas/chroma v0.10.0 // indirect
|
||||
github.com/ProtonMail/go-crypto v0.0.0-20230717121422-5aa5874ade95 // indirect
|
||||
github.com/acomagu/bufpipe v1.0.4 // indirect
|
||||
github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 // indirect
|
||||
github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 // indirect
|
||||
github.com/andybalholm/brotli v1.0.6 // indirect
|
||||
github.com/apache/arrow/go/v12 v12.0.1 // indirect
|
||||
github.com/apache/thrift v0.16.0 // indirect
|
||||
github.com/atotto/clipboard v0.1.4 // indirect
|
||||
github.com/aws/aws-sdk-go-v2 v1.17.7 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.10 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.13.18 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.59 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.31 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.25 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/v4a v1.0.23 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.11 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.26 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.25 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.14.0 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.31.0 // indirect
|
||||
github.com/aws/smithy-go v1.13.5 // indirect
|
||||
github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect
|
||||
github.com/aymerick/douceur v0.2.0 // indirect
|
||||
github.com/andybalholm/brotli v1.0.5 // indirect
|
||||
github.com/benbjohnson/clock v1.1.0 // indirect
|
||||
github.com/beorn7/perks v1.0.1 // indirect
|
||||
github.com/bodgit/plumbing v1.3.0 // indirect
|
||||
github.com/bodgit/sevenzip v1.4.5 // indirect
|
||||
github.com/bodgit/windows v1.0.1 // indirect
|
||||
github.com/bodgit/plumbing v1.2.0 // indirect
|
||||
github.com/bodgit/sevenzip v1.3.0 // indirect
|
||||
github.com/bodgit/windows v1.0.0 // indirect
|
||||
github.com/cespare/xxhash/v2 v2.2.0 // indirect
|
||||
github.com/cloudflare/circl v1.3.7 // indirect
|
||||
github.com/containerd/console v1.0.4-0.20230313162750-1ae8d489ac81 // indirect
|
||||
github.com/cloudflare/circl v1.3.3 // indirect
|
||||
github.com/connesc/cipherio v0.2.1 // indirect
|
||||
github.com/containerd/stargz-snapshotter/estargz v0.14.3 // indirect
|
||||
github.com/couchbase/gocbcore/v10 v10.3.0 // indirect
|
||||
github.com/couchbase/gocbcoreps v0.1.0 // indirect
|
||||
github.com/couchbase/goprotostellar v1.0.0 // indirect
|
||||
github.com/couchbaselabs/gocbconnstr/v2 v2.0.0-20230515165046-68b522a21131 // indirect
|
||||
github.com/cyphar/filepath-securejoin v0.2.4 // indirect
|
||||
github.com/danieljoos/wincred v1.1.2 // indirect
|
||||
github.com/couchbase/gocbcore/v10 v10.2.4 // indirect
|
||||
github.com/davecgh/go-spew v1.1.1 // indirect
|
||||
github.com/dimchansky/utfbom v1.1.1 // indirect
|
||||
github.com/dlclark/regexp2 v1.4.0 // indirect
|
||||
github.com/docker/cli v24.0.0+incompatible // indirect
|
||||
github.com/docker/cli v23.0.5+incompatible // indirect
|
||||
github.com/docker/distribution v2.8.2+incompatible // indirect
|
||||
github.com/docker/docker v24.0.7+incompatible // indirect
|
||||
github.com/docker/docker v23.0.5+incompatible // indirect
|
||||
github.com/docker/docker-credential-helpers v0.7.0 // indirect
|
||||
github.com/dsnet/compress v0.0.1 // indirect
|
||||
github.com/dvsekhvalnov/jose2go v1.6.0 // indirect
|
||||
github.com/emirpasic/gods v1.18.1 // indirect
|
||||
github.com/form3tech-oss/jwt-go v3.2.5+incompatible // indirect
|
||||
github.com/go-asn1-ber/asn1-ber v1.5.5 // indirect
|
||||
github.com/go-asn1-ber/asn1-ber v1.5.4 // indirect
|
||||
github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect
|
||||
github.com/go-git/go-billy/v5 v5.5.0 // indirect
|
||||
github.com/go-git/go-billy/v5 v5.4.1 // indirect
|
||||
github.com/go-ole/go-ole v1.2.6 // indirect
|
||||
github.com/goccy/go-json v0.10.0 // indirect
|
||||
github.com/godbus/dbus v0.0.0-20190726142602-4481cbc300e2 // indirect
|
||||
github.com/golang-jwt/jwt/v4 v4.5.0 // indirect
|
||||
github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe // indirect
|
||||
github.com/golang-sql/sqlexp v0.1.0 // indirect
|
||||
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
|
||||
github.com/golang/protobuf v1.5.3 // indirect
|
||||
github.com/golang/snappy v0.0.4 // indirect
|
||||
github.com/google/flatbuffers v23.1.21+incompatible // indirect
|
||||
github.com/google/go-github/v56 v56.0.0 // indirect
|
||||
github.com/google/go-github/v57 v57.0.0 // indirect
|
||||
github.com/google/go-github/v53 v53.2.0 // indirect
|
||||
github.com/google/go-querystring v1.1.0 // indirect
|
||||
github.com/google/pprof v0.0.0-20211214055906-6f57359322fd // indirect
|
||||
github.com/google/s2a-go v0.1.7 // indirect
|
||||
github.com/googleapis/enterprise-certificate-proxy v0.3.2 // indirect
|
||||
github.com/gorilla/css v1.0.0 // indirect
|
||||
github.com/gregjones/httpcache v0.0.0-20171119193500-2bcd89a1743f // indirect
|
||||
github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 // indirect
|
||||
github.com/grpc-ecosystem/grpc-gateway/v2 v2.15.2 // indirect
|
||||
github.com/gsterjov/go-libsecret v0.0.0-20161001094733-a6f4afe4910c // indirect
|
||||
github.com/google/s2a-go v0.1.4 // indirect
|
||||
github.com/google/uuid v1.3.0 // indirect
|
||||
github.com/googleapis/enterprise-certificate-proxy v0.2.5 // indirect
|
||||
github.com/h2non/parth v0.0.0-20190131123155-b4df798d6542 // indirect
|
||||
github.com/hashicorp/errwrap v1.0.0 // indirect
|
||||
github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
|
||||
github.com/hashicorp/go-multierror v1.1.1 // indirect
|
||||
github.com/hashicorp/golang-lru/v2 v2.0.4 // indirect
|
||||
github.com/imdario/mergo v0.3.15 // indirect
|
||||
github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect
|
||||
github.com/jmespath/go-jmespath v0.4.0 // indirect
|
||||
github.com/josharian/intern v1.0.0 // indirect
|
||||
github.com/jpillora/s3 v1.1.4 // indirect
|
||||
github.com/kevinburke/ssh_config v1.2.0 // indirect
|
||||
github.com/klauspost/asmfmt v1.3.2 // indirect
|
||||
github.com/klauspost/compress v1.17.4 // indirect
|
||||
github.com/klauspost/cpuid/v2 v2.2.3 // indirect
|
||||
github.com/klauspost/compress v1.16.5 // indirect
|
||||
github.com/klauspost/pgzip v1.2.5 // indirect
|
||||
github.com/launchdarkly/ccache v1.1.0 // indirect
|
||||
github.com/launchdarkly/eventsource v1.6.2 // indirect
|
||||
github.com/launchdarkly/go-jsonstream/v3 v3.0.0 // indirect
|
||||
github.com/launchdarkly/go-sdk-common/v3 v3.0.1 // indirect
|
||||
github.com/launchdarkly/go-sdk-events/v2 v2.0.2 // indirect
|
||||
github.com/launchdarkly/go-semver v1.0.2 // indirect
|
||||
github.com/launchdarkly/go-server-sdk-evaluation/v2 v2.0.2 // indirect
|
||||
github.com/lucasb-eyer/go-colorful v1.2.0 // indirect
|
||||
github.com/mailru/easyjson v0.7.7 // indirect
|
||||
github.com/mattn/go-colorable v0.1.13 // indirect
|
||||
github.com/mattn/go-localereader v0.0.1 // indirect
|
||||
github.com/mattn/go-runewidth v0.0.15 // indirect
|
||||
github.com/mattn/go-isatty v0.0.17 // indirect
|
||||
github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
|
||||
github.com/microcosm-cc/bluemonday v1.0.23 // indirect
|
||||
github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8 // indirect
|
||||
github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3 // indirect
|
||||
github.com/mitchellh/go-homedir v1.1.0 // indirect
|
||||
github.com/montanaflynn/stats v0.7.0 // indirect
|
||||
github.com/mtibben/percent v0.2.1 // indirect
|
||||
github.com/muesli/ansi v0.0.0-20211031195517-c9f0611b6c70 // indirect
|
||||
github.com/muesli/cancelreader v0.2.2 // indirect
|
||||
github.com/muesli/termenv v0.15.2 // indirect
|
||||
github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe // indirect
|
||||
github.com/nwaples/rardecode/v2 v2.0.0-beta.2 // indirect
|
||||
github.com/olekukonko/tablewriter v0.0.5 // indirect
|
||||
github.com/onsi/ginkgo v1.16.5 // indirect
|
||||
github.com/onsi/gomega v1.23.0 // indirect
|
||||
github.com/opencontainers/go-digest v1.0.0 // indirect
|
||||
github.com/opencontainers/image-spec v1.1.0-rc3 // indirect
|
||||
github.com/pierrec/lz4/v4 v4.1.19 // indirect
|
||||
github.com/pierrec/lz4/v4 v4.1.15 // indirect
|
||||
github.com/pjbgf/sha1cd v0.3.0 // indirect
|
||||
github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 // indirect
|
||||
github.com/pkg/diff v0.0.0-20200914180035-5b29258ca4f7 // indirect
|
||||
github.com/pmezard/go-difflib v1.0.0 // indirect
|
||||
github.com/prometheus/client_model v0.4.1-0.20230718164431-9a2bf3000d16 // indirect
|
||||
github.com/prometheus/common v0.44.0 // indirect
|
||||
github.com/prometheus/procfs v0.11.1 // indirect
|
||||
github.com/rivo/uniseg v0.4.2 // indirect
|
||||
github.com/sahilm/fuzzy v0.1.0 // indirect
|
||||
github.com/sirupsen/logrus v1.9.1 // indirect
|
||||
github.com/skeema/knownhosts v1.2.1 // indirect
|
||||
github.com/prometheus/client_model v0.3.0 // indirect
|
||||
github.com/prometheus/common v0.42.0 // indirect
|
||||
github.com/prometheus/procfs v0.10.1 // indirect
|
||||
github.com/rogpeppe/go-internal v1.10.0 // indirect
|
||||
github.com/sirupsen/logrus v1.9.0 // indirect
|
||||
github.com/skeema/knownhosts v1.2.0 // indirect
|
||||
github.com/therootcompany/xz v1.0.1 // indirect
|
||||
github.com/ulikunitz/xz v0.5.11 // indirect
|
||||
github.com/ulikunitz/xz v0.5.10 // indirect
|
||||
github.com/vbatts/tar-split v0.11.3 // indirect
|
||||
github.com/xanzy/ssh-agent v0.3.3 // indirect
|
||||
github.com/xdg-go/pbkdf2 v1.0.0 // indirect
|
||||
github.com/xdg-go/scram v1.1.2 // indirect
|
||||
github.com/xdg-go/stringprep v1.0.4 // indirect
|
||||
github.com/xhit/go-str2duration/v2 v2.1.0 // indirect
|
||||
github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d // indirect
|
||||
github.com/yuin/goldmark v1.5.2 // indirect
|
||||
github.com/yuin/goldmark-emoji v1.0.1 // indirect
|
||||
github.com/yusufpapurcu/wmi v1.2.2 // indirect
|
||||
github.com/zeebo/xxh3 v1.0.2 // indirect
|
||||
go.einride.tech/aip v0.60.0 // indirect
|
||||
go.opencensus.io v0.24.0 // indirect
|
||||
go.uber.org/multierr v1.11.0 // indirect
|
||||
go.uber.org/atomic v1.7.0 // indirect
|
||||
go.uber.org/multierr v1.6.0 // indirect
|
||||
go4.org v0.0.0-20200411211856-f5505b9728dd // indirect
|
||||
golang.org/x/mod v0.14.0 // indirect
|
||||
golang.org/x/sys v0.15.0 // indirect
|
||||
golang.org/x/term v0.15.0 // indirect
|
||||
golang.org/x/time v0.5.0 // indirect
|
||||
golang.org/x/tools v0.16.0 // indirect
|
||||
golang.org/x/mod v0.11.0 // indirect
|
||||
golang.org/x/net v0.12.0 // indirect
|
||||
golang.org/x/sys v0.10.0 // indirect
|
||||
golang.org/x/time v0.3.0 // indirect
|
||||
golang.org/x/tools v0.10.0 // indirect
|
||||
golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect
|
||||
google.golang.org/appengine v1.6.7 // indirect
|
||||
google.golang.org/genproto v0.0.0-20231106174013-bbf56f31fb17 // indirect
|
||||
google.golang.org/genproto/googleapis/api v0.0.0-20231106174013-bbf56f31fb17 // indirect
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20231120223509-83a465c0220f // indirect
|
||||
google.golang.org/grpc v1.59.0 // indirect
|
||||
gopkg.in/square/go-jose.v2 v2.6.0 // indirect
|
||||
google.golang.org/genproto v0.0.0-20230629202037-9506855d4529 // indirect
|
||||
google.golang.org/genproto/googleapis/api v0.0.0-20230629202037-9506855d4529 // indirect
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20230706204954-ccb25ca9f130 // indirect
|
||||
google.golang.org/grpc v1.56.2 // indirect
|
||||
gopkg.in/warnings.v0 v0.1.2 // indirect
|
||||
gopkg.in/yaml.v2 v2.4.0 // indirect
|
||||
gopkg.in/yaml.v3 v3.0.1 // indirect
|
||||
)
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
# trufflesecurity/protos:1.18-0
|
||||
|
||||
FROM golang:1.21-bullseye
|
||||
FROM golang:1.18-buster
|
||||
|
||||
ARG TARGETARCH
|
||||
ARG TARGETOS
|
||||
|
||||
@@ -44,18 +44,6 @@ If you think that something should be included outside of these guidelines, plea
|
||||
- Go 1.17+
|
||||
- Make
|
||||
|
||||
### Adding New Token Formats to an Existing Scanner
|
||||
|
||||
In some instances, services will update their token format, requiring a new regex to properly detect secrets in addition to supporting the previous token format. Accomodating this can be done without adding a net-new detector. [We provide a `Versioner` interface](https://github.com/trufflesecurity/trufflehog/blob/e18cfd5e0af1469a9f05b8d5732bcc94c39da49c/pkg/detectors/detectors.go#L30) that can be implemented.
|
||||
|
||||
1. Create a copy of the package and append `_v2` to the package and file names. Ex: `<packagename>/` -> `<packagename>_v2`, `<packagename>.go` -> `<packagename>_v2.go`
|
||||
|
||||
Note: Be sure to update the tests to reference the new secret values in GSM, or the tests will fail.
|
||||
|
||||
2. Implement the `Versioner` interface. [GitHub example implementation.](/pkg/detectors/github_old/github_old.go#L22)
|
||||
|
||||
3. Proceed from step 3 of [Creating a new Secret Scanner](#creating-a-new-secret-scanner)
|
||||
|
||||
### Creating a new Secret Scanner
|
||||
|
||||
1. Identify the Secret Detector name from the [/proto/detectors.proto](/proto/detectors.proto) `DetectorType` enum.
|
||||
@@ -108,7 +96,7 @@ Do not embed test credentials in the test code. Instead, use GCP Secrets Manager
|
||||
Note: `/tmp/s` is a valid path on Linux. You will need to change that for Windows or OSX, otherwise you will see an error. On Windows you will also need to install [WSL](https://docs.microsoft.com/en-us/windows/wsl/install).
|
||||
|
||||
```bash
|
||||
gcloud secrets versions access --project trufflehog-testing --secret detectors5 latest > /tmp/s
|
||||
gcloud secrets versions access --project trufflehog-testing --secret detectors3 latest > /tmp/s
|
||||
```
|
||||
|
||||
2. Add the secret that you need for testing.
|
||||
@@ -125,9 +113,8 @@ Do not embed test credentials in the test code. Instead, use GCP Secrets Manager
|
||||
3. Update the secret version with your modification.
|
||||
|
||||
```bash
|
||||
gcloud secrets versions add --project trufflehog-testing detectors5 --data-file /tmp/s
|
||||
gcloud secrets versions add --project trufflehog-testing detectors3 --data-file /tmp/s
|
||||
```
|
||||
Note: We increment the detectors file name `detectors(n+1)` once the previous one exceeds the max size allowed by GSM (65kb).
|
||||
|
||||
4. Access the secret value as shown in the [example code](pkg/detectors/heroku/heroku_test.go).
|
||||
|
||||
|
||||
@@ -15,9 +15,9 @@ The purpose of Secret Detectors is to discover secrets with exceptionally high s
|
||||
+ [Sourcing Guidelines](#sourcing-guidelines)
|
||||
+ [Development Guidelines](#development-guidelines)
|
||||
+ [Development Dependencies](#development-dependencies)
|
||||
+ [Creating a new Secret Detector](#creating-a-new-secret-detector)
|
||||
+ [Testing the Detector](#testing-the-detector)
|
||||
+ [Creating a new Secret Scanner](#creating-a-new-secret-scanner)
|
||||
* [Addendum](#addendum)
|
||||
+ [Using a test secret file](#using-a-test-secret-file)
|
||||
+ [Adding Protos in Windows](#adding-protos-in-windows)
|
||||
|
||||
## Getting Started
|
||||
@@ -40,89 +40,37 @@ If you think that something should be included outside of these guidelines, plea
|
||||
- Go 1.17+
|
||||
- Make
|
||||
|
||||
### Adding New Token Formats to an Existing Scanner
|
||||
### Creating a new Secret Scanner
|
||||
|
||||
In some instances, services will update their token format, requiring a new regex to properly detect secrets in addition to supporting the previous token format. Accomodating this can be done without adding a net-new detector. [We provide a `Versioner` interface](https://github.com/trufflesecurity/trufflehog/blob/e18cfd5e0af1469a9f05b8d5732bcc94c39da49c/pkg/detectors/detectors.go#L30) that can be implemented.
|
||||
1. Identify the Secret Detector name from the [/proto/detectors.proto](/proto/detectors.proto) `DetectorType` enum. If necessary, run `make protos` when adding new ones.
|
||||
|
||||
1. Create a copy of the package and append `_v2` to the package and file names. Ex: `<packagename>/` -> `<packagename>_v2`, `<packagename>.go` -> `<packagename>_v2.go`
|
||||
|
||||
Note: Be sure to update the tests to reference the new secret values in GSM, or the tests will fail.
|
||||
|
||||
2. Implement the `Versioner` interface. [GitHub example implementation.](/pkg/detectors/github_old/github_old.go#L22)
|
||||
|
||||
3. Proceed from step 3 of [Creating a new Secret Scanner](#creating-a-new-secret-scanner)
|
||||
|
||||
### Creating a new Secret Detector
|
||||
|
||||
1. Add a new Secret Detector enum to the [`DetectorType` list here](/proto/detectors.proto).
|
||||
|
||||
2. Run `make protos` to update the `.pb` files.
|
||||
|
||||
3. Generate the Secret Detector
|
||||
2. Generate the Secret Detector
|
||||
|
||||
```bash
|
||||
go run hack/generate/generate.go detector <DetectorType enum name>
|
||||
example: go run hack/generate/generate.go detector SampleAPI
|
||||
```
|
||||
4. Add the Secret Detector to TruffleHog's Default Detectors
|
||||
3. Add Secret Scanner
|
||||
|
||||
Add the secret scanner to the [`pkg/engine/defaults.go`](https://github.com/trufflesecurity/trufflehog/blob/main/pkg/engine/defaults.go) file like [`github.com/trufflesecurity/trufflehog/v3/pkg/detectors/<detector_name>`](https://github.com/trufflesecurity/trufflehog/blob/b71ea27a696bdf1c3141f637fda4ee4936c2f2d6/pkg/engine/defaults.go#L9) and
|
||||
[`<detector_name>.Scanner{}`](https://github.com/trufflesecurity/trufflehog/blob/b71ea27a696bdf1c3141f637fda4ee4936c2f2d6/pkg/engine/defaults.go#L1546)
|
||||
Add the secret scanner to the `pkg/engine/defaults.go` file like `github.com/trufflesecurity/trufflehog/v3/pkg/detectors/<detector_name>` and
|
||||
`<detector_name>.Scanner{},`
|
||||
|
||||
5. Complete the Secret Detector.
|
||||
4. Complete the secret detector.
|
||||
|
||||
The previous step templated a boilerplate + some example code as a package in the `pkg/detectors` folder for you to work on.
|
||||
The Secret Detector can be completed with these general steps:
|
||||
The secret detector can be completed with these general steps:
|
||||
|
||||
1. Update the pattern regex and keywords. Try iterating with [regex101.com](http://regex101.com/).
|
||||
2. Update the verifier code to use a non-destructive API call that can determine whether the secret is valid or not.
|
||||
1. Create a [test secrets file, and export the variable](#using-a-test-secret-file)
|
||||
2. Update the pattern regex and keywords. Try iterating with [regex101.com](http://regex101.com/).
|
||||
3. Update the verifier code to use a non-destructive API call that can determine whether the secret is valid or not.
|
||||
* Make sure you understand [verification indeterminacy](#verification-indeterminacy).
|
||||
3. Create a [test for the detector](#testing-the-detector)
|
||||
4. Create a pull request for review.
|
||||
|
||||
### Testing the Detector
|
||||
To ensure the quality of your PR, make sure your tests are passing with verified credentials.
|
||||
|
||||
1. Create a file called `.env` with this env file format:
|
||||
|
||||
```bash
|
||||
SECRET_TYPE_ONE=value
|
||||
SECRET_TYPE_ONE_INACTIVE=v@lue
|
||||
```
|
||||
|
||||
2. Export the `TEST_SECRET_FILE` variable, pointing to the env file:
|
||||
|
||||
```bash
|
||||
export TEST_SECRET_FILE=".env"
|
||||
```
|
||||
The `.env` file should be in the new detector's directory like this:
|
||||
```
|
||||
├── tailscale
|
||||
│ ├── .env
|
||||
│ ├── tailscale.go
|
||||
│ └── tailscale_test.go
|
||||
```
|
||||
Now that a `.env` file is present, the test file can load secrets locally.
|
||||
|
||||
3. Next, update the tests as necessary. A test file has already been generated by the `go run hack/generate/generate.go` command from earlier. There are 5 cases that have been generated:
|
||||
1. Found and verified (using a credential loaded from the .env file)
|
||||
2. Found and unverified (determinately, i.e. the secret is invalid)
|
||||
3. Found and unverified (indeterminately due to timeout)
|
||||
4. Found and unverified (indeterminately due to an unexpected API response)
|
||||
5. Not found
|
||||
|
||||
Make any necessary updates to the tests. Note there might not be any changes required as the tests generated by the `go run hack/generate/generate.go` command are pretty good.
|
||||
[Here is an exemplary test file for a detector which covers all 5 test cases](https://github.com/trufflesecurity/trufflehog/blob/6f9065b0aae981133a7fa3431c17a5c6213be226/pkg/detectors/browserstack/browserstack_test.go).
|
||||
|
||||
4. Now run the tests and check to make sure they are passing ✔️!
|
||||
```bash
|
||||
go test ./pkg/detectors/<detector> -tags=detectors
|
||||
```
|
||||
|
||||
If the tests are passing, feel free to open a PR!
|
||||
|
||||
|
||||
|
||||
4. Update the tests with these test cases at minimum:
|
||||
1. Found and verified (using a credential loaded from GCP Secrets)
|
||||
2. Found and unverified (determinately, i.e. the secret is invalid)
|
||||
3. Found and unverified (indeterminately due to timeout)
|
||||
4. Found and unverified (indeterminately due to an unexpected API response)
|
||||
5. Not found
|
||||
6. Any false positive cases that you come across
|
||||
5. Create a pull request for review.
|
||||
|
||||
## Addendum
|
||||
|
||||
@@ -139,6 +87,23 @@ For example, consider a hypothetical authentication endpoint that returns `200 O
|
||||
* A `403` response would indicate that verification failed **determinately** and no error object should be returned.
|
||||
* Any other response would indicate that verification failed **indeterminately** and an error object should be returned.
|
||||
|
||||
### Using a test secret file
|
||||
|
||||
1. Create a file called `.env` with this env file format:
|
||||
|
||||
```bash
|
||||
SECRET_TYPE_ONE=value
|
||||
SECRET_TYPE_ONE_INACTIVE=v@lue
|
||||
```
|
||||
|
||||
2. Export the `TEST_SECRET_FILE` variable, pointing to the env file:
|
||||
|
||||
```bash
|
||||
export TEST_SECRET_FILE=".env"
|
||||
```
|
||||
|
||||
Now, the detector test should attempt to load the given env key from that file.
|
||||
|
||||
### Adding Protos in Windows
|
||||
|
||||
1. Install Ubuntu App in Microsoft Store https://www.microsoft.com/en-us/p/ubuntu/9nblggh4msv6.
|
||||
@@ -156,3 +121,7 @@ For example, consider a hypothetical authentication endpoint that returns `200 O
|
||||
make protos
|
||||
```
|
||||
|
||||
### Testing a detector
|
||||
```bash
|
||||
go test ./pkg/detectors/<detector> -tags=detectors
|
||||
```
|
||||
|
||||
@@ -8,10 +8,10 @@ import (
|
||||
"strings"
|
||||
"text/template"
|
||||
|
||||
"github.com/alecthomas/kingpin/v2"
|
||||
"github.com/go-errors/errors"
|
||||
"golang.org/x/text/cases"
|
||||
"golang.org/x/text/language"
|
||||
"gopkg.in/alecthomas/kingpin.v2"
|
||||
)
|
||||
|
||||
var (
|
||||
|
||||
@@ -10,9 +10,9 @@ import (
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"github.com/alecthomas/kingpin/v2"
|
||||
"github.com/paulbellamy/ratecounter"
|
||||
"golang.org/x/sync/semaphore"
|
||||
"gopkg.in/alecthomas/kingpin.v2"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/context"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/decoders"
|
||||
@@ -74,7 +74,7 @@ func main() {
|
||||
selectedScanners := map[string]detectors.Detector{}
|
||||
allScanners := getAllScanners()
|
||||
|
||||
allDecoders := decoders.DefaultDecoders()
|
||||
decoders := decoders.DefaultDecoders()
|
||||
|
||||
input := strings.ToLower(*scanCmdDetector)
|
||||
if input == "all" {
|
||||
@@ -121,7 +121,7 @@ func main() {
|
||||
|
||||
for chunk := range chunksChan {
|
||||
for name, scanner := range selectedScanners {
|
||||
for _, dec := range allDecoders {
|
||||
for _, dec := range decoders {
|
||||
decoded := dec.FromChunk(&sources.Chunk{Data: chunk.Data})
|
||||
if decoded != nil {
|
||||
foundKeyword := false
|
||||
@@ -201,13 +201,10 @@ func main() {
|
||||
},
|
||||
},
|
||||
}
|
||||
},
|
||||
true,
|
||||
false,
|
||||
)
|
||||
})
|
||||
|
||||
logger.Info("scanning repo", "repo", r)
|
||||
err = s.ScanRepo(ctx, repo, path, git.NewScanOptions(), sources.ChanReporter{Ch: chunksChan})
|
||||
err = s.ScanRepo(ctx, repo, path, git.NewScanOptions(), chunksChan)
|
||||
if err != nil {
|
||||
logFatal(err, "error scanning repo")
|
||||
}
|
||||
|
||||
@@ -5,21 +5,17 @@ import (
|
||||
"net/http"
|
||||
_ "net/http/pprof"
|
||||
"os"
|
||||
"os/signal"
|
||||
"runtime"
|
||||
"strconv"
|
||||
"strings"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"github.com/alecthomas/kingpin/v2"
|
||||
"github.com/felixge/fgprof"
|
||||
"github.com/go-logr/logr"
|
||||
"github.com/jpillora/overseer"
|
||||
"github.com/mattn/go-isatty"
|
||||
"google.golang.org/protobuf/types/known/anypb"
|
||||
"gopkg.in/alecthomas/kingpin.v2"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/cleantemp"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/config"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/context"
|
||||
@@ -31,7 +27,7 @@ import (
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/output"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/tui"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/sources/git"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/updater"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/version"
|
||||
)
|
||||
@@ -50,14 +46,12 @@ var (
|
||||
noVerification = cli.Flag("no-verification", "Don't verify the results.").Bool()
|
||||
onlyVerified = cli.Flag("only-verified", "Only output verified results.").Bool()
|
||||
filterUnverified = cli.Flag("filter-unverified", "Only output first unverified result per chunk per detector if there are more than one results.").Bool()
|
||||
filterEntropy = cli.Flag("filter-entropy", "Filter unverified results with Shannon entropy. Start with 3.0.").Float64()
|
||||
configFilename = cli.Flag("config", "Path to configuration file.").ExistingFile()
|
||||
// rules = cli.Flag("rules", "Path to file with custom rules.").String()
|
||||
printAvgDetectorTime = cli.Flag("print-avg-detector-time", "Print the average time spent on each detector.").Bool()
|
||||
noUpdate = cli.Flag("no-update", "Don't check for updates.").Bool()
|
||||
fail = cli.Flag("fail", "Exit with code 183 if results are found.").Bool()
|
||||
verifiers = cli.Flag("verifier", "Set custom verification endpoints.").StringMap()
|
||||
customVerifiersOnly = cli.Flag("custom-verifiers-only", "Only use custom verification endpoints.").Bool()
|
||||
archiveMaxSize = cli.Flag("archive-max-size", "Maximum size of archive to scan. (Byte units eg. 512B, 2KB, 4MB)").Bytes()
|
||||
archiveMaxDepth = cli.Flag("archive-max-depth", "Maximum depth of archive to scan.").Int()
|
||||
archiveTimeout = cli.Flag("archive-timeout", "Maximum time to spend extracting an archive.").Duration()
|
||||
@@ -72,25 +66,21 @@ var (
|
||||
gitScanSinceCommit = gitScan.Flag("since-commit", "Commit to start scan from.").String()
|
||||
gitScanBranch = gitScan.Flag("branch", "Branch to scan.").String()
|
||||
gitScanMaxDepth = gitScan.Flag("max-depth", "Maximum depth of commits to scan.").Int()
|
||||
gitScanBare = gitScan.Flag("bare", "Scan bare repository (e.g. useful while using in pre-receive hooks)").Bool()
|
||||
_ = gitScan.Flag("allow", "No-op flag for backwards compat.").Bool()
|
||||
_ = gitScan.Flag("entropy", "No-op flag for backwards compat.").Bool()
|
||||
_ = gitScan.Flag("regex", "No-op flag for backwards compat.").Bool()
|
||||
|
||||
githubScan = cli.Command("github", "Find credentials in GitHub repositories.")
|
||||
githubScanEndpoint = githubScan.Flag("endpoint", "GitHub endpoint.").Default("https://api.github.com").String()
|
||||
githubScanRepos = githubScan.Flag("repo", `GitHub repository to scan. You can repeat this flag. Example: "https://github.com/dustin-decker/secretsandstuff"`).Strings()
|
||||
githubScanOrgs = githubScan.Flag("org", `GitHub organization to scan. You can repeat this flag. Example: "trufflesecurity"`).Strings()
|
||||
githubScanToken = githubScan.Flag("token", "GitHub token. Can be provided with environment variable GITHUB_TOKEN.").Envar("GITHUB_TOKEN").String()
|
||||
githubIncludeForks = githubScan.Flag("include-forks", "Include forks in scan.").Bool()
|
||||
githubIncludeMembers = githubScan.Flag("include-members", "Include organization member repositories in scan.").Bool()
|
||||
githubIncludeRepos = githubScan.Flag("include-repos", `Repositories to include in an org scan. This can also be a glob pattern. You can repeat this flag. Must use Github repo full name. Example: "trufflesecurity/trufflehog", "trufflesecurity/t*"`).Strings()
|
||||
githubExcludeRepos = githubScan.Flag("exclude-repos", `Repositories to exclude in an org scan. This can also be a glob pattern. You can repeat this flag. Must use Github repo full name. Example: "trufflesecurity/driftwood", "trufflesecurity/d*"`).Strings()
|
||||
githubScanIncludePaths = githubScan.Flag("include-paths", "Path to file with newline separated regexes for files to include in scan.").Short('i').String()
|
||||
githubScanExcludePaths = githubScan.Flag("exclude-paths", "Path to file with newline separated regexes for files to exclude in scan.").Short('x').String()
|
||||
githubScanIssueComments = githubScan.Flag("issue-comments", "Include issue descriptions and comments in scan.").Bool()
|
||||
githubScanPRComments = githubScan.Flag("pr-comments", "Include pull request descriptions and comments in scan.").Bool()
|
||||
githubScanGistComments = githubScan.Flag("gist-comments", "Include gist comments in scan.").Bool()
|
||||
githubScan = cli.Command("github", "Find credentials in GitHub repositories.")
|
||||
githubScanEndpoint = githubScan.Flag("endpoint", "GitHub endpoint.").Default("https://api.github.com").String()
|
||||
githubScanRepos = githubScan.Flag("repo", `GitHub repository to scan. You can repeat this flag. Example: "https://github.com/dustin-decker/secretsandstuff"`).Strings()
|
||||
githubScanOrgs = githubScan.Flag("org", `GitHub organization to scan. You can repeat this flag. Example: "trufflesecurity"`).Strings()
|
||||
githubScanToken = githubScan.Flag("token", "GitHub token. Can be provided with environment variable GITHUB_TOKEN.").Envar("GITHUB_TOKEN").String()
|
||||
githubIncludeForks = githubScan.Flag("include-forks", "Include forks in scan.").Bool()
|
||||
githubIncludeMembers = githubScan.Flag("include-members", "Include organization member repositories in scan.").Bool()
|
||||
githubIncludeRepos = githubScan.Flag("include-repos", `Repositories to include in an org scan. This can also be a glob pattern. You can repeat this flag. Must use Github repo full name. Example: "trufflesecurity/trufflehog", "trufflesecurity/t*"`).Strings()
|
||||
githubExcludeRepos = githubScan.Flag("exclude-repos", `Repositories to exclude in an org scan. This can also be a glob pattern. You can repeat this flag. Must use Github repo full name. Example: "trufflesecurity/driftwood", "trufflesecurity/d*"`).Strings()
|
||||
githubScanIncludePaths = githubScan.Flag("include-paths", "Path to file with newline separated regexes for files to include in scan.").Short('i').String()
|
||||
githubScanExcludePaths = githubScan.Flag("exclude-paths", "Path to file with newline separated regexes for files to exclude in scan.").Short('x').String()
|
||||
|
||||
gitlabScan = cli.Command("gitlab", "Find credentials in GitLab repositories.")
|
||||
// TODO: Add more GitLab options
|
||||
@@ -111,7 +101,6 @@ var (
|
||||
|
||||
s3Scan = cli.Command("s3", "Find credentials in S3 buckets.")
|
||||
s3ScanKey = s3Scan.Flag("key", "S3 key used to authenticate. Can be provided with environment variable AWS_ACCESS_KEY_ID.").Envar("AWS_ACCESS_KEY_ID").String()
|
||||
s3ScanRoleArns = s3Scan.Flag("role-arn", "Specify the ARN of an IAM role to assume for scanning. You can repeat this flag.").Strings()
|
||||
s3ScanSecret = s3Scan.Flag("secret", "S3 secret used to authenticate. Can be provided with environment variable AWS_SECRET_ACCESS_KEY.").Envar("AWS_SECRET_ACCESS_KEY").String()
|
||||
s3ScanSessionToken = s3Scan.Flag("session-token", "S3 session token used to authenticate temporary credentials. Can be provided with environment variable AWS_SESSION_TOKEN.").Envar("AWS_SESSION_TOKEN").String()
|
||||
s3ScanCloudEnv = s3Scan.Flag("cloud-environment", "Use IAM credentials in cloud environment.").Bool()
|
||||
@@ -142,9 +131,6 @@ var (
|
||||
|
||||
dockerScan = cli.Command("docker", "Scan Docker Image")
|
||||
dockerScanImages = dockerScan.Flag("image", "Docker image to scan. Use the file:// prefix to point to a local tarball, otherwise a image registry is assumed.").Required().Strings()
|
||||
|
||||
travisCiScan = cli.Command("travisci", "Scan TravisCI")
|
||||
travisCiScanToken = travisCiScan.Flag("token", "TravisCI token. Can also be provided with environment variable").Envar("TRAVISCI_TOKEN").Required().String()
|
||||
)
|
||||
|
||||
func init() {
|
||||
@@ -157,21 +143,6 @@ func init() {
|
||||
}
|
||||
|
||||
cli.Version("trufflehog " + version.BuildVersion)
|
||||
|
||||
// Support -h for help
|
||||
cli.HelpFlag.Short('h')
|
||||
|
||||
if len(os.Args) <= 1 && isatty.IsTerminal(os.Stdout.Fd()) {
|
||||
args := tui.Run()
|
||||
if len(args) == 0 {
|
||||
os.Exit(0)
|
||||
}
|
||||
|
||||
// Overwrite the Args slice so overseer works properly.
|
||||
os.Args = os.Args[:1]
|
||||
os.Args = append(os.Args, args...)
|
||||
}
|
||||
|
||||
cmd = kingpin.MustParse(cli.Parse(os.Args[1:]))
|
||||
|
||||
switch {
|
||||
@@ -222,36 +193,10 @@ func main() {
|
||||
}
|
||||
|
||||
func run(state overseer.State) {
|
||||
ctx, cancel := context.WithCancelCause(context.Background())
|
||||
defer cancel(nil)
|
||||
|
||||
go func() {
|
||||
if err := cleantemp.CleanTempArtifacts(ctx); err != nil {
|
||||
ctx.Logger().Error(err, "error cleaning temporary artifacts")
|
||||
}
|
||||
}()
|
||||
|
||||
ctx := context.Background()
|
||||
logger := ctx.Logger()
|
||||
logFatal := logFatalFunc(logger)
|
||||
|
||||
killSignal := make(chan os.Signal, 1)
|
||||
signal.Notify(killSignal, syscall.SIGINT, syscall.SIGTERM, syscall.SIGQUIT)
|
||||
go func() {
|
||||
<-killSignal
|
||||
logger.Info("Received signal, shutting down.")
|
||||
cancel(fmt.Errorf("canceling context due to signal"))
|
||||
|
||||
if err := cleantemp.CleanTempArtifacts(ctx); err != nil {
|
||||
logger.Error(err, "error cleaning temporary artifacts")
|
||||
} else {
|
||||
logger.Info("cleaned temporary artifacts")
|
||||
}
|
||||
|
||||
time.Sleep(time.Second * 10)
|
||||
logger.Info("10 seconds elapsed. Forcing shutdown.")
|
||||
os.Exit(0)
|
||||
}()
|
||||
|
||||
logger.V(2).Info(fmt.Sprintf("trufflehog %s", version.BuildVersion))
|
||||
|
||||
if *githubScanToken != "" {
|
||||
@@ -366,9 +311,8 @@ func run(state overseer.State) {
|
||||
"detector", id,
|
||||
)
|
||||
}
|
||||
if !*customVerifiersOnly || len(urls) == 0 {
|
||||
urls = append(urls, customizer.DefaultEndpoint())
|
||||
}
|
||||
// TODO: Add flag to ignore the default endpoint.
|
||||
urls = append(urls, customizer.DefaultEndpoint())
|
||||
if err := customizer.SetEndpoints(urls...); err != nil {
|
||||
logFatal(err, "failed configuring custom endpoint for detector", "detector", id)
|
||||
}
|
||||
@@ -391,16 +335,11 @@ func run(state overseer.State) {
|
||||
printer = new(output.PlainPrinter)
|
||||
}
|
||||
|
||||
if !*jsonLegacy && !*jsonOut {
|
||||
fmt.Fprintf(os.Stderr, "🐷🔑🐷 TruffleHog. Unearth your secrets. 🐷🔑🐷\n\n")
|
||||
}
|
||||
|
||||
e, err := engine.Start(ctx,
|
||||
engine.WithConcurrency(uint8(*concurrency)),
|
||||
engine.WithDecoders(decoders.DefaultDecoders()...),
|
||||
engine.WithDetectors(engine.DefaultDetectors()...),
|
||||
engine.WithDetectors(conf.Detectors...),
|
||||
engine.WithVerify(!*noVerification),
|
||||
engine.WithDetectors(!*noVerification, engine.DefaultDetectors()...),
|
||||
engine.WithDetectors(!*noVerification, conf.Detectors...),
|
||||
engine.WithFilterDetectors(includeFilter),
|
||||
engine.WithFilterDetectors(excludeFilter),
|
||||
engine.WithFilterDetectors(endpointCustomizer),
|
||||
@@ -408,23 +347,38 @@ func run(state overseer.State) {
|
||||
engine.WithOnlyVerified(*onlyVerified),
|
||||
engine.WithPrintAvgDetectorTime(*printAvgDetectorTime),
|
||||
engine.WithPrinter(printer),
|
||||
engine.WithFilterEntropy(*filterEntropy),
|
||||
)
|
||||
if err != nil {
|
||||
logFatal(err, "error initializing engine")
|
||||
}
|
||||
|
||||
var repoPath string
|
||||
var remote bool
|
||||
switch cmd {
|
||||
case gitScan.FullCommand():
|
||||
filter, err := common.FilterFromFiles(*gitScanIncludePaths, *gitScanExcludePaths)
|
||||
if err != nil {
|
||||
logFatal(err, "could not create filter")
|
||||
}
|
||||
repoPath, remote, err = git.PrepareRepoSinceCommit(ctx, *gitScanURI, *gitScanSinceCommit)
|
||||
if err != nil || repoPath == "" {
|
||||
logFatal(err, "error preparing git repo for scanning")
|
||||
}
|
||||
if remote {
|
||||
defer os.RemoveAll(repoPath)
|
||||
}
|
||||
excludedGlobs := []string{}
|
||||
if *gitScanExcludeGlobs != "" {
|
||||
excludedGlobs = strings.Split(*gitScanExcludeGlobs, ",")
|
||||
}
|
||||
|
||||
cfg := sources.GitConfig{
|
||||
URI: *gitScanURI,
|
||||
IncludePathsFile: *gitScanIncludePaths,
|
||||
ExcludePathsFile: *gitScanExcludePaths,
|
||||
HeadRef: *gitScanBranch,
|
||||
BaseRef: *gitScanSinceCommit,
|
||||
MaxDepth: *gitScanMaxDepth,
|
||||
Bare: *gitScanBare,
|
||||
ExcludeGlobs: *gitScanExcludeGlobs,
|
||||
RepoPath: repoPath,
|
||||
HeadRef: *gitScanBranch,
|
||||
BaseRef: *gitScanSinceCommit,
|
||||
MaxDepth: *gitScanMaxDepth,
|
||||
Filter: filter,
|
||||
ExcludeGlobs: excludedGlobs,
|
||||
}
|
||||
if err = e.ScanGit(ctx, cfg); err != nil {
|
||||
logFatal(err, "Failed to scan Git.")
|
||||
@@ -439,19 +393,16 @@ func run(state overseer.State) {
|
||||
}
|
||||
|
||||
cfg := sources.GithubConfig{
|
||||
Endpoint: *githubScanEndpoint,
|
||||
Token: *githubScanToken,
|
||||
IncludeForks: *githubIncludeForks,
|
||||
IncludeMembers: *githubIncludeMembers,
|
||||
Concurrency: *concurrency,
|
||||
ExcludeRepos: *githubExcludeRepos,
|
||||
IncludeRepos: *githubIncludeRepos,
|
||||
Repos: *githubScanRepos,
|
||||
Orgs: *githubScanOrgs,
|
||||
IncludeIssueComments: *githubScanIssueComments,
|
||||
IncludePullRequestComments: *githubScanPRComments,
|
||||
IncludeGistComments: *githubScanGistComments,
|
||||
Filter: filter,
|
||||
Endpoint: *githubScanEndpoint,
|
||||
Token: *githubScanToken,
|
||||
IncludeForks: *githubIncludeForks,
|
||||
IncludeMembers: *githubIncludeMembers,
|
||||
Concurrency: *concurrency,
|
||||
ExcludeRepos: *githubExcludeRepos,
|
||||
IncludeRepos: *githubIncludeRepos,
|
||||
Repos: *githubScanRepos,
|
||||
Orgs: *githubScanOrgs,
|
||||
Filter: filter,
|
||||
}
|
||||
if err := e.ScanGitHub(ctx, cfg); err != nil {
|
||||
logFatal(err, "Failed to scan Github.")
|
||||
@@ -472,6 +423,10 @@ func run(state overseer.State) {
|
||||
logFatal(err, "Failed to scan GitLab.")
|
||||
}
|
||||
case filesystemScan.FullCommand():
|
||||
filter, err := common.FilterFromFiles(*filesystemScanIncludePaths, *filesystemScanExcludePaths)
|
||||
if err != nil {
|
||||
logFatal(err, "could not create filter")
|
||||
}
|
||||
if len(*filesystemDirectories) > 0 {
|
||||
ctx.Logger().Info("--directory flag is deprecated, please pass directories as arguments")
|
||||
}
|
||||
@@ -479,9 +434,8 @@ func run(state overseer.State) {
|
||||
paths = append(paths, *filesystemPaths...)
|
||||
paths = append(paths, *filesystemDirectories...)
|
||||
cfg := sources.FilesystemConfig{
|
||||
Paths: paths,
|
||||
IncludePathsFile: *filesystemScanIncludePaths,
|
||||
ExcludePathsFile: *filesystemScanExcludePaths,
|
||||
Paths: paths,
|
||||
Filter: filter,
|
||||
}
|
||||
if err = e.ScanFileSystem(ctx, cfg); err != nil {
|
||||
logFatal(err, "Failed to scan filesystem")
|
||||
@@ -492,7 +446,6 @@ func run(state overseer.State) {
|
||||
Secret: *s3ScanSecret,
|
||||
SessionToken: *s3ScanSessionToken,
|
||||
Buckets: *s3ScanBuckets,
|
||||
Roles: *s3ScanRoleArns,
|
||||
CloudCred: *s3ScanCloudEnv,
|
||||
MaxObjectSize: int64(*s3ScanMaxObjectSize),
|
||||
}
|
||||
@@ -515,10 +468,6 @@ func run(state overseer.State) {
|
||||
if err := e.ScanCircleCI(ctx, *circleCiScanToken); err != nil {
|
||||
logFatal(err, "Failed to scan CircleCI.")
|
||||
}
|
||||
case travisCiScan.FullCommand():
|
||||
if err := e.ScanTravisCI(ctx, *travisCiScanToken); err != nil {
|
||||
logFatal(err, "Failed to scan TravisCI.")
|
||||
}
|
||||
case gcsScan.FullCommand():
|
||||
cfg := sources.GCSConfig{
|
||||
ProjectID: *gcsProjectID,
|
||||
@@ -552,6 +501,10 @@ func run(state overseer.State) {
|
||||
}
|
||||
}
|
||||
|
||||
if !*jsonLegacy && !*jsonOut {
|
||||
fmt.Fprintf(os.Stderr, "🐷🔑🐷 TruffleHog. Unearth your secrets. 🐷🔑🐷\n\n")
|
||||
}
|
||||
|
||||
// Wait for all workers to finish.
|
||||
if err = e.Finish(ctx); err != nil {
|
||||
logFatal(err, "engine failed to finish execution")
|
||||
@@ -564,7 +517,6 @@ func run(state overseer.State) {
|
||||
"bytes", metrics.BytesScanned,
|
||||
"verified_secrets", metrics.VerifiedSecretsFound,
|
||||
"unverified_secrets", metrics.UnverifiedSecretsFound,
|
||||
"scan_duration", metrics.ScanDuration.String(),
|
||||
)
|
||||
|
||||
if *printAvgDetectorTime {
|
||||
@@ -613,11 +565,11 @@ func printAverageDetectorTime(e *engine.Engine) {
|
||||
|
||||
// detectorTypeToSet is a helper function to convert a slice of detector IDs into a set.
|
||||
func detectorTypeToSet(detectors []config.DetectorID) map[config.DetectorID]struct{} {
|
||||
out := make(map[config.DetectorID]struct{}, len(detectors))
|
||||
output := make(map[config.DetectorID]struct{}, len(detectors))
|
||||
for _, d := range detectors {
|
||||
out[d] = struct{}{}
|
||||
output[d] = struct{}{}
|
||||
}
|
||||
return out
|
||||
return output
|
||||
}
|
||||
|
||||
// getWithDetectorID is a helper function to get a value from a map using a
|
||||
|
||||
Vendored
+3
-3
@@ -4,9 +4,9 @@ package cache
|
||||
// Cache is used to store key/value pairs.
|
||||
type Cache interface {
|
||||
// Set stores the given key/value pair.
|
||||
Set(string, any)
|
||||
Set(string, string)
|
||||
// Get returns the value for the given key and a boolean indicating if the key was found.
|
||||
Get(string) (any, bool)
|
||||
Get(string) (string, bool)
|
||||
// Exists returns true if the given key exists in the cache.
|
||||
Exists(string) bool
|
||||
// Delete the given key from the cache.
|
||||
@@ -18,7 +18,7 @@ type Cache interface {
|
||||
// Keys returns all keys in the cache.
|
||||
Keys() []string
|
||||
// Values returns all values in the cache.
|
||||
Values() []any
|
||||
Values() []string
|
||||
// Contents returns all keys in the cache encoded as a string.
|
||||
Contents() string
|
||||
}
|
||||
|
||||
Vendored
+27
-52
@@ -5,77 +5,52 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/patrickmn/go-cache"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/context"
|
||||
)
|
||||
|
||||
const (
|
||||
defaultExpirationInterval = 12 * time.Hour
|
||||
defaultPurgeInterval = 13 * time.Hour
|
||||
defaultExpiration = cache.DefaultExpiration
|
||||
expirationInterval = 12 * time.Hour
|
||||
purgeInterval = 13 * time.Hour
|
||||
defaultExpiration = cache.DefaultExpiration
|
||||
)
|
||||
|
||||
// Cache wraps the go-cache library to provide an in-memory key-value store.
|
||||
// Cache is a wrapper around the go-cache library.
|
||||
type Cache struct {
|
||||
c *cache.Cache
|
||||
expiration time.Duration
|
||||
purgeInterval time.Duration
|
||||
c *cache.Cache
|
||||
}
|
||||
|
||||
// CacheOption defines a function type used for configuring a Cache.
|
||||
type CacheOption func(*Cache)
|
||||
|
||||
// WithExpirationInterval returns a CacheOption to set the expiration interval of cache items.
|
||||
// The interval determines the duration a cached item remains in the cache before it is expired.
|
||||
func WithExpirationInterval(interval time.Duration) CacheOption {
|
||||
return func(c *Cache) { c.expiration = interval }
|
||||
}
|
||||
|
||||
// WithPurgeInterval returns a CacheOption to set the interval at which the cache purges expired items.
|
||||
// Regular purging helps in freeing up memory by removing stale entries.
|
||||
func WithPurgeInterval(interval time.Duration) CacheOption {
|
||||
return func(c *Cache) { c.purgeInterval = interval }
|
||||
}
|
||||
|
||||
// New constructs a new in-memory cache instance with optional configurations.
|
||||
// By default, it sets the expiration and purge intervals to 12 and 13 hours, respectively.
|
||||
// These defaults can be overridden using the functional options: WithExpirationInterval and WithPurgeInterval.
|
||||
func New(opts ...CacheOption) *Cache {
|
||||
return NewWithData(nil, opts...)
|
||||
}
|
||||
|
||||
// CacheEntry represents a single entry in the cache, consisting of a key and its corresponding value.
|
||||
type CacheEntry struct {
|
||||
// Key is the unique identifier for the entry.
|
||||
Key string
|
||||
// Value is the data stored in the entry.
|
||||
Value any
|
||||
// New constructs a new in-memory cache.
|
||||
func New() *Cache {
|
||||
c := cache.New(expirationInterval, purgeInterval)
|
||||
return &Cache{c: c}
|
||||
}
|
||||
|
||||
// NewWithData constructs a new in-memory cache with existing data.
|
||||
// It also accepts CacheOption parameters to override default configuration values.
|
||||
func NewWithData(data []CacheEntry, opts ...CacheOption) *Cache {
|
||||
instance := &Cache{expiration: defaultExpirationInterval, purgeInterval: defaultPurgeInterval}
|
||||
for _, opt := range opts {
|
||||
opt(instance)
|
||||
}
|
||||
func NewWithData(ctx context.Context, data []string) *Cache {
|
||||
ctx.Logger().V(3).Info("Loading cache", "num-items", len(data))
|
||||
|
||||
// Convert data slice to map required by go-cache.
|
||||
items := make(map[string]cache.Item, len(data))
|
||||
for _, d := range data {
|
||||
items[d.Key] = cache.Item{Object: d.Value, Expiration: int64(defaultExpiration)}
|
||||
items[d] = cache.Item{Object: d, Expiration: int64(defaultExpiration)}
|
||||
}
|
||||
|
||||
instance.c = cache.NewFrom(instance.expiration, instance.purgeInterval, items)
|
||||
return instance
|
||||
c := cache.NewFrom(expirationInterval, purgeInterval, items)
|
||||
return &Cache{c: c}
|
||||
}
|
||||
|
||||
// Set adds a key-value pair to the cache.
|
||||
func (c *Cache) Set(key string, value any) {
|
||||
func (c *Cache) Set(key, value string) {
|
||||
c.c.Set(key, value, defaultExpiration)
|
||||
}
|
||||
|
||||
// Get returns the value for the given key.
|
||||
func (c *Cache) Get(key string) (any, bool) {
|
||||
return c.c.Get(key)
|
||||
func (c *Cache) Get(key string) (string, bool) {
|
||||
res, ok := c.c.Get(key)
|
||||
if !ok {
|
||||
return "", ok
|
||||
}
|
||||
return res.(string), ok
|
||||
}
|
||||
|
||||
// Exists returns true if the given key exists in the cache.
|
||||
@@ -110,16 +85,16 @@ func (c *Cache) Keys() []string {
|
||||
}
|
||||
|
||||
// Values returns all values in the cache.
|
||||
func (c *Cache) Values() []any {
|
||||
func (c *Cache) Values() []string {
|
||||
items := c.c.Items()
|
||||
res := make([]any, 0, len(items))
|
||||
res := make([]string, 0, len(items))
|
||||
for _, v := range items {
|
||||
res = append(res, v.Object)
|
||||
res = append(res, v.Object.(string))
|
||||
}
|
||||
return res
|
||||
}
|
||||
|
||||
// Contents returns a comma-separated string containing all keys in the cache.
|
||||
// Contents returns all key-value pairs in the cache encodes as a string.
|
||||
func (c *Cache) Contents() string {
|
||||
items := c.c.Items()
|
||||
res := make([]string, 0, len(items))
|
||||
|
||||
Vendored
+6
-8
@@ -7,6 +7,8 @@ import (
|
||||
"testing"
|
||||
|
||||
"github.com/google/go-cmp/cmp"
|
||||
|
||||
logContext "github.com/trufflesecurity/trufflehog/v3/pkg/context"
|
||||
)
|
||||
|
||||
func TestCache(t *testing.T) {
|
||||
@@ -32,7 +34,7 @@ func TestCache(t *testing.T) {
|
||||
// Test delete.
|
||||
c.Delete("key1")
|
||||
v, ok = c.Get("key1")
|
||||
if ok || v != nil {
|
||||
if ok || v != "" {
|
||||
t.Fatalf("Unexpected value for key1 after delete: %v, %v", v, ok)
|
||||
}
|
||||
|
||||
@@ -40,7 +42,7 @@ func TestCache(t *testing.T) {
|
||||
c.Set("key10", "key10")
|
||||
c.Clear()
|
||||
v, ok = c.Get("key10")
|
||||
if ok || v != nil {
|
||||
if ok || v != "" {
|
||||
t.Fatalf("Unexpected value for key10 after clear: %v, %v", v, ok)
|
||||
}
|
||||
|
||||
@@ -58,10 +60,7 @@ func TestCache(t *testing.T) {
|
||||
}
|
||||
|
||||
// Test getting only the values.
|
||||
vals := make([]string, 0, c.Count())
|
||||
for _, v := range c.Values() {
|
||||
vals = append(vals, v.(string))
|
||||
}
|
||||
vals := c.Values()
|
||||
sort.Strings(vals)
|
||||
sort.Strings(values)
|
||||
if !cmp.Equal(values, vals) {
|
||||
@@ -83,8 +82,7 @@ func TestCache(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestCache_NewWithData(t *testing.T) {
|
||||
data := []CacheEntry{{"key1", "value1"}, {"key2", "value2"}, {"key3", "value3"}}
|
||||
c := NewWithData(data)
|
||||
c := NewWithData(logContext.Background(), []string{"key1", "key2", "key3"})
|
||||
|
||||
// Test the count.
|
||||
if c.Count() != 3 {
|
||||
|
||||
@@ -1,119 +0,0 @@
|
||||
package cleantemp
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/mitchellh/go-ps"
|
||||
|
||||
logContext "github.com/trufflesecurity/trufflehog/v3/pkg/context"
|
||||
)
|
||||
|
||||
const (
|
||||
defaultExecPath = "trufflehog"
|
||||
defaultArtifactPrefixFormat = "%s-%d-"
|
||||
)
|
||||
|
||||
// MkdirTemp returns a temporary directory path formatted as:
|
||||
// trufflehog-<pid>-<randint>
|
||||
func MkdirTemp() (string, error) {
|
||||
pid := os.Getpid()
|
||||
tmpdir := fmt.Sprintf(defaultArtifactPrefixFormat, defaultExecPath, pid)
|
||||
dir, err := os.MkdirTemp(os.TempDir(), tmpdir)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return dir, nil
|
||||
}
|
||||
|
||||
// Unlike MkdirTemp, we only want to generate the filename string.
|
||||
// The tempfile creation in trufflehog we're interested in
|
||||
// is generally handled by "github.com/trufflesecurity/disk-buffer-reader"
|
||||
func MkFilename() string {
|
||||
pid := os.Getpid()
|
||||
filename := fmt.Sprintf(defaultArtifactPrefixFormat, defaultExecPath, pid)
|
||||
return filename
|
||||
}
|
||||
|
||||
// Only compile during startup.
|
||||
var trufflehogRE = regexp.MustCompile(`^trufflehog-\d+-\d+$`)
|
||||
|
||||
// CleanTempArtifacts deletes orphaned temp directories and files that do not contain running PID values.
|
||||
func CleanTempArtifacts(ctx logContext.Context) error {
|
||||
executablePath, err := os.Executable()
|
||||
if err != nil {
|
||||
executablePath = defaultExecPath
|
||||
}
|
||||
execName := filepath.Base(executablePath)
|
||||
|
||||
var pids []string
|
||||
procs, err := ps.Processes()
|
||||
if err != nil {
|
||||
return fmt.Errorf("error getting jobs PIDs: %w", err)
|
||||
}
|
||||
|
||||
for _, proc := range procs {
|
||||
if proc.Executable() == execName {
|
||||
pids = append(pids, strconv.Itoa(proc.Pid()))
|
||||
}
|
||||
}
|
||||
|
||||
if len(pids) == 0 {
|
||||
ctx.Logger().V(5).Info("No trufflehog processes were found")
|
||||
return nil
|
||||
}
|
||||
|
||||
tempDir := os.TempDir()
|
||||
dir, err := os.Open(tempDir)
|
||||
if err != nil {
|
||||
return fmt.Errorf("error opening temp dir: %w", err)
|
||||
}
|
||||
defer dir.Close()
|
||||
|
||||
for {
|
||||
entries, err := dir.ReadDir(1) // read only one entry
|
||||
if err != nil {
|
||||
if err == io.EOF {
|
||||
break
|
||||
}
|
||||
continue
|
||||
}
|
||||
entry := entries[0]
|
||||
|
||||
if trufflehogRE.MatchString(entry.Name()) {
|
||||
|
||||
// Mark these artifacts initially as ones that should be deleted.
|
||||
shouldDelete := true
|
||||
// Check if the name matches any live PIDs.
|
||||
// Potential race condition here if a PID is started and creates tmp data after the initial check.
|
||||
for _, pidval := range pids {
|
||||
if strings.Contains(entry.Name(), fmt.Sprintf("-%s-", pidval)) {
|
||||
shouldDelete = false
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
if shouldDelete {
|
||||
path := filepath.Join(tempDir, entry.Name())
|
||||
isDir := entry.IsDir()
|
||||
if isDir {
|
||||
err = os.RemoveAll(path)
|
||||
} else {
|
||||
err = os.Remove(path)
|
||||
}
|
||||
if err != nil {
|
||||
return fmt.Errorf("error deleting temp artifact (dir: %v) %s: %w", isDir, path, err)
|
||||
}
|
||||
|
||||
ctx.Logger().V(4).Info("Deleted orphaned temp artifact", "artifact", path)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
@@ -1,30 +0,0 @@
|
||||
package cleantemp
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"github.com/mitchellh/go-ps"
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
func TestExecName(t *testing.T) {
|
||||
executablePath, err := os.Executable()
|
||||
assert.Nil(t, err)
|
||||
execName := filepath.Base(executablePath)
|
||||
assert.Equal(t, "cleantemp.test", execName)
|
||||
|
||||
procs, err := ps.Processes()
|
||||
assert.Nil(t, err)
|
||||
assert.NotEmpty(t, procs)
|
||||
|
||||
found := false
|
||||
for _, proc := range procs {
|
||||
if proc.Executable() == execName {
|
||||
found = true
|
||||
}
|
||||
}
|
||||
|
||||
assert.True(t, found)
|
||||
}
|
||||
@@ -16,14 +16,9 @@ func IsDone(ctx context.Context) bool {
|
||||
// write would succeed, either operation will be performed randomly.
|
||||
func CancellableWrite[T any](ctx context.Context, ch chan<- T, item T) error {
|
||||
select {
|
||||
case <-ctx.Done(): // priority to context cancellation
|
||||
case ch <- item:
|
||||
return nil
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
default:
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
case ch <- item:
|
||||
return nil
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,120 +0,0 @@
|
||||
package glob
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"github.com/gobwas/glob"
|
||||
)
|
||||
|
||||
// Filter is a generic filter for excluding and including globs (limited
|
||||
// regular expressions). Exclusion takes precedence if both include and exclude
|
||||
// lists are provided.
|
||||
type Filter struct {
|
||||
exclude []glob.Glob
|
||||
include []glob.Glob
|
||||
}
|
||||
|
||||
type globFilterOpt func(*Filter) error
|
||||
|
||||
// WithExcludeGlobs adds exclude globs to the filter.
|
||||
func WithExcludeGlobs(excludes ...string) globFilterOpt {
|
||||
return func(f *Filter) error {
|
||||
for _, exclude := range excludes {
|
||||
g, err := glob.Compile(exclude)
|
||||
if err != nil {
|
||||
return fmt.Errorf("invalid exclude glob %q: %w", exclude, err)
|
||||
}
|
||||
f.exclude = append(f.exclude, g)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
// WithIncludeGlobs adds include globs to the filter.
|
||||
func WithIncludeGlobs(includes ...string) globFilterOpt {
|
||||
return func(f *Filter) error {
|
||||
for _, include := range includes {
|
||||
g, err := glob.Compile(include)
|
||||
if err != nil {
|
||||
return fmt.Errorf("invalid include glob %q: %w", include, err)
|
||||
}
|
||||
f.include = append(f.include, g)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
// NewGlobFilter creates a new Filter with the provided options.
|
||||
func NewGlobFilter(opts ...globFilterOpt) (*Filter, error) {
|
||||
filter := &Filter{}
|
||||
for _, opt := range opts {
|
||||
if err := opt(filter); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
return filter, nil
|
||||
}
|
||||
|
||||
// ShouldInclude returns whether the object is in the include list or not in
|
||||
// the exclude list (exclude taking precedence).
|
||||
func (f *Filter) ShouldInclude(object string) bool {
|
||||
if f == nil {
|
||||
return true
|
||||
}
|
||||
exclude, include := len(f.exclude), len(f.include)
|
||||
if exclude == 0 && include == 0 {
|
||||
return true
|
||||
} else if exclude > 0 && include == 0 {
|
||||
return f.shouldIncludeFromExclude(object)
|
||||
} else if exclude == 0 && include > 0 {
|
||||
return f.shouldIncludeFromInclude(object)
|
||||
} else {
|
||||
if ok, err := f.shouldIncludeFromBoth(object); err == nil {
|
||||
return ok
|
||||
}
|
||||
// Ambiguous case.
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
// shouldIncludeFromExclude checks for explicitly excluded paths. This should
|
||||
// only be called when the include list is empty.
|
||||
func (f *Filter) shouldIncludeFromExclude(object string) bool {
|
||||
for _, g := range f.exclude {
|
||||
if g.Match(object) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// shouldIncludeFromInclude checks for explicitly included paths. This should
|
||||
// only be called when the exclude list is empty.
|
||||
func (f *Filter) shouldIncludeFromInclude(object string) bool {
|
||||
for _, g := range f.include {
|
||||
if g.Match(object) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// shouldIncludeFromBoth checks for either excluded or included paths. Exclusion
|
||||
// takes precedence. If neither list contains the object, true is returned.
|
||||
func (f *Filter) shouldIncludeFromBoth(object string) (bool, error) {
|
||||
// Exclude takes precedence. If we find the object in the exclude list,
|
||||
// we should not match.
|
||||
for _, g := range f.exclude {
|
||||
if g.Match(object) {
|
||||
return false, nil
|
||||
}
|
||||
}
|
||||
// If we find the object in the include list, we should match.
|
||||
for _, g := range f.include {
|
||||
if g.Match(object) {
|
||||
return true, nil
|
||||
}
|
||||
}
|
||||
// If we find it in neither, return an error to let the caller decide.
|
||||
return false, fmt.Errorf("ambiguous match")
|
||||
}
|
||||
@@ -1,138 +0,0 @@
|
||||
package glob
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"pgregory.net/rapid"
|
||||
)
|
||||
|
||||
type globTest struct {
|
||||
input string
|
||||
shouldInclude bool
|
||||
}
|
||||
|
||||
func testGlobs(t *testing.T, filter *Filter, tests ...globTest) {
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.input, func(t *testing.T) {
|
||||
// Invert because mentally it's easier to say whether an
|
||||
// input should be included.
|
||||
assert.Equal(t, tt.shouldInclude, filter.ShouldInclude(tt.input))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestGlobFilterExclude(t *testing.T) {
|
||||
filter, err := NewGlobFilter(WithExcludeGlobs("foo", "bar*"))
|
||||
assert.NoError(t, err)
|
||||
|
||||
testGlobs(t, filter,
|
||||
globTest{"foo", false},
|
||||
globTest{"bar", false},
|
||||
globTest{"bara", false},
|
||||
globTest{"barb", false},
|
||||
globTest{"barbosa", false},
|
||||
globTest{"foobar", true},
|
||||
globTest{"food", true},
|
||||
globTest{"anything else", true},
|
||||
)
|
||||
}
|
||||
|
||||
func TestGlobFilterInclude(t *testing.T) {
|
||||
filter, err := NewGlobFilter(WithIncludeGlobs("foo", "bar*"))
|
||||
assert.NoError(t, err)
|
||||
|
||||
testGlobs(t, filter,
|
||||
globTest{"foo", true},
|
||||
globTest{"bar", true},
|
||||
globTest{"bara", true},
|
||||
globTest{"barb", true},
|
||||
globTest{"barbosa", true},
|
||||
globTest{"foobar", false},
|
||||
globTest{"food", false},
|
||||
globTest{"anything else", false},
|
||||
)
|
||||
}
|
||||
|
||||
func TestGlobFilterEmpty(t *testing.T) {
|
||||
filter, err := NewGlobFilter()
|
||||
assert.NoError(t, err)
|
||||
|
||||
testGlobs(t, filter,
|
||||
globTest{"foo", true},
|
||||
globTest{"bar", true},
|
||||
globTest{"bara", true},
|
||||
globTest{"barb", true},
|
||||
globTest{"barbosa", true},
|
||||
globTest{"foobar", true},
|
||||
globTest{"food", true},
|
||||
globTest{"anything else", true},
|
||||
)
|
||||
}
|
||||
|
||||
func TestGlobFilterExcludeInclude(t *testing.T) {
|
||||
filter, err := NewGlobFilter(WithExcludeGlobs("/foo/bar/**"), WithIncludeGlobs("/foo/**"))
|
||||
assert.NoError(t, err)
|
||||
|
||||
testGlobs(t, filter,
|
||||
globTest{"/foo/a", true},
|
||||
globTest{"/foo/b", true},
|
||||
globTest{"/foo/c/d/e", true},
|
||||
globTest{"/foo/bar/a", false},
|
||||
globTest{"/foo/bar/b", false},
|
||||
globTest{"/foo/bar/c/d/e", false},
|
||||
globTest{"/any/other/path", false},
|
||||
)
|
||||
}
|
||||
|
||||
func TestGlobFilterExcludePrecedence(t *testing.T) {
|
||||
filter, err := NewGlobFilter(WithExcludeGlobs("foo"), WithIncludeGlobs("foo*"))
|
||||
assert.NoError(t, err)
|
||||
|
||||
testGlobs(t, filter,
|
||||
globTest{"foo", false},
|
||||
globTest{"foobar", true},
|
||||
)
|
||||
}
|
||||
|
||||
func TestGlobErrorContainsGlob(t *testing.T) {
|
||||
invalidGlob := "[this is invalid because it doesn't close the capture group"
|
||||
_, err := NewGlobFilter(WithExcludeGlobs(invalidGlob))
|
||||
assert.Error(t, err)
|
||||
assert.Contains(t, err.Error(), invalidGlob)
|
||||
}
|
||||
|
||||
// The filters in this test should be mutually exclusive because one includes
|
||||
// and the other excludes the same glob.
|
||||
func TestGlobInverse(t *testing.T) {
|
||||
for _, glob := range []string{
|
||||
"a",
|
||||
"a*",
|
||||
"a**",
|
||||
"*a",
|
||||
"**a",
|
||||
"*",
|
||||
} {
|
||||
include, err := NewGlobFilter(WithIncludeGlobs(glob))
|
||||
assert.NoError(t, err)
|
||||
exclude, err := NewGlobFilter(WithExcludeGlobs(glob))
|
||||
assert.NoError(t, err)
|
||||
rapid.Check(t, func(t *rapid.T) {
|
||||
input := rapid.String().Draw(t, "input")
|
||||
a, b := include.ShouldInclude(input), exclude.ShouldInclude(input)
|
||||
if a == b {
|
||||
t.Fatalf("Filter(Include(%q)) == Filter(Exclude(%q)) == %v for input %q", glob, glob, a, input)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestGlobDefaultFilters(t *testing.T) {
|
||||
for _, filter := range []*Filter{nil, {}} {
|
||||
rapid.Check(t, func(t *rapid.T) {
|
||||
if !filter.ShouldInclude(rapid.String().Draw(t, "input")) {
|
||||
t.Fatalf("filter %#v did not include input", filter)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
+13
-27
@@ -2,10 +2,8 @@ package common
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
"crypto/rand"
|
||||
"encoding/base32"
|
||||
"encoding/binary"
|
||||
"fmt"
|
||||
"io"
|
||||
"math/big"
|
||||
"strings"
|
||||
@@ -29,6 +27,18 @@ func RemoveStringSliceItem(item string, slice *[]string) {
|
||||
}
|
||||
}
|
||||
|
||||
func MinInt(a, b int) int {
|
||||
if a < b {
|
||||
return a
|
||||
}
|
||||
return b
|
||||
}
|
||||
|
||||
func BytesEqual(a, b []byte, numBytes int) bool {
|
||||
limit := MinInt(numBytes, MinInt(len(a), len(b))-1)
|
||||
return bytes.Equal(a[:limit], b[:limit])
|
||||
}
|
||||
|
||||
func ResponseContainsSubstring(reader io.ReadCloser, target string) (bool, error) {
|
||||
scanner := bufio.NewScanner(reader)
|
||||
for scanner.Scan() {
|
||||
@@ -54,27 +64,3 @@ func RandomID(length int) string {
|
||||
|
||||
return string(b)
|
||||
}
|
||||
|
||||
func GetAccountNumFromAWSID(AWSID string) (string, error) {
|
||||
// Function to get the account number from an AWS ID (no verification required)
|
||||
// Source: https://medium.com/@TalBeerySec/a-short-note-on-aws-key-id-f88cc4317489
|
||||
if len(AWSID) < 4 {
|
||||
return "", fmt.Errorf("AWSID is too short")
|
||||
}
|
||||
trimmedAWSID := AWSID[4:]
|
||||
decodedBytes, err := base32.StdEncoding.WithPadding(base32.NoPadding).DecodeString(strings.ToUpper(trimmedAWSID))
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
if len(decodedBytes) < 6 {
|
||||
return "", fmt.Errorf("Decoded AWSID is too short")
|
||||
}
|
||||
|
||||
data := make([]byte, 8)
|
||||
copy(data[2:], decodedBytes[0:6])
|
||||
z := binary.BigEndian.Uint64(data)
|
||||
const mask uint64 = 0x7fffffffff80
|
||||
accountNum := (z & mask) >> 7
|
||||
return fmt.Sprintf("%012d", accountNum), nil
|
||||
}
|
||||
|
||||
+1
-83
@@ -7,83 +7,7 @@ import (
|
||||
|
||||
var (
|
||||
KB, MB, GB, TB, PB = 1e3, 1e6, 1e9, 1e12, 1e15
|
||||
IgnoredExtensions = []string{
|
||||
// multimedia/containers
|
||||
"mp4",
|
||||
"avi",
|
||||
"mpeg",
|
||||
"mpg",
|
||||
"mov",
|
||||
"wmv",
|
||||
"m4p",
|
||||
"swf",
|
||||
"mp2",
|
||||
"flv",
|
||||
"vob",
|
||||
"webm",
|
||||
"hdv",
|
||||
"3gp",
|
||||
"ogg",
|
||||
"mp3",
|
||||
"wav",
|
||||
"flac",
|
||||
"webp",
|
||||
"pdf",
|
||||
|
||||
// images
|
||||
"png",
|
||||
"jpg",
|
||||
"jpeg",
|
||||
"gif",
|
||||
"tiff",
|
||||
|
||||
"fnt", // Windows font file
|
||||
"fon", // Generic font file
|
||||
"ttf", // TrueType font
|
||||
"otf", // OpenType font
|
||||
"woff", // Web Open Font Format
|
||||
"woff2", // Web Open Font Format 2
|
||||
"eot", // Embedded OpenType font
|
||||
"svgz", // Compressed Scalable Vector Graphics file
|
||||
"icns", // Apple icon image file
|
||||
"ico", // Icon file
|
||||
}
|
||||
|
||||
binaryExtensions = map[string]struct{}{
|
||||
// binaries
|
||||
// These can theoretically contain secrets, but need decoding for users to make sense of them, and we don't have
|
||||
// any such decoders right now.
|
||||
"class": {}, // Java bytecode class file
|
||||
"dll": {}, // Dynamic Link Library, Windows
|
||||
"jdo": {}, // Java Data Object, Java serialization format
|
||||
"jks": {}, // Java Key Store, Java keystore format
|
||||
"ser": {}, // Java serialization format
|
||||
"idx": {}, // Index file, often binary
|
||||
"hprof": {}, // Java heap dump format
|
||||
"exe": {}, // Executable, Windows
|
||||
"bin": {}, // Binary, often used for compiled source code
|
||||
"so": {}, // Shared object, Unix/Linux
|
||||
"o": {}, // Object file from compilation/ intermediate object file
|
||||
"a": {}, // Static library, Unix/Linux
|
||||
"dylib": {}, // Dynamic library, macOS
|
||||
"lib": {}, // Library, Unix/Linux
|
||||
"obj": {}, // Object file, typically from compiled source code
|
||||
"pdb": {}, // Program Database, Microsoft Visual Studio debugging format
|
||||
"dat": {}, // Generic data file, often binary but not always
|
||||
"elf": {}, // Executable and Linkable Format, common in Unix/Linux
|
||||
"dmg": {}, // Disk Image for macOS
|
||||
"iso": {}, // ISO image (optical disk image)
|
||||
"img": {}, // Disk image files
|
||||
"out": {}, // Common output file from compiled executable in Unix/Linux
|
||||
"com": {}, // DOS command file, executable
|
||||
"sys": {}, // Windows system file, often a driver
|
||||
"vxd": {}, // Virtual device driver in Windows
|
||||
"sfx": {}, // Self-extracting archive
|
||||
"bundle": {}, // Mac OS X application bundle
|
||||
"pyo": {}, // Compiled Python file
|
||||
"pyc": {}, // Compiled Python file
|
||||
"sym": {}, // Symbolic link, Unix/Linux
|
||||
}
|
||||
IgnoredExtensions = []string{"mp4", "avi", "mpeg", "mpg", "mov", "wmv", "m4p", "swf", "mp2", "flv", "vob", "webm", "hdv", "3gp", "ogg", "mp3", "wav", "flac", "webp"}
|
||||
)
|
||||
|
||||
func SkipFile(filename string) bool {
|
||||
@@ -94,9 +18,3 @@ func SkipFile(filename string) bool {
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// IsBinary returns true if the file extension is in the binaryExtensions list.
|
||||
func IsBinary(filename string) bool {
|
||||
_, ok := binaryExtensions[strings.ToLower(strings.TrimPrefix(filepath.Ext(filename), "."))]
|
||||
return ok
|
||||
}
|
||||
|
||||
@@ -34,36 +34,3 @@ func BenchmarkSkipFile(b *testing.B) {
|
||||
SkipFile("test.mp4")
|
||||
}
|
||||
}
|
||||
|
||||
func TestIsBinary(t *testing.T) {
|
||||
type testCase struct {
|
||||
file string
|
||||
want bool
|
||||
}
|
||||
|
||||
// Add a test case for each binary extension.
|
||||
testCases := make([]testCase, 0, len(binaryExtensions)+1)
|
||||
for ext := range binaryExtensions {
|
||||
testCases = append(testCases, testCase{
|
||||
file: "test." + ext,
|
||||
want: true,
|
||||
})
|
||||
}
|
||||
|
||||
// Add a test case for a file that should not be skipped.
|
||||
testCases = append(testCases, testCase{file: "test.txt", want: false})
|
||||
|
||||
for _, tt := range testCases {
|
||||
t.Run(tt.file, func(t *testing.T) {
|
||||
if got := IsBinary(tt.file); got != tt.want {
|
||||
t.Errorf("IsBinary(%v) got %v, want %v", tt.file, got, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkIsBinary(b *testing.B) {
|
||||
for i := 0; i < b.N; i++ {
|
||||
IsBinary("test.exe")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -31,15 +31,15 @@ func NewYAML(input []byte) (*Config, error) {
|
||||
return nil, err
|
||||
}
|
||||
// Convert the structured YAML into detectors.
|
||||
var d []detectors.Detector
|
||||
var detectors []detectors.Detector
|
||||
for _, detectorConfig := range messages.Detectors {
|
||||
detector, err := custom_detectors.NewWebhookCustomRegex(detectorConfig)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
d = append(d, detector)
|
||||
detectors = append(detectors, detector)
|
||||
}
|
||||
return &Config{
|
||||
Detectors: d,
|
||||
Detectors: detectors,
|
||||
}, nil
|
||||
}
|
||||
|
||||
@@ -66,15 +66,15 @@ func ParseDetectors(input string) ([]DetectorID, error) {
|
||||
if item == "" {
|
||||
continue
|
||||
}
|
||||
allDetectors, ok := specialGroups[strings.ToLower(item)]
|
||||
detectors, ok := specialGroups[strings.ToLower(item)]
|
||||
if !ok {
|
||||
var err error
|
||||
allDetectors, err = asRange(item)
|
||||
detectors, err = asRange(item)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
for _, d := range allDetectors {
|
||||
for _, d := range detectors {
|
||||
if _, ok := seenDetector[d]; ok {
|
||||
continue
|
||||
}
|
||||
|
||||
+44
-44
@@ -2,7 +2,10 @@ package context
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"runtime/debug"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/go-logr/logr"
|
||||
@@ -26,16 +29,17 @@ type Context interface {
|
||||
Logger() logr.Logger
|
||||
}
|
||||
|
||||
// CancelFunc and CancelCauseFunc are type aliases to allow use as if they are
|
||||
// the same types as the standard library variants.
|
||||
// CancelFunc is a type alias to context.CancelFunc to allow use as if they are
|
||||
// the same types.
|
||||
type CancelFunc = context.CancelFunc
|
||||
type CancelCauseFunc = context.CancelCauseFunc
|
||||
|
||||
// logCtx implements Context.
|
||||
type logCtx struct {
|
||||
// Embed context.Context to get all methods for free.
|
||||
context.Context
|
||||
log logr.Logger
|
||||
log logr.Logger
|
||||
err *error
|
||||
errLock *sync.Mutex
|
||||
}
|
||||
|
||||
// Logger returns a structured logger.
|
||||
@@ -43,6 +47,13 @@ func (l logCtx) Logger() logr.Logger {
|
||||
return l.log
|
||||
}
|
||||
|
||||
func (l logCtx) Err() error {
|
||||
if l.err != nil && *l.err != nil {
|
||||
return *l.err
|
||||
}
|
||||
return l.Context.Err()
|
||||
}
|
||||
|
||||
// Background returns context.Background with a default logger.
|
||||
func Background() Context {
|
||||
return logCtx{
|
||||
@@ -66,17 +77,7 @@ func WithCancel(parent Context) (Context, context.CancelFunc) {
|
||||
log: parent.Logger(),
|
||||
Context: ctx,
|
||||
}
|
||||
return lCtx, cancel
|
||||
}
|
||||
|
||||
// WithCancelCause returns context.WithCancelCause with the log object propagated.
|
||||
func WithCancelCause(parent Context) (Context, context.CancelCauseFunc) {
|
||||
ctx, cancel := context.WithCancelCause(parent)
|
||||
lCtx := logCtx{
|
||||
log: parent.Logger(),
|
||||
Context: ctx,
|
||||
}
|
||||
return lCtx, cancel
|
||||
return captureCancelCallstack(lCtx, cancel)
|
||||
}
|
||||
|
||||
// WithDeadline returns context.WithDeadline with the log object propagated and
|
||||
@@ -87,18 +88,7 @@ func WithDeadline(parent Context, d time.Time) (Context, context.CancelFunc) {
|
||||
log: parent.Logger().WithValues("deadline", d),
|
||||
Context: ctx,
|
||||
}
|
||||
return lCtx, cancel
|
||||
}
|
||||
|
||||
// WithDeadlineCause returns context.WithDeadlineCause with the log object
|
||||
// propagated and the deadline added to the structured log values.
|
||||
func WithDeadlineCause(parent Context, d time.Time, cause error) (Context, context.CancelFunc) {
|
||||
ctx, cancel := context.WithDeadlineCause(parent, d, cause)
|
||||
lCtx := logCtx{
|
||||
log: parent.Logger().WithValues("deadline", d),
|
||||
Context: ctx,
|
||||
}
|
||||
return lCtx, cancel
|
||||
return captureCancelCallstack(lCtx, cancel)
|
||||
}
|
||||
|
||||
// WithTimeout returns context.WithTimeout with the log object propagated and
|
||||
@@ -109,23 +99,7 @@ func WithTimeout(parent Context, timeout time.Duration) (Context, context.Cancel
|
||||
log: parent.Logger().WithValues("timeout", timeout),
|
||||
Context: ctx,
|
||||
}
|
||||
return lCtx, cancel
|
||||
}
|
||||
|
||||
// WithTimeoutCause returns context.WithTimeoutCause with the log object
|
||||
// propagated and the timeout added to the structured log values.
|
||||
func WithTimeoutCause(parent Context, timeout time.Duration, cause error) (Context, context.CancelFunc) {
|
||||
ctx, cancel := context.WithTimeoutCause(parent, timeout, cause)
|
||||
lCtx := logCtx{
|
||||
log: parent.Logger().WithValues("timeout", timeout),
|
||||
Context: ctx,
|
||||
}
|
||||
return lCtx, cancel
|
||||
}
|
||||
|
||||
// Cause returns the context.Cause of the context.
|
||||
func Cause(ctx context.Context) error {
|
||||
return context.Cause(ctx)
|
||||
return captureCancelCallstack(lCtx, cancel)
|
||||
}
|
||||
|
||||
// WithValue returns context.WithValue with the log object propagated and
|
||||
@@ -176,3 +150,29 @@ func AddLogger(parent context.Context) Context {
|
||||
func SetDefaultLogger(l logr.Logger) {
|
||||
defaultLogger = l
|
||||
}
|
||||
|
||||
// captureCancelCallstack is a helper function to capture the callstack where
|
||||
// the cancel function was first called.
|
||||
func captureCancelCallstack(ctx logCtx, f context.CancelFunc) (Context, context.CancelFunc) {
|
||||
if ctx.err == nil {
|
||||
var err error
|
||||
ctx.err = &err
|
||||
ctx.errLock = &sync.Mutex{}
|
||||
}
|
||||
return ctx, func() {
|
||||
ctx.errLock.Lock()
|
||||
defer ctx.errLock.Unlock()
|
||||
// We must check Err() before calling f() since f() sets the error.
|
||||
// If there's already an error, do nothing special.
|
||||
if ctx.Err() != nil {
|
||||
f()
|
||||
return
|
||||
}
|
||||
f()
|
||||
// Set the error with the stacktrace if the err pointer is non-nil.
|
||||
*ctx.err = fmt.Errorf(
|
||||
"%w (canceled at %v\n%s)",
|
||||
ctx.Err(), time.Now(), string(debug.Stack()),
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
+44
-11
@@ -3,7 +3,6 @@ package context
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
@@ -171,17 +170,51 @@ func TestDefaultLogger(t *testing.T) {
|
||||
ctx.Logger().Info("this shouldn't panic")
|
||||
}
|
||||
|
||||
func TestRace(t *testing.T) {
|
||||
ctx, cancel := WithCancel(Background())
|
||||
go cancel()
|
||||
go func() { _ = ctx.Err() }()
|
||||
func TestErrCallstack(t *testing.T) {
|
||||
c, cancel := WithCancel(Background())
|
||||
ctx := c.(logCtx)
|
||||
cancel()
|
||||
_ = ctx.Err()
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
assert.Contains(t, ctx.Err().Error(), "TestErrCallstack")
|
||||
case <-time.After(1 * time.Second):
|
||||
assert.Fail(t, "context should be done")
|
||||
}
|
||||
}
|
||||
|
||||
func TestCause(t *testing.T) {
|
||||
ctx, cancel := WithCancelCause(Background())
|
||||
err := fmt.Errorf("oh no")
|
||||
cancel(err)
|
||||
assert.Equal(t, err, Cause(ctx))
|
||||
func TestErrCallstackTimeout(t *testing.T) {
|
||||
ctx, cancel := WithTimeout(Background(), 10*time.Millisecond)
|
||||
defer cancel()
|
||||
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
// Deadline exceeded errors will not have a callstack from the cancel
|
||||
// function.
|
||||
assert.NotContains(t, ctx.Err().Error(), "TestErrCallstackTimeout")
|
||||
case <-time.After(1 * time.Second):
|
||||
assert.Fail(t, "context should be done")
|
||||
}
|
||||
}
|
||||
|
||||
func TestErrCallstackTimeoutCancel(t *testing.T) {
|
||||
ctx, cancel := WithTimeout(Background(), 10*time.Millisecond)
|
||||
|
||||
var err error
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
err = ctx.Err()
|
||||
case <-time.After(1 * time.Second):
|
||||
assert.Fail(t, "context should be done")
|
||||
}
|
||||
|
||||
// Calling cancel after deadline exceeded should not overwrite the original
|
||||
// error.
|
||||
cancel()
|
||||
assert.Equal(t, err, ctx.Err())
|
||||
}
|
||||
|
||||
func TestRace(t *testing.T) {
|
||||
_, cancel := WithCancel(Background())
|
||||
go cancel()
|
||||
cancel()
|
||||
}
|
||||
|
||||
@@ -20,20 +20,20 @@ import (
|
||||
// for poorly defined regexps.
|
||||
const maxTotalMatches = 100
|
||||
|
||||
// CustomRegexWebhook is a CustomRegex with webhook validation that is
|
||||
// customRegexWebhook is a CustomRegex with webhook validation that is
|
||||
// guaranteed to be valid (assuming the data is not changed after
|
||||
// initialization).
|
||||
type CustomRegexWebhook struct {
|
||||
type customRegexWebhook struct {
|
||||
*custom_detectorspb.CustomRegex
|
||||
}
|
||||
|
||||
// Ensure the Scanner satisfies the interface at compile time.
|
||||
var _ detectors.Detector = (*CustomRegexWebhook)(nil)
|
||||
var _ detectors.Detector = (*customRegexWebhook)(nil)
|
||||
|
||||
// NewWebhookCustomRegex initializes and validates a CustomRegexWebhook. An
|
||||
// NewWebhookCustomRegex initializes and validates a customRegexWebhook. An
|
||||
// unexported type is intentionally returned here to ensure the values have
|
||||
// been validated.
|
||||
func NewWebhookCustomRegex(pb *custom_detectorspb.CustomRegex) (*CustomRegexWebhook, error) {
|
||||
func NewWebhookCustomRegex(pb *custom_detectorspb.CustomRegex) (*customRegexWebhook, error) {
|
||||
// TODO: Return all validation errors.
|
||||
if err := ValidateKeywords(pb.Keywords); err != nil {
|
||||
return nil, err
|
||||
@@ -52,12 +52,12 @@ func NewWebhookCustomRegex(pb *custom_detectorspb.CustomRegex) (*CustomRegexWebh
|
||||
}
|
||||
|
||||
// TODO: Copy only necessary data out of pb.
|
||||
return &CustomRegexWebhook{pb}, nil
|
||||
return &customRegexWebhook{pb}, nil
|
||||
}
|
||||
|
||||
var httpClient = common.SaneHttpClient()
|
||||
|
||||
func (c *CustomRegexWebhook) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
|
||||
func (c *customRegexWebhook) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
|
||||
dataStr := string(data)
|
||||
regexMatches := make(map[string][][]string, len(c.GetRegex()))
|
||||
|
||||
@@ -109,7 +109,7 @@ func (c *CustomRegexWebhook) FromData(ctx context.Context, verify bool, data []b
|
||||
return results, nil
|
||||
}
|
||||
|
||||
func (c *CustomRegexWebhook) createResults(ctx context.Context, match map[string][]string, verify bool, results chan<- detectors.Result) error {
|
||||
func (c *customRegexWebhook) createResults(ctx context.Context, match map[string][]string, verify bool, results chan<- detectors.Result) error {
|
||||
if common.IsDone(ctx) {
|
||||
// TODO: Log we're possibly leaving out results.
|
||||
return ctx.Err()
|
||||
@@ -180,7 +180,7 @@ func (c *CustomRegexWebhook) createResults(ctx context.Context, match map[string
|
||||
}
|
||||
}
|
||||
|
||||
func (c *CustomRegexWebhook) Keywords() []string {
|
||||
func (c *customRegexWebhook) Keywords() []string {
|
||||
return c.GetKeywords()
|
||||
}
|
||||
|
||||
@@ -245,6 +245,6 @@ func permutateMatches(regexMatches map[string][][]string) []map[string][]string
|
||||
return matches
|
||||
}
|
||||
|
||||
func (c *CustomRegexWebhook) Type() detectorspb.DetectorType {
|
||||
func (c *customRegexWebhook) Type() detectorspb.DetectorType {
|
||||
return detectorspb.DetectorType_CustomRegex
|
||||
}
|
||||
|
||||
@@ -105,7 +105,7 @@ func TestCustomDetectorsParsing(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestFromData_InvalidRegEx(t *testing.T) {
|
||||
c := &CustomRegexWebhook{
|
||||
c := &customRegexWebhook{
|
||||
&custom_detectorspb.CustomRegex{
|
||||
Name: "Internal bi tool",
|
||||
Keywords: []string{"secret_v1_", "pat_v2_"},
|
||||
|
||||
@@ -4,7 +4,6 @@ import (
|
||||
"bytes"
|
||||
"encoding/base64"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
|
||||
)
|
||||
|
||||
@@ -26,8 +25,7 @@ func init() {
|
||||
}
|
||||
}
|
||||
|
||||
func (d *Base64) FromChunk(chunk *sources.Chunk) *DecodableChunk {
|
||||
decodableChunk := &DecodableChunk{Chunk: chunk, DecoderType: detectorspb.DecoderType_BASE64}
|
||||
func (d *Base64) FromChunk(chunk *sources.Chunk) *sources.Chunk {
|
||||
encodedSubstrings := getSubstringsOfCharacterSet(chunk.Data, 20, b64CharsetMapping, b64EndChars)
|
||||
decodedSubstrings := make(map[string][]byte)
|
||||
|
||||
@@ -63,7 +61,7 @@ func (d *Base64) FromChunk(chunk *sources.Chunk) *DecodableChunk {
|
||||
}
|
||||
result.Write(chunk.Data[start:])
|
||||
chunk.Data = result.Bytes()
|
||||
return decodableChunk
|
||||
return chunk
|
||||
}
|
||||
|
||||
return nil
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
package decoders
|
||||
|
||||
import (
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
|
||||
)
|
||||
|
||||
@@ -14,15 +13,8 @@ func DefaultDecoders() []Decoder {
|
||||
}
|
||||
}
|
||||
|
||||
// DecodableChunk is a chunk that includes the type of decoder used.
|
||||
// This allows us to avoid a type assertion on each decoder.
|
||||
type DecodableChunk struct {
|
||||
*sources.Chunk
|
||||
DecoderType detectorspb.DecoderType
|
||||
}
|
||||
|
||||
type Decoder interface {
|
||||
FromChunk(chunk *sources.Chunk) *DecodableChunk
|
||||
FromChunk(chunk *sources.Chunk) *sources.Chunk
|
||||
}
|
||||
|
||||
// Fuzz is an entrypoint for go-fuzz, which is an AFL-style fuzzing tool.
|
||||
|
||||
@@ -5,24 +5,22 @@ import (
|
||||
"encoding/binary"
|
||||
"unicode/utf8"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
|
||||
)
|
||||
|
||||
type UTF16 struct{}
|
||||
|
||||
func (d *UTF16) FromChunk(chunk *sources.Chunk) *DecodableChunk {
|
||||
func (d *UTF16) FromChunk(chunk *sources.Chunk) *sources.Chunk {
|
||||
if chunk == nil || len(chunk.Data) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
decodableChunk := &DecodableChunk{Chunk: chunk, DecoderType: detectorspb.DecoderType_UTF16}
|
||||
if utf16Data, err := utf16ToUTF8(chunk.Data); err == nil {
|
||||
if len(utf16Data) == 0 {
|
||||
return nil
|
||||
}
|
||||
chunk.Data = utf16Data
|
||||
return decodableChunk
|
||||
return chunk
|
||||
}
|
||||
|
||||
return nil
|
||||
|
||||
@@ -4,25 +4,22 @@ import (
|
||||
"bytes"
|
||||
"unicode/utf8"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
|
||||
)
|
||||
|
||||
type UTF8 struct{}
|
||||
|
||||
func (d *UTF8) FromChunk(chunk *sources.Chunk) *DecodableChunk {
|
||||
func (d *UTF8) FromChunk(chunk *sources.Chunk) *sources.Chunk {
|
||||
if chunk == nil || len(chunk.Data) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
decodableChunk := &DecodableChunk{Chunk: chunk, DecoderType: detectorspb.DecoderType_PLAIN}
|
||||
|
||||
if !utf8.Valid(chunk.Data) {
|
||||
chunk.Data = extractSubstrings(chunk.Data)
|
||||
return decodableChunk
|
||||
return chunk
|
||||
}
|
||||
|
||||
return decodableChunk
|
||||
return chunk
|
||||
}
|
||||
|
||||
// extractSubstrings performs similarly to the strings binutil,
|
||||
|
||||
@@ -108,7 +108,6 @@ func BenchmarkFromData(benchmark *testing.B) {
|
||||
s := Scanner{}
|
||||
for name, data := range detectors.MustGetBenchmarkData() {
|
||||
benchmark.Run(name, func(b *testing.B) {
|
||||
b.ResetTimer()
|
||||
for n := 0; n < b.N; n++ {
|
||||
_, err := s.FromData(ctx, false, data)
|
||||
if err != nil {
|
||||
|
||||
@@ -108,7 +108,6 @@ func BenchmarkFromData(benchmark *testing.B) {
|
||||
s := Scanner{}
|
||||
for name, data := range detectors.MustGetBenchmarkData() {
|
||||
benchmark.Run(name, func(b *testing.B) {
|
||||
b.ResetTimer()
|
||||
for n := 0; n < b.N; n++ {
|
||||
_, err := s.FromData(ctx, false, data)
|
||||
if err != nil {
|
||||
|
||||
@@ -108,7 +108,6 @@ func BenchmarkFromData(benchmark *testing.B) {
|
||||
s := Scanner{}
|
||||
for name, data := range detectors.MustGetBenchmarkData() {
|
||||
benchmark.Run(name, func(b *testing.B) {
|
||||
b.ResetTimer()
|
||||
for n := 0; n < b.N; n++ {
|
||||
_, err := s.FromData(ctx, false, data)
|
||||
if err != nil {
|
||||
|
||||
@@ -108,7 +108,6 @@ func BenchmarkFromData(benchmark *testing.B) {
|
||||
s := Scanner{}
|
||||
for name, data := range detectors.MustGetBenchmarkData() {
|
||||
benchmark.Run(name, func(b *testing.B) {
|
||||
b.ResetTimer()
|
||||
for n := 0; n < b.N; n++ {
|
||||
_, err := s.FromData(ctx, false, data)
|
||||
if err != nil {
|
||||
|
||||
@@ -108,7 +108,6 @@ func BenchmarkFromData(benchmark *testing.B) {
|
||||
s := Scanner{}
|
||||
for name, data := range detectors.MustGetBenchmarkData() {
|
||||
benchmark.Run(name, func(b *testing.B) {
|
||||
b.ResetTimer()
|
||||
for n := 0; n < b.N; n++ {
|
||||
_, err := s.FromData(ctx, false, data)
|
||||
if err != nil {
|
||||
|
||||
@@ -109,7 +109,6 @@ func BenchmarkFromData(benchmark *testing.B) {
|
||||
s := Scanner{}
|
||||
for name, data := range detectors.MustGetBenchmarkData() {
|
||||
benchmark.Run(name, func(b *testing.B) {
|
||||
b.ResetTimer()
|
||||
for n := 0; n < b.N; n++ {
|
||||
_, err := s.FromData(ctx, false, data)
|
||||
if err != nil {
|
||||
|
||||
@@ -109,7 +109,6 @@ func BenchmarkFromData(benchmark *testing.B) {
|
||||
s := Scanner{}
|
||||
for name, data := range detectors.MustGetBenchmarkData() {
|
||||
benchmark.Run(name, func(b *testing.B) {
|
||||
b.ResetTimer()
|
||||
for n := 0; n < b.N; n++ {
|
||||
_, err := s.FromData(ctx, false, data)
|
||||
if err != nil {
|
||||
|
||||
@@ -110,7 +110,6 @@ func BenchmarkFromData(benchmark *testing.B) {
|
||||
s := Scanner{}
|
||||
for name, data := range detectors.MustGetBenchmarkData() {
|
||||
benchmark.Run(name, func(b *testing.B) {
|
||||
b.ResetTimer()
|
||||
for n := 0; n < b.N; n++ {
|
||||
s.FromData(ctx, false, data)
|
||||
}
|
||||
|
||||
@@ -109,7 +109,6 @@ func BenchmarkFromData(benchmark *testing.B) {
|
||||
s := Scanner{}
|
||||
for name, data := range detectors.MustGetBenchmarkData() {
|
||||
benchmark.Run(name, func(b *testing.B) {
|
||||
b.ResetTimer()
|
||||
for n := 0; n < b.N; n++ {
|
||||
_, err := s.FromData(ctx, false, data)
|
||||
if err != nil {
|
||||
|
||||
@@ -10,7 +10,6 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/kylelemons/godebug/pretty"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
@@ -109,7 +108,6 @@ func BenchmarkFromData(benchmark *testing.B) {
|
||||
s := Scanner{}
|
||||
for name, data := range detectors.MustGetBenchmarkData() {
|
||||
benchmark.Run(name, func(b *testing.B) {
|
||||
b.ResetTimer()
|
||||
for n := 0; n < b.N; n++ {
|
||||
_, err := s.FromData(ctx, false, data)
|
||||
if err != nil {
|
||||
|
||||
@@ -55,9 +55,6 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
Raw: []byte(resMatch),
|
||||
RawV2: []byte(resMatch + resIdMatch),
|
||||
}
|
||||
s1.ExtraData = map[string]string{
|
||||
"rotation_guide": "https://howtorotate.com/docs/tutorials/airbrake/",
|
||||
}
|
||||
|
||||
if verify {
|
||||
payload := strings.NewReader(`{"environment":"production","username":"john","email":"john@smith.com","repository":"https://github.com/airbrake/airbrake","revision":"38748467ea579e7ae64f7815452307c9d05e05c5","version":"v2.0"}`)
|
||||
|
||||
@@ -109,7 +109,6 @@ func BenchmarkFromData(benchmark *testing.B) {
|
||||
s := Scanner{}
|
||||
for name, data := range detectors.MustGetBenchmarkData() {
|
||||
benchmark.Run(name, func(b *testing.B) {
|
||||
b.ResetTimer()
|
||||
for n := 0; n < b.N; n++ {
|
||||
_, err := s.FromData(ctx, false, data)
|
||||
if err != nil {
|
||||
|
||||
@@ -45,9 +45,6 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
DetectorType: detectorspb.DetectorType_AirbrakeUserKey,
|
||||
Raw: []byte(resMatch),
|
||||
}
|
||||
s1.ExtraData = map[string]string{
|
||||
"rotation_guide": "https://howtorotate.com/docs/tutorials/airbrake/",
|
||||
}
|
||||
|
||||
if verify {
|
||||
req, err := http.NewRequestWithContext(ctx, "GET", "https://api.airbrake.io/api/v4/projects?key="+resMatch, nil)
|
||||
|
||||
@@ -108,7 +108,6 @@ func BenchmarkFromData(benchmark *testing.B) {
|
||||
s := Scanner{}
|
||||
for name, data := range detectors.MustGetBenchmarkData() {
|
||||
benchmark.Run(name, func(b *testing.B) {
|
||||
b.ResetTimer()
|
||||
for n := 0; n < b.N; n++ {
|
||||
_, err := s.FromData(ctx, false, data)
|
||||
if err != nil {
|
||||
|
||||
@@ -108,7 +108,6 @@ func BenchmarkFromData(benchmark *testing.B) {
|
||||
s := Scanner{}
|
||||
for name, data := range detectors.MustGetBenchmarkData() {
|
||||
benchmark.Run(name, func(b *testing.B) {
|
||||
b.ResetTimer()
|
||||
for n := 0; n < b.N; n++ {
|
||||
_, err := s.FromData(ctx, false, data)
|
||||
if err != nil {
|
||||
|
||||
@@ -2,7 +2,6 @@ package airtableapikey
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"regexp"
|
||||
@@ -21,9 +20,8 @@ var _ detectors.Detector = (*Scanner)(nil)
|
||||
var (
|
||||
client = common.SaneHttpClient()
|
||||
|
||||
appPat = regexp.MustCompile(`(app[a-zA-Z0-9_-]{14})`) // could be part of url
|
||||
keyPat = regexp.MustCompile(`\b(key[a-zA-Z0-9_-]{14})\b`)
|
||||
personalPat = regexp.MustCompile(`(\bpat[[:alnum:]]{14}\.[[:alnum:]]{64}\b)`)
|
||||
appPat = regexp.MustCompile(`(app[a-zA-Z0-9_-]{14})`) // could be part of url
|
||||
keyPat = regexp.MustCompile(`\b(key[a-zA-Z0-9_-]{14})\b`)
|
||||
)
|
||||
|
||||
// Keywords are used for efficiently pre-filtering chunks.
|
||||
@@ -32,31 +30,17 @@ func (s Scanner) Keywords() []string {
|
||||
return []string{"airtable"}
|
||||
}
|
||||
|
||||
type response struct {
|
||||
Error struct {
|
||||
Type string `json:"type"`
|
||||
Message string `json:"message"`
|
||||
} `json:"error"`
|
||||
}
|
||||
|
||||
// FromData will find and optionally verify AirtableApiKey secrets in a given set of bytes.
|
||||
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
|
||||
dataStr := string(data)
|
||||
|
||||
appMatches := appPat.FindAllStringSubmatch(dataStr, -1)
|
||||
keyMatches := keyPat.FindAllStringSubmatch(dataStr, -1)
|
||||
personalKeyMatches := personalPat.FindAllStringSubmatch(dataStr, -1)
|
||||
|
||||
if len(keyMatches) == 0 {
|
||||
keyMatches = personalKeyMatches
|
||||
|
||||
}
|
||||
|
||||
for _, keyMatch := range keyMatches {
|
||||
if len(keyMatch) != 2 {
|
||||
continue
|
||||
}
|
||||
|
||||
keyRes := strings.TrimSpace(keyMatch[1])
|
||||
|
||||
for _, appMatch := range appMatches {
|
||||
@@ -83,15 +67,6 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
defer res.Body.Close()
|
||||
if res.StatusCode >= 200 && res.StatusCode < 300 {
|
||||
s1.Verified = true
|
||||
} else if res.StatusCode == 403 {
|
||||
var resp response
|
||||
if err = json.NewDecoder(res.Body).Decode(&resp); err == nil {
|
||||
// check if the error is due to invalid permissions or model not found
|
||||
if resp.Error.Type == "INVALID_PERMISSIONS_OR_MODEL_NOT_FOUND" {
|
||||
// The key is verified as it works, but the user must enumerate the tables or permissions for the key.
|
||||
s1.Verified = true
|
||||
}
|
||||
}
|
||||
} else {
|
||||
if detectors.IsKnownFalsePositive(keyRes, detectors.DefaultFalsePositives, true) {
|
||||
continue
|
||||
|
||||
@@ -9,10 +9,9 @@ import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/kylelemons/godebug/pretty"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
|
||||
"github.com/google/go-cmp/cmp"
|
||||
"github.com/google/go-cmp/cmp/cmpopts"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
|
||||
)
|
||||
@@ -100,9 +99,8 @@ func TestAirtableApiKey_FromChunk(t *testing.T) {
|
||||
}
|
||||
got[i].Raw = nil
|
||||
}
|
||||
ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "RawV2", "verificationError")
|
||||
if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {
|
||||
t.Errorf("Airtable.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
|
||||
if diff := pretty.Compare(got, tt.want); diff != "" {
|
||||
t.Errorf("AirtableApiKey.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
|
||||
}
|
||||
})
|
||||
}
|
||||
@@ -113,7 +111,6 @@ func BenchmarkFromData(benchmark *testing.B) {
|
||||
s := Scanner{}
|
||||
for name, data := range detectors.MustGetBenchmarkData() {
|
||||
benchmark.Run(name, func(b *testing.B) {
|
||||
b.ResetTimer()
|
||||
for n := 0; n < b.N; n++ {
|
||||
_, err := s.FromData(ctx, false, data)
|
||||
if err != nil {
|
||||
|
||||
@@ -108,7 +108,6 @@ func BenchmarkFromData(benchmark *testing.B) {
|
||||
s := Scanner{}
|
||||
for name, data := range detectors.MustGetBenchmarkData() {
|
||||
benchmark.Run(name, func(b *testing.B) {
|
||||
b.ResetTimer()
|
||||
for n := 0; n < b.N; n++ {
|
||||
_, err := s.FromData(ctx, false, data)
|
||||
if err != nil {
|
||||
|
||||
@@ -108,7 +108,6 @@ func BenchmarkFromData(benchmark *testing.B) {
|
||||
s := Scanner{}
|
||||
for name, data := range detectors.MustGetBenchmarkData() {
|
||||
benchmark.Run(name, func(b *testing.B) {
|
||||
b.ResetTimer()
|
||||
for n := 0; n < b.N; n++ {
|
||||
_, err := s.FromData(ctx, false, data)
|
||||
if err != nil {
|
||||
|
||||
@@ -65,11 +65,10 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
} else if res.StatusCode == 401 {
|
||||
// The secret is determinately not verified (nothing to do)
|
||||
} else {
|
||||
err = fmt.Errorf("unexpected HTTP response status %d", res.StatusCode)
|
||||
s1.SetVerificationError(err, resMatch)
|
||||
s1.VerificationError = fmt.Errorf("request to %v returned unexpected status %d", res.Request.URL, res.StatusCode)
|
||||
}
|
||||
} else {
|
||||
s1.SetVerificationError(err, resMatch)
|
||||
s1.VerificationError = err
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -9,9 +9,7 @@ import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/google/go-cmp/cmp"
|
||||
"github.com/google/go-cmp/cmp/cmpopts"
|
||||
|
||||
"github.com/kylelemons/godebug/pretty"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
@@ -21,7 +19,7 @@ import (
|
||||
func TestAlchemy_FromChunk(t *testing.T) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
|
||||
defer cancel()
|
||||
testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")
|
||||
testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors4")
|
||||
if err != nil {
|
||||
t.Fatalf("could not get test secrets from GCP: %s", err)
|
||||
}
|
||||
@@ -133,12 +131,13 @@ func TestAlchemy_FromChunk(t *testing.T) {
|
||||
if len(got[i].Raw) == 0 {
|
||||
t.Fatalf("no raw secret present: \n %+v", got[i])
|
||||
}
|
||||
if (got[i].VerificationError() != nil) != tt.wantVerificationErr {
|
||||
t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError())
|
||||
got[i].Raw = nil
|
||||
if (got[i].VerificationError != nil) != tt.wantVerificationErr {
|
||||
t.Fatalf("verification error = %v, wantVerificationError %v", got[i].VerificationError, tt.wantVerificationErr)
|
||||
}
|
||||
got[i].VerificationError = nil
|
||||
}
|
||||
ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "verificationError")
|
||||
if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {
|
||||
if diff := pretty.Compare(got, tt.want); diff != "" {
|
||||
t.Errorf("Alchemy.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
|
||||
}
|
||||
})
|
||||
@@ -150,7 +149,6 @@ func BenchmarkFromData(benchmark *testing.B) {
|
||||
s := Scanner{}
|
||||
for name, data := range detectors.MustGetBenchmarkData() {
|
||||
benchmark.Run(name, func(b *testing.B) {
|
||||
b.ResetTimer()
|
||||
for n := 0; n < b.N; n++ {
|
||||
_, err := s.FromData(ctx, false, data)
|
||||
if err != nil {
|
||||
|
||||
@@ -108,7 +108,6 @@ func BenchmarkFromData(benchmark *testing.B) {
|
||||
s := Scanner{}
|
||||
for name, data := range detectors.MustGetBenchmarkData() {
|
||||
benchmark.Run(name, func(b *testing.B) {
|
||||
b.ResetTimer()
|
||||
for n := 0; n < b.N; n++ {
|
||||
_, err := s.FromData(ctx, false, data)
|
||||
if err != nil {
|
||||
|
||||
@@ -109,7 +109,6 @@ func BenchmarkFromData(benchmark *testing.B) {
|
||||
s := Scanner{}
|
||||
for name, data := range detectors.MustGetBenchmarkData() {
|
||||
benchmark.Run(name, func(b *testing.B) {
|
||||
b.ResetTimer()
|
||||
for n := 0; n < b.N; n++ {
|
||||
_, err := s.FromData(ctx, false, data)
|
||||
if err != nil {
|
||||
|
||||
@@ -108,7 +108,6 @@ func BenchmarkFromData(benchmark *testing.B) {
|
||||
s := Scanner{}
|
||||
for name, data := range detectors.MustGetBenchmarkData() {
|
||||
benchmark.Run(name, func(b *testing.B) {
|
||||
b.ResetTimer()
|
||||
for n := 0; n < b.N; n++ {
|
||||
_, err := s.FromData(ctx, false, data)
|
||||
if err != nil {
|
||||
|
||||
@@ -109,7 +109,6 @@ func BenchmarkFromData(benchmark *testing.B) {
|
||||
s := Scanner{}
|
||||
for name, data := range detectors.MustGetBenchmarkData() {
|
||||
benchmark.Run(name, func(b *testing.B) {
|
||||
b.ResetTimer()
|
||||
for n := 0; n < b.N; n++ {
|
||||
_, err := s.FromData(ctx, false, data)
|
||||
if err != nil {
|
||||
|
||||
@@ -87,7 +87,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
}
|
||||
|
||||
if verify {
|
||||
req, err := http.NewRequestWithContext(ctx, "GET", "https://ecs.aliyuncs.com/?", nil)
|
||||
req, err := http.NewRequestWithContext(ctx, "GET", "http://ecs.aliyuncs.com/?", nil)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
|
||||
@@ -110,7 +110,6 @@ func BenchmarkFromData(benchmark *testing.B) {
|
||||
s := Scanner{}
|
||||
for name, data := range detectors.MustGetBenchmarkData() {
|
||||
benchmark.Run(name, func(b *testing.B) {
|
||||
b.ResetTimer()
|
||||
for n := 0; n < b.N; n++ {
|
||||
_, err := s.FromData(ctx, false, data)
|
||||
if err != nil {
|
||||
|
||||
@@ -108,7 +108,6 @@ func BenchmarkFromData(benchmark *testing.B) {
|
||||
s := Scanner{}
|
||||
for name, data := range detectors.MustGetBenchmarkData() {
|
||||
benchmark.Run(name, func(b *testing.B) {
|
||||
b.ResetTimer()
|
||||
for n := 0; n < b.N; n++ {
|
||||
_, err := s.FromData(ctx, false, data)
|
||||
if err != nil {
|
||||
|
||||
@@ -108,7 +108,6 @@ func BenchmarkFromData(benchmark *testing.B) {
|
||||
s := Scanner{}
|
||||
for name, data := range detectors.MustGetBenchmarkData() {
|
||||
benchmark.Run(name, func(b *testing.B) {
|
||||
b.ResetTimer()
|
||||
for n := 0; n < b.N; n++ {
|
||||
_, err := s.FromData(ctx, false, data)
|
||||
if err != nil {
|
||||
|
||||
@@ -109,7 +109,6 @@ func BenchmarkFromData(benchmark *testing.B) {
|
||||
s := Scanner{}
|
||||
for name, data := range detectors.MustGetBenchmarkData() {
|
||||
benchmark.Run(name, func(b *testing.B) {
|
||||
b.ResetTimer()
|
||||
for n := 0; n < b.N; n++ {
|
||||
_, err := s.FromData(ctx, false, data)
|
||||
if err != nil {
|
||||
|
||||
@@ -108,7 +108,6 @@ func BenchmarkFromData(benchmark *testing.B) {
|
||||
s := Scanner{}
|
||||
for name, data := range detectors.MustGetBenchmarkData() {
|
||||
benchmark.Run(name, func(b *testing.B) {
|
||||
b.ResetTimer()
|
||||
for n := 0; n < b.N; n++ {
|
||||
_, err := s.FromData(ctx, false, data)
|
||||
if err != nil {
|
||||
|
||||
@@ -112,7 +112,6 @@ func BenchmarkFromData(benchmark *testing.B) {
|
||||
s := Scanner{}
|
||||
for name, data := range detectors.MustGetBenchmarkData() {
|
||||
benchmark.Run(name, func(b *testing.B) {
|
||||
b.ResetTimer()
|
||||
for n := 0; n < b.N; n++ {
|
||||
_, err := s.FromData(ctx, false, data)
|
||||
if err != nil {
|
||||
|
||||
@@ -1,120 +0,0 @@
|
||||
package anthropic
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
|
||||
)
|
||||
|
||||
type Scanner struct {
|
||||
client *http.Client
|
||||
}
|
||||
|
||||
// Ensure the Scanner satisfies the interface at compile time.
|
||||
var _ detectors.Detector = (*Scanner)(nil)
|
||||
|
||||
var (
|
||||
defaultClient = common.SaneHttpClient()
|
||||
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
|
||||
keyPat = regexp.MustCompile(`\b(sk-ant-api03-[\w\-]{93}AA)\b`)
|
||||
)
|
||||
|
||||
// Keywords are used for efficiently pre-filtering chunks.
|
||||
// Use identifiers in the secret preferably, or the provider name.
|
||||
func (s Scanner) Keywords() []string {
|
||||
return []string{"sk-ant-api03"}
|
||||
}
|
||||
|
||||
type response struct {
|
||||
Error struct {
|
||||
Type string `json:"type"`
|
||||
Message string `json:"message"`
|
||||
} `json:"error"`
|
||||
}
|
||||
|
||||
// FromData will find and optionally verify Anthropic secrets in a given set of bytes.
|
||||
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
|
||||
dataStr := string(data)
|
||||
|
||||
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
|
||||
|
||||
for _, match := range matches {
|
||||
if len(match) != 2 {
|
||||
continue
|
||||
}
|
||||
resMatch := strings.TrimSpace(match[1])
|
||||
|
||||
s1 := detectors.Result{
|
||||
DetectorType: detectorspb.DetectorType_Anthropic,
|
||||
Raw: []byte(resMatch),
|
||||
}
|
||||
|
||||
if verify {
|
||||
client := s.client
|
||||
if client == nil {
|
||||
client = defaultClient
|
||||
}
|
||||
body := map[string]interface{}{
|
||||
"prompt": "test",
|
||||
}
|
||||
|
||||
bodyBytes, err := json.Marshal(body)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
req, err := http.NewRequestWithContext(ctx, "POST", "https://api.anthropic.com/v1/complete", bytes.NewReader(bodyBytes))
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
req.Header.Set("x-api-key", resMatch)
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
res, err := client.Do(req)
|
||||
if err == nil {
|
||||
defer res.Body.Close()
|
||||
if res.StatusCode == http.StatusBadRequest {
|
||||
var resp response
|
||||
if err = json.NewDecoder(res.Body).Decode(&resp); err != nil {
|
||||
err = fmt.Errorf("unexpected HTTP response body: %w", err)
|
||||
s1.SetVerificationError(err, resMatch)
|
||||
continue
|
||||
}
|
||||
if resp.Error.Message == "max_tokens_to_sample: field required" {
|
||||
// The secret is verified
|
||||
// Anthropic returns 400 on a request containing a valid API key,
|
||||
// when not containing a valid model field
|
||||
s1.Verified = true
|
||||
}
|
||||
} else if res.StatusCode == 401 {
|
||||
// The secret is determinately not verified (nothing to do)
|
||||
// Anthropic returns 401 on all requests not containing a valid x-api-key header
|
||||
} else {
|
||||
err = fmt.Errorf("unexpected HTTP response status %d", res.StatusCode)
|
||||
s1.SetVerificationError(err, resMatch)
|
||||
}
|
||||
} else {
|
||||
s1.SetVerificationError(err, resMatch)
|
||||
}
|
||||
}
|
||||
|
||||
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
|
||||
if !s1.Verified && detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
|
||||
continue
|
||||
}
|
||||
|
||||
results = append(results, s1)
|
||||
}
|
||||
|
||||
return results, nil
|
||||
}
|
||||
|
||||
func (s Scanner) Type() detectorspb.DetectorType {
|
||||
return detectorspb.DetectorType_Anthropic
|
||||
}
|
||||
@@ -1,162 +0,0 @@
|
||||
//go:build detectors
|
||||
// +build detectors
|
||||
|
||||
package anthropic
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/google/go-cmp/cmp"
|
||||
"github.com/google/go-cmp/cmp/cmpopts"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
|
||||
)
|
||||
|
||||
func TestAnthropic_FromChunk(t *testing.T) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
|
||||
defer cancel()
|
||||
testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")
|
||||
if err != nil {
|
||||
t.Fatalf("could not get test secrets from GCP: %s", err)
|
||||
}
|
||||
secret := testSecrets.MustGetField("ANTHROPIC")
|
||||
inactiveSecret := testSecrets.MustGetField("ANTHROPIC_INACTIVE")
|
||||
|
||||
type args struct {
|
||||
ctx context.Context
|
||||
data []byte
|
||||
verify bool
|
||||
}
|
||||
tests := []struct {
|
||||
name string
|
||||
s Scanner
|
||||
args args
|
||||
want []detectors.Result
|
||||
wantErr bool
|
||||
wantVerificationErr bool
|
||||
}{
|
||||
{
|
||||
name: "found, verified",
|
||||
s: Scanner{},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a anthropic secret %s within", secret)),
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_Anthropic,
|
||||
Verified: true,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
wantVerificationErr: false,
|
||||
},
|
||||
{
|
||||
name: "found, unverified",
|
||||
s: Scanner{},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a anthropic secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_Anthropic,
|
||||
Verified: false,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
wantVerificationErr: false,
|
||||
},
|
||||
{
|
||||
name: "not found",
|
||||
s: Scanner{},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte("You cannot find the secret within"),
|
||||
verify: true,
|
||||
},
|
||||
want: nil,
|
||||
wantErr: false,
|
||||
wantVerificationErr: false,
|
||||
},
|
||||
{
|
||||
name: "found, would be verified if not for timeout",
|
||||
s: Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a anthropic secret %s within", secret)),
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_Anthropic,
|
||||
Verified: false,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
wantVerificationErr: true,
|
||||
},
|
||||
{
|
||||
name: "found, verified but unexpected api surface",
|
||||
s: Scanner{client: common.ConstantResponseHttpClient(404, "")},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a anthropic secret %s within", secret)),
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_Anthropic,
|
||||
Verified: false,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
wantVerificationErr: true,
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
|
||||
if (err != nil) != tt.wantErr {
|
||||
t.Errorf("Anthropic.FromData() error = %v, wantErr %v", err, tt.wantErr)
|
||||
return
|
||||
}
|
||||
for i := range got {
|
||||
if len(got[i].Raw) == 0 {
|
||||
t.Fatalf("no raw secret present: \n %+v", got[i])
|
||||
}
|
||||
if (got[i].VerificationError() != nil) != tt.wantVerificationErr {
|
||||
t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError())
|
||||
}
|
||||
}
|
||||
ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "verificationError")
|
||||
if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {
|
||||
t.Errorf("Anthropic.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkFromData(benchmark *testing.B) {
|
||||
ctx := context.Background()
|
||||
s := Scanner{}
|
||||
for name, data := range detectors.MustGetBenchmarkData() {
|
||||
benchmark.Run(name, func(b *testing.B) {
|
||||
b.ResetTimer()
|
||||
for n := 0; n < b.N; n++ {
|
||||
_, err := s.FromData(ctx, false, data)
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -109,7 +109,6 @@ func BenchmarkFromData(benchmark *testing.B) {
|
||||
s := Scanner{}
|
||||
for name, data := range detectors.MustGetBenchmarkData() {
|
||||
benchmark.Run(name, func(b *testing.B) {
|
||||
b.ResetTimer()
|
||||
for n := 0; n < b.N; n++ {
|
||||
_, err := s.FromData(ctx, false, data)
|
||||
if err != nil {
|
||||
|
||||
@@ -108,7 +108,6 @@ func BenchmarkFromData(benchmark *testing.B) {
|
||||
s := Scanner{}
|
||||
for name, data := range detectors.MustGetBenchmarkData() {
|
||||
benchmark.Run(name, func(b *testing.B) {
|
||||
b.ResetTimer()
|
||||
for n := 0; n < b.N; n++ {
|
||||
_, err := s.FromData(ctx, false, data)
|
||||
if err != nil {
|
||||
|
||||
@@ -108,7 +108,6 @@ func BenchmarkFromData(benchmark *testing.B) {
|
||||
s := Scanner{}
|
||||
for name, data := range detectors.MustGetBenchmarkData() {
|
||||
benchmark.Run(name, func(b *testing.B) {
|
||||
b.ResetTimer()
|
||||
for n := 0; n < b.N; n++ {
|
||||
_, err := s.FromData(ctx, false, data)
|
||||
if err != nil {
|
||||
|
||||
@@ -109,7 +109,6 @@ func BenchmarkFromData(benchmark *testing.B) {
|
||||
s := Scanner{}
|
||||
for name, data := range detectors.MustGetBenchmarkData() {
|
||||
benchmark.Run(name, func(b *testing.B) {
|
||||
b.ResetTimer()
|
||||
for n := 0; n < b.N; n++ {
|
||||
_, err := s.FromData(ctx, false, data)
|
||||
if err != nil {
|
||||
|
||||
@@ -109,7 +109,6 @@ func BenchmarkFromData(benchmark *testing.B) {
|
||||
s := Scanner{}
|
||||
for name, data := range detectors.MustGetBenchmarkData() {
|
||||
benchmark.Run(name, func(b *testing.B) {
|
||||
b.ResetTimer()
|
||||
for n := 0; n < b.N; n++ {
|
||||
_, err := s.FromData(ctx, false, data)
|
||||
if err != nil {
|
||||
|
||||
@@ -109,7 +109,6 @@ func BenchmarkFromData(benchmark *testing.B) {
|
||||
s := Scanner{}
|
||||
for name, data := range detectors.MustGetBenchmarkData() {
|
||||
benchmark.Run(name, func(b *testing.B) {
|
||||
b.ResetTimer()
|
||||
for n := 0; n < b.N; n++ {
|
||||
_, err := s.FromData(ctx, false, data)
|
||||
if err != nil {
|
||||
|
||||
@@ -108,7 +108,6 @@ func BenchmarkFromData(benchmark *testing.B) {
|
||||
s := Scanner{}
|
||||
for name, data := range detectors.MustGetBenchmarkData() {
|
||||
benchmark.Run(name, func(b *testing.B) {
|
||||
b.ResetTimer()
|
||||
for n := 0; n < b.N; n++ {
|
||||
_, err := s.FromData(ctx, false, data)
|
||||
if err != nil {
|
||||
|
||||
@@ -105,7 +105,6 @@ func BenchmarkFromData(benchmark *testing.B) {
|
||||
s := Scanner{}
|
||||
for name, data := range detectors.MustGetBenchmarkData() {
|
||||
benchmark.Run(name, func(b *testing.B) {
|
||||
b.ResetTimer()
|
||||
for n := 0; n < b.N; n++ {
|
||||
_, err := s.FromData(ctx, false, data)
|
||||
if err != nil {
|
||||
|
||||
@@ -109,7 +109,6 @@ func BenchmarkFromData(benchmark *testing.B) {
|
||||
s := Scanner{}
|
||||
for name, data := range detectors.MustGetBenchmarkData() {
|
||||
benchmark.Run(name, func(b *testing.B) {
|
||||
b.ResetTimer()
|
||||
for n := 0; n < b.N; n++ {
|
||||
_, err := s.FromData(ctx, false, data)
|
||||
if err != nil {
|
||||
|
||||
@@ -108,7 +108,6 @@ func BenchmarkFromData(benchmark *testing.B) {
|
||||
s := Scanner{}
|
||||
for name, data := range detectors.MustGetBenchmarkData() {
|
||||
benchmark.Run(name, func(b *testing.B) {
|
||||
b.ResetTimer()
|
||||
for n := 0; n < b.N; n++ {
|
||||
_, err := s.FromData(ctx, false, data)
|
||||
if err != nil {
|
||||
|
||||
@@ -108,7 +108,6 @@ func BenchmarkFromData(benchmark *testing.B) {
|
||||
s := Scanner{}
|
||||
for name, data := range detectors.MustGetBenchmarkData() {
|
||||
benchmark.Run(name, func(b *testing.B) {
|
||||
b.ResetTimer()
|
||||
for n := 0; n < b.N; n++ {
|
||||
_, err := s.FromData(ctx, false, data)
|
||||
if err != nil {
|
||||
|
||||
@@ -108,7 +108,6 @@ func BenchmarkFromData(benchmark *testing.B) {
|
||||
s := Scanner{}
|
||||
for name, data := range detectors.MustGetBenchmarkData() {
|
||||
benchmark.Run(name, func(b *testing.B) {
|
||||
b.ResetTimer()
|
||||
for n := 0; n < b.N; n++ {
|
||||
_, err := s.FromData(ctx, false, data)
|
||||
if err != nil {
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user