Compare commits

..

2 Commits

Author SHA1 Message Date
ahrav 1065d95d79 Merge branch 'feature-bytification' into detectors-35 2023-09-11 05:56:15 -07:00
Ahrav Dutta 8d2211e1c9 PR 35 of 40 2023-08-25 21:41:33 -07:00
379 changed files with 7851 additions and 23905 deletions
+15
View File
@@ -0,0 +1,15 @@
# To get started with Dependabot version updates, you'll need to specify which
# package ecosystems to update and where the package manifests are located.
# Please see the documentation for all configuration options:
# https://help.github.com/github/administering-a-repository/configuration-options-for-dependency-updates
version: 2
updates:
- package-ecosystem: "gomod" # See documentation for possible values
directory: "/" # Location of package manifests
schedule:
interval: "monthly"
- package-ecosystem: "github-actions"
directory: "/"
schedule:
interval: "monthly"
-8
View File
@@ -1,8 +0,0 @@
{
"$schema": "https://docs.renovatebot.com/renovate-schema.json",
"extends": [
"config:base"
],
"prConcurrentLimit": 3,
"prHourlyLimit": 2
}
+1 -1
View File
@@ -38,7 +38,7 @@ jobs:
steps:
- name: Checkout repository
uses: actions/checkout@v4
uses: actions/checkout@v3
# Initializes the CodeQL tools for scanning.
- name: Initialize CodeQL
-46
View File
@@ -1,46 +0,0 @@
name: detector test aggregation
on:
workflow_dispatch:
schedule:
- cron: "0 8 * * *"
jobs:
test-detectors:
runs-on: ubuntu-latest
permissions:
actions: "read"
contents: "read"
id-token: "write"
steps:
- name: Install Go
uses: actions/setup-go@v4
with:
go-version: "1.21"
- name: Checkout code
uses: actions/checkout@v4
- id: "auth"
uses: "google-github-actions/auth@v2"
with:
workload_identity_provider: "projects/811013774421/locations/global/workloadIdentityPools/github-pool/providers/github-provider"
service_account: "github-ci-external@trufflehog-testing.iam.gserviceaccount.com"
- name: Test
run: make test-integration
- name: Set up gotestsum
run: |
go install gotest.tools/gotestsum@latest
mkdir -p tmp/test-results
- name: Test
run: |
CGO_ENABLED=1 gotestsum --junitfile tmp/test-results/test.xml --raw-command -- go test -json -tags=detectors -timeout=15m $(go list ./... | grep pkg/detectors)
- name: Upload test results to BuildPulse for flaky test detection
if: ${{ !cancelled() }} # Run this step even when the tests fail. Skip if the workflow is cancelled.
uses: buildpulse/buildpulse-action@main
with:
account: 79229934
repository: 694446374
path: |
tmp/test-results/*.xml
key: ${{ secrets.BUILDPULSE_DETECTORS_ACCESS_KEY_ID }}
secret: ${{ secrets.BUILDPULSE_DETECTORS_SECRET_ACCESS_KEY }}
tags: detectors
+1 -1
View File
@@ -18,7 +18,7 @@ jobs:
- uses: actions/setup-go@v4
with:
go-version: '1.21'
- uses: actions/checkout@v4
- uses: actions/checkout@v3
- name: golangci-lint
uses: golangci/golangci-lint-action@v3
with:
+2 -2
View File
@@ -16,7 +16,7 @@ jobs:
go-version: '1.21'
- name: Checkout code
uses: actions/checkout@v4
uses: actions/checkout@v3
with:
fetch-depth: 0
ref: ${{ github.head_ref }}
@@ -52,7 +52,7 @@ jobs:
echo PREVIOUS_TAG=$(cat previous_tag.txt) >> $GITHUB_ENV
- name: Checkout code
uses: actions/checkout@v4
uses: actions/checkout@v3
with:
fetch-depth: 0
ref: ${{ env.PREVIOUS_TAG }}
+5 -8
View File
@@ -8,7 +8,6 @@ on:
permissions:
contents: write
packages: write
id-token: write
jobs:
Release:
@@ -17,18 +16,18 @@ jobs:
DOCKER_CLI_EXPERIMENTAL: "enabled"
steps:
- name: Checkout
uses: actions/checkout@v4
uses: actions/checkout@v3
with:
fetch-depth: 0
- name: Set up QEMU
uses: docker/setup-qemu-action@v3
uses: docker/setup-qemu-action@v2
- name: Docker Login to DockerHub
uses: docker/login-action@v3
uses: docker/login-action@v2
with:
username: ${{ secrets.DOCKERHUB_USERNAME }}
password: ${{ secrets.DOCKERHUB_TOKEN }}
- name: Docker Login to GitHub Container Registry
uses: docker/login-action@v3
uses: docker/login-action@v2
with:
registry: ghcr.io
username: ${{ github.repository_owner }}
@@ -37,10 +36,8 @@ jobs:
uses: actions/setup-go@v4
with:
go-version: '1.21'
- name: Cosign install
uses: sigstore/cosign-installer@1fc5bd396d372bee37d608f955b336615edf79c8 # v3.2.0
- name: Run GoReleaser
uses: goreleaser/goreleaser-action@v5
uses: goreleaser/goreleaser-action@v4
with:
distribution: goreleaser
version: latest
+1 -1
View File
@@ -18,7 +18,7 @@ jobs:
with:
go-version: '1.21'
- name: Checkout code
uses: actions/checkout@v4
uses: actions/checkout@v3
with:
fetch-depth: 0
ref: ${{ github.head_ref }}
+1 -1
View File
@@ -13,7 +13,7 @@ jobs:
with:
go-version: '1.21'
- name: Checkout code
uses: actions/checkout@v4
uses: actions/checkout@v3
- name: Smoke
run: |
set -e
+1 -1
View File
@@ -19,6 +19,6 @@ jobs:
- uses: actions/setup-go@v4
with:
go-version: '1.21'
- uses: actions/checkout@v4
- uses: actions/checkout@v3
- name: Run Snifftest
run: make snifftest
+49 -48
View File
@@ -1,3 +1,4 @@
name: Test
on:
@@ -13,57 +14,57 @@ jobs:
if: ${{ ! github.event.pull_request.head.repo.fork }}
runs-on: ubuntu-latest
permissions:
actions: "read"
contents: "read"
id-token: "write"
actions: 'read'
contents: 'read'
id-token: 'write'
steps:
- name: Install Go
uses: actions/setup-go@v4
with:
go-version: "1.21"
- name: Checkout code
uses: actions/checkout@v4
- id: "auth"
uses: "google-github-actions/auth@v2"
with:
workload_identity_provider: "projects/811013774421/locations/global/workloadIdentityPools/github-pool/providers/github-provider"
service_account: "github-ci-external@trufflehog-testing.iam.gserviceaccount.com"
- name: Set up gotestsum
run: |
go install gotest.tools/gotestsum@latest
mkdir -p tmp/test-results
- name: Test
run: |
CGO_ENABLED=1 gotestsum --junitfile tmp/test-results/test.xml --raw-command -- go test -json -tags=sources $(go list ./... | grep -v /vendor/ | grep -v pkg/detectors)
if: ${{ success() || failure() }} # always run this step, even if there were previous errors
- name: Upload test results to BuildPulse for flaky test detection
if: ${{ !cancelled() }} # Run this step even when the tests fail. Skip if the workflow is cancelled.
uses: buildpulse/buildpulse-action@main
with:
account: 79229934
repository: 77726177
path: |
tmp/test-results/*.xml
key: ${{ secrets.BUILDPULSE_ACCESS_KEY_ID }}
secret: ${{ secrets.BUILDPULSE_SECRET_ACCESS_KEY }}
tags: integration
- name: Annotate test results
uses: mikepenz/action-junit-report@v4
if: success() || failure() # always run even if the previous step fails
with:
report_paths: "tmp/test-results/*.xml"
- name: Install Go
uses: actions/setup-go@v4
with:
go-version: '1.21'
- name: Checkout code
uses: actions/checkout@v3
- id: 'auth'
uses: 'google-github-actions/auth@v1'
with:
workload_identity_provider: 'projects/811013774421/locations/global/workloadIdentityPools/github-pool/providers/github-provider'
service_account: 'github-ci-external@trufflehog-testing.iam.gserviceaccount.com'
- name: Test
run: make test-integration
test-detectors:
if: ${{ ! github.event.pull_request.head.repo.fork }}
runs-on: ubuntu-latest
permissions:
actions: 'read'
contents: 'read'
id-token: 'write'
steps:
- name: Install Go
uses: actions/setup-go@v4
with:
go-version: '1.21'
- name: Checkout code
uses: actions/checkout@v3
- id: 'auth'
uses: 'google-github-actions/auth@v1'
with:
workload_identity_provider: 'projects/811013774421/locations/global/workloadIdentityPools/github-pool/providers/github-provider'
service_account: 'github-ci-external@trufflehog-testing.iam.gserviceaccount.com'
- name: Test
run: make test-detectors
continue-on-error: true
test-community:
if: ${{ github.event.pull_request.head.repo.fork }}
runs-on: ubuntu-latest
permissions:
actions: "read"
contents: "read"
actions: 'read'
contents: 'read'
steps:
- name: Install Go
uses: actions/setup-go@v4
with:
go-version: "1.21"
- name: Checkout code
uses: actions/checkout@v4
- name: Test
run: make test-community
- name: Install Go
uses: actions/setup-go@v4
with:
go-version: '1.21'
- name: Checkout code
uses: actions/checkout@v3
- name: Test
run: make test-community
-12
View File
@@ -96,15 +96,3 @@ brews:
homepage: "https://github.com/trufflesecurity/trufflehog"
install: |
bin.install "trufflehog"
signs:
- cmd: cosign
signature: "${artifact}.sig"
certificate: "${artifact}.pem"
args:
- "sign-blob"
- "--oidc-issuer=https://token.actions.githubusercontent.com"
- "--output-certificate=${certificate}"
- "--output-signature=${signature}"
- "${artifact}"
- "--yes"
artifacts: checksum
+3 -3
View File
@@ -8,9 +8,9 @@ RUN --mount=type=cache,target=/go/pkg/mod \
--mount=type=cache,target=/root/.cache/go-build \
GOOS=${TARGETOS} GOARCH=${TARGETARCH} go build -o trufflehog .
FROM alpine:3.18
RUN apk add --no-cache bash git openssh-client ca-certificates rpm2cpio binutils cpio \
&& rm -rf /var/cache/apk/* && update-ca-certificates
FROM alpine:3.15
RUN apk add --no-cache bash git openssh-client ca-certificates \
&& update-ca-certificates
COPY --from=builder /build/trufflehog /usr/bin/trufflehog
COPY entrypoint.sh /etc/entrypoint.sh
RUN chmod +x /etc/entrypoint.sh
+4 -3
View File
@@ -1,7 +1,8 @@
FROM alpine:3.18
FROM alpine:3.15
RUN apk add --no-cache bash git openssh-client ca-certificates rpm2cpio binutils cpio \
&& rm -rf /var/cache/apk/* && update-ca-certificates
RUN apk add --no-cache bash git openssh-client ca-certificates \
&& rm -rf /var/cache/apk/* && \
update-ca-certificates
WORKDIR /usr/bin/
COPY trufflehog .
COPY entrypoint.sh /etc/entrypoint.sh
+4 -4
View File
@@ -1,4 +1,4 @@
PROTOS_IMAGE ?= trufflesecurity/protos:1.21-0
PROTOS_IMAGE ?= trufflesecurity/protos:1.18-0
.PHONY: check
.PHONY: lint
@@ -52,14 +52,14 @@ run-debug:
CGO_ENABLED=0 go run . git file://. --json --debug
protos:
docker run --rm -u "$(shell id -u)" -v "$(shell dirname $(realpath $(firstword $(MAKEFILE_LIST))))":/pwd "${PROTOS_IMAGE}" bash -c "cd /pwd; /pwd/scripts/gen_proto.sh"
docker run -u "$(shell id -u)" -v "$(shell dirname $(realpath $(firstword $(MAKEFILE_LIST))))":/pwd "${PROTOS_IMAGE}" bash -c "cd /pwd; /pwd/scripts/gen_proto.sh"
protos-windows:
docker run --rm -v "$(shell cygpath -w $(shell pwd))":/pwd "${PROTOS_IMAGE}" bash -c "cd /pwd; ./scripts/gen_proto.sh"
docker run -v "$(shell cygpath -w $(shell pwd))":/pwd "${PROTOS_IMAGE}" bash -c "cd /pwd; ./scripts/gen_proto.sh"
release-protos-image:
docker buildx build --push --platform=linux/amd64,linux/arm64 \
-t trufflesecurity/protos:1.21-0 -f hack/Dockerfile.protos .
-t trufflesecurity/protos:1.18-0 -f hack/Dockerfile.protos .
snifftest:
./hack/snifftest/snifftest.sh
+65 -143
View File
@@ -16,7 +16,7 @@
---
# :mag*right: \_Now Scanning*
# :mag_right: _Now Scanning_
<div align="center">
@@ -45,7 +45,6 @@ docker run --rm -it -v "$PWD:/pwd" trufflesecurity/trufflehog:latest github --or
# :floppy_disk: Installation
Several options available for you:
```bash
# MacOS users
brew install trufflesecurity/trufflehog/trufflehog
@@ -62,47 +61,8 @@ Download and unpack from https://github.com/trufflesecurity/trufflehog/releases
# Compile from source
git clone https://github.com/trufflesecurity/trufflehog.git
cd trufflehog; go install
# Using installation script
curl -sSfL https://raw.githubusercontent.com/trufflesecurity/trufflehog/main/scripts/install.sh | sh -s -- -b /usr/local/bin
# Using installation script to install a specific version
curl -sSfL https://raw.githubusercontent.com/trufflesecurity/trufflehog/main/scripts/install.sh | sh -s -- -b /usr/local/bin <ReleaseTag like v3.56.0>
```
# :closed_lock_with_key: Verifying the artifacts
Checksums are applied to all artifacts, and the resulting checksum file is signed using cosign.
You need the following tool to verify signature:
- [Cosign](https://docs.sigstore.dev/cosign/installation/)
Verification steps are as follow:
1. Download the artifact files you want, and the following files from the [releases](https://github.com/trufflesecurity/trufflehog/releases) page.
- trufflehog\_{version}\_checksums.txt
- trufflehog\_{version}\_checksums.txt.pem
- trufflehog\_{version}\_checksums.txt.sig
2. Verify the signature:
```shell
cosign verify-blob <path to trufflehog_{version}_checksums.txt> \
--certificate <path to trufflehog_{version}_checksums.txt.pem> \
--signature <path to trufflehog_{version}_checksums.txt.sig> \
--certificate-identity-regexp 'https://github\.com/trufflesecurity/trufflehog/\.github/workflows/.+' \
--certificate-oidc-issuer "https://token.actions.githubusercontent.com"
```
3. Once the signature is confirmed as valid, you can proceed to validate that the SHA256 sums align with the downloaded artifact:
```shell
sha256sum --ignore-missing -c trufflehog_{version}_checksums.txt
```
Replace `{version}` with the downloaded files version
# :rocket: Quick Start
## 1: Scan a repo for only verified secrets
@@ -114,7 +74,6 @@ trufflehog git https://github.com/trufflesecurity/test_keys --only-verified
```
Expected output:
```
🐷🔑🐷 TruffleHog. Unearth your secrets. 🐷🔑🐷
@@ -146,49 +105,42 @@ trufflehog git https://github.com/trufflesecurity/test_keys --only-verified --js
```
Expected output:
```
{"SourceMetadata":{"Data":{"Git":{"commit":"fbc14303ffbf8fb1c2c1914e8dda7d0121633aca","file":"keys","email":"counter \u003ccounter@counters-MacBook-Air.local\u003e","repository":"https://github.com/trufflesecurity/test_keys","timestamp":"2022-06-16 10:17:40 -0700 PDT","line":4}}},"SourceID":0,"SourceType":16,"SourceName":"trufflehog - git","DetectorType":2,"DetectorName":"AWS","DecoderName":"PLAIN","Verified":true,"Raw":"AKIAYVP4CIPPERUVIFXG","Redacted":"AKIAYVP4CIPPERUVIFXG","ExtraData":{"account":"595918472158","arn":"arn:aws:iam::595918472158:user/canarytokens.com@@mirux23ppyky6hx3l6vclmhnj","user_id":"AIDAYVP4CIPPJ5M54LRCY"},"StructuredData":null}
...
```
## 4: Scan a GitHub Repo + its Issues and Pull Requests.
```bash
trufflehog github --repo=https://github.com/trufflesecurity/test_keys --issue-comments --pr-comments
```
## 5: Scan an S3 bucket for verified keys
## 4: Scan an S3 bucket for verified keys
```bash
trufflehog s3 --bucket=<bucket name> --only-verified
```
## 6: Scan S3 buckets using IAM Roles
## 5: Scan S3 buckets using IAM Roles
```bash
trufflehog s3 --role-arn=<iam role arn>
```
## 7: Scan a Github Repo using SSH authentication in docker
## 6: Scan a Github Repo using SSH authentication in docker
```bash
docker run --rm -v "$HOME/.ssh:/root/.ssh:ro" trufflesecurity/trufflehog:latest git ssh://github.com/trufflesecurity/test_keys
```
## 8: Scan individual files or directories
## 7: Scan individual files or directories
```bash
trufflehog filesystem path/to/file1.txt path/to/file2.txt path/to/dir
```
## 9: Scan GCS buckets for verified secrets.
## 8: Scan GCS buckets for verified secrets.
```bash
trufflehog gcs --project-id=<project-ID> --cloud-environment --only-verified
```
## 10: Scan a Docker image for verified secrets.
## 9: Scan a Docker image for verified secrets.
Use the `--image` flag multiple times to scan multiple images.
@@ -196,37 +148,30 @@ Use the `--image` flag multiple times to scan multiple images.
trufflehog docker --image trufflesecurity/secrets --only-verified
```
## 11: Scan in CI
Set the `--since-commit` flag to your default branch that people merge into (ex: "main"). Set the `--branch` flag to your PR's branch name (ex: "feature-1"). Depending on the CI/CD platform you use, this value can be pulled in dynamically (ex: [CIRCLE_BRANCH in Circle CI](https://circleci.com/docs/variables/) and [TRAVIS_PULL_REQUEST_BRANCH in Travis CI](https://docs.travis-ci.com/user/environment-variables/)). If the repo is cloned and the target branch is already checked out during the CI/CD workflow, then `--branch HEAD` should be sufficient. The `--fail` flag will return an 183 error code if valid credentials are found.
```bash
trufflehog git file://. --since-commit main --branch feature-1 --only-verified --fail
```
# :question: FAQ
- All I see is `🐷🔑🐷 TruffleHog. Unearth your secrets. 🐷🔑🐷` and the program exits, what gives?
- That means no secrets were detected
- Why is the scan taking a long time when I scan a GitHub org
- Unauthenticated GitHub scans have rate limits. To improve your rate limits, include the `--token` flag with a personal access token
- It says a private key was verified, what does that mean?
- Check out our Driftwood blog post to learn how to do this, in short we've confirmed the key can be used live for SSH or SSL [Blog post](https://trufflesecurity.com/blog/driftwood-know-if-private-keys-are-sensitive/)
- Is there an easy way to ignore specific secrets?
- If the scanned source [supports line numbers](https://github.com/trufflesecurity/trufflehog/blob/d6375ba92172fd830abb4247cca15e3176448c5d/pkg/engine/engine.go#L358-L365), then you can add a `trufflehog:ignore` comment on the line containing the secret to ignore that secrets.
+ All I see is `🐷🔑🐷 TruffleHog. Unearth your secrets. 🐷🔑🐷` and the program exits, what gives?
+ That means no secrets were detected
+ Why is the scan is taking a long time when I scan a GitHub org
+ Unauthenticated GitHub scans have rate limits. To improve your rate limits, include the `--token` flag with a personal access token
+ It says a private key was verified, what does that mean?
+ Check out our Driftwood blog post to learn how to do this, in short we've confirmed the key can be used live for SSH or SSL [Blog post](https://trufflesecurity.com/blog/driftwood-know-if-private-keys-are-sensitive/)
+ Is there an easy way to ignore specific secrets?
+ If the scanned source [supports line numbers](https://github.com/trufflesecurity/trufflehog/blob/d6375ba92172fd830abb4247cca15e3176448c5d/pkg/engine/engine.go#L358-L365), then you can add a `trufflehog:ignore` comment on the line containing the secret to ignore that secrets.
# :newspaper: What's new in v3?
TruffleHog v3 is a complete rewrite in Go with many new powerful features.
- We've **added over 700 credential detectors that support active verification against their respective APIs**.
- We've also added native **support for scanning GitHub, GitLab, Docker, filesystems, S3, GCS, Circle CI and Travis CI**.
- We've also added native **support for scanning GitHub, GitLab, filesystems, S3, GCS and Circle CI**.
- **Instantly verify private keys** against millions of github users and **billions** of TLS certificates using our [Driftwood](https://trufflesecurity.com/blog/driftwood) technology.
- Scan binaries, documents, and other file formats
- Scan binaries and other file formats
- Available as a GitHub Action and a pre-commit hook
## What is credential verification?
## What is credential verification?
For every potential credential that is detected, we've painstakingly implemented programmatic verification against the API that we think it belongs to. Verification eliminates false positives. For example, the [AWS credential detector](pkg/detectors/aws/aws.go) performs a `GetCallerIdentity` API call against the AWS API to verify if an AWS credential is active.
# :memo: Usage
@@ -241,8 +186,8 @@ TruffleHog has a sub-command for each source of data that you may want to scan:
- filesystem (files and directories)
- syslog
- circleci
- travisci
- GCS (Google Cloud Storage)
- stdin (coming soon)
Each subcommand can have options that you can see with the `--help` flag provided to the sub command:
@@ -253,58 +198,33 @@ usage: TruffleHog git [<flags>] <uri>
Find credentials in git repositories.
Flags:
-h, --help Show context-sensitive help (also try --help-long and --help-man).
--debug Run in debug mode.
--trace Run in trace mode.
--profile Enables profiling and sets a pprof and fgprof server on :18066.
-j, --json Output in JSON format.
--json-legacy Use the pre-v3.0 JSON format. Only works with git, gitlab, and github sources.
--github-actions Output in GitHub Actions format.
--concurrency=8 Number of concurrent workers.
--no-verification Don't verify the results.
--only-verified Only output verified results.
--filter-unverified Only output first unverified result per chunk per detector if there are more than one results.
--filter-entropy=FILTER-ENTROPY
Filter unverified results with Shannon entropy. Start with 3.0.
--help Show context-sensitive help (also try --help-long and --help-man).
--debug Run in debug mode.
--trace Run in trace mode.
--profile Enables profiling and sets a pprof and fgprof server on :18066.
-j, --json Output in JSON format.
--json-legacy Use the pre-v3.0 JSON format. Only works with git, gitlab, and github sources.
--concurrency=10 Number of concurrent workers.
--no-verification Don't verify the results.
--only-verified Only output verified results.
--filter-unverified Only output first unverified result per chunk per detector if there are more than one results.
--config=CONFIG Path to configuration file.
--print-avg-detector-time
Print the average time spent on each detector.
--no-update Don't check for updates.
--fail Exit with code 183 if results are found.
--verifier=VERIFIER ... Set custom verification endpoints.
--archive-max-size=ARCHIVE-MAX-SIZE
Maximum size of archive to scan. (Byte units eg. 512B, 2KB, 4MB)
--archive-max-depth=ARCHIVE-MAX-DEPTH
Maximum depth of archive to scan.
--archive-timeout=ARCHIVE-TIMEOUT
Maximum time to spend extracting an archive.
--include-detectors="all" Comma separated list of detector types to include. Protobuf name or IDs may be used, as well as ranges.
--exclude-detectors=EXCLUDE-DETECTORS
Comma separated list of detector types to exclude. Protobuf name or IDs may be used, as well as ranges. IDs defined here take precedence over the include list.
--version Show application version.
-i, --include-paths=INCLUDE-PATHS
Path to file with newline separated regexes for files to include in scan.
-x, --exclude-paths=EXCLUDE-PATHS
Path to file with newline separated regexes for files to exclude in scan.
--exclude-globs=EXCLUDE-GLOBS
Comma separated list of globs to exclude in scan. This option filters at the `git log` level, resulting in faster scans.
--since-commit=SINCE-COMMIT
Commit to start scan from.
--branch=BRANCH Branch to scan.
--max-depth=MAX-DEPTH Maximum depth of commits to scan.
--bare Scan bare repository (e.g. useful while using in pre-receive hooks)
--print-avg-detector-time Print the average time spent on each detector.
--no-update Don't check for updates.
--fail Exit with code 183 if results are found.
--version Show application version.
Args:
<uri> Git repository URL. https://, file://, or ssh:// schema expected.
```
For example, to scan a `git` repository, start with
For example, to scan a `git` repository, start with
```
$ trufflehog git https://github.com/trufflesecurity/trufflehog.git
```
## S3
## S3
The S3 source supports assuming IAM roles for scanning in addition to IAM users. This makes it easier for users to scan multiple AWS accounts without needing to rely on hardcoded credentials for each account.
@@ -329,12 +249,11 @@ trufflehog s3 --role-arn=<iam-role-arn-1> --role-arn=<iam-role-arn-2>
```
Exit Codes:
- 0: No errors and no results were found.
- 1: An error was encountered. Sources may not have completed scans.
- 183: No errors were encountered, but results were found. Will only be returned if `--fail` flag is used.
## :octocat: TruffleHog Github Action
# :octocat: TruffleHog Github Action
```yaml
- name: TruffleHog
@@ -354,7 +273,6 @@ The TruffleHog OSS Github Action can be used to scan a range of commits for leak
any results are found.
For example, to scan the contents of pull requests you could use the following workflow:
```yaml
name: TruffleHog Secrets Scan
on: [pull_request]
@@ -375,26 +293,26 @@ jobs:
extra_args: --debug --only-verified
```
## Pre-commit Hook
# Pre-commit Hook
Trufflehog can be used in a pre-commit hook to prevent credentials from leaking before they ever leave your computer.
An example `.pre-commit-config.yaml` is provided (see [pre-commit.com](https://pre-commit.com/) for installation).
```yaml
repos:
- repo: local
hooks:
- id: trufflehog
name: TruffleHog
description: Detect secrets in your data.
entry: bash -c 'trufflehog git file://. --since-commit HEAD --only-verified --fail'
# For running trufflehog in docker, use the following entry instead:
# entry: bash -c 'docker run --rm -v "$(pwd):/workdir" -i --rm trufflesecurity/trufflehog:latest git file:///workdir --since-commit HEAD --only-verified --fail'
language: system
stages: ["commit", "push"]
- repo: local
hooks:
- id: trufflehog
name: TruffleHog
description: Detect secrets in your data.
entry: bash -c 'trufflehog git file://. --since-commit HEAD --only-verified --fail'
# For running trufflehog in docker, use the following entry instead:
# entry: bash -c 'docker run --rm -v "$(pwd):/workdir" -i --rm trufflesecurity/trufflehog:latest git file:///workdir --since-commit HEAD --only-verified --fail'
language: system
stages: ["commit", "push"]
```
## Regex Detector (alpha)
# Regex Detector (alpha)
Trufflehog supports detection and verification of custom regular expressions.
For detection, at least one **regular expression** and **keyword** is required.
@@ -413,17 +331,17 @@ status code, the secret is considered verified.
```yaml
# config.yaml
detectors:
- name: hog detector
keywords:
- hog
regex:
adjective: hogs are (\S+)
verify:
- endpoint: http://localhost:8000/
# unsafe must be set if the endpoint is HTTP
unsafe: true
headers:
- "Authorization: super secret authorization header"
- name: hog detector
keywords:
- hog
regex:
adjective: hogs are (\S+)
verify:
- endpoint: http://localhost:8000/
# unsafe must be set if the endpoint is HTTP
unsafe: true
headers:
- 'Authorization: super secret authorization header'
```
```
@@ -491,29 +409,33 @@ with HTTPServer(('', 8000), Verifier) as server:
This project exists thanks to all the people who contribute. [[Contribute](CONTRIBUTING.md)].
<a href="https://github.com/trufflesecurity/trufflehog/graphs/contributors">
<img src="https://contrib.rocks/image?repo=trufflesecurity/trufflehog" />
</a>
# :computer: Contributing
Contributions are very welcome! Please see our [contribution guidelines first](CONTRIBUTING.md).
We no longer accept contributions to TruffleHog v2, but that code is available in the `v2` branch.
## Adding new secret detectors
We have published some [documentation and tooling to get started on adding new secret detectors](hack/docs/Adding_Detectors_external.md). Let's improve detection together!
# Use as a library
Currently, trufflehog is in heavy development and no guarantees can be made on
the stability of the public APIs at this time.
# License Change
Since v3.0, TruffleHog is released under a AGPL 3 license, included in [`LICENSE`](LICENSE). TruffleHog v3.0 uses none of the previous codebase, but care was taken to preserve backwards compatibility on the command line interface. The work previous to this release is still available licensed under GPL 2.0 in the history of this repository and the previous package releases and tags. A completed CLA is required for us to accept contributions going forward.
# :money_with_wings: Enterprise product
Are you interested in continuously monitoring your Git, Jira, Slack, Confluence, etc.. for credentials? We have an enterprise product that can help. Reach out here to learn more https://trufflesecurity.com/contact/
File diff suppressed because one or more lines are too long

Before

Width:  |  Height:  |  Size: 83 KiB

After

Width:  |  Height:  |  Size: 11 KiB

-14
View File
@@ -1,14 +0,0 @@
# Examples
This folder contains various examples like custom detectors, scripts, etc. Feel free to contribute!
### Generic Detector
An often requested feature for TruffleHog is a generic detector. By default, we do not support generic detection as it would result in lots of false positives. However, if you want to attempt detect generic secrets you can use a custom detector.
#### Try it out:
```
wget https://raw.githubusercontent.com/trufflesecurity/trufflehog/main/examples/generic.yml
trufflehog filesystem --config=$PWD/generic.yml $PWD
# to filter so that _only_ generic credentials are logged:
trufflehog filesystem --config=$PWD/generic.yml --json --no-verification $PWD | awk '/generic-api-key/{print $0}'
```
-15
View File
@@ -1,15 +0,0 @@
detectors:
- name: generic-api-key
keywords:
- key
- api
- token
- secret
- client
- passwd
- password
- auth
- access
regex:
# borrowing the gitleaks generic-api-key regex
generic-api-key: "(?i)(?:key|api|token|secret|client|passwd|password|auth|access)(?:[0-9a-z\\-_\\t .]{0,20})(?:[\\s|']|[\\s|\"]){0,3}(?:=|>|:{1,3}=|\\|\\|:|<=|=>|:|\\?=)(?:'|\"|\\s|=|\\x60){0,5}([0-9a-z\\-_.=]{10,150})(?:['|\"|\\n|\\r|\\s|\\x60|;]|$)"
+91 -99
View File
@@ -2,99 +2,91 @@ module github.com/trufflesecurity/trufflehog/v3
go 1.21
replace github.com/jpillora/overseer => github.com/trufflesecurity/overseer v1.2.7
replace github.com/jpillora/overseer => github.com/trufflesecurity/overseer v1.1.7-custom5
// go wants to pull in the latest but after v10.2.5 has a breaking API change
replace github.com/couchbase/gocbcore/v10 => github.com/couchbase/gocbcore/v10 v10.2.4
require (
cloud.google.com/go/secretmanager v1.11.4
cloud.google.com/go/storage v1.35.1
github.com/Azure/go-autorest/autorest/azure/auth v0.5.12
github.com/AzureAD/microsoft-authentication-library-for-go v1.2.0
cloud.google.com/go/secretmanager v1.11.1
cloud.google.com/go/storage v1.31.0
github.com/Azure/go-autorest/autorest/azure/auth v0.5.11
github.com/BobuSumisu/aho-corasick v1.0.3
github.com/TheZeroSlave/zapsentry v1.19.0
github.com/alecthomas/kingpin/v2 v2.4.0
github.com/aws/aws-sdk-go v1.48.12
github.com/aymanbagabas/go-osc52 v1.2.2
github.com/TheZeroSlave/zapsentry v1.17.0
github.com/aws/aws-sdk-go v1.44.83
github.com/aymanbagabas/go-osc52 v1.2.1
github.com/bill-rich/disk-buffer-reader v0.1.7
github.com/bill-rich/go-syslog v0.0.0-20220413021637-49edb52a574c
github.com/bitfinexcom/bitfinex-api-go v0.0.0-20210608095005-9e0b26f200fb
github.com/bradleyfalzon/ghinstallation/v2 v2.8.0
github.com/bradleyfalzon/ghinstallation/v2 v2.6.0
github.com/charmbracelet/bubbles v0.16.1
github.com/charmbracelet/bubbletea v0.24.2
github.com/charmbracelet/bubbletea v0.24.1
github.com/charmbracelet/glamour v0.6.0
github.com/charmbracelet/lipgloss v0.9.1
github.com/coinbase/waas-client-library-go v1.0.8
github.com/couchbase/gocb/v2 v2.7.0
github.com/charmbracelet/lipgloss v0.7.1
github.com/couchbase/gocb/v2 v2.6.3
github.com/crewjam/rfc5424 v0.1.0
github.com/denisenkom/go-mssqldb v0.12.3
github.com/envoyproxy/protoc-gen-validate v1.0.2
github.com/fatih/color v1.16.0
github.com/fatih/color v1.15.0
github.com/felixge/fgprof v0.9.3
github.com/getsentry/sentry-go v0.25.0
github.com/go-errors/errors v1.5.1
github.com/go-git/go-git/v5 v5.10.1
github.com/go-ldap/ldap/v3 v3.4.6
github.com/go-logr/logr v1.3.0
github.com/go-logr/zapr v1.3.0
github.com/getsentry/sentry-go v0.22.0
github.com/go-errors/errors v1.4.2
github.com/go-git/go-git/v5 v5.8.1
github.com/go-ldap/ldap/v3 v3.4.5
github.com/go-logr/logr v1.2.4
github.com/go-logr/zapr v1.2.4
github.com/go-redis/redis v6.15.9+incompatible
github.com/go-sql-driver/mysql v1.7.1
github.com/gobwas/glob v0.2.3
github.com/golang-jwt/jwt v3.2.2+incompatible
github.com/google/go-cmp v0.6.0
github.com/google/go-containerregistry v0.17.0
github.com/google/go-cmp v0.5.9
github.com/google/go-containerregistry v0.15.2
github.com/google/go-github/v42 v42.0.0
github.com/google/go-github/v56 v56.0.0
github.com/google/go-github/v57 v57.0.0
github.com/google/uuid v1.4.0
github.com/googleapis/gax-go/v2 v2.12.0
github.com/h2non/filetype v1.1.3
github.com/hashicorp/go-retryablehttp v0.7.5
github.com/hashicorp/golang-lru v0.6.0
github.com/hashicorp/golang-lru/v2 v2.0.7
github.com/hashicorp/go-retryablehttp v0.7.4
github.com/hashicorp/golang-lru v0.5.1
github.com/jlaffaye/ftp v0.2.0
github.com/joho/godotenv v1.5.1
github.com/jpillora/overseer v1.1.6
github.com/kylelemons/godebug v1.1.0
github.com/launchdarkly/go-server-sdk/v6 v6.1.1
github.com/launchdarkly/go-server-sdk/v6 v6.1.0
github.com/lib/pq v1.10.9
github.com/lrstanley/bubblezone v0.0.0-20230911164824-e3824f1adde9
github.com/marusama/semaphore/v2 v2.5.0
github.com/mattn/go-isatty v0.0.20
github.com/mattn/go-sqlite3 v1.14.18
github.com/lrstanley/bubblezone v0.0.0-20221222153816-e95291e2243e
github.com/mattn/go-isatty v0.0.18
github.com/mattn/go-sqlite3 v1.14.17
github.com/mholt/archiver/v4 v4.0.0-alpha.8
github.com/mitchellh/go-ps v1.0.0
github.com/muesli/reflow v0.3.0
github.com/patrickmn/go-cache v2.1.0+incompatible
github.com/paulbellamy/ratecounter v0.2.0
github.com/pkg/errors v0.9.1
github.com/prometheus/client_golang v1.17.0
github.com/rabbitmq/amqp091-go v1.9.0
github.com/prometheus/client_golang v1.16.0
github.com/rabbitmq/amqp091-go v1.8.1
github.com/sergi/go-diff v1.3.1
github.com/shuheiktgw/go-travis v0.3.1
github.com/snowflakedb/gosnowflake v1.6.25
github.com/snowflakedb/gosnowflake v1.6.23
github.com/stretchr/testify v1.8.4
github.com/tailscale/depaware v0.0.0-20210622194025-720c4b409502
github.com/trufflesecurity/disk-buffer-reader v0.2.1
github.com/xanzy/go-gitlab v0.94.0
go.mongodb.org/mongo-driver v1.12.1
go.uber.org/mock v0.3.0
go.uber.org/zap v1.26.0
golang.org/x/crypto v0.16.0
golang.org/x/exp v0.0.0-20231127185646-65229373498e
golang.org/x/oauth2 v0.15.0
golang.org/x/sync v0.5.0
golang.org/x/text v0.14.0
google.golang.org/api v0.152.0
github.com/xanzy/go-gitlab v0.88.0
go.mongodb.org/mongo-driver v1.12.0
go.uber.org/mock v0.2.0
go.uber.org/zap v1.24.0
golang.org/x/crypto v0.12.0
golang.org/x/exp v0.0.0-20230206171751-46f607a40771
golang.org/x/oauth2 v0.10.0
golang.org/x/sync v0.3.0
golang.org/x/text v0.12.0
google.golang.org/api v0.131.0
google.golang.org/protobuf v1.31.0
gopkg.in/alecthomas/kingpin.v2 v2.2.6
gopkg.in/h2non/gock.v1 v1.1.2
pgregory.net/rapid v1.1.0
sigs.k8s.io/yaml v1.4.0
sigs.k8s.io/yaml v1.3.0
)
require (
cloud.google.com/go v0.110.10 // indirect
cloud.google.com/go/compute v1.23.3 // indirect
cloud.google.com/go v0.110.2 // indirect
cloud.google.com/go/compute v1.20.1 // indirect
cloud.google.com/go/compute/metadata v0.2.3 // indirect
cloud.google.com/go/iam v1.1.5 // indirect
cloud.google.com/go/longrunning v0.5.4 // indirect
cloud.google.com/go/iam v1.1.0 // indirect
dario.cat/mergo v1.0.0 // indirect
github.com/99designs/go-keychain v0.0.0-20191008050251-8e49817e8af4 // indirect
github.com/99designs/keyring v1.2.2 // indirect
@@ -111,9 +103,10 @@ require (
github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 // indirect
github.com/JohnCGriffin/overflow v0.0.0-20211019200055-46fa312c352c // indirect
github.com/Microsoft/go-winio v0.6.1 // indirect
github.com/ProtonMail/go-crypto v0.0.0-20230828082145-3c4c8a2d2371 // indirect
github.com/ProtonMail/go-crypto v0.0.0-20230717121422-5aa5874ade95 // indirect
github.com/acomagu/bufpipe v1.0.4 // indirect
github.com/alecthomas/chroma v0.10.0 // indirect
github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 // indirect
github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 // indirect
github.com/andybalholm/brotli v1.0.5 // indirect
github.com/apache/arrow/go/v12 v12.0.1 // indirect
@@ -134,35 +127,33 @@ require (
github.com/aws/smithy-go v1.13.5 // indirect
github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect
github.com/aymerick/douceur v0.2.0 // indirect
github.com/benbjohnson/clock v1.1.0 // indirect
github.com/beorn7/perks v1.0.1 // indirect
github.com/bodgit/plumbing v1.3.0 // indirect
github.com/bodgit/sevenzip v1.4.3 // indirect
github.com/bodgit/windows v1.0.1 // indirect
github.com/bodgit/plumbing v1.2.0 // indirect
github.com/bodgit/sevenzip v1.3.0 // indirect
github.com/bodgit/windows v1.0.0 // indirect
github.com/cespare/xxhash/v2 v2.2.0 // indirect
github.com/cloudflare/circl v1.3.3 // indirect
github.com/connesc/cipherio v0.2.1 // indirect
github.com/containerd/console v1.0.4-0.20230313162750-1ae8d489ac81 // indirect
github.com/containerd/stargz-snapshotter/estargz v0.14.3 // indirect
github.com/couchbase/gocbcore/v10 v10.3.0 // indirect
github.com/couchbase/gocbcoreps v0.1.0 // indirect
github.com/couchbase/goprotostellar v1.0.0 // indirect
github.com/couchbaselabs/gocbconnstr/v2 v2.0.0-20230515165046-68b522a21131 // indirect
github.com/cyphar/filepath-securejoin v0.2.4 // indirect
github.com/couchbase/gocbcore/v10 v10.2.4 // indirect
github.com/danieljoos/wincred v1.1.2 // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/dimchansky/utfbom v1.1.1 // indirect
github.com/dlclark/regexp2 v1.4.0 // indirect
github.com/docker/cli v24.0.0+incompatible // indirect
github.com/docker/cli v23.0.5+incompatible // indirect
github.com/docker/distribution v2.8.2+incompatible // indirect
github.com/docker/docker v24.0.0+incompatible // indirect
github.com/docker/docker v23.0.5+incompatible // indirect
github.com/docker/docker-credential-helpers v0.7.0 // indirect
github.com/dsnet/compress v0.0.1 // indirect
github.com/dvsekhvalnov/jose2go v1.5.0 // indirect
github.com/emirpasic/gods v1.18.1 // indirect
github.com/form3tech-oss/jwt-go v3.2.5+incompatible // indirect
github.com/gabriel-vasile/mimetype v1.4.2 // indirect
github.com/go-asn1-ber/asn1-ber v1.5.5 // indirect
github.com/go-asn1-ber/asn1-ber v1.5.4 // indirect
github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect
github.com/go-git/go-billy/v5 v5.5.0 // indirect
github.com/go-git/go-billy/v5 v5.4.1 // indirect
github.com/go-ole/go-ole v1.2.6 // indirect
github.com/goccy/go-json v0.10.0 // indirect
github.com/godbus/dbus v0.0.0-20190726142602-4481cbc300e2 // indirect
@@ -173,14 +164,14 @@ require (
github.com/golang/protobuf v1.5.3 // indirect
github.com/golang/snappy v0.0.4 // indirect
github.com/google/flatbuffers v23.1.21+incompatible // indirect
github.com/google/go-github/v53 v53.2.0 // indirect
github.com/google/go-querystring v1.1.0 // indirect
github.com/google/pprof v0.0.0-20211214055906-6f57359322fd // indirect
github.com/google/s2a-go v0.1.7 // indirect
github.com/googleapis/enterprise-certificate-proxy v0.3.2 // indirect
github.com/google/s2a-go v0.1.4 // indirect
github.com/google/uuid v1.3.0 // indirect
github.com/googleapis/enterprise-certificate-proxy v0.2.5 // indirect
github.com/gorilla/css v1.0.0 // indirect
github.com/gregjones/httpcache v0.0.0-20171119193500-2bcd89a1743f // indirect
github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 // indirect
github.com/grpc-ecosystem/grpc-gateway/v2 v2.15.2 // indirect
github.com/gsterjov/go-libsecret v0.0.0-20161001094733-a6f4afe4910c // indirect
github.com/h2non/parth v0.0.0-20190131123155-b4df798d6542 // indirect
github.com/hashicorp/errwrap v1.0.0 // indirect
@@ -192,78 +183,79 @@ require (
github.com/jpillora/s3 v1.1.4 // indirect
github.com/kevinburke/ssh_config v1.2.0 // indirect
github.com/klauspost/asmfmt v1.3.2 // indirect
github.com/klauspost/compress v1.16.6 // indirect
github.com/klauspost/compress v1.16.5 // indirect
github.com/klauspost/cpuid/v2 v2.2.3 // indirect
github.com/klauspost/pgzip v1.2.5 // indirect
github.com/launchdarkly/ccache v1.1.0 // indirect
github.com/launchdarkly/eventsource v1.6.2 // indirect
github.com/launchdarkly/go-jsonstream/v3 v3.0.0 // indirect
github.com/launchdarkly/go-sdk-common/v3 v3.0.1 // indirect
github.com/launchdarkly/go-sdk-events/v2 v2.0.2 // indirect
github.com/launchdarkly/go-sdk-events/v2 v2.0.1 // indirect
github.com/launchdarkly/go-semver v1.0.2 // indirect
github.com/launchdarkly/go-server-sdk-evaluation/v2 v2.0.2 // indirect
github.com/lucasb-eyer/go-colorful v1.2.0 // indirect
github.com/mailru/easyjson v0.7.7 // indirect
github.com/mattn/go-colorable v0.1.13 // indirect
github.com/mattn/go-localereader v0.0.1 // indirect
github.com/mattn/go-runewidth v0.0.15 // indirect
github.com/mattn/go-runewidth v0.0.14 // indirect
github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
github.com/microcosm-cc/bluemonday v1.0.23 // indirect
github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8 // indirect
github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3 // indirect
github.com/mitchellh/go-homedir v1.1.0 // indirect
github.com/montanaflynn/stats v0.7.0 // indirect
github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe // indirect
github.com/mtibben/percent v0.2.1 // indirect
github.com/muesli/ansi v0.0.0-20211031195517-c9f0611b6c70 // indirect
github.com/muesli/cancelreader v0.2.2 // indirect
github.com/muesli/termenv v0.15.2 // indirect
github.com/muesli/termenv v0.15.1 // indirect
github.com/nwaples/rardecode/v2 v2.0.0-beta.2 // indirect
github.com/olekukonko/tablewriter v0.0.5 // indirect
github.com/onsi/ginkgo v1.16.5 // indirect
github.com/onsi/gomega v1.23.0 // indirect
github.com/opencontainers/go-digest v1.0.0 // indirect
github.com/opencontainers/image-spec v1.1.0-rc3 // indirect
github.com/pierrec/lz4/v4 v4.1.18 // indirect
github.com/pierrec/lz4/v4 v4.1.17 // indirect
github.com/pjbgf/sha1cd v0.3.0 // indirect
github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 // indirect
github.com/pkg/diff v0.0.0-20200914180035-5b29258ca4f7 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/prometheus/client_model v0.4.1-0.20230718164431-9a2bf3000d16 // indirect
github.com/prometheus/common v0.44.0 // indirect
github.com/prometheus/procfs v0.11.1 // indirect
github.com/prometheus/client_model v0.3.0 // indirect
github.com/prometheus/common v0.42.0 // indirect
github.com/prometheus/procfs v0.10.1 // indirect
github.com/rivo/uniseg v0.4.2 // indirect
github.com/rogpeppe/go-internal v1.10.0 // indirect
github.com/sahilm/fuzzy v0.1.0 // indirect
github.com/sirupsen/logrus v1.9.1 // indirect
github.com/skeema/knownhosts v1.2.1 // indirect
github.com/sirupsen/logrus v1.9.0 // indirect
github.com/skeema/knownhosts v1.2.0 // indirect
github.com/therootcompany/xz v1.0.1 // indirect
github.com/ulikunitz/xz v0.5.11 // indirect
github.com/ulikunitz/xz v0.5.10 // indirect
github.com/vbatts/tar-split v0.11.3 // indirect
github.com/xanzy/ssh-agent v0.3.3 // indirect
github.com/xdg-go/pbkdf2 v1.0.0 // indirect
github.com/xdg-go/scram v1.1.2 // indirect
github.com/xdg-go/stringprep v1.0.4 // indirect
github.com/xhit/go-str2duration/v2 v2.1.0 // indirect
github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d // indirect
github.com/yuin/goldmark v1.5.2 // indirect
github.com/yuin/goldmark-emoji v1.0.1 // indirect
github.com/yusufpapurcu/wmi v1.2.2 // indirect
github.com/zeebo/xxh3 v1.0.2 // indirect
go.einride.tech/aip v0.60.0 // indirect
go.opencensus.io v0.24.0 // indirect
go.uber.org/multierr v1.11.0 // indirect
go.uber.org/atomic v1.7.0 // indirect
go.uber.org/multierr v1.6.0 // indirect
go4.org v0.0.0-20200411211856-f5505b9728dd // indirect
golang.org/x/mod v0.14.0 // indirect
golang.org/x/net v0.19.0 // indirect
golang.org/x/sys v0.15.0 // indirect
golang.org/x/term v0.15.0 // indirect
golang.org/x/time v0.5.0 // indirect
golang.org/x/tools v0.16.0 // indirect
golang.org/x/mod v0.12.0 // indirect
golang.org/x/net v0.14.0 // indirect
golang.org/x/sys v0.11.0 // indirect
golang.org/x/term v0.11.0 // indirect
golang.org/x/time v0.3.0 // indirect
golang.org/x/tools v0.12.0 // indirect
golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect
google.golang.org/appengine v1.6.7 // indirect
google.golang.org/genproto v0.0.0-20231106174013-bbf56f31fb17 // indirect
google.golang.org/genproto/googleapis/api v0.0.0-20231106174013-bbf56f31fb17 // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20231120223509-83a465c0220f // indirect
google.golang.org/grpc v1.59.0 // indirect
gopkg.in/square/go-jose.v2 v2.6.0 // indirect
google.golang.org/genproto v0.0.0-20230629202037-9506855d4529 // indirect
google.golang.org/genproto/googleapis/api v0.0.0-20230629202037-9506855d4529 // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20230706204954-ccb25ca9f130 // indirect
google.golang.org/grpc v1.56.2 // indirect
gopkg.in/warnings.v0 v0.1.2 // indirect
gopkg.in/yaml.v2 v2.4.0 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
)
+193 -318
View File
File diff suppressed because it is too large Load Diff
+1 -1
View File
@@ -1,6 +1,6 @@
# trufflesecurity/protos:1.18-0
FROM golang:1.21-bullseye
FROM golang:1.18-buster
ARG TARGETARCH
ARG TARGETOS
+42 -61
View File
@@ -15,9 +15,9 @@ The purpose of Secret Detectors is to discover secrets with exceptionally high s
+ [Sourcing Guidelines](#sourcing-guidelines)
+ [Development Guidelines](#development-guidelines)
+ [Development Dependencies](#development-dependencies)
+ [Creating a new Secret Detector](#creating-a-new-secret-detector)
+ [Testing the Detector](#testing-the-detector)
+ [Creating a new Secret Scanner](#creating-a-new-secret-scanner)
* [Addendum](#addendum)
+ [Using a test secret file](#using-a-test-secret-file)
+ [Adding Protos in Windows](#adding-protos-in-windows)
## Getting Started
@@ -52,77 +52,37 @@ Note: Be sure to update the tests to reference the new secret values in GSM, or
3. Proceed from step 3 of [Creating a new Secret Scanner](#creating-a-new-secret-scanner)
### Creating a new Secret Detector
### Creating a new Secret Scanner
1. Add a new Secret Detector enum to the [`DetectorType` list here](/proto/detectors.proto).
1. Identify the Secret Detector name from the [/proto/detectors.proto](/proto/detectors.proto) `DetectorType` enum. If necessary, run `make protos` when adding new ones.
2. Run `make protos` to update the `.pb` files.
3. Generate the Secret Detector
2. Generate the Secret Detector
```bash
go run hack/generate/generate.go detector <DetectorType enum name>
example: go run hack/generate/generate.go detector SampleAPI
```
4. Add the Secret Detector to TruffleHog's Default Detectors
3. Add Secret Scanner
Add the secret scanner to the [`pkg/engine/defaults.go`](https://github.com/trufflesecurity/trufflehog/blob/main/pkg/engine/defaults.go) file like [`github.com/trufflesecurity/trufflehog/v3/pkg/detectors/<detector_name>`](https://github.com/trufflesecurity/trufflehog/blob/b71ea27a696bdf1c3141f637fda4ee4936c2f2d6/pkg/engine/defaults.go#L9) and
[`<detector_name>.Scanner{}`](https://github.com/trufflesecurity/trufflehog/blob/b71ea27a696bdf1c3141f637fda4ee4936c2f2d6/pkg/engine/defaults.go#L1546)
Add the secret scanner to the `pkg/engine/defaults.go` file like `github.com/trufflesecurity/trufflehog/v3/pkg/detectors/<detector_name>` and
`<detector_name>.Scanner{},`
5. Complete the Secret Detector.
4. Complete the secret detector.
The previous step templated a boilerplate + some example code as a package in the `pkg/detectors` folder for you to work on.
The Secret Detector can be completed with these general steps:
The secret detector can be completed with these general steps:
1. Update the pattern regex and keywords. Try iterating with [regex101.com](http://regex101.com/).
2. Update the verifier code to use a non-destructive API call that can determine whether the secret is valid or not.
1. Create a [test secrets file, and export the variable](#using-a-test-secret-file)
2. Update the pattern regex and keywords. Try iterating with [regex101.com](http://regex101.com/).
3. Update the verifier code to use a non-destructive API call that can determine whether the secret is valid or not.
* Make sure you understand [verification indeterminacy](#verification-indeterminacy).
3. Create a [test for the detector](#testing-the-detector)
4. Create a pull request for review.
### Testing the Detector
To ensure the quality of your PR, make sure your tests are passing with verified credentials.
1. Create a file called `.env` with this env file format:
```bash
SECRET_TYPE_ONE=value
SECRET_TYPE_ONE_INACTIVE=v@lue
```
2. Export the `TEST_SECRET_FILE` variable, pointing to the env file:
```bash
export TEST_SECRET_FILE=".env"
```
The `.env` file should be in the new detector's directory like this:
```
├── tailscale
│   ├── .env
│   ├── tailscale.go
│   └── tailscale_test.go
```
Now that a `.env` file is present, the test file can load secrets locally.
3. Next, update the tests as necessary. A test file has already been generated by the `go run hack/generate/generate.go` command from earlier. There are 5 cases that have been generated:
1. Found and verified (using a credential loaded from the .env file)
2. Found and unverified (determinately, i.e. the secret is invalid)
3. Found and unverified (indeterminately due to timeout)
4. Found and unverified (indeterminately due to an unexpected API response)
5. Not found
Make any necessary updates to the tests. Note there might not be any changes required as the tests generated by the `go run hack/generate/generate.go` command are pretty good.
[Here is an exemplary test file for a detector which covers all 5 test cases](https://github.com/trufflesecurity/trufflehog/blob/6f9065b0aae981133a7fa3431c17a5c6213be226/pkg/detectors/browserstack/browserstack_test.go).
4. Now run the tests and check to make sure they are passing ✔️!
```bash
go test ./pkg/detectors/<detector> -tags=detectors
```
If the tests are passing, feel free to open a PR!
4. Update the tests with these test cases at minimum:
1. Found and verified (using a credential loaded from GCP Secrets)
2. Found and unverified (determinately, i.e. the secret is invalid)
3. Found and unverified (indeterminately due to timeout)
4. Found and unverified (indeterminately due to an unexpected API response)
5. Not found
6. Any false positive cases that you come across
5. Create a pull request for review.
## Addendum
@@ -139,6 +99,23 @@ For example, consider a hypothetical authentication endpoint that returns `200 O
* A `403` response would indicate that verification failed **determinately** and no error object should be returned.
* Any other response would indicate that verification failed **indeterminately** and an error object should be returned.
### Using a test secret file
1. Create a file called `.env` with this env file format:
```bash
SECRET_TYPE_ONE=value
SECRET_TYPE_ONE_INACTIVE=v@lue
```
2. Export the `TEST_SECRET_FILE` variable, pointing to the env file:
```bash
export TEST_SECRET_FILE=".env"
```
Now, the detector test should attempt to load the given env key from that file.
### Adding Protos in Windows
1. Install Ubuntu App in Microsoft Store https://www.microsoft.com/en-us/p/ubuntu/9nblggh4msv6.
@@ -156,3 +133,7 @@ For example, consider a hypothetical authentication endpoint that returns `200 O
make protos
```
### Testing a detector
```bash
go test ./pkg/detectors/<detector> -tags=detectors
```
+1 -1
View File
@@ -8,10 +8,10 @@ import (
"strings"
"text/template"
"github.com/alecthomas/kingpin/v2"
"github.com/go-errors/errors"
"golang.org/x/text/cases"
"golang.org/x/text/language"
"gopkg.in/alecthomas/kingpin.v2"
)
var (
+4 -4
View File
@@ -10,9 +10,9 @@ import (
"sync/atomic"
"time"
"github.com/alecthomas/kingpin/v2"
"github.com/paulbellamy/ratecounter"
"golang.org/x/sync/semaphore"
"gopkg.in/alecthomas/kingpin.v2"
"github.com/trufflesecurity/trufflehog/v3/pkg/context"
"github.com/trufflesecurity/trufflehog/v3/pkg/decoders"
@@ -74,7 +74,7 @@ func main() {
selectedScanners := map[string]detectors.Detector{}
allScanners := getAllScanners()
allDecoders := decoders.DefaultDecoders()
decoders := decoders.DefaultDecoders()
input := strings.ToLower(*scanCmdDetector)
if input == "all" {
@@ -121,7 +121,7 @@ func main() {
for chunk := range chunksChan {
for name, scanner := range selectedScanners {
for _, dec := range allDecoders {
for _, dec := range decoders {
decoded := dec.FromChunk(&sources.Chunk{Data: chunk.Data})
if decoded != nil {
foundKeyword := false
@@ -204,7 +204,7 @@ func main() {
})
logger.Info("scanning repo", "repo", r)
err = s.ScanRepo(ctx, repo, path, git.NewScanOptions(), sources.ChanReporter{Ch: chunksChan})
err = s.ScanRepo(ctx, repo, path, git.NewScanOptions(), chunksChan)
if err != nil {
logFatal(err, "error scanning repo")
}
+44 -45
View File
@@ -10,14 +10,13 @@ import (
"strings"
"syscall"
"github.com/alecthomas/kingpin/v2"
"github.com/felixge/fgprof"
"github.com/go-logr/logr"
"github.com/jpillora/overseer"
"github.com/mattn/go-isatty"
"google.golang.org/protobuf/types/known/anypb"
"gopkg.in/alecthomas/kingpin.v2"
"github.com/trufflesecurity/trufflehog/v3/pkg/cleantemp"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/config"
"github.com/trufflesecurity/trufflehog/v3/pkg/context"
@@ -29,6 +28,7 @@ import (
"github.com/trufflesecurity/trufflehog/v3/pkg/output"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
"github.com/trufflesecurity/trufflehog/v3/pkg/sources/git"
"github.com/trufflesecurity/trufflehog/v3/pkg/tui"
"github.com/trufflesecurity/trufflehog/v3/pkg/updater"
"github.com/trufflesecurity/trufflehog/v3/pkg/version"
@@ -48,7 +48,6 @@ var (
noVerification = cli.Flag("no-verification", "Don't verify the results.").Bool()
onlyVerified = cli.Flag("only-verified", "Only output verified results.").Bool()
filterUnverified = cli.Flag("filter-unverified", "Only output first unverified result per chunk per detector if there are more than one results.").Bool()
filterEntropy = cli.Flag("filter-entropy", "Filter unverified results with Shannon entropy. Start with 3.0.").Float64()
configFilename = cli.Flag("config", "Path to configuration file.").ExistingFile()
// rules = cli.Flag("rules", "Path to file with custom rules.").String()
printAvgDetectorTime = cli.Flag("print-avg-detector-time", "Print the average time spent on each detector.").Bool()
@@ -85,8 +84,8 @@ var (
githubExcludeRepos = githubScan.Flag("exclude-repos", `Repositories to exclude in an org scan. This can also be a glob pattern. You can repeat this flag. Must use Github repo full name. Example: "trufflesecurity/driftwood", "trufflesecurity/d*"`).Strings()
githubScanIncludePaths = githubScan.Flag("include-paths", "Path to file with newline separated regexes for files to include in scan.").Short('i').String()
githubScanExcludePaths = githubScan.Flag("exclude-paths", "Path to file with newline separated regexes for files to exclude in scan.").Short('x').String()
githubScanIssueComments = githubScan.Flag("issue-comments", "Include issue descriptions and comments in scan.").Bool()
githubScanPRComments = githubScan.Flag("pr-comments", "Include pull request descriptions and comments in scan.").Bool()
githubScanIssueComments = githubScan.Flag("issue-comments", "Include issue comments in scan.").Bool()
githubScanPRComments = githubScan.Flag("pr-comments", "Include pull request comments in scan.").Bool()
githubScanGistComments = githubScan.Flag("gist-comments", "Include gist comments in scan.").Bool()
gitlabScan = cli.Command("gitlab", "Find credentials in GitLab repositories.")
@@ -106,11 +105,11 @@ var (
filesystemScanIncludePaths = filesystemScan.Flag("include-paths", "Path to file with newline separated regexes for files to include in scan.").Short('i').String()
filesystemScanExcludePaths = filesystemScan.Flag("exclude-paths", "Path to file with newline separated regexes for files to exclude in scan.").Short('x').String()
s3Scan = cli.Command("s3", "Find credentials in S3 buckets.")
s3ScanKey = s3Scan.Flag("key", "S3 key used to authenticate. Can be provided with environment variable AWS_ACCESS_KEY_ID.").Envar("AWS_ACCESS_KEY_ID").String()
s3ScanRoleArns = s3Scan.Flag("role-arn", "Specify the ARN of an IAM role to assume for scanning. You can repeat this flag.").Strings()
s3ScanSecret = s3Scan.Flag("secret", "S3 secret used to authenticate. Can be provided with environment variable AWS_SECRET_ACCESS_KEY.").Envar("AWS_SECRET_ACCESS_KEY").String()
s3ScanSessionToken = s3Scan.Flag("session-token", "S3 session token used to authenticate temporary credentials. Can be provided with environment variable AWS_SESSION_TOKEN.").Envar("AWS_SESSION_TOKEN").String()
s3Scan = cli.Command("s3", "Find credentials in S3 buckets.")
s3ScanKey = s3Scan.Flag("key", "S3 key used to authenticate. Can be provided with environment variable AWS_ACCESS_KEY_ID.").Envar("AWS_ACCESS_KEY_ID").String()
s3ScanRoleArns = s3Scan.Flag("role-arn", "Specify the ARN of an IAM role to assume for scanning. You can repeat this flag.").Strings()
s3ScanSecret = s3Scan.Flag("secret", "S3 secret used to authenticate. Can be provided with environment variable AWS_SECRET_ACCESS_KEY.").Envar("AWS_SECRET_ACCESS_KEY").String()
s3ScanSessionToken = s3Scan.Flag("session-token", "S3 session token used to authenticate temporary credentials. Can be provided with environment variable AWS_SESSION_TOKEN.").Envar("AWS_SESSION_TOKEN").String()
s3ScanCloudEnv = s3Scan.Flag("cloud-environment", "Use IAM credentials in cloud environment.").Bool()
s3ScanBuckets = s3Scan.Flag("bucket", "Name of S3 bucket to scan. You can repeat this flag.").Strings()
s3ScanMaxObjectSize = s3Scan.Flag("max-object-size", "Maximum size of objects to scan. Objects larger than this will be skipped. (Byte units eg. 512B, 2KB, 4MB)").Default("250MB").Bytes()
@@ -139,9 +138,6 @@ var (
dockerScan = cli.Command("docker", "Scan Docker Image")
dockerScanImages = dockerScan.Flag("image", "Docker image to scan. Use the file:// prefix to point to a local tarball, otherwise a image registry is assumed.").Required().Strings()
travisCiScan = cli.Command("travisci", "Scan TravisCI")
travisCiScanToken = travisCiScan.Flag("token", "TravisCI token. Can also be provided with environment variable").Envar("TRAVISCI_TOKEN").Required().String()
)
func init() {
@@ -155,9 +151,6 @@ func init() {
cli.Version("trufflehog " + version.BuildVersion)
// Support -h for help
cli.HelpFlag.Short('h')
if len(os.Args) <= 1 && isatty.IsTerminal(os.Stdout.Fd()) {
args := tui.Run()
if len(args) == 0 {
@@ -216,11 +209,6 @@ func main() {
if err != nil {
logFatal(err, "error occurred with trufflehog updater 🐷")
}
ctx := context.Background()
go cleantemp.RunCleanupLoop(ctx)
}
func run(state overseer.State) {
@@ -366,16 +354,11 @@ func run(state overseer.State) {
printer = new(output.PlainPrinter)
}
if !*jsonLegacy && !*jsonOut {
fmt.Fprintf(os.Stderr, "🐷🔑🐷 TruffleHog. Unearth your secrets. 🐷🔑🐷\n\n")
}
e, err := engine.Start(ctx,
engine.WithConcurrency(uint8(*concurrency)),
engine.WithDecoders(decoders.DefaultDecoders()...),
engine.WithDetectors(engine.DefaultDetectors()...),
engine.WithDetectors(conf.Detectors...),
engine.WithVerify(!*noVerification),
engine.WithDetectors(!*noVerification, engine.DefaultDetectors()...),
engine.WithDetectors(!*noVerification, conf.Detectors...),
engine.WithFilterDetectors(includeFilter),
engine.WithFilterDetectors(excludeFilter),
engine.WithFilterDetectors(endpointCustomizer),
@@ -383,23 +366,39 @@ func run(state overseer.State) {
engine.WithOnlyVerified(*onlyVerified),
engine.WithPrintAvgDetectorTime(*printAvgDetectorTime),
engine.WithPrinter(printer),
engine.WithFilterEntropy(*filterEntropy),
)
if err != nil {
logFatal(err, "error initializing engine")
}
var repoPath string
var remote bool
switch cmd {
case gitScan.FullCommand():
filter, err := common.FilterFromFiles(*gitScanIncludePaths, *gitScanExcludePaths)
if err != nil {
logFatal(err, "could not create filter")
}
repoPath, remote, err = git.PrepareRepoSinceCommit(ctx, *gitScanURI, *gitScanSinceCommit)
if err != nil || repoPath == "" {
logFatal(err, "error preparing git repo for scanning")
}
if remote {
defer os.RemoveAll(repoPath)
}
excludedGlobs := []string{}
if *gitScanExcludeGlobs != "" {
excludedGlobs = strings.Split(*gitScanExcludeGlobs, ",")
}
cfg := sources.GitConfig{
URI: *gitScanURI,
IncludePathsFile: *gitScanIncludePaths,
ExcludePathsFile: *gitScanExcludePaths,
HeadRef: *gitScanBranch,
BaseRef: *gitScanSinceCommit,
MaxDepth: *gitScanMaxDepth,
Bare: *gitScanBare,
ExcludeGlobs: *gitScanExcludeGlobs,
RepoPath: repoPath,
HeadRef: *gitScanBranch,
BaseRef: *gitScanSinceCommit,
MaxDepth: *gitScanMaxDepth,
Bare: *gitScanBare,
Filter: filter,
ExcludeGlobs: excludedGlobs,
}
if err = e.ScanGit(ctx, cfg); err != nil {
logFatal(err, "Failed to scan Git.")
@@ -470,7 +469,7 @@ func run(state overseer.State) {
Secret: *s3ScanSecret,
SessionToken: *s3ScanSessionToken,
Buckets: *s3ScanBuckets,
Roles: *s3ScanRoleArns,
Roles: *s3ScanRoleArns,
CloudCred: *s3ScanCloudEnv,
MaxObjectSize: int64(*s3ScanMaxObjectSize),
}
@@ -493,10 +492,6 @@ func run(state overseer.State) {
if err := e.ScanCircleCI(ctx, *circleCiScanToken); err != nil {
logFatal(err, "Failed to scan CircleCI.")
}
case travisCiScan.FullCommand():
if err := e.ScanTravisCI(ctx, *travisCiScanToken); err != nil {
logFatal(err, "Failed to scan TravisCI.")
}
case gcsScan.FullCommand():
cfg := sources.GCSConfig{
ProjectID: *gcsProjectID,
@@ -530,6 +525,10 @@ func run(state overseer.State) {
}
}
if !*jsonLegacy && !*jsonOut {
fmt.Fprintf(os.Stderr, "🐷🔑🐷 TruffleHog. Unearth your secrets. 🐷🔑🐷\n\n")
}
// Wait for all workers to finish.
if err = e.Finish(ctx); err != nil {
logFatal(err, "engine failed to finish execution")
@@ -591,11 +590,11 @@ func printAverageDetectorTime(e *engine.Engine) {
// detectorTypeToSet is a helper function to convert a slice of detector IDs into a set.
func detectorTypeToSet(detectors []config.DetectorID) map[config.DetectorID]struct{} {
out := make(map[config.DetectorID]struct{}, len(detectors))
output := make(map[config.DetectorID]struct{}, len(detectors))
for _, d := range detectors {
out[d] = struct{}{}
output[d] = struct{}{}
}
return out
return output
}
// getWithDetectorID is a helper function to get a value from a map using a
+1 -1
View File
@@ -94,7 +94,7 @@ func (c *Cache) Values() []string {
return res
}
// Contents returns a comma-separated string containing all keys in the cache.
// Contents returns all key-value pairs in the cache encodes as a string.
func (c *Cache) Contents() string {
items := c.c.Items()
res := make([]string, 0, len(items))
-115
View File
@@ -1,115 +0,0 @@
package cleantemp
import (
"context"
"fmt"
"os"
"path/filepath"
"regexp"
"strconv"
"strings"
"time"
"github.com/mitchellh/go-ps"
logContext "github.com/trufflesecurity/trufflehog/v3/pkg/context"
)
// MkdirTemp returns a temporary directory path formatted as:
// trufflehog-<pid>-<randint>
func MkdirTemp() (string, error) {
pid := os.Getpid()
tmpdir := fmt.Sprintf("%s-%d-", "trufflehog", pid)
dir, err := os.MkdirTemp(os.TempDir(), tmpdir)
if err != nil {
return "", err
}
return dir, nil
}
// CleanTemp is used to remove orphaned artifacts from aborted scans.
type CleanTemp interface {
// CleanTempDir removes orphaned directories from sources. ex: Git
CleanTempDir(ctx logContext.Context, dirName string, pid int) error
// CleanTempFiles removes orphaned files/artifacts from sources. ex: Artifactory
CleanTempFiles(ctx context.Context, fileName string, pid int) error
}
// Only compile during startup.
var trufflehogRE = regexp.MustCompile(`^trufflehog-\d+-\d+$`)
// CleanTempDir removes orphaned temp directories that do not contain running PID values.
func CleanTempDir(ctx logContext.Context) error {
const defaultExecPath = "trufflehog"
executablePath, err := os.Executable()
if err != nil {
executablePath = defaultExecPath
}
execName := filepath.Base(executablePath)
// Finds other trufflehog PIDs that may be running
var pids []string
procs, err := ps.Processes()
if err != nil {
return fmt.Errorf("error getting jobs PIDs: %w", err)
}
for _, proc := range procs {
if proc.Executable() == execName {
pids = append(pids, strconv.Itoa(proc.Pid()))
}
}
tempDir := os.TempDir()
dirs, err := os.ReadDir(tempDir)
if err != nil {
return fmt.Errorf("error reading temp dir: %w", err)
}
for _, dir := range dirs {
// Ensure that all directories match the pattern.
if trufflehogRE.MatchString(dir.Name()) {
// Mark these directories initially as ones that should be deleted.
shouldDelete := true
// If they match any live PIDs, mark as should not delete.
for _, pidval := range pids {
if strings.Contains(dir.Name(), fmt.Sprintf("-%s-", pidval)) {
shouldDelete = false
// break out so we can still delete directories even if no other Trufflehog processes are running.
break
}
}
if shouldDelete {
dirPath := filepath.Join(tempDir, dir.Name())
if err := os.RemoveAll(dirPath); err != nil {
return fmt.Errorf("error deleting temp directory: %s", dirPath)
}
ctx.Logger().V(1).Info("Deleted directory", "directory", dirPath)
}
}
}
return nil
}
// RunCleanupLoop runs a loop that cleans up orphaned directories every 15 seconds
func RunCleanupLoop(ctx logContext.Context) {
if err := CleanTempDir(ctx); err != nil {
ctx.Logger().Error(err, "error cleaning up orphaned directories ")
}
const cleanupLoopInterval = 15 * time.Second
ticker := time.NewTicker(cleanupLoopInterval)
defer ticker.Stop()
for {
select {
case <-ticker.C:
if err := CleanTempDir(ctx); err != nil {
ctx.Logger().Error(err, "error cleaning up orphaned directories")
}
case <-ctx.Done():
ctx.Logger().Info("Cleanup loop exiting due to context cancellation")
return
}
}
}
-30
View File
@@ -1,30 +0,0 @@
package cleantemp
import (
"os"
"path/filepath"
"testing"
"github.com/mitchellh/go-ps"
"github.com/stretchr/testify/assert"
)
func TestExecName(t *testing.T) {
executablePath, err := os.Executable()
assert.Nil(t, err)
execName := filepath.Base(executablePath)
assert.Equal(t, "cleantemp.test", execName)
procs, err := ps.Processes()
assert.Nil(t, err)
assert.NotEmpty(t, procs)
found := false
for _, proc := range procs {
if proc.Executable() == execName {
found = true
}
}
assert.True(t, found)
}
-120
View File
@@ -1,120 +0,0 @@
package glob
import (
"fmt"
"github.com/gobwas/glob"
)
// Filter is a generic filter for excluding and including globs (limited
// regular expressions). Exclusion takes precedence if both include and exclude
// lists are provided.
type Filter struct {
exclude []glob.Glob
include []glob.Glob
}
type globFilterOpt func(*Filter) error
// WithExcludeGlobs adds exclude globs to the filter.
func WithExcludeGlobs(excludes ...string) globFilterOpt {
return func(f *Filter) error {
for _, exclude := range excludes {
g, err := glob.Compile(exclude)
if err != nil {
return fmt.Errorf("invalid exclude glob %q: %w", exclude, err)
}
f.exclude = append(f.exclude, g)
}
return nil
}
}
// WithIncludeGlobs adds include globs to the filter.
func WithIncludeGlobs(includes ...string) globFilterOpt {
return func(f *Filter) error {
for _, include := range includes {
g, err := glob.Compile(include)
if err != nil {
return fmt.Errorf("invalid include glob %q: %w", include, err)
}
f.include = append(f.include, g)
}
return nil
}
}
// NewGlobFilter creates a new Filter with the provided options.
func NewGlobFilter(opts ...globFilterOpt) (*Filter, error) {
filter := &Filter{}
for _, opt := range opts {
if err := opt(filter); err != nil {
return nil, err
}
}
return filter, nil
}
// ShouldInclude returns whether the object is in the include list or not in
// the exclude list (exclude taking precedence).
func (f *Filter) ShouldInclude(object string) bool {
if f == nil {
return true
}
exclude, include := len(f.exclude), len(f.include)
if exclude == 0 && include == 0 {
return true
} else if exclude > 0 && include == 0 {
return f.shouldIncludeFromExclude(object)
} else if exclude == 0 && include > 0 {
return f.shouldIncludeFromInclude(object)
} else {
if ok, err := f.shouldIncludeFromBoth(object); err == nil {
return ok
}
// Ambiguous case.
return false
}
}
// shouldIncludeFromExclude checks for explicitly excluded paths. This should
// only be called when the include list is empty.
func (f *Filter) shouldIncludeFromExclude(object string) bool {
for _, g := range f.exclude {
if g.Match(object) {
return false
}
}
return true
}
// shouldIncludeFromInclude checks for explicitly included paths. This should
// only be called when the exclude list is empty.
func (f *Filter) shouldIncludeFromInclude(object string) bool {
for _, g := range f.include {
if g.Match(object) {
return true
}
}
return false
}
// shouldIncludeFromBoth checks for either excluded or included paths. Exclusion
// takes precedence. If neither list contains the object, true is returned.
func (f *Filter) shouldIncludeFromBoth(object string) (bool, error) {
// Exclude takes precedence. If we find the object in the exclude list,
// we should not match.
for _, g := range f.exclude {
if g.Match(object) {
return false, nil
}
}
// If we find the object in the include list, we should match.
for _, g := range f.include {
if g.Match(object) {
return true, nil
}
}
// If we find it in neither, return an error to let the caller decide.
return false, fmt.Errorf("ambiguous match")
}
-138
View File
@@ -1,138 +0,0 @@
package glob
import (
"testing"
"github.com/stretchr/testify/assert"
"pgregory.net/rapid"
)
type globTest struct {
input string
shouldInclude bool
}
func testGlobs(t *testing.T, filter *Filter, tests ...globTest) {
for _, tt := range tests {
t.Run(tt.input, func(t *testing.T) {
// Invert because mentally it's easier to say whether an
// input should be included.
assert.Equal(t, tt.shouldInclude, filter.ShouldInclude(tt.input))
})
}
}
func TestGlobFilterExclude(t *testing.T) {
filter, err := NewGlobFilter(WithExcludeGlobs("foo", "bar*"))
assert.NoError(t, err)
testGlobs(t, filter,
globTest{"foo", false},
globTest{"bar", false},
globTest{"bara", false},
globTest{"barb", false},
globTest{"barbosa", false},
globTest{"foobar", true},
globTest{"food", true},
globTest{"anything else", true},
)
}
func TestGlobFilterInclude(t *testing.T) {
filter, err := NewGlobFilter(WithIncludeGlobs("foo", "bar*"))
assert.NoError(t, err)
testGlobs(t, filter,
globTest{"foo", true},
globTest{"bar", true},
globTest{"bara", true},
globTest{"barb", true},
globTest{"barbosa", true},
globTest{"foobar", false},
globTest{"food", false},
globTest{"anything else", false},
)
}
func TestGlobFilterEmpty(t *testing.T) {
filter, err := NewGlobFilter()
assert.NoError(t, err)
testGlobs(t, filter,
globTest{"foo", true},
globTest{"bar", true},
globTest{"bara", true},
globTest{"barb", true},
globTest{"barbosa", true},
globTest{"foobar", true},
globTest{"food", true},
globTest{"anything else", true},
)
}
func TestGlobFilterExcludeInclude(t *testing.T) {
filter, err := NewGlobFilter(WithExcludeGlobs("/foo/bar/**"), WithIncludeGlobs("/foo/**"))
assert.NoError(t, err)
testGlobs(t, filter,
globTest{"/foo/a", true},
globTest{"/foo/b", true},
globTest{"/foo/c/d/e", true},
globTest{"/foo/bar/a", false},
globTest{"/foo/bar/b", false},
globTest{"/foo/bar/c/d/e", false},
globTest{"/any/other/path", false},
)
}
func TestGlobFilterExcludePrecedence(t *testing.T) {
filter, err := NewGlobFilter(WithExcludeGlobs("foo"), WithIncludeGlobs("foo*"))
assert.NoError(t, err)
testGlobs(t, filter,
globTest{"foo", false},
globTest{"foobar", true},
)
}
func TestGlobErrorContainsGlob(t *testing.T) {
invalidGlob := "[this is invalid because it doesn't close the capture group"
_, err := NewGlobFilter(WithExcludeGlobs(invalidGlob))
assert.Error(t, err)
assert.Contains(t, err.Error(), invalidGlob)
}
// The filters in this test should be mutually exclusive because one includes
// and the other excludes the same glob.
func TestGlobInverse(t *testing.T) {
for _, glob := range []string{
"a",
"a*",
"a**",
"*a",
"**a",
"*",
} {
include, err := NewGlobFilter(WithIncludeGlobs(glob))
assert.NoError(t, err)
exclude, err := NewGlobFilter(WithExcludeGlobs(glob))
assert.NoError(t, err)
rapid.Check(t, func(t *rapid.T) {
input := rapid.String().Draw(t, "input")
a, b := include.ShouldInclude(input), exclude.ShouldInclude(input)
if a == b {
t.Fatalf("Filter(Include(%q)) == Filter(Exclude(%q)) == %v for input %q", glob, glob, a, input)
}
})
}
}
func TestGlobDefaultFilters(t *testing.T) {
for _, filter := range []*Filter{nil, {}} {
rapid.Check(t, func(t *rapid.T) {
if !filter.ShouldInclude(rapid.String().Draw(t, "input")) {
t.Fatalf("filter %#v did not include input", filter)
}
})
}
}
+13 -27
View File
@@ -2,10 +2,8 @@ package common
import (
"bufio"
"bytes"
"crypto/rand"
"encoding/base32"
"encoding/binary"
"fmt"
"io"
"math/big"
"strings"
@@ -29,6 +27,18 @@ func RemoveStringSliceItem(item string, slice *[]string) {
}
}
func MinInt(a, b int) int {
if a < b {
return a
}
return b
}
func BytesEqual(a, b []byte, numBytes int) bool {
limit := MinInt(numBytes, MinInt(len(a), len(b))-1)
return bytes.Equal(a[:limit], b[:limit])
}
func ResponseContainsSubstring(reader io.ReadCloser, target string) (bool, error) {
scanner := bufio.NewScanner(reader)
for scanner.Scan() {
@@ -54,27 +64,3 @@ func RandomID(length int) string {
return string(b)
}
func GetAccountNumFromAWSID(AWSID string) (string, error) {
// Function to get the account number from an AWS ID (no verification required)
// Source: https://medium.com/@TalBeerySec/a-short-note-on-aws-key-id-f88cc4317489
if len(AWSID) < 4 {
return "", fmt.Errorf("AWSID is too short")
}
trimmedAWSID := AWSID[4:]
decodedBytes, err := base32.StdEncoding.WithPadding(base32.NoPadding).DecodeString(strings.ToUpper(trimmedAWSID))
if err != nil {
return "", err
}
if len(decodedBytes) < 6 {
return "", fmt.Errorf("Decoded AWSID is too short")
}
data := make([]byte, 8)
copy(data[2:], decodedBytes[0:6])
z := binary.BigEndian.Uint64(data)
const mask uint64 = 0x7fffffffff80
accountNum := (z & mask) >> 7
return fmt.Sprintf("%012d", accountNum), nil
}
+1 -44
View File
@@ -7,50 +7,7 @@ import (
var (
KB, MB, GB, TB, PB = 1e3, 1e6, 1e9, 1e12, 1e15
IgnoredExtensions = []string{
// multimedia/containers
"mp4",
"avi",
"mpeg",
"mpg",
"mov",
"wmv",
"m4p",
"swf",
"mp2",
"flv",
"vob",
"webm",
"hdv",
"3gp",
"ogg",
"mp3",
"wav",
"flac",
"webp",
// images
"png",
"jpg",
"jpeg",
"gif",
"tiff",
// binaries
// These can theoretically contain secrets, but need decoding for users to make sense of them, and we don't have
// any such decoders right now.
"class",
"dll",
"xsb",
"jdo",
"jar",
"ear",
"war",
"jks",
"ser",
"idx",
"hprof",
}
IgnoredExtensions = []string{"mp4", "avi", "mpeg", "mpg", "mov", "wmv", "m4p", "swf", "mp2", "flv", "vob", "webm", "hdv", "3gp", "ogg", "mp3", "wav", "flac", "webp"}
)
func SkipFile(filename string) bool {
+3 -3
View File
@@ -31,15 +31,15 @@ func NewYAML(input []byte) (*Config, error) {
return nil, err
}
// Convert the structured YAML into detectors.
var d []detectors.Detector
var detectors []detectors.Detector
for _, detectorConfig := range messages.Detectors {
detector, err := custom_detectors.NewWebhookCustomRegex(detectorConfig)
if err != nil {
return nil, err
}
d = append(d, detector)
detectors = append(detectors, detector)
}
return &Config{
Detectors: d,
Detectors: detectors,
}, nil
}
+3 -3
View File
@@ -66,15 +66,15 @@ func ParseDetectors(input string) ([]DetectorID, error) {
if item == "" {
continue
}
allDetectors, ok := specialGroups[strings.ToLower(item)]
detectors, ok := specialGroups[strings.ToLower(item)]
if !ok {
var err error
allDetectors, err = asRange(item)
detectors, err = asRange(item)
if err != nil {
return nil, err
}
}
for _, d := range allDetectors {
for _, d := range detectors {
if _, ok := seenDetector[d]; ok {
continue
}
+10 -10
View File
@@ -20,20 +20,20 @@ import (
// for poorly defined regexps.
const maxTotalMatches = 100
// CustomRegexWebhook is a CustomRegex with webhook validation that is
// customRegexWebhook is a CustomRegex with webhook validation that is
// guaranteed to be valid (assuming the data is not changed after
// initialization).
type CustomRegexWebhook struct {
type customRegexWebhook struct {
*custom_detectorspb.CustomRegex
}
// Ensure the Scanner satisfies the interface at compile time.
var _ detectors.Detector = (*CustomRegexWebhook)(nil)
var _ detectors.Detector = (*customRegexWebhook)(nil)
// NewWebhookCustomRegex initializes and validates a CustomRegexWebhook. An
// NewWebhookCustomRegex initializes and validates a customRegexWebhook. An
// unexported type is intentionally returned here to ensure the values have
// been validated.
func NewWebhookCustomRegex(pb *custom_detectorspb.CustomRegex) (*CustomRegexWebhook, error) {
func NewWebhookCustomRegex(pb *custom_detectorspb.CustomRegex) (*customRegexWebhook, error) {
// TODO: Return all validation errors.
if err := ValidateKeywords(pb.Keywords); err != nil {
return nil, err
@@ -52,12 +52,12 @@ func NewWebhookCustomRegex(pb *custom_detectorspb.CustomRegex) (*CustomRegexWebh
}
// TODO: Copy only necessary data out of pb.
return &CustomRegexWebhook{pb}, nil
return &customRegexWebhook{pb}, nil
}
var httpClient = common.SaneHttpClient()
func (c *CustomRegexWebhook) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
func (c *customRegexWebhook) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
dataStr := string(data)
regexMatches := make(map[string][][]string, len(c.GetRegex()))
@@ -109,7 +109,7 @@ func (c *CustomRegexWebhook) FromData(ctx context.Context, verify bool, data []b
return results, nil
}
func (c *CustomRegexWebhook) createResults(ctx context.Context, match map[string][]string, verify bool, results chan<- detectors.Result) error {
func (c *customRegexWebhook) createResults(ctx context.Context, match map[string][]string, verify bool, results chan<- detectors.Result) error {
if common.IsDone(ctx) {
// TODO: Log we're possibly leaving out results.
return ctx.Err()
@@ -180,7 +180,7 @@ func (c *CustomRegexWebhook) createResults(ctx context.Context, match map[string
}
}
func (c *CustomRegexWebhook) Keywords() []string {
func (c *customRegexWebhook) Keywords() []string {
return c.GetKeywords()
}
@@ -245,6 +245,6 @@ func permutateMatches(regexMatches map[string][][]string) []map[string][]string
return matches
}
func (c *CustomRegexWebhook) Type() detectorspb.DetectorType {
func (c *customRegexWebhook) Type() detectorspb.DetectorType {
return detectorspb.DetectorType_CustomRegex
}
@@ -105,7 +105,7 @@ func TestCustomDetectorsParsing(t *testing.T) {
}
func TestFromData_InvalidRegEx(t *testing.T) {
c := &CustomRegexWebhook{
c := &customRegexWebhook{
&custom_detectorspb.CustomRegex{
Name: "Internal bi tool",
Keywords: []string{"secret_v1_", "pat_v2_"},
+2 -4
View File
@@ -4,7 +4,6 @@ import (
"bytes"
"encoding/base64"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
)
@@ -26,8 +25,7 @@ func init() {
}
}
func (d *Base64) FromChunk(chunk *sources.Chunk) *DecodableChunk {
decodableChunk := &DecodableChunk{Chunk: chunk, DecoderType: detectorspb.DecoderType_BASE64}
func (d *Base64) FromChunk(chunk *sources.Chunk) *sources.Chunk {
encodedSubstrings := getSubstringsOfCharacterSet(chunk.Data, 20, b64CharsetMapping, b64EndChars)
decodedSubstrings := make(map[string][]byte)
@@ -63,7 +61,7 @@ func (d *Base64) FromChunk(chunk *sources.Chunk) *DecodableChunk {
}
result.Write(chunk.Data[start:])
chunk.Data = result.Bytes()
return decodableChunk
return chunk
}
return nil
+1 -9
View File
@@ -1,7 +1,6 @@
package decoders
import (
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
)
@@ -14,15 +13,8 @@ func DefaultDecoders() []Decoder {
}
}
// DecodableChunk is a chunk that includes the type of decoder used.
// This allows us to avoid a type assertion on each decoder.
type DecodableChunk struct {
*sources.Chunk
DecoderType detectorspb.DecoderType
}
type Decoder interface {
FromChunk(chunk *sources.Chunk) *DecodableChunk
FromChunk(chunk *sources.Chunk) *sources.Chunk
}
// Fuzz is an entrypoint for go-fuzz, which is an AFL-style fuzzing tool.
+2 -4
View File
@@ -5,24 +5,22 @@ import (
"encoding/binary"
"unicode/utf8"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
)
type UTF16 struct{}
func (d *UTF16) FromChunk(chunk *sources.Chunk) *DecodableChunk {
func (d *UTF16) FromChunk(chunk *sources.Chunk) *sources.Chunk {
if chunk == nil || len(chunk.Data) == 0 {
return nil
}
decodableChunk := &DecodableChunk{Chunk: chunk, DecoderType: detectorspb.DecoderType_UTF16}
if utf16Data, err := utf16ToUTF8(chunk.Data); err == nil {
if len(utf16Data) == 0 {
return nil
}
chunk.Data = utf16Data
return decodableChunk
return chunk
}
return nil
+3 -6
View File
@@ -4,25 +4,22 @@ import (
"bytes"
"unicode/utf8"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
)
type UTF8 struct{}
func (d *UTF8) FromChunk(chunk *sources.Chunk) *DecodableChunk {
func (d *UTF8) FromChunk(chunk *sources.Chunk) *sources.Chunk {
if chunk == nil || len(chunk.Data) == 0 {
return nil
}
decodableChunk := &DecodableChunk{Chunk: chunk, DecoderType: detectorspb.DecoderType_PLAIN}
if !utf8.Valid(chunk.Data) {
chunk.Data = extractSubstrings(chunk.Data)
return decodableChunk
return chunk
}
return decodableChunk
return chunk
}
// extractSubstrings performs similarly to the strings binutil,
@@ -55,9 +55,6 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
Raw: []byte(resMatch),
RawV2: []byte(resMatch + resIdMatch),
}
s1.ExtraData = map[string]string{
"rotation_guide": "https://howtorotate.com/docs/tutorials/airbrake/",
}
if verify {
payload := strings.NewReader(`{"environment":"production","username":"john","email":"john@smith.com","repository":"https://github.com/airbrake/airbrake","revision":"38748467ea579e7ae64f7815452307c9d05e05c5","version":"v2.0"}`)
@@ -45,9 +45,6 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
DetectorType: detectorspb.DetectorType_AirbrakeUserKey,
Raw: []byte(resMatch),
}
s1.ExtraData = map[string]string{
"rotation_guide": "https://howtorotate.com/docs/tutorials/airbrake/",
}
if verify {
req, err := http.NewRequestWithContext(ctx, "GET", "https://api.airbrake.io/api/v4/projects?key="+resMatch, nil)
+2 -27
View File
@@ -2,7 +2,6 @@ package airtableapikey
import (
"context"
"encoding/json"
"fmt"
"net/http"
"regexp"
@@ -21,9 +20,8 @@ var _ detectors.Detector = (*Scanner)(nil)
var (
client = common.SaneHttpClient()
appPat = regexp.MustCompile(`(app[a-zA-Z0-9_-]{14})`) // could be part of url
keyPat = regexp.MustCompile(`\b(key[a-zA-Z0-9_-]{14})\b`)
personalPat = regexp.MustCompile(`(\bpat[[:alnum:]]{14}\.[[:alnum:]]{64}\b)`)
appPat = regexp.MustCompile(`(app[a-zA-Z0-9_-]{14})`) // could be part of url
keyPat = regexp.MustCompile(`\b(key[a-zA-Z0-9_-]{14})\b`)
)
// Keywords are used for efficiently pre-filtering chunks.
@@ -32,31 +30,17 @@ func (s Scanner) Keywords() []string {
return []string{"airtable"}
}
type response struct {
Error struct {
Type string `json:"type"`
Message string `json:"message"`
} `json:"error"`
}
// FromData will find and optionally verify AirtableApiKey secrets in a given set of bytes.
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
dataStr := string(data)
appMatches := appPat.FindAllStringSubmatch(dataStr, -1)
keyMatches := keyPat.FindAllStringSubmatch(dataStr, -1)
personalKeyMatches := personalPat.FindAllStringSubmatch(dataStr, -1)
if len(keyMatches) == 0 {
keyMatches = personalKeyMatches
}
for _, keyMatch := range keyMatches {
if len(keyMatch) != 2 {
continue
}
keyRes := strings.TrimSpace(keyMatch[1])
for _, appMatch := range appMatches {
@@ -83,15 +67,6 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
defer res.Body.Close()
if res.StatusCode >= 200 && res.StatusCode < 300 {
s1.Verified = true
} else if res.StatusCode == 403 {
var resp response
if err = json.NewDecoder(res.Body).Decode(&resp); err == nil {
// check if the error is due to invalid permissions or model not found
if resp.Error.Type == "INVALID_PERMISSIONS_OR_MODEL_NOT_FOUND" {
// The key is verified as it works, but the user must enumerate the tables or permissions for the key.
s1.Verified = true
}
}
} else {
if detectors.IsKnownFalsePositive(keyRes, detectors.DefaultFalsePositives, true) {
continue
@@ -9,10 +9,9 @@ import (
"testing"
"time"
"github.com/kylelemons/godebug/pretty"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/google/go-cmp/cmp"
"github.com/google/go-cmp/cmp/cmpopts"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
)
@@ -100,9 +99,8 @@ func TestAirtableApiKey_FromChunk(t *testing.T) {
}
got[i].Raw = nil
}
ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "RawV2", "verificationError")
if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {
t.Errorf("Airtable.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
if diff := pretty.Compare(got, tt.want); diff != "" {
t.Errorf("AirtableApiKey.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
}
})
}
+2 -3
View File
@@ -65,11 +65,10 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
} else if res.StatusCode == 401 {
// The secret is determinately not verified (nothing to do)
} else {
err = fmt.Errorf("unexpected HTTP response status %d", res.StatusCode)
s1.SetVerificationError(err, resMatch)
s1.VerificationError = fmt.Errorf("unexpected HTTP response status %d", res.StatusCode)
}
} else {
s1.SetVerificationError(err, resMatch)
s1.VerificationError = err
}
}
+6 -7
View File
@@ -6,11 +6,10 @@ package alchemy
import (
"context"
"fmt"
"testing"
"time"
"github.com/google/go-cmp/cmp"
"github.com/google/go-cmp/cmp/cmpopts"
"testing"
"time"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
@@ -21,7 +20,7 @@ import (
func TestAlchemy_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")
testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors4")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
@@ -133,11 +132,11 @@ func TestAlchemy_FromChunk(t *testing.T) {
if len(got[i].Raw) == 0 {
t.Fatalf("no raw secret present: \n %+v", got[i])
}
if (got[i].VerificationError() != nil) != tt.wantVerificationErr {
t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError())
if (got[i].VerificationError != nil) != tt.wantVerificationErr {
t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError)
}
}
ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "verificationError")
ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "VerificationError")
if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {
t.Errorf("Alchemy.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
}
-120
View File
@@ -1,120 +0,0 @@
package anthropic
import (
"bytes"
"context"
"encoding/json"
"fmt"
"net/http"
"regexp"
"strings"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
)
type Scanner struct {
client *http.Client
}
// Ensure the Scanner satisfies the interface at compile time.
var _ detectors.Detector = (*Scanner)(nil)
var (
defaultClient = common.SaneHttpClient()
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
keyPat = regexp.MustCompile(`\b(sk-ant-api03-[\w\-]{93}AA)\b`)
)
// Keywords are used for efficiently pre-filtering chunks.
// Use identifiers in the secret preferably, or the provider name.
func (s Scanner) Keywords() []string {
return []string{"sk-ant-api03"}
}
type response struct {
Error struct {
Type string `json:"type"`
Message string `json:"message"`
} `json:"error"`
}
// FromData will find and optionally verify Anthropic secrets in a given set of bytes.
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
dataStr := string(data)
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
for _, match := range matches {
if len(match) != 2 {
continue
}
resMatch := strings.TrimSpace(match[1])
s1 := detectors.Result{
DetectorType: detectorspb.DetectorType_Anthropic,
Raw: []byte(resMatch),
}
if verify {
client := s.client
if client == nil {
client = defaultClient
}
body := map[string]interface{}{
"prompt": "test",
}
bodyBytes, err := json.Marshal(body)
if err != nil {
continue
}
req, err := http.NewRequestWithContext(ctx, "POST", "https://api.anthropic.com/v1/complete", bytes.NewReader(bodyBytes))
if err != nil {
continue
}
req.Header.Set("x-api-key", resMatch)
req.Header.Set("Content-Type", "application/json")
res, err := client.Do(req)
if err == nil {
defer res.Body.Close()
if res.StatusCode == http.StatusBadRequest {
var resp response
if err = json.NewDecoder(res.Body).Decode(&resp); err != nil {
err = fmt.Errorf("unexpected HTTP response body: %w", err)
s1.SetVerificationError(err, resMatch)
continue
}
if resp.Error.Message == "max_tokens_to_sample: field required" {
// The secret is verified
// Anthropic returns 400 on a request containing a valid API key,
// when not containing a valid model field
s1.Verified = true
}
} else if res.StatusCode == 401 {
// The secret is determinately not verified (nothing to do)
// Anthropic returns 401 on all requests not containing a valid x-api-key header
} else {
err = fmt.Errorf("unexpected HTTP response status %d", res.StatusCode)
s1.SetVerificationError(err, resMatch)
}
} else {
s1.SetVerificationError(err, resMatch)
}
}
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
if !s1.Verified && detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
continue
}
results = append(results, s1)
}
return results, nil
}
func (s Scanner) Type() detectorspb.DetectorType {
return detectorspb.DetectorType_Anthropic
}
-162
View File
@@ -1,162 +0,0 @@
//go:build detectors
// +build detectors
package anthropic
import (
"context"
"fmt"
"testing"
"time"
"github.com/google/go-cmp/cmp"
"github.com/google/go-cmp/cmp/cmpopts"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
)
func TestAnthropic_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
secret := testSecrets.MustGetField("ANTHROPIC")
inactiveSecret := testSecrets.MustGetField("ANTHROPIC_INACTIVE")
type args struct {
ctx context.Context
data []byte
verify bool
}
tests := []struct {
name string
s Scanner
args args
want []detectors.Result
wantErr bool
wantVerificationErr bool
}{
{
name: "found, verified",
s: Scanner{},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a anthropic secret %s within", secret)),
verify: true,
},
want: []detectors.Result{
{
DetectorType: detectorspb.DetectorType_Anthropic,
Verified: true,
},
},
wantErr: false,
wantVerificationErr: false,
},
{
name: "found, unverified",
s: Scanner{},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a anthropic secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
verify: true,
},
want: []detectors.Result{
{
DetectorType: detectorspb.DetectorType_Anthropic,
Verified: false,
},
},
wantErr: false,
wantVerificationErr: false,
},
{
name: "not found",
s: Scanner{},
args: args{
ctx: context.Background(),
data: []byte("You cannot find the secret within"),
verify: true,
},
want: nil,
wantErr: false,
wantVerificationErr: false,
},
{
name: "found, would be verified if not for timeout",
s: Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a anthropic secret %s within", secret)),
verify: true,
},
want: []detectors.Result{
{
DetectorType: detectorspb.DetectorType_Anthropic,
Verified: false,
},
},
wantErr: false,
wantVerificationErr: true,
},
{
name: "found, verified but unexpected api surface",
s: Scanner{client: common.ConstantResponseHttpClient(404, "")},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a anthropic secret %s within", secret)),
verify: true,
},
want: []detectors.Result{
{
DetectorType: detectorspb.DetectorType_Anthropic,
Verified: false,
},
},
wantErr: false,
wantVerificationErr: true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
if (err != nil) != tt.wantErr {
t.Errorf("Anthropic.FromData() error = %v, wantErr %v", err, tt.wantErr)
return
}
for i := range got {
if len(got[i].Raw) == 0 {
t.Fatalf("no raw secret present: \n %+v", got[i])
}
if (got[i].VerificationError() != nil) != tt.wantVerificationErr {
t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError())
}
}
ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "verificationError")
if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {
t.Errorf("Anthropic.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
}
})
}
}
func BenchmarkFromData(benchmark *testing.B) {
ctx := context.Background()
s := Scanner{}
for name, data := range detectors.MustGetBenchmarkData() {
benchmark.Run(name, func(b *testing.B) {
b.ResetTimer()
for n := 0; n < b.N; n++ {
_, err := s.FromData(ctx, false, data)
if err != nil {
b.Fatal(err)
}
}
})
}
}
+7 -3
View File
@@ -2,6 +2,8 @@ package appfollow
import (
"context"
b64 "encoding/base64"
"fmt"
"net/http"
"regexp"
"strings"
@@ -20,7 +22,7 @@ var (
client = common.SaneHttpClient()
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"appfollow"}) + `\b(eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9\.[0-9A-Za-z]{74}\.[0-9A-Z-a-z\-_]{43})\b`)
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"appfollow"}) + `\b([0-9A-Za-z]{20})\b`)
)
// Keywords are used for efficiently pre-filtering chunks.
@@ -47,11 +49,13 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
}
if verify {
req, err := http.NewRequestWithContext(ctx, "GET", "https://api.appfollow.io/api/v2/account/users", nil)
data := fmt.Sprintf("%s:", resMatch)
sEnc := b64.StdEncoding.EncodeToString([]byte(data))
req, err := http.NewRequestWithContext(ctx, "GET", "https://api.appfollow.io/test", nil)
if err != nil {
continue
}
req.Header.Add("X-AppFollow-API-Token", resMatch)
req.Header.Add("Authorization", fmt.Sprintf("Basic %s", sEnc))
res, err := client.Do(req)
if err == nil {
defer res.Body.Close()
-97
View File
@@ -1,97 +0,0 @@
package appoptics
import (
"context"
b64 "encoding/base64"
"fmt"
"net/http"
"regexp"
"strings"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
)
type Scanner struct {
client *http.Client
}
// Ensure the Scanner satisfies the interface at compile time.
var _ detectors.Detector = (*Scanner)(nil)
var (
defaultClient = common.SaneHttpClient()
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"appoptics"}) + `\b([0-9a-zA-Z_-]{71})\b`)
)
// Keywords are used for efficiently pre-filtering chunks.
// Use identifiers in the secret preferably, or the provider name.
func (s Scanner) Keywords() []string {
return []string{"appoptics"}
}
// FromData will find and optionally verify Appoptics secrets in a given set of bytes.
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
dataStr := string(data)
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
for _, match := range matches {
if len(match) != 2 {
continue
}
resMatch := strings.TrimSpace(match[1])
s1 := detectors.Result{
DetectorType: detectorspb.DetectorType_AppOptics,
Raw: []byte(resMatch),
}
if verify {
client := s.client
if client == nil {
client = defaultClient
}
req, err := http.NewRequestWithContext(ctx, "GET", "https://api.appoptics.com/v1/metrics", nil)
if err != nil {
continue
}
data := fmt.Sprintf("%s:", resMatch)
sEnc := b64.StdEncoding.EncodeToString([]byte(data))
req.Header.Add("Content-Type", "application/json")
req.Header.Add("Authorization", fmt.Sprintf("Basic %s", sEnc))
res, err := client.Do(req)
if err == nil {
defer res.Body.Close()
if res.StatusCode >= 200 && res.StatusCode < 300 {
s1.Verified = true
} else if res.StatusCode == 401 {
// The secret is determinately not verified (nothing to do)
} else {
err = fmt.Errorf("unexpected HTTP response status %d", res.StatusCode)
s1.SetVerificationError(err, resMatch)
}
} else {
s1.SetVerificationError(err, resMatch)
}
}
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
if !s1.Verified && detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
continue
}
results = append(results, s1)
}
return results, nil
}
func (s Scanner) Type() detectorspb.DetectorType {
return detectorspb.DetectorType_AppOptics
}
-128
View File
@@ -1,128 +0,0 @@
//go:build detectors
// +build detectors
package appoptics
import (
"context"
"fmt"
"testing"
"time"
"github.com/google/go-cmp/cmp"
"github.com/google/go-cmp/cmp/cmpopts"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
)
func TestAppoptics_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
secret := testSecrets.MustGetField("APPOPTICS")
inactiveSecret := testSecrets.MustGetField("APPOPTICS_INACTIVE")
type args struct {
ctx context.Context
data []byte
verify bool
}
tests := []struct {
name string
s Scanner
args args
want []detectors.Result
wantErr bool
wantVerificationErr bool
}{
{
name: "found, verified",
s: Scanner{},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a appoptics secret %s within", secret)),
verify: true,
},
want: []detectors.Result{
{
DetectorType: detectorspb.DetectorType_AppOptics,
Verified: true,
},
},
wantErr: false,
wantVerificationErr: false,
},
{
name: "found, unverified",
s: Scanner{},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a appoptics secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
verify: true,
},
want: []detectors.Result{
{
DetectorType: detectorspb.DetectorType_AppOptics,
Verified: false,
},
},
wantErr: false,
wantVerificationErr: false,
},
{
name: "not found",
s: Scanner{},
args: args{
ctx: context.Background(),
data: []byte("You cannot find the secret within"),
verify: true,
},
want: nil,
wantErr: false,
wantVerificationErr: false,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
if (err != nil) != tt.wantErr {
t.Errorf("Appoptics.FromData() error = %v, wantErr %v", err, tt.wantErr)
return
}
for i := range got {
if len(got[i].Raw) == 0 {
t.Fatalf("no raw secret present: \n %+v", got[i])
}
if (got[i].VerificationError() != nil) != tt.wantVerificationErr {
t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError())
}
}
ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "verificationError")
if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {
t.Errorf("Appoptics.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
}
})
}
}
func BenchmarkFromData(benchmark *testing.B) {
ctx := context.Background()
s := Scanner{}
for name, data := range detectors.MustGetBenchmarkData() {
benchmark.Run(name, func(b *testing.B) {
b.ResetTimer()
for n := 0; n < b.N; n++ {
_, err := s.FromData(ctx, false, data)
if err != nil {
b.Fatal(err)
}
}
})
}
}
+109 -168
View File
@@ -22,22 +22,6 @@ type scanner struct {
skipIDs map[string]struct{}
}
// resourceTypes derived from: https://docs.aws.amazon.com/IAM/latest/UserGuide/reference_identifiers.html#identifiers-unique-ids
var resourceTypes = map[string]string{
"ABIA": "AWS STS service bearer token",
"ACCA": "Context-specific credential",
"AGPA": "User group",
"AIDA": "IAM user",
"AIPA": "Amazon EC2 instance profile",
"AKIA": "Access key",
"ANPA": "Managed policy",
"ANVA": "Version in a managed policy",
"APKA": "Public key",
"AROA": "Role",
"ASCA": "Certificate",
"ASIA": "Temporary (AWS STS) access key IDs",
}
func New(opts ...func(*scanner)) *scanner {
scanner := &scanner{
skipIDs: map[string]struct{}{},
@@ -69,7 +53,7 @@ var (
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
// Key types are from this list https://docs.aws.amazon.com/IAM/latest/UserGuide/reference_identifiers.html#identifiers-unique-ids
idPat = regexp.MustCompile(`\b((AKIA|ABIA|ACCA)[0-9A-Z]{16})\b`)
idPat = regexp.MustCompile(`\b((?:AKIA|ABIA|ACCA|ASIA)[0-9A-Z]{16})\b`)
secretPat = regexp.MustCompile(`[^A-Za-z0-9+\/]{0,1}([A-Za-z0-9+\/]{40})[^A-Za-z0-9+\/]{0,1}`)
// Hashes, like those for git, do technically match the secret pattern.
// But they are extremely unlikely to be generated as an actual AWS secret.
@@ -109,7 +93,7 @@ func (s scanner) FromData(ctx context.Context, verify bool, data []byte) (result
secretMatches := secretPat.FindAllStringSubmatch(dataStr, -1)
for _, idMatch := range idMatches {
if len(idMatch) != 3 {
if len(idMatch) != 2 {
continue
}
resIDMatch := strings.TrimSpace(idMatch[1])
@@ -131,47 +115,123 @@ func (s scanner) FromData(ctx context.Context, verify bool, data []byte) (result
Raw: []byte(resIDMatch),
Redacted: resIDMatch,
RawV2: []byte(resIDMatch + resSecretMatch),
ExtraData: map[string]string{
"resource_type": resourceTypes[idMatch[2]],
},
}
if verify {
isVerified, extraData, verificationErr := s.verifyMatch(ctx, resIDMatch, resSecretMatch, true)
s1.Verified = isVerified
// It'd be good to log when calculated account value does not match
// the account value from verification. Should only be edge cases at most.
// if extraData["account"] != s1.ExtraData["account"] && extraData["account"] != "" {//log here}
// REQUEST VALUES.
method := "GET"
service := "sts"
host := "sts.amazonaws.com"
region := "us-east-1"
endpoint := "https://sts.amazonaws.com"
now := time.Now().UTC()
datestamp := now.Format("20060102")
amzDate := now.Format("20060102T150405Z0700")
// Append the extraData to the existing ExtraData map.
// This will overwrite with the new verified values.
for k, v := range extraData {
s1.ExtraData[k] = v
}
if verificationErr != nil {
s1.SetVerificationError(verificationErr, resSecretMatch)
}
}
if !s1.Verified {
// Unverified results that contain common test words are probably not secrets
if detectors.IsKnownFalsePositive(resSecretMatch, detectors.DefaultFalsePositives, true) {
req, err := http.NewRequestWithContext(ctx, method, endpoint, nil)
if err != nil {
continue
}
// Unverified results that look like hashes are probably not secrets
if falsePositiveSecretCheck.MatchString(resSecretMatch) {
continue
}
}
req.Header.Set("Accept", "application/json")
// If we haven't already found an account number for this ID (via API), calculate one.
if _, ok := s1.ExtraData["account"]; !ok {
account, err := common.GetAccountNumFromAWSID(resIDMatch)
// TASK 1: CREATE A CANONICAL REQUEST.
// http://docs.aws.amazon.com/general/latest/gr/sigv4-create-canonical-request.html
canonicalURI := "/"
canonicalHeaders := "host:" + host + "\n"
signedHeaders := "host"
algorithm := "AWS4-HMAC-SHA256"
credentialScope := fmt.Sprintf("%s/%s/%s/aws4_request", datestamp, region, service)
params := req.URL.Query()
params.Add("Action", "GetCallerIdentity")
params.Add("Version", "2011-06-15")
params.Add("X-Amz-Algorithm", algorithm)
params.Add("X-Amz-Credential", resIDMatch+"/"+credentialScope)
params.Add("X-Amz-Date", amzDate)
params.Add("X-Amz-Expires", "30")
params.Add("X-Amz-SignedHeaders", signedHeaders)
canonicalQuerystring := params.Encode()
payloadHash := GetHash("") // empty payload
canonicalRequest := method + "\n" + canonicalURI + "\n" + canonicalQuerystring + "\n" + canonicalHeaders + "\n" + signedHeaders + "\n" + payloadHash
// TASK 2: CREATE THE STRING TO SIGN.
stringToSign := algorithm + "\n" + amzDate + "\n" + credentialScope + "\n" + GetHash(canonicalRequest)
// TASK 3: CALCULATE THE SIGNATURE.
// https://docs.aws.amazon.com/general/latest/gr/sigv4-calculate-signature.html
hash := GetHMAC([]byte(fmt.Sprintf("AWS4%s", resSecretMatch)), []byte(datestamp))
hash = GetHMAC(hash, []byte(region))
hash = GetHMAC(hash, []byte(service))
hash = GetHMAC(hash, []byte("aws4_request"))
signature2 := GetHMAC(hash, []byte(stringToSign)) // Get Signature HMAC SHA256
signature := hex.EncodeToString(signature2)
// TASK 4: ADD SIGNING INFORMATION TO THE REQUEST.
params.Add("X-Amz-Signature", signature)
req.Header.Add("Content-type", "application/x-www-form-urlencoded; charset=utf-8")
req.URL.RawQuery = params.Encode()
client := s.verificationClient
if client == nil {
client = defaultVerificationClient
}
res, err := client.Do(req)
if err == nil {
s1.ExtraData["account"] = account
if res.StatusCode >= 200 && res.StatusCode < 300 {
identityInfo := identityRes{}
err := json.NewDecoder(res.Body).Decode(&identityInfo)
if err == nil {
s1.Verified = true
s1.ExtraData = map[string]string{
"account": identityInfo.GetCallerIdentityResponse.GetCallerIdentityResult.Account,
"user_id": identityInfo.GetCallerIdentityResponse.GetCallerIdentityResult.UserID,
"arn": identityInfo.GetCallerIdentityResponse.GetCallerIdentityResult.Arn,
}
} else {
s1.VerificationError = err
}
res.Body.Close()
} else {
// This function will check false positives for common test words, but also it will make sure the key appears "random" enough to be a real key.
if detectors.IsKnownFalsePositive(resSecretMatch, detectors.DefaultFalsePositives, true) {
continue
}
if res.StatusCode == 403 {
var body awsErrorResponseBody
err = json.NewDecoder(res.Body).Decode(&body)
res.Body.Close()
if err == nil {
// All instances of the code I've seen in the wild are PascalCased but this check is
// case-insensitive out of an abundance of caution
if strings.EqualFold(body.Error.Code, "InvalidClientTokenId") {
// determinate failure - nothing to do
} else {
// We see a surprising number of false-negative signature mismatch errors here
// (The official SDK somehow elicits even more than just making the request
// ourselves)
s1.VerificationError = fmt.Errorf("request to %v returned status %d with an unexpected reason (%s: %s)", res.Request.URL, res.StatusCode, body.Error.Code, body.Error.Message)
}
} else {
s1.VerificationError = fmt.Errorf("couldn't parse the sts response body (%v)", err)
}
} else {
s1.VerificationError = fmt.Errorf("request to %v returned unexpected status %d", res.Request.URL, res.StatusCode)
}
}
} else {
s1.VerificationError = err
}
}
// If the result is unverified and matches something like a git hash, don't include it in the results.
if !s1.Verified && falsePositiveSecretCheck.MatchString(resSecretMatch) {
continue
}
results = append(results, s1)
// If we've found a verified match with this ID, we don't need to look for any more. So move on to the next ID.
if s1.Verified {
@@ -182,125 +242,6 @@ func (s scanner) FromData(ctx context.Context, verify bool, data []byte) (result
return awsCustomCleanResults(results), nil
}
func (s scanner) verifyMatch(ctx context.Context, resIDMatch, resSecretMatch string, retryOn403 bool) (bool, map[string]string, error) {
// REQUEST VALUES.
method := "GET"
service := "sts"
host := "sts.amazonaws.com"
region := "us-east-1"
endpoint := "https://sts.amazonaws.com"
now := time.Now().UTC()
datestamp := now.Format("20060102")
amzDate := now.Format("20060102T150405Z0700")
req, err := http.NewRequestWithContext(ctx, method, endpoint, nil)
if err != nil {
return false, nil, err
}
req.Header.Set("Accept", "application/json")
// TASK 1: CREATE A CANONICAL REQUEST.
// http://docs.aws.amazon.com/general/latest/gr/sigv4-create-canonical-request.html
canonicalURI := "/"
canonicalHeaders := "host:" + host + "\n"
signedHeaders := "host"
algorithm := "AWS4-HMAC-SHA256"
credentialScope := fmt.Sprintf("%s/%s/%s/aws4_request", datestamp, region, service)
params := req.URL.Query()
params.Add("Action", "GetCallerIdentity")
params.Add("Version", "2011-06-15")
params.Add("X-Amz-Algorithm", algorithm)
params.Add("X-Amz-Credential", resIDMatch+"/"+credentialScope)
params.Add("X-Amz-Date", amzDate)
params.Add("X-Amz-Expires", "30")
params.Add("X-Amz-SignedHeaders", signedHeaders)
canonicalQuerystring := params.Encode()
payloadHash := GetHash("") // empty payload
canonicalRequest := method + "\n" + canonicalURI + "\n" + canonicalQuerystring + "\n" + canonicalHeaders + "\n" + signedHeaders + "\n" + payloadHash
// TASK 2: CREATE THE STRING TO SIGN.
stringToSign := algorithm + "\n" + amzDate + "\n" + credentialScope + "\n" + GetHash(canonicalRequest)
// TASK 3: CALCULATE THE SIGNATURE.
// https://docs.aws.amazon.com/general/latest/gr/sigv4-calculate-signature.html
hash := GetHMAC([]byte(fmt.Sprintf("AWS4%s", resSecretMatch)), []byte(datestamp))
hash = GetHMAC(hash, []byte(region))
hash = GetHMAC(hash, []byte(service))
hash = GetHMAC(hash, []byte("aws4_request"))
signature2 := GetHMAC(hash, []byte(stringToSign)) // Get Signature HMAC SHA256
signature := hex.EncodeToString(signature2)
// TASK 4: ADD SIGNING INFORMATION TO THE REQUEST.
params.Add("X-Amz-Signature", signature)
req.Header.Add("Content-type", "application/x-www-form-urlencoded; charset=utf-8")
req.URL.RawQuery = params.Encode()
client := s.verificationClient
if client == nil {
client = defaultVerificationClient
}
extraData := map[string]string{
"rotation_guide": "https://howtorotate.com/docs/tutorials/aws/",
}
res, err := client.Do(req)
if err == nil {
defer res.Body.Close()
if res.StatusCode >= 200 && res.StatusCode < 300 {
identityInfo := identityRes{}
err := json.NewDecoder(res.Body).Decode(&identityInfo)
if err == nil {
extraData["account"] = identityInfo.GetCallerIdentityResponse.GetCallerIdentityResult.Account
extraData["user_id"] = identityInfo.GetCallerIdentityResponse.GetCallerIdentityResult.UserID
extraData["arn"] = identityInfo.GetCallerIdentityResponse.GetCallerIdentityResult.Arn
return true, extraData, nil
} else {
return false, nil, err
}
} else if res.StatusCode == 403 {
// Experimentation has indicated that if you make two GetCallerIdentity requests within five seconds that
// share a key ID but are signed with different secrets the second one will be rejected with a 403 that
// carries a SignatureDoesNotMatch code in its body. This happens even if the second ID-secret pair is
// valid. Since this is exactly our access pattern, we need to work around it.
//
// Fortunately, experimentation has also revealed a workaround: simply resubmit the second request. The
// response to the resubmission will be as expected. But there's a caveat: You can't have closed the body of
// the response to the original second request, or read to its end, or the resubmission will also yield a
// SignatureDoesNotMatch. For this reason, we have to re-request all 403s. We can't re-request only
// SignatureDoesNotMatch responses, because we can only tell whether a given 403 is a SignatureDoesNotMatch
// after decoding its response body, which requires reading the entire response body, which disables the
// workaround.
//
// We are clearly deep in the guts of AWS implementation details here, so this all might change with no
// notice. If you're here because something in this detector broke, you have my condolences.
if retryOn403 {
return s.verifyMatch(ctx, resIDMatch, resSecretMatch, false)
}
var body awsErrorResponseBody
err = json.NewDecoder(res.Body).Decode(&body)
if err == nil {
// All instances of the code I've seen in the wild are PascalCased but this check is
// case-insensitive out of an abundance of caution
if strings.EqualFold(body.Error.Code, "InvalidClientTokenId") {
return false, nil, nil
} else {
return false, nil, fmt.Errorf("request returned status %d with an unexpected reason (%s: %s)", res.StatusCode, body.Error.Code, body.Error.Message)
}
} else {
return false, nil, fmt.Errorf("couldn't parse the sts response body (%v)", err)
}
} else {
return false, nil, fmt.Errorf("request to %v returned unexpected status %d", res.Request.URL, res.StatusCode)
}
} else {
return false, nil, err
}
}
func awsCustomCleanResults(results []detectors.Result) []detectors.Result {
if len(results) == 0 {
return results
@@ -321,7 +262,7 @@ func awsCustomCleanResults(results []detectors.Result) []detectors.Result {
}
}
var out []detectors.Result
out := []detectors.Result{}
for _, r := range idResults {
out = append(out, r)
}
+13 -66
View File
@@ -61,11 +61,9 @@ func TestAWS_FromChunk(t *testing.T) {
Verified: true,
Redacted: "AKIASP2TPHJSQH3FJRUX",
ExtraData: map[string]string{
"resource_type": "Access key",
"rotation_guide": "https://howtorotate.com/docs/tutorials/aws/",
"account": "171436882533",
"arn": "arn:aws:iam::171436882533:user/canarytokens.com@@4dxkh0pdeop3bzu9zx5wob793",
"user_id": "AIDASP2TPHJSUFRSTTZX4",
"account": "171436882533",
"arn": "arn:aws:iam::171436882533:user/canarytokens.com@@4dxkh0pdeop3bzu9zx5wob793",
"user_id": "AIDASP2TPHJSUFRSTTZX4",
},
},
},
@@ -84,10 +82,7 @@ func TestAWS_FromChunk(t *testing.T) {
DetectorType: detectorspb.DetectorType_AWS,
Verified: false,
Redacted: "AKIASP2TPHJSQH3FJRUX",
ExtraData: map[string]string{
"resource_type": "Access key",
"account": "171436882533",
},
ExtraData: nil,
},
},
wantErr: false,
@@ -116,21 +111,15 @@ func TestAWS_FromChunk(t *testing.T) {
DetectorType: detectorspb.DetectorType_AWS,
Verified: false,
Redacted: "AKIASP2TPHJSQH3FJXYZ",
ExtraData: map[string]string{
"resource_type": "Access key",
"account": "171436882533",
},
},
{
DetectorType: detectorspb.DetectorType_AWS,
Verified: true,
Redacted: "AKIASP2TPHJSQH3FJRUX",
ExtraData: map[string]string{
"resource_type": "Access key",
"rotation_guide": "https://howtorotate.com/docs/tutorials/aws/",
"account": "171436882533",
"arn": "arn:aws:iam::171436882533:user/canarytokens.com@@4dxkh0pdeop3bzu9zx5wob793",
"user_id": "AIDASP2TPHJSUFRSTTZX4",
"account": "171436882533",
"arn": "arn:aws:iam::171436882533:user/canarytokens.com@@4dxkh0pdeop3bzu9zx5wob793",
"user_id": "AIDASP2TPHJSUFRSTTZX4",
},
},
},
@@ -161,11 +150,9 @@ func TestAWS_FromChunk(t *testing.T) {
Verified: true,
Redacted: "AKIASP2TPHJSQH3FJRUX",
ExtraData: map[string]string{
"resource_type": "Access key",
"rotation_guide": "https://howtorotate.com/docs/tutorials/aws/",
"account": "171436882533",
"arn": "arn:aws:iam::171436882533:user/canarytokens.com@@4dxkh0pdeop3bzu9zx5wob793",
"user_id": "AIDASP2TPHJSUFRSTTZX4",
"account": "171436882533",
"arn": "arn:aws:iam::171436882533:user/canarytokens.com@@4dxkh0pdeop3bzu9zx5wob793",
"user_id": "AIDASP2TPHJSUFRSTTZX4",
},
},
{
@@ -189,10 +176,6 @@ func TestAWS_FromChunk(t *testing.T) {
DetectorType: detectorspb.DetectorType_AWS,
Verified: false,
Redacted: "AKIASP2TPHJSQH3FJRUX",
ExtraData: map[string]string{
"resource_type": "Access key",
"account": "171436882533",
},
},
},
wantErr: false,
@@ -224,10 +207,6 @@ func TestAWS_FromChunk(t *testing.T) {
DetectorType: detectorspb.DetectorType_AWS,
Verified: false,
Redacted: "AKIASP2TPHJSQH3FJRUX",
ExtraData: map[string]string{
"resource_type": "Access key",
"account": "171436882533",
},
},
},
wantErr: false,
@@ -246,10 +225,6 @@ func TestAWS_FromChunk(t *testing.T) {
DetectorType: detectorspb.DetectorType_AWS,
Verified: false,
Redacted: "AKIASP2TPHJSQH3FJRUX",
ExtraData: map[string]string{
"resource_type": "Access key",
"account": "171436882533",
},
},
},
wantErr: false,
@@ -268,39 +243,11 @@ func TestAWS_FromChunk(t *testing.T) {
DetectorType: detectorspb.DetectorType_AWS,
Verified: false,
Redacted: "AKIASP2TPHJSQH3FJRUX",
ExtraData: map[string]string{
"resource_type": "Access key",
"account": "171436882533",
},
},
},
wantErr: false,
wantVerificationError: true,
},
{
name: "verified secret checked directly after unverified secret with same key id",
s: scanner{},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("%s\n%s\n%s", inactiveSecret, id, secret)),
verify: true,
},
want: []detectors.Result{
{
DetectorType: detectorspb.DetectorType_AWS,
Verified: true,
Redacted: "AKIASP2TPHJSQH3FJRUX",
ExtraData: map[string]string{
"resource_type": "Access key",
"rotation_guide": "https://howtorotate.com/docs/tutorials/aws/",
"account": "171436882533",
"arn": "arn:aws:iam::171436882533:user/canarytokens.com@@4dxkh0pdeop3bzu9zx5wob793",
"user_id": "AIDASP2TPHJSUFRSTTZX4",
},
},
},
wantErr: false,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
@@ -314,11 +261,11 @@ func TestAWS_FromChunk(t *testing.T) {
if len(got[i].Raw) == 0 {
t.Fatalf("no raw secret present: \n %+v", got[i])
}
if (got[i].VerificationError() != nil) != tt.wantVerificationError {
t.Fatalf("wantVerificationError %v, verification error = %v", tt.wantVerificationError, got[i].VerificationError())
if (got[i].VerificationError != nil) != tt.wantVerificationError {
t.Fatalf("wantVerificationError %v, verification error = %v", tt.wantVerificationError, got[i].VerificationError)
}
}
ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "RawV2", "Raw", "verificationError")
ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "RawV2", "Raw", "VerificationError")
if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {
t.Errorf("AWS.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
}
@@ -1,339 +0,0 @@
package awssessionkey
import (
"context"
"crypto/hmac"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"fmt"
"net/http"
"regexp"
"strings"
"time"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
)
type scanner struct {
verificationClient *http.Client
skipIDs map[string]struct{}
}
func New(opts ...func(*scanner)) *scanner {
scanner := &scanner{
skipIDs: map[string]struct{}{},
}
for _, opt := range opts {
opt(scanner)
}
return scanner
}
func WithSkipIDs(skipIDs []string) func(*scanner) {
return func(s *scanner) {
ids := map[string]struct{}{}
for _, id := range skipIDs {
ids[id] = struct{}{}
}
s.skipIDs = ids
}
}
// Ensure the scanner satisfies the interface at compile time.
var _ detectors.Detector = (*scanner)(nil)
var (
defaultVerificationClient = common.SaneHttpClient()
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
// Key types are from this list https://docs.aws.amazon.com/IAM/latest/UserGuide/reference_identifiers.html#identifiers-unique-ids
idPat = regexp.MustCompile(`\b((?:ASIA)[0-9A-Z]{16})\b`)
secretPat = regexp.MustCompile(`\b[^A-Za-z0-9+\/]{0,1}([A-Za-z0-9+\/]{40})[^A-Za-z0-9+\/]{0,1}\b`)
sessionPat = regexp.MustCompile(`\b[^A-Za-z0-9+\/]{0,1}([A-Za-z0-9+=\/]{41,1000})[^A-Za-z0-9+=\/]{0,1}\b`)
// Hashes, like those for git, do technically match the secret pattern.
// But they are extremely unlikely to be generated as an actual AWS secret.
// So when we find them, if they're not verified, we should ignore the result.
falsePositiveSecretCheck = regexp.MustCompile(`[a-f0-9]{40}`)
)
// Keywords are used for efficiently pre-filtering chunks.
// Use identifiers in the secret preferably, or the provider name.
func (s scanner) Keywords() []string {
return []string{
"ASIA",
}
}
func GetHash(input string) string {
data := []byte(input)
hasher := sha256.New()
hasher.Write(data)
return hex.EncodeToString(hasher.Sum(nil))
}
func GetHMAC(key []byte, data []byte) []byte {
hasher := hmac.New(sha256.New, key)
hasher.Write(data)
return hasher.Sum(nil)
}
func checkSessionToken(sessionToken string, secret string) bool {
if !strings.Contains(sessionToken, "YXdz") || strings.Contains(sessionToken, secret) {
// Handle error if the sessionToken is not a valid base64 string
return false
}
return true
}
// FromData will find and optionally verify AWS secrets in a given set of bytes.
func (s scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
dataStr := string(data)
idMatches := idPat.FindAllStringSubmatch(dataStr, -1)
secretMatches := secretPat.FindAllStringSubmatch(dataStr, -1)
sessionMatches := sessionPat.FindAllStringSubmatch(dataStr, -1)
for _, idMatch := range idMatches {
if len(idMatch) != 2 {
continue
}
resIDMatch := strings.TrimSpace(idMatch[1])
if s.skipIDs != nil {
if _, ok := s.skipIDs[resIDMatch]; ok {
continue
}
}
for _, secretMatch := range secretMatches {
if len(secretMatch) != 2 {
continue
}
resSecretMatch := strings.TrimSpace(secretMatch[1])
for _, sessionMatch := range sessionMatches {
if len(sessionMatch) != 2 {
continue
}
resSessionMatch := strings.TrimSpace(sessionMatch[1])
if !checkSessionToken(resSessionMatch, resSecretMatch) {
continue
}
s1 := detectors.Result{
DetectorType: detectorspb.DetectorType_AWSSessionKey,
Raw: []byte(resIDMatch),
Redacted: resIDMatch,
RawV2: []byte(resIDMatch + resSecretMatch + resSessionMatch),
ExtraData: make(map[string]string),
}
if verify {
isVerified, extraData, verificationErr := s.verifyMatch(ctx, resIDMatch, resSecretMatch, resSessionMatch, true)
s1.Verified = isVerified
if extraData != nil {
s1.ExtraData = extraData
}
s1.SetVerificationError(verificationErr, resSecretMatch)
}
if !s1.Verified {
// Unverified results that contain common test words are probably not secrets
if detectors.IsKnownFalsePositive(resSecretMatch, detectors.DefaultFalsePositives, true) {
continue
}
// Unverified results that look like hashes are probably not secrets
if falsePositiveSecretCheck.MatchString(resSecretMatch) {
continue
}
}
// If we haven't already found an account number for this ID (via API), calculate one.
if _, ok := s1.ExtraData["account"]; !ok {
account, err := common.GetAccountNumFromAWSID(resIDMatch)
if err == nil {
s1.ExtraData["account"] = account
}
}
results = append(results, s1)
// If we've found a verified match with this ID, we don't need to look for any more. So move on to the next ID.
if s1.Verified {
break
}
}
}
}
return awsCustomCleanResults(results), nil
}
func (s scanner) verifyMatch(ctx context.Context, resIDMatch, resSecretMatch string, resSessionMatch string, retryOn403 bool) (bool, map[string]string, error) {
// REQUEST VALUES.
method := "GET"
service := "sts"
host := "sts.amazonaws.com"
region := "us-east-1"
endpoint := "https://sts.amazonaws.com"
now := time.Now().UTC()
datestamp := now.Format("20060102")
amzDate := now.Format("20060102T150405Z")
req, err := http.NewRequestWithContext(ctx, method, endpoint, nil)
if err != nil {
return false, nil, err
}
req.Header.Set("Accept", "application/json")
canonicalURI := "/"
canonicalHeaders := "host:" + host + "\n" + "x-amz-date:" + amzDate + "\n" + "x-amz-security-token:" + resSessionMatch + "\n"
signedHeaders := "host;x-amz-date;x-amz-security-token"
algorithm := "AWS4-HMAC-SHA256"
credentialScope := fmt.Sprintf("%s/%s/%s/aws4_request", datestamp, region, service)
params := req.URL.Query()
params.Add("Action", "GetCallerIdentity")
params.Add("Version", "2011-06-15")
canonicalQuerystring := params.Encode()
payloadHash := GetHash("") // empty payload
canonicalRequest := method + "\n" + canonicalURI + "\n" + canonicalQuerystring + "\n" + canonicalHeaders + "\n" + signedHeaders + "\n" + payloadHash
stringToSign := algorithm + "\n" + amzDate + "\n" + credentialScope + "\n" + GetHash(canonicalRequest)
hash := GetHMAC([]byte(fmt.Sprintf("AWS4%s", resSecretMatch)), []byte(datestamp))
hash = GetHMAC(hash, []byte(region))
hash = GetHMAC(hash, []byte(service))
hash = GetHMAC(hash, []byte("aws4_request"))
signature2 := GetHMAC(hash, []byte(stringToSign)) // Get Signature HMAC SHA256
signature := hex.EncodeToString(signature2)
authorizationHeader := fmt.Sprintf("%s Credential=%s/%s, SignedHeaders=%s, Signature=%s",
algorithm, resIDMatch, credentialScope, signedHeaders, signature)
req.Header.Add("Authorization", authorizationHeader)
req.Header.Add("x-amz-date", amzDate)
req.Header.Add("x-amz-security-token", resSessionMatch)
req.URL.RawQuery = params.Encode()
client := s.verificationClient
if client == nil {
client = defaultVerificationClient
}
extraData := map[string]string{
"rotation_guide": "https://howtorotate.com/docs/tutorials/aws/",
}
res, err := client.Do(req)
if err == nil {
defer res.Body.Close()
if res.StatusCode >= 200 && res.StatusCode < 300 {
identityInfo := identityRes{}
err := json.NewDecoder(res.Body).Decode(&identityInfo)
if err == nil {
extraData["account"] = identityInfo.GetCallerIdentityResponse.GetCallerIdentityResult.Account
extraData["user_id"] = identityInfo.GetCallerIdentityResponse.GetCallerIdentityResult.UserID
extraData["arn"] = identityInfo.GetCallerIdentityResponse.GetCallerIdentityResult.Arn
return true, extraData, nil
} else {
return false, nil, err
}
} else if res.StatusCode == 403 {
// Experimentation has indicated that if you make two GetCallerIdentity requests within five seconds that
// share a key ID but are signed with different secrets the second one will be rejected with a 403 that
// carries a SignatureDoesNotMatch code in its body. This happens even if the second ID-secret pair is
// valid. Since this is exactly our access pattern, we need to work around it.
//
// Fortunately, experimentation has also revealed a workaround: simply resubmit the second request. The
// response to the resubmission will be as expected. But there's a caveat: You can't have closed the body of
// the response to the original second request, or read to its end, or the resubmission will also yield a
// SignatureDoesNotMatch. For this reason, we have to re-request all 403s. We can't re-request only
// SignatureDoesNotMatch responses, because we can only tell whether a given 403 is a SignatureDoesNotMatch
// after decoding its response body, which requires reading the entire response body, which disables the
// workaround.
//
// We are clearly deep in the guts of AWS implementation details here, so this all might change with no
// notice. If you're here because something in this detector broke, you have my condolences.
if retryOn403 {
return s.verifyMatch(ctx, resIDMatch, resSecretMatch, resSessionMatch, false)
}
var body awsErrorResponseBody
err = json.NewDecoder(res.Body).Decode(&body)
if err == nil {
// All instances of the code I've seen in the wild are PascalCased but this check is
// case-insensitive out of an abundance of caution
if strings.EqualFold(body.Error.Code, "InvalidClientTokenId") {
return false, nil, nil
} else {
return false, nil, fmt.Errorf("request to %v returned status %d with an unexpected reason (%s: %s)", res.Request.URL, res.StatusCode, body.Error.Code, body.Error.Message)
}
} else {
return false, nil, fmt.Errorf("couldn't parse the sts response body (%v)", err)
}
} else {
return false, nil, fmt.Errorf("request to %v returned unexpected status %d", res.Request.URL, res.StatusCode)
}
} else {
return false, nil, err
}
}
func awsCustomCleanResults(results []detectors.Result) []detectors.Result {
if len(results) == 0 {
return results
}
// For every ID, we want at most one result, preferably verified.
idResults := map[string]detectors.Result{}
for _, result := range results {
// Always accept the verified result as the result for the given ID.
if result.Verified {
idResults[result.Redacted] = result
continue
}
// Only include an unverified result if we don't already have a result for a given ID.
if _, exist := idResults[result.Redacted]; !exist {
idResults[result.Redacted] = result
}
}
var out []detectors.Result
for _, r := range idResults {
out = append(out, r)
}
return out
}
type awsError struct {
Code string `json:"Code"`
Message string `json:"Message"`
}
type awsErrorResponseBody struct {
Error awsError `json:"Error"`
}
type identityRes struct {
GetCallerIdentityResponse struct {
GetCallerIdentityResult struct {
Account string `json:"Account"`
Arn string `json:"Arn"`
UserID string `json:"UserId"`
} `json:"GetCallerIdentityResult"`
ResponseMetadata struct {
RequestID string `json:"RequestId"`
} `json:"ResponseMetadata"`
} `json:"GetCallerIdentityResponse"`
}
func (s scanner) Type() detectorspb.DetectorType {
return detectorspb.DetectorType_AWSSessionKey
}
-4
View File
@@ -57,10 +57,6 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
RawV2: []byte(clientID[2] + clientSecret[2] + tenantID[2]),
Redacted: clientID[2],
}
// Set the RotationGuideURL in the ExtraData
s.ExtraData = map[string]string{
"rotation_guide": "https://howtorotate.com/docs/tutorials/azure/",
}
if verify {
cred := auth.NewClientCredentialsConfig(clientID[2], clientSecret[2], tenantID[2])
-114
View File
@@ -1,114 +0,0 @@
package azurebatch
import (
"context"
"crypto/hmac"
"crypto/sha256"
"encoding/base64"
"fmt"
"net/http"
"regexp"
"strings"
"time"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
)
type Scanner struct {
client *http.Client
}
// Ensure the Scanner satisfies the interface at compile time.
var _ detectors.Detector = (*Scanner)(nil)
var (
defaultClient = common.SaneHttpClient()
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
urlPat = regexp.MustCompile(`https://(.{1,50})\.(.{1,50})\.batch\.azure\.com`)
secretPat = regexp.MustCompile(`[A-Za-z0-9+/=]{88}`)
)
// Keywords are used for efficiently pre-filtering chunks.
// Use identifiers in the secret preferably, or the provider name.
func (s Scanner) Keywords() []string {
return []string{".batch.azure.com"}
}
// FromData will find and optionally verify Azurebatch secrets in a given set of bytes.
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
dataStr := string(data)
urlMatches := urlPat.FindAllStringSubmatch(dataStr, -1)
secretMatches := secretPat.FindAllStringSubmatch(dataStr, -1)
for _, urlMatch := range urlMatches {
for _, secretMatch := range secretMatches {
endpoint := urlMatch[0]
accountName := urlMatch[1]
accountKey := secretMatch[0]
s1 := detectors.Result{
DetectorType: detectorspb.DetectorType_AzureBatch,
Raw: []byte(endpoint),
Redacted: endpoint,
RawV2: []byte(endpoint + accountKey),
}
if verify {
client := s.client
if client == nil {
client = defaultClient
}
url := fmt.Sprintf("%s/applications?api-version=2020-09-01.12.0", endpoint)
date := time.Now().UTC().Format(http.TimeFormat)
stringToSign := fmt.Sprintf(
"GET\n\n\n\n\napplication/json\n%s\n\n\n\n\n\n%s\napi-version:%s",
date,
strings.ToLower(fmt.Sprintf("/%s/applications", accountName)),
"2020-09-01.12.0",
)
key, _ := base64.StdEncoding.DecodeString(accountKey)
h := hmac.New(sha256.New, key)
h.Write([]byte(stringToSign))
signature := base64.StdEncoding.EncodeToString(h.Sum(nil))
req, err := http.NewRequest("GET", url, nil)
if err != nil {
continue
}
req.Header.Set("Content-Type", "application/json")
req.Header.Set("Authorization", fmt.Sprintf("SharedKey %s:%s", accountName, signature))
req.Header.Set("Date", date)
resp, err := client.Do(req)
if err != nil {
continue
}
defer resp.Body.Close()
if resp.StatusCode == http.StatusOK {
s1.Verified = true
}
}
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
if !s1.Verified && detectors.IsKnownFalsePositive(accountKey, detectors.DefaultFalsePositives, true) {
continue
}
results = append(results, s1)
if s1.Verified {
break
}
}
}
return results, nil
}
func (s Scanner) Type() detectorspb.DetectorType {
return detectorspb.DetectorType_AzureBatch
}
-130
View File
@@ -1,130 +0,0 @@
//go:build detectors
// +build detectors
package azurebatch
import (
"context"
"fmt"
"testing"
"time"
"github.com/google/go-cmp/cmp"
"github.com/google/go-cmp/cmp/cmpopts"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
)
func TestAzurebatch_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
url := testSecrets.MustGetField("AZUREBATCH_URL")
secret := testSecrets.MustGetField("AZUREBATCH_KEY")
inactiveSecret := testSecrets.MustGetField("AZUREBATCH_KEY_INACTIVE")
type args struct {
ctx context.Context
data []byte
verify bool
}
tests := []struct {
name string
s Scanner
args args
want []detectors.Result
wantErr bool
wantVerificationErr bool
}{
{
name: "found, verified",
s: Scanner{},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a azurebatch secret %s and %s within", url, secret)),
verify: true,
},
want: []detectors.Result{
{
DetectorType: detectorspb.DetectorType_AzureBatch,
Verified: true,
},
},
wantErr: false,
wantVerificationErr: false,
},
{
name: "found, unverified",
s: Scanner{},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a azurebatch secret %s and %s within but not valid", url, inactiveSecret)), // the secret would satisfy the regex but not pass validation
verify: true,
},
want: []detectors.Result{
{
DetectorType: detectorspb.DetectorType_AzureBatch,
Verified: false,
},
},
wantErr: false,
wantVerificationErr: false,
},
{
name: "not found",
s: Scanner{},
args: args{
ctx: context.Background(),
data: []byte("You cannot find the secret within"),
verify: true,
},
want: nil,
wantErr: false,
wantVerificationErr: false,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
if (err != nil) != tt.wantErr {
t.Errorf("AzureBatch.FromData() error = %v, wantErr %v", err, tt.wantErr)
return
}
for i := range got {
if len(got[i].Raw) == 0 {
t.Fatalf("no raw secret present: \n %+v", got[i])
}
if (got[i].VerificationError() != nil) != tt.wantVerificationErr {
t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError())
}
}
ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "RawV2", "Raw", "Redacted", "verificationError")
if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {
t.Errorf("AzureBatch.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
}
})
}
}
func BenchmarkFromData(benchmark *testing.B) {
ctx := context.Background()
s := Scanner{}
for name, data := range detectors.MustGetBenchmarkData() {
benchmark.Run(name, func(b *testing.B) {
b.ResetTimer()
for n := 0; n < b.N; n++ {
_, err := s.FromData(ctx, false, data)
if err != nil {
b.Fatal(err)
}
}
})
}
}
@@ -1,102 +0,0 @@
package azurecontainerregistry
import (
"context"
"encoding/base64"
"fmt"
"net/http"
"regexp"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
)
type Scanner struct {
client *http.Client
}
// Ensure the Scanner satisfies the interface at compile time.
var _ detectors.Detector = (*Scanner)(nil)
var (
defaultClient = common.SaneHttpClient()
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
url = regexp.MustCompile(`([a-zA-Z0-9-]{1,100})\.azurecr\.io`)
password = regexp.MustCompile(`\b[A-Za-z0-9+/=]{52}\b`)
)
// Keywords are used for efficiently pre-filtering chunks.
// Use identifiers in the secret preferably, or the provider name.
func (s Scanner) Keywords() []string {
return []string{".azurecr.io"}
}
// FromData will find and optionally verify Azurecontainerregistry secrets in a given set of bytes.
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
dataStr := string(data)
urlMatches := url.FindAllStringSubmatch(dataStr, -1)
passwordMatches := password.FindAllStringSubmatch(dataStr, -1)
for _, urlMatch := range urlMatches {
for _, passwordMatch := range passwordMatches {
endpoint := urlMatch[0]
username := urlMatch[1]
password := passwordMatch[0]
s1 := detectors.Result{
DetectorType: detectorspb.DetectorType_AzureContainerRegistry,
Raw: []byte(endpoint),
Redacted: endpoint,
RawV2: []byte(endpoint + password),
}
if verify {
client := s.client
if client == nil {
client = defaultClient
}
auth := base64.StdEncoding.EncodeToString([]byte(fmt.Sprintf("%s:%s", username, password)))
url := fmt.Sprintf("https://%s/v2/", endpoint)
req, err := http.NewRequest("GET", url, nil)
if err != nil {
continue
}
req.Header.Set("Authorization", fmt.Sprintf("Basic %s", auth))
res, err := client.Do(req)
if err == nil {
defer res.Body.Close()
if res.StatusCode >= 200 && res.StatusCode < 300 {
s1.Verified = true
} else if res.StatusCode == 401 {
// The secret is determinately not verified (nothing to do)
} else {
err = fmt.Errorf("unexpected HTTP response status %d", res.StatusCode)
s1.SetVerificationError(err, password)
}
} else {
s1.SetVerificationError(err, username, password)
}
}
if !s1.Verified && detectors.IsKnownFalsePositive(password, detectors.DefaultFalsePositives, true) {
continue
}
results = append(results, s1)
if s1.Verified {
break
}
}
}
return results, nil
}
func (s Scanner) Type() detectorspb.DetectorType {
return detectorspb.DetectorType_AzureContainerRegistry
}
@@ -1,129 +0,0 @@
//go:build detectors
// +build detectors
package azurecontainerregistry
import (
"context"
"fmt"
"testing"
"time"
"github.com/google/go-cmp/cmp"
"github.com/google/go-cmp/cmp/cmpopts"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
)
func TestAzureContainerRegistry_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
azureHost := testSecrets.MustGetField("AZURE_CR_HOST")
password := testSecrets.MustGetField("AZURE_CR_PASSWORD")
passwordInactive := testSecrets.MustGetField("AZURE_CR_PASSWORD_INACTIVE")
type args struct {
ctx context.Context
data []byte
verify bool
}
tests := []struct {
name string
s Scanner
args args
want []detectors.Result
wantErr bool
wantVerificationErr bool
}{
{
name: "found, verified",
s: Scanner{},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a azurecontainerregistry secret %s and %s within", azureHost, password)),
verify: true,
},
want: []detectors.Result{
{
DetectorType: detectorspb.DetectorType_AzureContainerRegistry,
Verified: true,
},
},
wantErr: false,
wantVerificationErr: false,
},
{
name: "found, unverified",
s: Scanner{},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a azurecontainerregistry secret %s and %s within but not valid", azureHost, passwordInactive)), // the secret would satisfy the regex but not pass validation
verify: true,
},
want: []detectors.Result{
{
DetectorType: detectorspb.DetectorType_AzureContainerRegistry,
Verified: false,
},
},
wantErr: false,
wantVerificationErr: false,
},
{
name: "not found",
s: Scanner{},
args: args{
ctx: context.Background(),
data: []byte("You cannot find the secret within"),
verify: true,
},
want: nil,
wantErr: false,
wantVerificationErr: false,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
if (err != nil) != tt.wantErr {
t.Errorf("AzureContainerRegistry.FromData() error = %v, wantErr %v", err, tt.wantErr)
return
}
for i := range got {
if len(got[i].Raw) == 0 {
t.Fatalf("no raw secret present: \n %+v", got[i])
}
if (got[i].VerificationError() != nil) != tt.wantVerificationErr {
t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError())
}
}
ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "RawV2", "Raw", "Redacted", "verificationError")
if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {
t.Errorf("AzureContainerRegistry.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
}
})
}
}
func BenchmarkFromData(benchmark *testing.B) {
ctx := context.Background()
s := Scanner{}
for name, data := range detectors.MustGetBenchmarkData() {
benchmark.Run(name, func(b *testing.B) {
b.ResetTimer()
for n := 0; n < b.N; n++ {
_, err := s.FromData(ctx, false, data)
if err != nil {
b.Fatal(err)
}
}
})
}
}
@@ -1,98 +0,0 @@
package azurestorage
import (
"context"
"crypto/hmac"
"crypto/sha256"
"encoding/base64"
"fmt"
"net/http"
"regexp"
"strings"
"time"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
)
type Scanner struct {
client *http.Client
}
var _ detectors.Detector = (*Scanner)(nil)
var (
defaultClient = http.DefaultClient
keyPat = regexp.MustCompile(`DefaultEndpointsProtocol=https;AccountName=(?P<account_name>[^;]+);AccountKey=(?P<account_key>[^;]+);EndpointSuffix=core\.windows\.net`)
)
func (s Scanner) Keywords() []string {
return []string{"DefaultEndpointsProtocol=https;AccountName="}
}
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
dataStr := string(data)
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
for _, match := range matches {
if len(match) != 3 {
continue
}
accountName := strings.TrimSpace(match[1])
accountKey := strings.TrimSpace(match[2])
s1 := detectors.Result{
DetectorType: detectorspb.DetectorType_AzureStorage,
Raw: []byte(accountName),
}
if verify {
client := s.client
if client == nil {
client = defaultClient
}
now := time.Now().UTC().Format(http.TimeFormat)
stringToSign := "GET\n\n\n\n\n\n\n\n\n\n\n\nx-ms-date:" + now + "\nx-ms-version:2019-12-12\n/" + accountName + "/\ncomp:list"
accountKeyBytes, _ := base64.StdEncoding.DecodeString(accountKey)
h := hmac.New(sha256.New, accountKeyBytes)
h.Write([]byte(stringToSign))
signature := base64.StdEncoding.EncodeToString(h.Sum(nil))
url := "https://" + accountName + ".blob.core.windows.net/?comp=list"
req, err := http.NewRequestWithContext(ctx, "GET", url, nil)
if err != nil {
continue
}
req.Header.Set("x-ms-date", now)
req.Header.Set("x-ms-version", "2019-12-12")
req.Header.Set("Authorization", "SharedKey "+accountName+":"+signature)
if err != nil {
continue
}
res, err := client.Do(req)
if err == nil {
defer res.Body.Close()
if res.StatusCode >= 200 && res.StatusCode < 300 {
s1.Verified = true
} else if res.StatusCode == 403 {
} else {
err = fmt.Errorf("unexpected HTTP response status %d", res.StatusCode)
s1.SetVerificationError(err, accountKey)
}
} else {
s1.SetVerificationError(err, accountKey)
}
}
results = append(results, s1)
}
return results, nil
}
func (s Scanner) Type() detectorspb.DetectorType {
return detectorspb.DetectorType_AzureStorage
}
@@ -1,162 +0,0 @@
//go:build detectors
// +build detectors
package azurestorage
import (
"context"
"fmt"
"testing"
"time"
"github.com/google/go-cmp/cmp"
"github.com/google/go-cmp/cmp/cmpopts"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
)
func TestAzurestorage_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
secret := testSecrets.MustGetField("AZURE_STORAGE")
inactiveSecret := testSecrets.MustGetField("AZURE_STORAGE_INACTIVE")
type args struct {
ctx context.Context
data []byte
verify bool
}
tests := []struct {
name string
s Scanner
args args
want []detectors.Result
wantErr bool
wantVerificationErr bool
}{
{
name: "found, verified",
s: Scanner{},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a azurestorage secret %s within", secret)),
verify: true,
},
want: []detectors.Result{
{
DetectorType: detectorspb.DetectorType_AzureStorage,
Verified: true,
},
},
wantErr: false,
wantVerificationErr: false,
},
{
name: "found, unverified",
s: Scanner{},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a azurestorage secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
verify: true,
},
want: []detectors.Result{
{
DetectorType: detectorspb.DetectorType_AzureStorage,
Verified: false,
},
},
wantErr: false,
wantVerificationErr: false,
},
{
name: "not found",
s: Scanner{},
args: args{
ctx: context.Background(),
data: []byte("You cannot find the secret within"),
verify: true,
},
want: nil,
wantErr: false,
wantVerificationErr: false,
},
{
name: "found, would be verified if not for timeout",
s: Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a azurestorage secret %s within", secret)),
verify: true,
},
want: []detectors.Result{
{
DetectorType: detectorspb.DetectorType_AzureStorage,
Verified: false,
},
},
wantErr: false,
wantVerificationErr: true,
},
{
name: "found, verified but unexpected api surface",
s: Scanner{client: common.ConstantResponseHttpClient(404, "")},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a azurestorage secret %s within", secret)),
verify: true,
},
want: []detectors.Result{
{
DetectorType: detectorspb.DetectorType_AzureStorage,
Verified: false,
},
},
wantErr: false,
wantVerificationErr: true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
if (err != nil) != tt.wantErr {
t.Errorf("Azuretorage.FromData() error = %v, wantErr %v", err, tt.wantErr)
return
}
for i := range got {
if len(got[i].Raw) == 0 {
t.Fatalf("no raw secret present: \n %+v", got[i])
}
if (got[i].VerificationError() != nil) != tt.wantVerificationErr {
t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError())
}
}
ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "verificationError")
if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {
t.Errorf("Azurestorage.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
}
})
}
}
func BenchmarkFromData(benchmark *testing.B) {
ctx := context.Background()
s := Scanner{}
for name, data := range detectors.MustGetBenchmarkData() {
benchmark.Run(name, func(b *testing.B) {
b.ResetTimer()
for n := 0; n < b.N; n++ {
_, err := s.FromData(ctx, false, data)
if err != nil {
b.Fatal(err)
}
}
})
}
}
@@ -1,8 +1,9 @@
package web3storage
package baseapiio
import (
"context"
"fmt"
"io"
"net/http"
"regexp"
"strings"
@@ -12,26 +13,25 @@ import (
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
)
type Scanner struct {
client *http.Client
}
type Scanner struct{}
// Ensure the Scanner satisfies the interface at compile time.
var _ detectors.Detector = (*Scanner)(nil)
var (
defaultClient = common.SaneHttpClient()
client = common.SaneHttpClient()
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"web3"}) + `\b(eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9\.eyJ[A-Za-z0-9-_]{100,300}\.[A-Za-z0-9-_]{25,100})\b`)
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"baseapi", "base-api"}) + `\b([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})\b`)
)
// Keywords are used for efficiently pre-filtering chunks.
// Use identifiers in the secret preferably, or the provider name.
func (s Scanner) Keywords() []string {
return []string{"web3"}
return []string{"baseapi", "base-api"}
}
// FromData will find and optionally verify Web3storage secrets in a given set of bytes.
// FromData will find and optionally verify BaseApiIO secrets in a given set of bytes.
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
dataStr := string(data)
@@ -44,16 +44,12 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
resMatch := strings.TrimSpace(match[1])
s1 := detectors.Result{
DetectorType: detectorspb.DetectorType_Web3Storage,
DetectorType: detectorspb.DetectorType_BaseApiIO,
Raw: []byte(resMatch),
}
if verify {
client := s.client
if client == nil {
client = defaultClient
}
req, err := http.NewRequestWithContext(ctx, "GET", "https://api.web3.storage/user/uploads", nil)
req, err := http.NewRequestWithContext(ctx, "GET", "https://api.base-api.io/v1/forms?page=1&per_page=10", nil)
if err != nil {
continue
}
@@ -61,22 +57,21 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
res, err := client.Do(req)
if err == nil {
defer res.Body.Close()
if res.StatusCode >= 200 && res.StatusCode < 300 {
s1.Verified = true
} else if res.StatusCode == 401 {
// The secret is determinately not verified (nothing to do)
} else {
err := fmt.Errorf("unexpected HTTP response status %d", res.StatusCode)
s1.SetVerificationError(err, resMatch)
bodyBytes, err := io.ReadAll(res.Body)
if err != nil {
continue
}
} else {
s1.SetVerificationError(err, resMatch)
}
}
body := string(bodyBytes)
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
if !s1.Verified && detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
continue
if strings.Contains(body, "items") {
s1.Verified = true
} else {
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
continue
}
}
}
}
results = append(results, s1)
@@ -86,5 +81,5 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
}
func (s Scanner) Type() detectorspb.DetectorType {
return detectorspb.DetectorType_Web3Storage
return detectorspb.DetectorType_BaseApiIO
}
@@ -1,7 +1,7 @@
//go:build detectors
// +build detectors
package instamojo
package baseapiio
import (
"context"
@@ -16,16 +16,15 @@ import (
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
)
func TestInstamojo_FromChunk(t *testing.T) {
func TestBaseApiIO_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")
testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
id := testSecrets.MustGetField("INSTAMOJO_CLIENT_ID")
secret := testSecrets.MustGetField("INSTAMOJO_SECRET")
inactiveSecret := testSecrets.MustGetField("INSTAMOJO_INACTIVE")
secret := testSecrets.MustGetField("BASEAPIIO")
inactiveSecret := testSecrets.MustGetField("BASEAPIIO_INACTIVE")
type args struct {
ctx context.Context
@@ -33,45 +32,43 @@ func TestInstamojo_FromChunk(t *testing.T) {
verify bool
}
tests := []struct {
name string
s Scanner
args args
want []detectors.Result
wantErr bool
wantVerificationErr bool
name string
s Scanner
args args
want []detectors.Result
wantErr bool
}{
{
name: "found, verified",
s: Scanner{},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a instamojo secret %s within id %s", secret, id)),
data: []byte(fmt.Sprintf("You can find a baseapiio secret %s within", secret)),
verify: true,
},
want: []detectors.Result{
{
DetectorType: detectorspb.DetectorType_Instamojo,
DetectorType: detectorspb.DetectorType_BaseApiIO,
Verified: true,
},
},
wantErr: false,
wantVerificationErr: false,
wantErr: false,
},
{
name: "found, unverified",
s: Scanner{},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a instamojo secret %s within but not valid, within id %s", inactiveSecret, id)), // the secret would satisfy the regex but not pass validation
data: []byte(fmt.Sprintf("You can find a baseapiio secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
verify: true,
},
want: []detectors.Result{
{
DetectorType: detectorspb.DetectorType_Instamojo,
DetectorType: detectorspb.DetectorType_BaseApiIO,
Verified: false,
},
},
wantErr: false,
wantVerificationErr: true,
wantErr: false,
},
{
name: "not found",
@@ -81,16 +78,16 @@ func TestInstamojo_FromChunk(t *testing.T) {
data: []byte("You cannot find the secret within"),
verify: true,
},
want: nil,
wantErr: false,
wantVerificationErr: true,
want: nil,
wantErr: false,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
s := Scanner{}
got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
if (err != nil) != tt.wantErr {
t.Errorf("Instamojo.FromData() error = %v, wantErr %v", err, tt.wantErr)
t.Errorf("BaseApiIO.FromData() error = %v, wantErr %v", err, tt.wantErr)
return
}
for i := range got {
@@ -100,7 +97,7 @@ func TestInstamojo_FromChunk(t *testing.T) {
got[i].Raw = nil
}
if diff := pretty.Compare(got, tt.want); diff != "" {
t.Errorf("Instamojo.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
t.Errorf("BaseApiIO.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
}
})
}
-90
View File
@@ -1,90 +0,0 @@
package betterstack
import (
"context"
"fmt"
"net/http"
"regexp"
"strings"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
)
type Scanner struct {
client *http.Client
}
// Ensure the Scanner satisfies the interface at compile time.
var _ detectors.Detector = (*Scanner)(nil)
var (
defaultClient = common.SaneHttpClient()
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"betterstack"}) + `\b([0-9a-zA-Z]{24})\b`)
)
// Keywords are used for efficiently pre-filtering chunks.
// Use identifiers in the secret preferably, or the provider name.
func (s Scanner) Keywords() []string {
return []string{"betterstack"}
}
// FromData will find and optionally verify Betterstack secrets in a given set of bytes.
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
dataStr := string(data)
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
for _, match := range matches {
if len(match) != 2 {
continue
}
resMatch := strings.TrimSpace(match[1])
s1 := detectors.Result{
DetectorType: detectorspb.DetectorType_BetterStack,
Raw: []byte(resMatch),
}
if verify {
client := s.client
if client == nil {
client = defaultClient
}
req, err := http.NewRequestWithContext(ctx, "GET", "https://uptime.betterstack.com/api/v2/monitors", nil)
if err != nil {
continue
}
req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", resMatch))
res, err := client.Do(req)
if err == nil {
defer res.Body.Close()
if res.StatusCode >= 200 && res.StatusCode < 300 {
s1.Verified = true
} else if res.StatusCode == 401 {
// The secret is determinately not verified (nothing to do)
} else {
err = fmt.Errorf("unexpected HTTP response status %d", res.StatusCode)
s1.SetVerificationError(err, resMatch)
}
} else {
s1.SetVerificationError(err, resMatch)
}
}
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
if !s1.Verified && detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
continue
}
results = append(results, s1)
}
return results, nil
}
func (s Scanner) Type() detectorspb.DetectorType {
return detectorspb.DetectorType_BetterStack
}
@@ -1,128 +0,0 @@
//go:build detectors
// +build detectors
package betterstack
import (
"context"
"fmt"
"testing"
"time"
"github.com/google/go-cmp/cmp"
"github.com/google/go-cmp/cmp/cmpopts"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
)
func TestBetterstack_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
secret := testSecrets.MustGetField("BETTERSTACK")
inactiveSecret := testSecrets.MustGetField("BETTERSTACK_INACTIVE")
type args struct {
ctx context.Context
data []byte
verify bool
}
tests := []struct {
name string
s Scanner
args args
want []detectors.Result
wantErr bool
wantVerificationErr bool
}{
{
name: "found, verified",
s: Scanner{},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a betterstack secret %s within", secret)),
verify: true,
},
want: []detectors.Result{
{
DetectorType: detectorspb.DetectorType_BetterStack,
Verified: true,
},
},
wantErr: false,
wantVerificationErr: false,
},
{
name: "found, unverified",
s: Scanner{},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a betterstack secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
verify: true,
},
want: []detectors.Result{
{
DetectorType: detectorspb.DetectorType_BetterStack,
Verified: false,
},
},
wantErr: false,
wantVerificationErr: false,
},
{
name: "not found",
s: Scanner{},
args: args{
ctx: context.Background(),
data: []byte("You cannot find the secret within"),
verify: true,
},
want: nil,
wantErr: false,
wantVerificationErr: false,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
if (err != nil) != tt.wantErr {
t.Errorf("Betterstack.FromData() error = %v, wantErr %v", err, tt.wantErr)
return
}
for i := range got {
if len(got[i].Raw) == 0 {
t.Fatalf("no raw secret present: \n %+v", got[i])
}
if (got[i].VerificationError() != nil) != tt.wantVerificationErr {
t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError())
}
}
ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "verificationError")
if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {
t.Errorf("Betterstack.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
}
})
}
}
func BenchmarkFromData(benchmark *testing.B) {
ctx := context.Background()
s := Scanner{}
for name, data := range detectors.MustGetBenchmarkData() {
benchmark.Run(name, func(b *testing.B) {
b.ResetTimer()
for n := 0; n < b.N; n++ {
_, err := s.FromData(ctx, false, data)
if err != nil {
b.Fatal(err)
}
}
})
}
}
+2 -15
View File
@@ -2,7 +2,6 @@ package bitcoinaverage
import (
"context"
"encoding/json"
"net/http"
"regexp"
"strings"
@@ -30,11 +29,6 @@ func (s Scanner) Keywords() []string {
return []string{"bitcoinaverage"}
}
type response struct {
Msg string `json:"msg"`
Success bool `json:"success"`
}
// FromData will find and optionally verify BitcoinAverage secrets in a given set of bytes.
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
dataStr := string(data)
@@ -52,7 +46,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
Raw: []byte(resMatch),
}
if verify {
req, err := http.NewRequestWithContext(ctx, "GET", "https://apiv2.bitcoinaverage.com/websocket/v3/get_ticket", nil)
req, err := http.NewRequestWithContext(ctx, "GET", "https://apiv2.bitcoinaverage.com/websocket/get_ticket", nil)
if err != nil {
continue
}
@@ -61,14 +55,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
if err == nil {
defer res.Body.Close()
if res.StatusCode >= 200 && res.StatusCode < 300 {
resp := &response{}
if err = json.NewDecoder(res.Body).Decode(resp); err != nil {
s1.SetVerificationError(err, resMatch)
continue
}
if resp.Success {
s1.Verified = true
}
s1.Verified = true
} else {
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
@@ -1,4 +1,4 @@
package fullstory_v2
package blablabus
import (
"context"
@@ -16,24 +16,21 @@ type Scanner struct{}
// Ensure the Scanner satisfies the interface at compile time.
var _ detectors.Detector = (*Scanner)(nil)
var _ detectors.Versioner = (*Scanner)(nil)
func (Scanner) Version() int { return 2 }
var (
client = common.SaneHttpClient()
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
keyPat = regexp.MustCompile(`\b(na1\.[A-Za-z0-9\+\/]{100})\b`)
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"blablabus"}) + `\b([0-9A-Za-z]{22})\b`)
)
// Keywords are used for efficiently pre-filtering chunks.
// Use identifiers in the secret preferably, or the provider name.
func (s Scanner) Keywords() []string {
return []string{"fullstory"}
return []string{"blablabus"}
}
// FromData will find and optionally verify Fullstory secrets in a given set of bytes.
// FromData will find and optionally verify Blablabus secrets in a given set of bytes.
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
dataStr := string(data)
@@ -46,16 +43,17 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
resMatch := strings.TrimSpace(match[1])
s1 := detectors.Result{
DetectorType: detectorspb.DetectorType_Fullstory,
DetectorType: detectorspb.DetectorType_Blablabus,
Raw: []byte(resMatch),
}
if verify {
req, err := http.NewRequestWithContext(ctx, "GET", "https://api.fullstory.com/v2/users", nil)
req, err := http.NewRequestWithContext(ctx, "GET", "https://api.idbus.com/v3/stops", nil)
if err != nil {
continue
}
req.Header.Add("Authorization", fmt.Sprintf("Basic %s", resMatch))
req.Header.Add("Content-Type", "application/json")
req.Header.Add("Authorization", fmt.Sprintf("Token %s", resMatch))
res, err := client.Do(req)
if err == nil {
defer res.Body.Close()
@@ -77,5 +75,5 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
}
func (s Scanner) Type() detectorspb.DetectorType {
return detectorspb.DetectorType_Fullstory
return detectorspb.DetectorType_Blablabus
}
@@ -1,7 +1,7 @@
//go:build detectors
// +build detectors
package fullstory_v2
package blablabus
import (
"context"
@@ -16,15 +16,15 @@ import (
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
)
func TestFullstory_FromChunk(t *testing.T) {
func TestBlablabus_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
secret := testSecrets.MustGetField("FULLSTORY_V2")
inactiveSecret := testSecrets.MustGetField("FULLSTORY_V2_INACTIVE")
secret := testSecrets.MustGetField("BLABLABUS")
inactiveSecret := testSecrets.MustGetField("BLABLABUS_INACTIVE")
type args struct {
ctx context.Context
@@ -43,12 +43,12 @@ func TestFullstory_FromChunk(t *testing.T) {
s: Scanner{},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a fullstory secret %s within", secret)),
data: []byte(fmt.Sprintf("You can find a blablabus secret %s within", secret)),
verify: true,
},
want: []detectors.Result{
{
DetectorType: detectorspb.DetectorType_Fullstory,
DetectorType: detectorspb.DetectorType_Blablabus,
Verified: true,
},
},
@@ -59,12 +59,12 @@ func TestFullstory_FromChunk(t *testing.T) {
s: Scanner{},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a fullstory secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
data: []byte(fmt.Sprintf("You can find a blablabus secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
verify: true,
},
want: []detectors.Result{
{
DetectorType: detectorspb.DetectorType_Fullstory,
DetectorType: detectorspb.DetectorType_Blablabus,
Verified: false,
},
},
@@ -87,7 +87,7 @@ func TestFullstory_FromChunk(t *testing.T) {
s := Scanner{}
got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
if (err != nil) != tt.wantErr {
t.Errorf("Fullstory.FromData() error = %v, wantErr %v", err, tt.wantErr)
t.Errorf("Blablabus.FromData() error = %v, wantErr %v", err, tt.wantErr)
return
}
for i := range got {
@@ -97,7 +97,7 @@ func TestFullstory_FromChunk(t *testing.T) {
got[i].Raw = nil
}
if diff := pretty.Compare(got, tt.want); diff != "" {
t.Errorf("Fullstory.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
t.Errorf("Blablabus.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
}
})
}
@@ -2,7 +2,6 @@ package braintreepayments
import (
"context"
"fmt"
"io"
"net/http"
"regexp"
@@ -13,21 +12,14 @@ import (
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
)
type Scanner struct {
client *http.Client
useTestURL bool
}
type Scanner struct{}
// Ensure the Scanner satisfies the interface at compile time
var _ detectors.Detector = (*Scanner)(nil)
const (
verifyURL = "https://payments.braintree-api.com/graphql"
verifyTestURL = "https://payments.sandbox.braintree-api.com/graphql"
)
var (
defaultClient = common.SaneHttpClient()
client = common.SaneHttpClient()
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"braintree"}) + `\b([0-9a-f]{32})\b`)
idPat = regexp.MustCompile(detectors.PrefixRegex([]string{"braintree"}) + `\b([0-9a-z]{16})\b`)
@@ -56,6 +48,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
if len(idMatch) != 2 {
continue
}
resIdMatch := strings.TrimSpace(idMatch[1])
s1 := detectors.Result{
@@ -64,16 +57,34 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
}
if verify {
client := s.getClient()
url := s.getBraintreeURL()
isVerified, verificationErr := verifyBraintree(ctx, client, url, resIdMatch, resMatch)
s1.Verified = isVerified
s1.SetVerificationError(verificationErr, resMatch)
}
payload := strings.NewReader(`{"query": "query { ping }"}`)
req, err := http.NewRequestWithContext(ctx, "POST", "https://payments.braintree-api.com/graphql", payload)
if err != nil {
continue
}
req.Header.Add("Content-Type", "application/json")
req.Header.Add("Braintree-Version", "2019-01-01")
req.SetBasicAuth(resIdMatch, resMatch)
res, err := client.Do(req)
if err == nil {
bodyBytes, err := io.ReadAll(res.Body)
if err != nil {
continue
}
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key
if !s1.Verified && detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
continue
bodyString := string(bodyBytes)
validResponse := strings.Contains(bodyString, `"data":{`)
defer res.Body.Close()
if res.StatusCode >= 200 && res.StatusCode < 300 && validResponse {
s1.Verified = true
} else {
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
continue
}
}
}
}
results = append(results, s1)
@@ -83,53 +94,6 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
return results, nil
}
func (s Scanner) getBraintreeURL() string {
if s.useTestURL {
return verifyTestURL
}
return verifyURL
}
func (s Scanner) getClient() *http.Client {
if s.client != nil {
return s.client
}
return defaultClient
}
func verifyBraintree(ctx context.Context, client *http.Client, url, pubKey, privKey string) (bool, error) {
payload := strings.NewReader(`{"query": "query { ping }"}`)
req, err := http.NewRequestWithContext(ctx, http.MethodPost, url, payload)
if err != nil {
return false, err
}
req.Header.Add("Content-Type", "application/json")
req.Header.Add("Braintree-Version", "2019-01-01")
req.SetBasicAuth(pubKey, privKey)
res, err := client.Do(req)
if err != nil {
return false, err
}
bodyBytes, err := io.ReadAll(res.Body)
if err != nil {
return false, err
}
defer res.Body.Close()
bodyString := string(bodyBytes)
if !(res.StatusCode == http.StatusOK) {
return false, fmt.Errorf("unexpected HTTP response status %d", res.StatusCode)
}
validResponse := `"data":{`
if strings.Contains(bodyString, validResponse) {
return true, nil
}
return false, nil
}
func (s Scanner) Type() detectorspb.DetectorType {
return detectorspb.DetectorType_BraintreePayments
}
@@ -6,11 +6,10 @@ import (
"testing"
"time"
"github.com/google/go-cmp/cmp"
"github.com/google/go-cmp/cmp/cmpopts"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/kylelemons/godebug/pretty"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
)
@@ -31,16 +30,15 @@ func TestBraintreePayments_FromChunk(t *testing.T) {
verify bool
}
tests := []struct {
name string
s Scanner
args args
want []detectors.Result
wantErr bool
wantVerificationErr bool
name string
s Scanner
args args
want []detectors.Result
wantErr bool
}{
{
name: "found, verified",
s: Scanner{useTestURL: true},
s: Scanner{},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a braintree secret %s within braintree %s", secret, id)),
@@ -54,47 +52,9 @@ func TestBraintreePayments_FromChunk(t *testing.T) {
},
wantErr: false,
},
{
name: "found, verified but unexpected api surface",
s: Scanner{
client: common.ConstantResponseHttpClient(404, ""),
},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a braintree secret %s within braintree %s", secret, id)),
verify: true,
},
want: []detectors.Result{
{
DetectorType: detectorspb.DetectorType_BraintreePayments,
Verified: false,
},
},
wantErr: false,
wantVerificationErr: true,
},
{
name: "found, would be verified if not for timeout",
s: Scanner{
client: common.SaneHttpClientTimeOut(1 * time.Microsecond),
},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a braintree secret %s within braintree %s", secret, id)),
verify: true,
},
want: []detectors.Result{
{
DetectorType: detectorspb.DetectorType_BraintreePayments,
Verified: false,
},
},
wantErr: false,
wantVerificationErr: true,
},
{
name: "found, unverified",
s: Scanner{useTestURL: true},
s: Scanner{},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a braintree secret %s within braintree %s but not valid", inactiveSecret, id)), // the secret would satisfy the regex but not pass validation
@@ -122,7 +82,8 @@ func TestBraintreePayments_FromChunk(t *testing.T) {
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
s := Scanner{}
got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
if (err != nil) != tt.wantErr {
t.Errorf("BraintreePayments.FromData() error = %v, wantErr %v", err, tt.wantErr)
return
@@ -131,13 +92,9 @@ func TestBraintreePayments_FromChunk(t *testing.T) {
if len(got[i].Raw) == 0 {
t.Fatalf("no raw secret present: \n %+v", got[i])
}
if (got[i].VerificationError() != nil) != tt.wantVerificationErr {
t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError())
}
got[i].Raw = nil
}
ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "verificationError")
if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {
if diff := pretty.Compare(got, tt.want); diff != "" {
t.Errorf("BraintreePayments.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
}
})
+21 -44
View File
@@ -2,7 +2,6 @@ package browserstack
import (
"context"
"fmt"
"net/http"
"regexp"
"strings"
@@ -12,21 +11,17 @@ import (
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
)
type Scanner struct {
client *http.Client
}
type Scanner struct{}
// Ensure the Scanner satisfies the interface at compile time.
var _ detectors.Detector = (*Scanner)(nil)
const browserStackAPIURL = "https://www.browserstack.com/automate/plan.json"
var (
defaultClient = common.SaneHttpClient()
client = common.SaneHttpClient()
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"hub-cloud.browserstack.com", "accessKey", "\"access_Key\":", "ACCESS_KEY", "key", "browserstackKey", "BS_AUTHKEY", "BROWSERSTACK_ACCESS_KEY"}) + `\b([0-9a-zA-Z]{20})\b`)
userPat = regexp.MustCompile(detectors.PrefixRegex([]string{"hub-cloud.browserstack.com", "userName", "\"username\":", "USER_NAME", "user", "browserstackUser", "BS_USERNAME", "BROWSERSTACK_USERNAME"}) + `\b([a-zA-Z\d]{3,18}[._-]?[a-zA-Z\d]{6,11})\b`)
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"https://", "http://", "hub-cloud.browserstack.com", "accessKey", "\"access_Key\":", "ACCESS_KEY", "key", "browserstackKey", "BS_AUTHKEY", "BROWSERSTACK_ACCESS_KEY"}) + `\b([0-9a-zA-Z]{20})\b`)
userPat = regexp.MustCompile(detectors.PrefixRegex([]string{"https://", "http://", "hub-cloud.browserstack.com", "userName", "\"username\":", "USER_NAME", "user", "browserstackUser", "BS_USERNAME", "BROWSERSTACK_USERNAME"}) + `\b([a-zA-Z\d]{3,18}[._-]+[a-zA-Z\d]{6})\b`)
)
// Keywords are used for efficiently pre-filtering chunks.
@@ -61,50 +56,32 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
}
if verify {
client := s.client
if client == nil {
client = defaultClient
req, err := http.NewRequestWithContext(ctx, "GET", "https://www.browserstack.com/automate/plan.json", nil)
if err != nil {
continue
}
req.Header.Add("Content-Type", "application/json")
req.SetBasicAuth(resUserMatch, resMatch)
res, err := client.Do(req)
if err == nil {
defer res.Body.Close()
if res.StatusCode >= 200 && res.StatusCode < 300 {
s1.Verified = true
} else {
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
continue
}
}
}
isVerified, verificationErr := verifyBrowserStackCredentials(ctx, client, resUserMatch, resMatch)
s1.Verified = isVerified
s1.SetVerificationError(verificationErr, resMatch)
}
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
if !s1.Verified && detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
continue
}
results = append(results, s1)
}
}
return results, nil
}
func verifyBrowserStackCredentials(ctx context.Context, client *http.Client, username, accessKey string) (bool, error) {
req, err := http.NewRequestWithContext(ctx, http.MethodGet, browserStackAPIURL, nil)
if err != nil {
return false, err
}
req.Header.Add("Content-Type", "application/json")
req.SetBasicAuth(username, accessKey)
res, err := client.Do(req)
if err != nil {
return false, err
}
defer res.Body.Close()
if res.StatusCode >= 200 && res.StatusCode < 300 {
return true, nil
} else if res.StatusCode != 401 {
return false, fmt.Errorf("unexpected HTTP response status %d", res.StatusCode)
}
return false, nil
}
func (s Scanner) Type() detectorspb.DetectorType {
return detectorspb.DetectorType_BrowserStack
}
+12 -52
View File
@@ -9,8 +9,7 @@ import (
"testing"
"time"
"github.com/google/go-cmp/cmp"
"github.com/google/go-cmp/cmp/cmpopts"
"github.com/kylelemons/godebug/pretty"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
@@ -34,19 +33,18 @@ func TestBrowserStack_FromChunk(t *testing.T) {
verify bool
}
tests := []struct {
name string
s Scanner
args args
want []detectors.Result
wantErr bool
wantVerificationErr bool
name string
s Scanner
args args
want []detectors.Result
wantErr bool
}{
{
name: "found, verified",
s: Scanner{},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a BROWSERSTACK_ACCESS_KEY %s within BROWSERSTACK_USERNAME %s", secret, secretUser)),
data: []byte(fmt.Sprintf("You can find a browserstack secret %s within browserstack %s", secret, secretUser)),
verify: true,
},
want: []detectors.Result{
@@ -63,7 +61,7 @@ func TestBrowserStack_FromChunk(t *testing.T) {
s: Scanner{},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a BROWSERSTACK_ACCESS_KEY %s within BROWSERSTACK_USERNAME %s but not valid", inactiveSecret, secretUser)), // the secret would satisfy the regex but not pass validation
data: []byte(fmt.Sprintf("You can find a browserstack secret %s within browserstack %s but not valid", inactiveSecret, secretUser)), // the secret would satisfy the regex but not pass validation
verify: true,
},
want: []detectors.Result{
@@ -75,42 +73,6 @@ func TestBrowserStack_FromChunk(t *testing.T) {
},
wantErr: false,
},
{
name: "found, would be verified if not for timeout",
s: Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a BROWSERSTACK_ACCESS_KEY %s within BROWSERSTACK_USERNAME %s", secret, secretUser)),
verify: true,
},
want: []detectors.Result{
{
DetectorType: detectorspb.DetectorType_BrowserStack,
RawV2: []byte(fmt.Sprintf("%s%s", secret, secretUser)),
Verified: false,
},
},
wantErr: false,
wantVerificationErr: true,
},
{
name: "found, verified but unexpected api surface",
s: Scanner{client: common.ConstantResponseHttpClient(404, "")},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a BROWSERSTACK_ACCESS_KEY %s within BROWSERSTACK_USERNAME %s", secret, secretUser)),
verify: true,
},
want: []detectors.Result{
{
DetectorType: detectorspb.DetectorType_BrowserStack,
Verified: false,
RawV2: []byte(fmt.Sprintf("%s%s", secret, secretUser)),
},
},
wantErr: false,
wantVerificationErr: true,
},
{
name: "not found",
s: Scanner{},
@@ -125,7 +87,8 @@ func TestBrowserStack_FromChunk(t *testing.T) {
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
s := Scanner{}
got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
if (err != nil) != tt.wantErr {
t.Errorf("BrowserStack.FromData() error = %v, wantErr %v", err, tt.wantErr)
return
@@ -134,12 +97,9 @@ func TestBrowserStack_FromChunk(t *testing.T) {
if len(got[i].Raw) == 0 {
t.Fatalf("no raw secret present: \n %+v", got[i])
}
if (got[i].VerificationError() != nil) != tt.wantVerificationErr {
t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError())
}
got[i].Raw = nil
}
ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "verificationError")
if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {
if diff := pretty.Compare(got, tt.want); diff != "" {
t.Errorf("BrowserStack.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
}
})
-97
View File
@@ -1,97 +0,0 @@
package budibase
import (
"context"
"fmt"
"net/http"
"regexp"
"strings"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
)
type Scanner struct {
client *http.Client
}
// Ensure the Scanner satisfies the interface at compile time.
var _ detectors.Detector = (*Scanner)(nil)
var (
defaultClient = common.SaneHttpClient()
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"budibase"}) + `\b([a-f0-9]{32}-[a-f0-9]{78,80})\b`)
)
// Keywords are used for efficiently pre-filtering chunks.
// Use identifiers in the secret preferably, or the provider name.
func (s Scanner) Keywords() []string {
return []string{"budibase"}
}
// FromData will find and optionally verify Budibase secrets in a given set of bytes.
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
dataStr := string(data)
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
for _, match := range matches {
if len(match) != 2 {
continue
}
resMatch := strings.TrimSpace(match[1])
s1 := detectors.Result{
DetectorType: detectorspb.DetectorType_Budibase,
Raw: []byte(resMatch),
}
if verify {
client := s.client
if client == nil {
client = defaultClient
}
// URL: https://docs.budibase.com/reference/appsearch
// API searches for the app with given name, since we only need to check api key, sending any appname will work.
payload := strings.NewReader(`{"name":"qwerty"}`)
req, err := http.NewRequestWithContext(ctx, "POST", "https://budibase.app/api/public/v1/applications/search", payload)
if err != nil {
continue
}
req.Header.Add("Content-Type", "application/json")
req.Header.Add("x-budibase-api-key", resMatch)
res, err := client.Do(req)
if err == nil {
defer res.Body.Close()
if res.StatusCode >= 200 && res.StatusCode < 300 {
s1.Verified = true
} else if res.StatusCode == 401 {
// The secret is determinately not verified (nothing to do)
} else {
err = fmt.Errorf("unexpected HTTP response status %d", res.StatusCode)
s1.SetVerificationError(err, resMatch)
}
} else {
s1.SetVerificationError(err, resMatch)
}
}
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
if !s1.Verified && detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
continue
}
results = append(results, s1)
}
return results, nil
}
func (s Scanner) Type() detectorspb.DetectorType {
return detectorspb.DetectorType_Budibase
}
-130
View File
@@ -1,130 +0,0 @@
//go:build detectors
// +build detectors
package budibase
import (
"context"
"fmt"
"testing"
"time"
"github.com/google/go-cmp/cmp"
"github.com/google/go-cmp/cmp/cmpopts"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
)
func TestBudibase_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
secret := testSecrets.MustGetField("BUDIBASE")
inactiveSecret := testSecrets.MustGetField("BUDIBASE_INACTIVE")
type args struct {
ctx context.Context
data []byte
verify bool
}
tests := []struct {
name string
s Scanner
args args
want []detectors.Result
wantErr bool
wantVerificationErr bool
}{
{
name: "found, verified",
s: Scanner{},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a budibase secret %s within", secret)),
verify: true,
},
want: []detectors.Result{
{
DetectorType: detectorspb.DetectorType_Budibase,
Verified: true,
},
},
wantErr: false,
wantVerificationErr: false,
},
{
name: "found, unverified",
s: Scanner{},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a budibase secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
verify: true,
},
want: func() []detectors.Result {
r := detectors.Result{
DetectorType: detectorspb.DetectorType_Budibase,
Verified: true,
}
r.SetVerificationError(fmt.Errorf("unexpected HTTP response status 403"))
return []detectors.Result{r}
}(),
wantErr: false,
wantVerificationErr: true,
},
{
name: "not found",
s: Scanner{},
args: args{
ctx: context.Background(),
data: []byte("You cannot find the secret within"),
verify: true,
},
want: nil,
wantErr: false,
wantVerificationErr: false,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
if (err != nil) != tt.wantErr {
t.Errorf("Budibase.FromData() error = %v, wantErr %v", err, tt.wantErr)
return
}
for i := range got {
if len(got[i].Raw) == 0 {
t.Fatalf("no raw secret present: \n %+v", got[i])
}
if (got[i].VerificationError() != nil) != tt.wantVerificationErr {
t.Fatalf("wantVerificationError = %v, verification error = %v", tt.want[i].VerificationError(), got[i].VerificationError())
}
}
ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "verificationError")
if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {
t.Errorf("Budibase.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
}
})
}
}
func BenchmarkFromData(benchmark *testing.B) {
ctx := context.Background()
s := Scanner{}
for name, data := range detectors.MustGetBenchmarkData() {
benchmark.Run(name, func(b *testing.B) {
b.ResetTimer()
for n := 0; n < b.N; n++ {
_, err := s.FromData(ctx, false, data)
if err != nil {
b.Fatal(err)
}
}
})
}
}
@@ -20,7 +20,7 @@ var (
client = common.SaneHttpClient()
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
keyPat = regexp.MustCompile(`\b(pk_[0-9]{7,8}_[0-9A-Z]{32})\b`)
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"clickup"}) + `\b(pk_[0-9]{8}_[0-9A-Z]{32})\b`)
)
// Keywords are used for efficiently pre-filtering chunks.
+1 -13
View File
@@ -2,7 +2,6 @@ package cloudsmith
import (
"context"
"encoding/json"
"net/http"
"regexp"
"strings"
@@ -30,10 +29,6 @@ func (s Scanner) Keywords() []string {
return []string{"cloudsmith"}
}
type response struct {
Authenticated bool `json:"authenticated"`
}
// FromData will find and optionally verify Cloudsmith secrets in a given set of bytes.
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
dataStr := string(data)
@@ -62,14 +57,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
if err == nil {
defer res.Body.Close()
if res.StatusCode >= 200 && res.StatusCode < 300 {
var r response
if err := json.NewDecoder(res.Body).Decode(&r); err != nil {
s1.SetVerificationError(err, resMatch)
continue
}
if r.Authenticated {
s1.Verified = true
}
s1.Verified = true
} else {
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
-90
View File
@@ -1,90 +0,0 @@
package coda
import (
"context"
"fmt"
"net/http"
"regexp"
"strings"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
)
type Scanner struct {
client *http.Client
}
// Ensure the Scanner satisfies the interface at compile time.
var _ detectors.Detector = (*Scanner)(nil)
var (
defaultClient = common.SaneHttpClient()
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"coda"}) + `\b([0-9a-z]{8}-[0-9a-z]{4}-[0-9a-z]{4}-[0-9a-z]{4}-[0-9a-z]{12})\b`)
)
// Keywords are used for efficiently pre-filtering chunks.
// Use identifiers in the secret preferably, or the provider name.
func (s Scanner) Keywords() []string {
return []string{"coda"}
}
// FromData will find and optionally verify Coda secrets in a given set of bytes.
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
dataStr := string(data)
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
for _, match := range matches {
if len(match) != 2 {
continue
}
resMatch := strings.TrimSpace(match[1])
s1 := detectors.Result{
DetectorType: detectorspb.DetectorType_Coda,
Raw: []byte(resMatch),
}
if verify {
client := s.client
if client == nil {
client = defaultClient
}
req, err := http.NewRequestWithContext(ctx, "GET", "https://coda.io/apis/v1/whoami", nil)
if err != nil {
continue
}
req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", resMatch))
res, err := client.Do(req)
if err == nil {
defer res.Body.Close()
if res.StatusCode >= 200 && res.StatusCode < 300 {
s1.Verified = true
} else if res.StatusCode == 401 {
// The secret is determinately not verified (nothing to do)
} else {
err = fmt.Errorf("unexpected HTTP response status %d", res.StatusCode)
s1.SetVerificationError(err, resMatch)
}
} else {
s1.SetVerificationError(err, resMatch)
}
}
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
if !s1.Verified && detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
continue
}
results = append(results, s1)
}
return results, nil
}
func (s Scanner) Type() detectorspb.DetectorType {
return detectorspb.DetectorType_Coda
}
-162
View File
@@ -1,162 +0,0 @@
//go:build detectors
// +build detectors
package coda
import (
"context"
"fmt"
"testing"
"time"
"github.com/google/go-cmp/cmp"
"github.com/google/go-cmp/cmp/cmpopts"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
)
func TestCoda_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
secret := testSecrets.MustGetField("CODA")
inactiveSecret := testSecrets.MustGetField("CODA_INACTIVE")
type args struct {
ctx context.Context
data []byte
verify bool
}
tests := []struct {
name string
s Scanner
args args
want []detectors.Result
wantErr bool
wantVerificationErr bool
}{
{
name: "found, verified",
s: Scanner{},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a coda secret %s within", secret)),
verify: true,
},
want: []detectors.Result{
{
DetectorType: detectorspb.DetectorType_Coda,
Verified: true,
},
},
wantErr: false,
wantVerificationErr: false,
},
{
name: "found, unverified",
s: Scanner{},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a coda secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
verify: true,
},
want: []detectors.Result{
{
DetectorType: detectorspb.DetectorType_Coda,
Verified: false,
},
},
wantErr: false,
wantVerificationErr: false,
},
{
name: "not found",
s: Scanner{},
args: args{
ctx: context.Background(),
data: []byte("You cannot find the secret within"),
verify: true,
},
want: nil,
wantErr: false,
wantVerificationErr: false,
},
{
name: "found, would be verified if not for timeout",
s: Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a coda secret %s within", secret)),
verify: true,
},
want: []detectors.Result{
{
DetectorType: detectorspb.DetectorType_Coda,
Verified: false,
},
},
wantErr: false,
wantVerificationErr: true,
},
{
name: "found, verified but unexpected api surface",
s: Scanner{client: common.ConstantResponseHttpClient(404, "")},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a coda secret %s within", secret)),
verify: true,
},
want: []detectors.Result{
{
DetectorType: detectorspb.DetectorType_Coda,
Verified: false,
},
},
wantErr: false,
wantVerificationErr: true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
if (err != nil) != tt.wantErr {
t.Errorf("Coda.FromData() error = %v, wantErr %v", err, tt.wantErr)
return
}
for i := range got {
if len(got[i].Raw) == 0 {
t.Fatalf("no raw secret present: \n %+v", got[i])
}
if (got[i].VerificationError() != nil) != tt.wantVerificationErr {
t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError())
}
}
ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "verificationError")
if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {
t.Errorf("Coda.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
}
})
}
}
func BenchmarkFromData(benchmark *testing.B) {
ctx := context.Background()
s := Scanner{}
for name, data := range detectors.MustGetBenchmarkData() {
benchmark.Run(name, func(b *testing.B) {
b.ResetTimer()
for n := 0; n < b.N; n++ {
_, err := s.FromData(ctx, false, data)
if err != nil {
b.Fatal(err)
}
}
})
}
}
+1 -15
View File
@@ -2,7 +2,6 @@ package codeclimate
import (
"context"
"encoding/json"
"fmt"
"net/http"
"regexp"
@@ -31,12 +30,6 @@ func (s Scanner) Keywords() []string {
return []string{"codeclimate"}
}
type response struct {
Data struct {
Id string `json:"id"`
} `json:"data"`
}
// FromData will find and optionally verify Codeclimate secrets in a given set of bytes.
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
dataStr := string(data)
@@ -65,14 +58,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
if err == nil {
defer res.Body.Close()
if res.StatusCode >= 200 && res.StatusCode < 300 {
var r response
if err := json.NewDecoder(res.Body).Decode(&r); err != nil {
s1.SetVerificationError(err, resMatch)
continue
}
if r.Data.Id != "" {
s1.Verified = true
}
s1.Verified = true
} else {
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
@@ -1,143 +0,0 @@
package coinbase_waas
import (
"context"
"crypto/ecdsa"
"crypto/x509"
"encoding/pem"
"errors"
"net/http"
"regexp"
"strings"
"github.com/coinbase/waas-client-library-go/auth"
"github.com/coinbase/waas-client-library-go/clients"
v1clients "github.com/coinbase/waas-client-library-go/clients/v1"
v1 "github.com/coinbase/waas-client-library-go/gen/go/coinbase/cloud/pools/v1"
"github.com/google/uuid"
"google.golang.org/api/googleapi"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
)
type Scanner struct {
client *http.Client
}
// Ensure the Scanner satisfies the interface at compile time.
var _ detectors.Detector = (*Scanner)(nil)
var (
// Reference: https://docs.cloud.coinbase.com/waas/docs/auth
keyNamePat = regexp.MustCompile(`(organizations\\*/\w{8}-\w{4}-\w{4}-\w{4}-\w{12}\\*/apiKeys\\*/\w{8}-\w{4}-\w{4}-\w{4}-\w{12})`)
privKeyPat = regexp.MustCompile(`(-----BEGIN EC(?:DSA)? PRIVATE KEY-----(?:\r|\n|\\+r|\\+n)(?:[a-zA-Z0-9+/]+={0,2}(?:\r|\n|\\+r|\\+n))+-----END EC(?:DSA)? PRIVATE KEY-----(?:\r|\n|\\+r|\\+n)?)`)
nameReplacer = strings.NewReplacer("\\", "")
keyReplacer = strings.NewReplacer(
"\r\n", "\n",
"\\r\\n", "\n",
"\\n", "\n",
"\\r", "\n",
)
)
// Keywords are used for efficiently pre-filtering chunks.
// Use identifiers in the secret preferably, or the provider name.
func (s Scanner) Keywords() []string {
return []string{"organizations", "apiKeys", "begin ec"}
}
// FromData will find and optionally verify CoinbaseWaaS secrets in a given set of bytes.
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
dataStr := string(data)
keyNameMatches := keyNamePat.FindAllStringSubmatch(dataStr, -1)
privKeyMatches := privKeyPat.FindAllStringSubmatch(dataStr, -1)
for _, keyNameMatch := range keyNameMatches {
resKeyNameMatch := nameReplacer.Replace(strings.TrimSpace(keyNameMatch[1]))
for _, privKeyMatch := range privKeyMatches {
resPrivKeyMatch := keyReplacer.Replace(strings.TrimSpace(privKeyMatch[1]))
if !isValidECPrivateKey([]byte(resPrivKeyMatch)) {
continue
}
s1 := detectors.Result{
DetectorType: detectorspb.DetectorType_CoinbaseWaaS,
Raw: []byte(resPrivKeyMatch),
RawV2: []byte(resKeyNameMatch + ":" + resPrivKeyMatch),
}
if verify {
isVerified, verificationErr := s.verifyMatch(ctx, resKeyNameMatch, resPrivKeyMatch)
s1.Verified = isVerified
s1.SetVerificationError(verificationErr, resPrivKeyMatch)
}
results = append(results, s1)
// If we've found a verified match with this ID, we don't need to look for anymore. So move on to the next ID.
if s1.Verified {
break
}
}
}
return results, nil
}
func isValidECPrivateKey(pemKey []byte) bool {
block, _ := pem.Decode(pemKey)
if block == nil {
return false
}
key, err := x509.ParseECPrivateKey(block.Bytes)
if err != nil {
return false
}
// Check the key type
if _, ok := key.Public().(*ecdsa.PublicKey); !ok {
return false
}
return true
}
func (s Scanner) verifyMatch(ctx context.Context, apiKeyName, privKey string) (bool, error) {
authOpt := clients.WithAPIKey(&auth.APIKey{
Name: apiKeyName,
PrivateKey: privKey,
})
clientOpt := clients.WithHTTPClient(s.client)
client, err := v1clients.NewPoolServiceClient(ctx, authOpt, clientOpt)
if err != nil {
return false, err
}
// Lookup an arbitrary pool name that shouldn't exist.
_, err = client.GetPool(ctx, &v1.GetPoolRequest{Name: uuid.New().String()})
if err != nil {
var apiErr *googleapi.Error
if errors.As(err, &apiErr) {
if apiErr.Code == 401 {
// Invalid |Name| or |PrivateKey|
return false, nil
} else if apiErr.Code == 404 {
// Valid |Name| and |PrivateKey| but the pool doesn't exist (expected).
return true, nil
}
}
// Unhandled error.
return false, err
}
// In theory this will never happen, but it also indicates a valid key.
return true, nil
}
func (s Scanner) Type() detectorspb.DetectorType {
return detectorspb.DetectorType_CoinbaseWaaS
}
@@ -1,304 +0,0 @@
//go:build detectors
// +build detectors
package coinbase_waas
import (
"context"
"encoding/base64"
"fmt"
"testing"
"time"
"github.com/google/go-cmp/cmp"
"github.com/google/go-cmp/cmp/cmpopts"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
)
func TestCoinbaseWaaS_Pattern(t *testing.T) {
tests := []struct {
name string
data string
shouldMatch bool
match string
}{
// True positives
// https://github.com/coinbase/waas-client-library-go/issues/41
{
name: "valid_result1",
data: `{ "name": "organizations/14d1742b-3575-4490-b9bc-a8a9c7e4973d/apiKeys/7473d38c-80c6-4a69-a715-1ea8fd950f6f", "principal": "8feb538e-137b-5864-b12a-7c75b60fa20a", "principalType": "USER", "publicKey": "-----BEGIN EC PUBLIC KEY-----\nMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEzR0G+CW0uVJFrpLUELqB+DlsmGmO\nA03Az8Fpv7azpgjAy87ibgQTThaQy1C1BccbCDkPoEs6mOnDkOebkybAKQ==\n-----END EC PUBLIC KEY-----\n", "privateKey": "-----BEGIN EC PRIVATE KEY-----\nMHcCAQEEIBddyynZ9Ya7op1B9nu1Dxyc1T6xLy72t45J2Smv9oXNoAoGCCqGSM49\nAwEHoUQDQgAEzR0G+CW0uVJFrpLUELqB+DlsmGmOA03Az8Fpv7azpgjAy87ibgQT\nThaQy1C1BccbCDkPoEs6mOnDkOebkybAKQ==\n-----END EC PRIVATE KEY-----\n", "createTime": "2023-08-19T12:29:08.938421763Z", "projectId": "5970e137-9c3d-4adc-b65d-58d33af2432d" }`,
shouldMatch: true,
match: "organizations/14d1742b-3575-4490-b9bc-a8a9c7e4973d/apiKeys/7473d38c-80c6-4a69-a715-1ea8fd950f6f:-----BEGIN EC PRIVATE KEY-----\nMHcCAQEEIBddyynZ9Ya7op1B9nu1Dxyc1T6xLy72t45J2Smv9oXNoAoGCCqGSM49\nAwEHoUQDQgAEzR0G+CW0uVJFrpLUELqB+DlsmGmOA03Az8Fpv7azpgjAy87ibgQT\nThaQy1C1BccbCDkPoEs6mOnDkOebkybAKQ==\n-----END EC PRIVATE KEY-----\n",
},
// https://github.com/coinbase/waas-client-library-go/pull/32#issuecomment-1666415017
{
name: "valid_result2_name_slashes",
data: `{
"name": "organizations\/d3f266dc-0d36-4cd0-91c3-e3a292b0b4b3\/apiKeys\/032c4fdf-d763-4b0c-9ed3-ff41a873bcc8",
"principal": "5d5c9f00-3224-52a7-a1f7-9e6ce3ada40c",
"principalType": "USER",
"publicKey": "-----BEGIN EC PUBLIC KEY-----\nMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEAjw43hwOqS2PF4gAFbhoxIJqCHAP\niqLdg5GFVn9QAS/0oY4/fJGrCn9rpQGOvHxHf1mtQ6j4bIWN1AtHvA/3uw==\n-----END EC PUBLIC KEY-----\n",
"privateKey": "-----BEGIN EC PRIVATE KEY-----\nMHcCAQEEIFkA1kU4DlNu36wTTHycWy6n1rsUH0UT8mfAKNtOukXHoAoGCCqGSM49\nAwEHoUQDQgAEAjw43hwOqS2PF4gAFbhoxIJqCHAPiqLdg5GFVn9QAS/0oY4/fJGr\nCn9rpQGOvHxHf1mtQ6j4bIWN1AtHvA/3uw==\n-----END EC PRIVATE KEY-----\n",
"createTime": "2023-08-05T06:34:40.265235553Z",
"projectId": "64b3f391-c69d-4c59-91a2-75816c1a0738"
}`,
shouldMatch: true,
match: "organizations/d3f266dc-0d36-4cd0-91c3-e3a292b0b4b3/apiKeys/032c4fdf-d763-4b0c-9ed3-ff41a873bcc8:-----BEGIN EC PRIVATE KEY-----\nMHcCAQEEIFkA1kU4DlNu36wTTHycWy6n1rsUH0UT8mfAKNtOukXHoAoGCCqGSM49\nAwEHoUQDQgAEAjw43hwOqS2PF4gAFbhoxIJqCHAPiqLdg5GFVn9QAS/0oY4/fJGr\nCn9rpQGOvHxHf1mtQ6j4bIWN1AtHvA/3uw==\n-----END EC PRIVATE KEY-----\n",
},
{
name: "valid_result3",
data: `name: "organizations/7eead2d5-fa48-4423-8f40-c70d8ce398ae/apiKeys/7b9516b6-d82e-44e8-bed5-89b160452ed9",
description: "principal": "775fb863-004f-5412-8e4c-e9449c612563" and install dependencies
runs: "principalType": "USER",
using: composite
steps:"publicKey": "-----BEGIN EC PUBLIC KEY-----\nMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEvHsvI08kox+n/8wSMFwCbK5hEf5b\n/g82Lmz3HpATKFmrICcOBX2lRHo99JWRrupmjUGxnD8i4sj4mZafTEokhA==\n-----END EC PUBLIC KEY-----\n",
- name: Setup Node.js
uses: actions/setup-node@v3
with: "privateKey": "-----BEGIN EC PRIVATE KEY-----\nMHcCAQEEIKOQ7lvGL0EiUzZ23pmH/NBPRwVV8yZsqofds5bSR9qFoAoGCCqGSM49\nAwEHoUQDQgAEvHsvI08kox+n/8wSMFwCbK5hEf5b/g82Lmz3HpATKFmrICcOBX2l\nRHo99JWRrupmjUGxnD8i4sj4mZafTEokhA==\n-----END EC PRIVATE KEY-----\n",
node-version-file: .nvmrc
- name: Cache dependencies`,
shouldMatch: true,
match: "organizations/7eead2d5-fa48-4423-8f40-c70d8ce398ae/apiKeys/7b9516b6-d82e-44e8-bed5-89b160452ed9:-----BEGIN EC PRIVATE KEY-----\nMHcCAQEEIKOQ7lvGL0EiUzZ23pmH/NBPRwVV8yZsqofds5bSR9qFoAoGCCqGSM49\nAwEHoUQDQgAEvHsvI08kox+n/8wSMFwCbK5hEf5b/g82Lmz3HpATKFmrICcOBX2l\nRHo99JWRrupmjUGxnD8i4sj4mZafTEokhA==\n-----END EC PRIVATE KEY-----\n",
},
{
name: "valid_result_ecdsa",
data: `{
"name": "organizations/7eead2d5-fa48-4423-8f40-c70d8ce398ae/apiKeys/7b9516b6-d82e-44e8-bed5-89b160452ed9",
"privateKey": "-----BEGIN ECDSA PRIVATE KEY-----\nMHcCAQEEINQdZMbF2r07KF0mxfLYt9Y1PNaC0C6UpZ31MxD4NEE8oAoGCCqGSM49\nAwEHoUQDQgAEeRFgMrQEHI/APWaziRH90jN7EozjdbPVxvzc1F4zqWTeCtLASwqA\nqnMugYX2epqsFhGn82xNXu2NwgORc6embQ==\n-----END ECDSA PRIVATE KEY-----\n"
}`,
shouldMatch: true,
match: "organizations/7eead2d5-fa48-4423-8f40-c70d8ce398ae/apiKeys/7b9516b6-d82e-44e8-bed5-89b160452ed9:-----BEGIN ECDSA PRIVATE KEY-----\nMHcCAQEEINQdZMbF2r07KF0mxfLYt9Y1PNaC0C6UpZ31MxD4NEE8oAoGCCqGSM49\nAwEHoUQDQgAEeRFgMrQEHI/APWaziRH90jN7EozjdbPVxvzc1F4zqWTeCtLASwqA\nqnMugYX2epqsFhGn82xNXu2NwgORc6embQ==\n-----END ECDSA PRIVATE KEY-----\n",
},
// TODO: Is it worth supporting case-insensitve headers?
// https://github.com/coinbase/waas-sdk-react-native/blob/bbaf597e73d02ecaf64161061e71b85d9eeeb9d6/example/src/.coinbase_cloud_api_key.json#L4
// {
// name: "valid_result_case_insensitive",
// data: `{
// "name": "organizations/7eead2d5-fa48-4423-8f40-c70d8ce398ae/apiKeys/7b9516b6-d82e-44e8-bed5-89b160452ed9",
// "privateKey": "-----BEGIN ECDSA private key-----\nMIGHAgEAMBMGByqGSM49AgEGCCqGSM49AwEHBG0wawIBAQQg8id7yCfmNp0ppczu\nDhjB1pesdDB6Uwuz6KxARrenNfyhRANCAASI6DBntdr+XSOaK55J++x8ORuDxn81\nENa0RmGFjTwu4vQcWcx5rrIWNh6b7FPxy6mrZl0n3rswEtZmUci8Y5HX\n-----END ECDSA PRIVATE KEY-----\n"
//}`,
// shouldMatch: true,
// match: "organizations/7eegad2d5-fa48-4423-8f40-c70d8ce398ae/apiKeys/7b9516b6-d82e-44e8-bed5-89b160452ed9:-----BEGIN ECDSA PRIVATE KEY-----\nMIGHAgEAMBMGByqGSM49AgEGCCqGSM49AwEHBG0wawIBAQQg8id7yCfmNp0ppczu\nDhjB1pesdDB6Uwuz6KxARrenNfyhRANCAASI6DBntdr+XSOaK55J++x8ORuDxn81\nENa0RmGFjTwu4vQcWcx5rrIWNh6b7FPxy6mrZl0n3rswEtZmUci8Y5HX\n-----END ECDSA PRIVATE KEY-----\n",
// },
// False positives
// https://github.com/coinbase/waas-client-library-go/blob/main/example.go
{
name: `invalid_key_name1`,
data: `const (
// apiKeyName is the name of the API Key to use. Fill this out before running the main function.
apiKeyName = "organizations/my-organization/apiKeys/my-api-key"
// privKeyTemplate is the private key of the API Key to use. Fill this out before running the main function.
privKeyTemplate = "-----BEGIN EC PRIVATE KEY-----\nmy-private-key\n-----END EC PRIVATE KEY-----\n"
)`,
shouldMatch: false,
},
// https://github.com/coinbase/waas-sdk-react-native/blob/bbaf597e73d02ecaf64161061e71b85d9eeeb9d6/example/src/.coinbase_cloud_api_key.json#L4
{
name: `invalid_key_name2`,
data: `{
"name": "organizations/organizationID/apiKeys/apiKeyName",
"privateKey": "-----BEGIN ECDSA Private Key-----ExamplePrivateKey-----END ECDSA Private Key-----\n"
}`,
},
{
name: `invalid_private_key`,
data: `{ "name": "organizations/14d1742b-3575-4490-b9bc-a8a9c7e4973d/apiKeys/7473d38c-80c6-4a69-a715-1ea8fd950f6f", "principal": "8feb538e-137b-5864-b12a-7c75b60fa20a", "principalType": "USER", "publicKey": "-----BEGIN EC PUBLIC KEY-----\ninvalid\n-----END EC PUBLIC KEY-----\n", "privateKey": "-----BEGIN EC PRIVATE KEY-----\ninvalid\n-----END EC PRIVATE KEY-----\n", "createTime": "2023-08-19T12:29:08.938421763Z", "projectId": "5970e137-9c3d-4adc-b65d-58d33af2432d" }`,
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
s := Scanner{}
results, err := s.FromData(context.Background(), false, []byte(test.data))
if err != nil {
t.Errorf("CoinbaseWaaS.FromData() error = %v", err)
return
}
if test.shouldMatch {
if len(results) == 0 {
t.Errorf("%s: did not receive a match for '%v' when one was expected", test.name, test.data)
return
}
expected := test.data
if test.match != "" {
expected = test.match
}
result := results[0]
resultData := string(result.RawV2)
if resultData != expected {
t.Errorf("%s: did not receive expected match.\n\texpected: '%s'\n\t actual: '%s'", test.name, expected, resultData)
return
}
} else {
if len(results) > 0 {
t.Errorf("%s: received a match for '%v' when one wasn't wanted", test.name, test.data)
return
}
}
})
}
}
func TestCoinbaseWaaS_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
secretb64 := testSecrets.MustGetField("COINBASE_WAAS")
secretB, err := base64.StdEncoding.DecodeString(secretb64)
if err != nil {
t.Fatalf("could not decode secret: %s", err)
}
secret := string(secretB)
inactiveSecretb64 := testSecrets.MustGetField("COINBASE_WAAS_INACTIVE")
inactiveSecretB, err := base64.StdEncoding.DecodeString(inactiveSecretb64)
if err != nil {
t.Fatalf("could not decode secret: %s", err)
}
inactiveSecret := string(inactiveSecretB)
type args struct {
ctx context.Context
data []byte
verify bool
}
tests := []struct {
name string
s Scanner
args args
want []detectors.Result
wantErr bool
wantVerificationErr bool
}{
{
name: "found, verified",
s: Scanner{},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a coinbase_waas secret %s within", secret)),
verify: true,
},
want: []detectors.Result{
{
DetectorType: detectorspb.DetectorType_CoinbaseWaaS,
Verified: true,
},
},
wantErr: false,
wantVerificationErr: false,
},
{
name: "found, unverified",
s: Scanner{},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a coinbase_waas secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
verify: true,
},
want: []detectors.Result{
{
DetectorType: detectorspb.DetectorType_CoinbaseWaaS,
Verified: false,
},
},
wantErr: false,
wantVerificationErr: false,
},
{
name: "not found",
s: Scanner{},
args: args{
ctx: context.Background(),
data: []byte("You cannot find the secret within"),
verify: true,
},
want: nil,
wantErr: false,
wantVerificationErr: false,
},
{
name: "found, would be verified if not for timeout",
s: Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a coinbase_waas secret %s within", secret)),
verify: true,
},
want: []detectors.Result{
{
DetectorType: detectorspb.DetectorType_CoinbaseWaaS,
Verified: false,
},
},
wantErr: false,
wantVerificationErr: true,
},
{
name: "found, verified but unexpected api surface",
s: Scanner{client: common.ConstantResponseHttpClient(500, "")},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a coinbase_waas secret %s within", secret)),
verify: true,
},
want: []detectors.Result{
{
DetectorType: detectorspb.DetectorType_CoinbaseWaaS,
Verified: false,
},
},
wantErr: false,
wantVerificationErr: true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
if (err != nil) != tt.wantErr {
t.Errorf("Coinbasewaas.FromData() error = %v, wantErr %v", err, tt.wantErr)
return
}
for i := range got {
if len(got[i].Raw) == 0 {
t.Fatalf("no raw secret present: \n %+v", got[i])
}
if (got[i].VerificationError() != nil) != tt.wantVerificationErr {
t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError())
}
}
ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "RawV2", "verificationError")
if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {
t.Errorf("Coinbasewaas.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
}
})
}
}
func BenchmarkFromData(benchmark *testing.B) {
ctx := context.Background()
s := Scanner{}
for name, data := range detectors.MustGetBenchmarkData() {
benchmark.Run(name, func(b *testing.B) {
b.ResetTimer()
for n := 0; n < b.N; n++ {
_, err := s.FromData(ctx, false, data)
if err != nil {
b.Fatal(err)
}
}
})
}
}
+17 -22
View File
@@ -2,7 +2,6 @@ package currencycloud
import (
"context"
"fmt"
"io"
"net/http"
"regexp"
@@ -55,33 +54,29 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
DetectorType: detectorspb.DetectorType_CurrencyCloud,
Raw: []byte(resMatch),
}
environments := []string{"devapi", "api"}
if verify {
for _, env := range environments {
// Get authentication token
payload := strings.NewReader(`{"login_id":"` + resEmailMatch + `","api_key":"` + resMatch + `"`)
req, err := http.NewRequestWithContext(ctx, "POST", "https://"+env+".currencycloud.com/v2/authenticate/api", payload)
// Get authentication token
payload := strings.NewReader(`{"login_id":"` + resEmailMatch + `","api_key":"` + resMatch + `"`)
req, err := http.NewRequestWithContext(ctx, "POST", "https://devapi.currencycloud.com/v2/authenticate/api", payload)
if err != nil {
continue
}
req.Header.Add("Content-Type", "application/json")
res, err := client.Do(req)
if err == nil {
defer res.Body.Close()
bodyBytes, err := io.ReadAll(res.Body)
if err != nil {
continue
}
req.Header.Add("Content-Type", "application/json")
res, err := client.Do(req)
if err == nil {
defer res.Body.Close()
bodyBytes, err := io.ReadAll(res.Body)
if err != nil {
body := string(bodyBytes)
if strings.Contains(body, "auth_token") {
s1.Verified = true
} else {
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
continue
}
body := string(bodyBytes)
if strings.Contains(body, "auth_token") {
s1.Verified = true
s1.ExtraData = map[string]string{"environment": fmt.Sprintf("https://%s.currencycloud.com", env)}
break
} else {
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
continue
}
}
}
}
}
@@ -12,19 +12,17 @@ import (
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
)
type Scanner struct {
client *http.Client
}
type Scanner struct{}
// Ensure the Scanner satisfies the interface at compile time.
var _ detectors.Detector = (*Scanner)(nil)
var (
defaultClient = common.SaneHttpClient()
client = common.SaneHttpClient()
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
domain = regexp.MustCompile(`\b([a-z0-9-]+(?:\.[a-z0-9-]+)*\.(cloud\.databricks\.com|gcp\.databricks\.com|azurewebsites\.net))\b`)
keyPat = regexp.MustCompile(`\b(dapi[0-9a-f]{32})(-\d)?\b`)
domain = regexp.MustCompile(`\b(https:\/\/[a-z0-9-]+\.cloud\.databricks\.com)\b`)
keyPat = regexp.MustCompile(`\b(dapi[a-z0-9]{32})\b`)
)
// Keywords are used for efficiently pre-filtering chunks.
@@ -41,9 +39,15 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
domainMatches := domain.FindAllStringSubmatch(dataStr, -1)
for _, match := range matches {
if len(match) != 2 {
continue
}
resMatch := strings.TrimSpace(match[1])
for _, domainmatch := range domainMatches {
if len(domainmatch) != 2 {
continue
}
resDomainMatch := strings.TrimSpace(domainmatch[1])
s1 := detectors.Result{
@@ -53,11 +57,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
}
if verify {
client := s.client
if client == nil {
client = defaultClient
}
req, err := http.NewRequestWithContext(ctx, "GET", "https://"+resDomainMatch+"/api/2.0/clusters/list", nil)
req, err := http.NewRequestWithContext(ctx, "GET", resDomainMatch + "/api/2.0/clusters/list", nil)
if err != nil {
continue
}
@@ -67,19 +67,14 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
defer res.Body.Close()
if res.StatusCode >= 200 && res.StatusCode < 300 {
s1.Verified = true
} else if res.StatusCode == 403 {
// nothing to do here
} else {
err = fmt.Errorf("unexpected HTTP response status %d", res.StatusCode)
s1.SetVerificationError(err, resMatch)
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
continue
}
}
} else {
s1.SetVerificationError(err, resMatch)
}
}
if !s1.Verified && detectors.IsKnownFalsePositive(string(s1.Raw), detectors.DefaultFalsePositives, true) {
continue
}
results = append(results, s1)
}
@@ -9,16 +9,14 @@ import (
"testing"
"time"
"github.com/google/go-cmp/cmp"
"github.com/google/go-cmp/cmp/cmpopts"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/kylelemons/godebug/pretty"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
)
func TestDatabricksToken_FromChunk(t *testing.T) {
func TestDatabrickstoken_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")
@@ -27,7 +25,6 @@ func TestDatabricksToken_FromChunk(t *testing.T) {
}
secret := testSecrets.MustGetField("DATABRICKSTOKEN")
inactiveSecret := testSecrets.MustGetField("DATABRICKSTOKEN_INACTIVE")
domain := testSecrets.MustGetField("DATABRICKSTOKEN_DOMAIN")
type args struct {
ctx context.Context
@@ -35,19 +32,18 @@ func TestDatabricksToken_FromChunk(t *testing.T) {
verify bool
}
tests := []struct {
name string
s Scanner
args args
want []detectors.Result
wantErr bool
wantVerificationErr bool
name string
s Scanner
args args
want []detectors.Result
wantErr bool
}{
{
name: "found, verified",
s: Scanner{},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a databrickstoken secret %s within %s", secret, domain)),
data: []byte(fmt.Sprintf("You can find a databrickstoken secret %s within", secret)),
verify: true,
},
want: []detectors.Result{
@@ -56,15 +52,14 @@ func TestDatabricksToken_FromChunk(t *testing.T) {
Verified: true,
},
},
wantErr: false,
wantVerificationErr: false,
wantErr: false,
},
{
name: "found, unverified",
s: Scanner{},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a databrickstoken secret %s within %s but not valid", inactiveSecret, domain)), // the secret would satisfy the regex but not pass validation
data: []byte(fmt.Sprintf("You can find a databrickstoken secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
verify: true,
},
want: []detectors.Result{
@@ -73,8 +68,7 @@ func TestDatabricksToken_FromChunk(t *testing.T) {
Verified: false,
},
},
wantErr: false,
wantVerificationErr: false,
wantErr: false,
},
{
name: "not found",
@@ -84,48 +78,14 @@ func TestDatabricksToken_FromChunk(t *testing.T) {
data: []byte("You cannot find the secret within"),
verify: true,
},
want: nil,
wantErr: false,
wantVerificationErr: false,
},
{
name: "found, would be verified if not for timeout",
s: Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a databrickstoken secret %s within %s", secret, domain)),
verify: true,
},
want: []detectors.Result{
{
DetectorType: detectorspb.DetectorType_DatabricksToken,
Verified: false,
},
},
wantErr: false,
wantVerificationErr: true,
},
{
name: "found, verified but unexpected api surface",
s: Scanner{client: common.ConstantResponseHttpClient(404, "")},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a databrickstoken secret %s within %s", secret, domain)),
verify: true,
},
want: []detectors.Result{
{
DetectorType: detectorspb.DetectorType_DatabricksToken,
Verified: false,
},
},
wantErr: false,
wantVerificationErr: true,
want: nil,
wantErr: false,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
s := Scanner{}
got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
if (err != nil) != tt.wantErr {
t.Errorf("Databrickstoken.FromData() error = %v, wantErr %v", err, tt.wantErr)
return
@@ -134,13 +94,10 @@ func TestDatabricksToken_FromChunk(t *testing.T) {
if len(got[i].Raw) == 0 {
t.Fatalf("no raw secret present: \n %+v", got[i])
}
if (got[i].VerificationError() != nil) != tt.wantVerificationErr {
t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError())
}
got[i].Raw = nil
}
ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "RawV2", "verificationError")
if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {
t.Errorf("DatabricksToken.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
if diff := pretty.Compare(got, tt.want); diff != "" {
t.Errorf("Databrickstoken.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
}
})
}
@@ -1,4 +1,4 @@
package ip2location
package datafire
import (
"context"
@@ -19,17 +19,18 @@ var _ detectors.Detector = (*Scanner)(nil)
var (
client = common.SaneHttpClient()
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"ip2location"}) + `\b([0-9A-Z]{32})\b`)
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"datafire"}) + `\b([a-z0-9\S]{175,190})\b`)
)
// Keywords are used for efficiently pre-filtering chunks.
// Use identifiers in the secret preferably, or the provider name.
func (s Scanner) Keywords() []string {
return []string{"ip2location"}
return []string{"datafire"}
}
// FromData will find and optionally verify Ip2location secrets in a given set of bytes.
// FromData will find and optionally verify DataFire secrets in a given set of bytes.
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
dataStr := string(data)
@@ -42,34 +43,29 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
resMatch := strings.TrimSpace(match[1])
s1 := detectors.Result{
DetectorType: detectorspb.DetectorType_Ip2location,
DetectorType: detectorspb.DetectorType_DataFire,
Raw: []byte(resMatch),
}
if verify {
req, err := http.NewRequestWithContext(ctx, "GET", "https://api.ip2location.io/?key="+resMatch, nil)
req, err := http.NewRequestWithContext(ctx, "GET", "https://api.datafire.io/projects/", nil)
if err != nil {
continue
}
req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", resMatch))
res, err := client.Do(req)
if err == nil {
defer res.Body.Close()
if res.StatusCode >= 200 && res.StatusCode < 300 {
s1.Verified = true
} else if res.StatusCode == 401 {
// The secret is determinately not verified (nothing to do)
} else {
err = fmt.Errorf("unexpected HTTP response status %d", res.StatusCode)
s1.SetVerificationError(err, resMatch)
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
continue
}
}
} else {
s1.SetVerificationError(err, resMatch)
}
}
if !s1.Verified && detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
continue
}
results = append(results, s1)
}
@@ -78,5 +74,5 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
}
func (s Scanner) Type() detectorspb.DetectorType {
return detectorspb.DetectorType_Ip2location
return detectorspb.DetectorType_DataFire
}
+120
View File
@@ -0,0 +1,120 @@
//go:build detectors
// +build detectors
package datafire
import (
"context"
"fmt"
"testing"
"time"
"github.com/kylelemons/godebug/pretty"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
)
func TestDataFire_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
secret := testSecrets.MustGetField("DATAFIRE")
inactiveSecret := testSecrets.MustGetField("DATAFIRE_INACTIVE")
type args struct {
ctx context.Context
data []byte
verify bool
}
tests := []struct {
name string
s Scanner
args args
want []detectors.Result
wantErr bool
}{
{
name: "found, verified",
s: Scanner{},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a datafire secret %s within", secret)),
verify: true,
},
want: []detectors.Result{
{
DetectorType: detectorspb.DetectorType_DataFire,
Verified: true,
},
},
wantErr: false,
},
{
name: "found, unverified",
s: Scanner{},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a datafire secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
verify: true,
},
want: []detectors.Result{
{
DetectorType: detectorspb.DetectorType_DataFire,
Verified: false,
},
},
wantErr: false,
},
{
name: "not found",
s: Scanner{},
args: args{
ctx: context.Background(),
data: []byte("You cannot find the secret within"),
verify: true,
},
want: nil,
wantErr: false,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
s := Scanner{}
got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
if (err != nil) != tt.wantErr {
t.Errorf("DataFire.FromData() error = %v, wantErr %v", err, tt.wantErr)
return
}
for i := range got {
if len(got[i].Raw) == 0 {
t.Fatalf("no raw secret present: \n %+v", got[i])
}
got[i].Raw = nil
}
if diff := pretty.Compare(got, tt.want); diff != "" {
t.Errorf("DataFire.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
}
})
}
}
func BenchmarkFromData(benchmark *testing.B) {
ctx := context.Background()
s := Scanner{}
for name, data := range detectors.MustGetBenchmarkData() {
benchmark.Run(name, func(b *testing.B) {
b.ResetTimer()
for n := 0; n < b.N; n++ {
_, err := s.FromData(ctx, false, data)
if err != nil {
b.Fatal(err)
}
}
})
}
}
-108
View File
@@ -1,108 +0,0 @@
package denodeploy
import (
"context"
"encoding/json"
"fmt"
"io"
"net/http"
"regexp"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
)
type Scanner struct {
client *http.Client
}
// Ensure the Scanner satisfies the interface at compile time.
var _ detectors.Detector = (*Scanner)(nil)
var (
defaultClient = common.SaneHttpClient()
tokenPat = regexp.MustCompile(`\b(dd[pw]_[a-zA-Z0-9]{36})\b`)
)
// Keywords are used for efficiently pre-filtering chunks.
// Use identifiers in the secret preferably, or the provider name.
func (s Scanner) Keywords() []string {
return []string{"ddp_", "ddw_"}
}
type userResponse struct {
Login string `json:"login"`
}
// FromData will find and optionally verify DenoDeploy secrets in a given set of bytes.
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
dataStr := string(data)
tokenMatches := tokenPat.FindAllStringSubmatch(dataStr, -1)
for _, tokenMatch := range tokenMatches {
token := tokenMatch[1]
s1 := detectors.Result{
DetectorType: s.Type(),
Raw: []byte(token),
}
if verify {
client := s.client
if client == nil {
client = defaultClient
}
req, err := http.NewRequestWithContext(ctx, "GET", "https://api.deno.com/user", nil)
req.Header.Set("Authorization", "Bearer "+token)
if err != nil {
continue
}
res, err := client.Do(req)
if err == nil {
defer res.Body.Close()
if res.StatusCode == 200 {
s1.Verified = true
body, err := io.ReadAll(res.Body)
if err != nil {
s1.SetVerificationError(err, token)
} else {
var user userResponse
if err := json.Unmarshal(body, &user); err != nil {
fmt.Printf("Unmarshal error: %v\n", err)
s1.SetVerificationError(err, token)
} else {
s1.ExtraData = map[string]string{
"login": user.Login,
}
}
}
} else if res.StatusCode == 401 {
// The secret is determinately not verified (nothing to do)
} else {
err = fmt.Errorf("unexpected HTTP response status %d", res.StatusCode)
s1.SetVerificationError(err, token)
}
} else {
s1.SetVerificationError(err, token)
}
}
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
if !s1.Verified && detectors.IsKnownFalsePositive(token, detectors.DefaultFalsePositives, true) {
continue
}
results = append(results, s1)
}
return results, nil
}
func (s Scanner) Type() detectorspb.DetectorType {
return detectorspb.DetectorType_DenoDeploy
}
-255
View File
@@ -1,255 +0,0 @@
//go:build detectors
// +build detectors
package denodeploy
import (
"context"
"fmt"
"testing"
"time"
"github.com/google/go-cmp/cmp"
"github.com/google/go-cmp/cmp/cmpopts"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
)
func TestDenoDeploy_Pattern(t *testing.T) {
tests := []struct {
name string
data string
shouldMatch bool
match string
}{
// True positives
{
name: `valid_deployctl`,
data: ` "tasks": {
"d": "deployctl deploy --prod --import-map=import_map.json --project=o88 main.ts --token ddp_eg5DjUmbR5lHZ3LiN9MajMk2tA1GxL2NRdvc",
"start": "deno run -A --unstable --watch=static/,routes/ dev.ts"
},`,
shouldMatch: true,
match: `ddp_eg5DjUmbR5lHZ3LiN9MajMk2tA1GxL2NRdvc`,
},
{
name: `valid_dotenv`,
data: `DENO_KV_ACCESS_TOKEN=ddp_hn029Cl2dIN4Jb0BF0L1V9opokoPVC30ddGk`,
shouldMatch: true,
match: `ddp_hn029Cl2dIN4Jb0BF0L1V9opokoPVC30ddGk`,
},
{
name: `valid_dotfile`,
data: `# deno
export DENO_INSTALL="/home/khushal/.deno"
export PATH="$DENO_INSTALL/bin:$PATH"
export DENO_DEPLOY_TOKEN="ddp_QLbDfRlMKpXSf3oCz20Hp8wVVxThDwlwhFbV""`,
shouldMatch: true,
match: `ddp_QLbDfRlMKpXSf3oCz20Hp8wVVxThDwlwhFbV`,
},
{
name: `valid_webtoken`,
data: ` // headers: { Authorization: 'Bearer ddw_ebahKKeZqiZVXOad7KJRHskLeP79Lf0OJXlj' }`,
shouldMatch: true,
match: `ddw_ebahKKeZqiZVXOad7KJRHskLeP79Lf0OJXlj`,
},
// False positives
{
name: `invalid_token1`,
data: ` "summoner2Id": 4,
"summonerId": "oljqJ1Ddp_LJm5s6ONPAJXIl97Bi6pcKMywYLG496a58rA",
"summonerLevel": 146,`,
shouldMatch: false,
},
{
name: `invalid_token2`,
data: ` "image_thumbnail_url": "https://encrypted-tbn0.gstatic.com/images?q=tbn:ANd9GcQFq6zzTXpXtRDdP_JbNkS58loAyCvhhZ1WWONaUkJoWbHsgwIJBw",`,
shouldMatch: false,
},
{
name: `invalid_token3`,
data: `matplotlib/backends/_macosx.cpython-37m-darwin.so,sha256=DDw_KRE5yTUEY5iDBwBW7KvDcTkDmrIu0N18i8I3FvA,90140`,
shouldMatch: false,
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
s := Scanner{}
results, err := s.FromData(context.Background(), false, []byte(test.data))
if err != nil {
t.Errorf("DenoDeploy.FromData() error = %v", err)
return
}
if test.shouldMatch {
if len(results) == 0 {
t.Errorf("%s: did not receive a match for '%v' when one was expected", test.name, test.data)
return
}
expected := test.data
if test.match != "" {
expected = test.match
}
result := results[0]
resultData := string(result.Raw)
if resultData != expected {
t.Errorf("%s: did not receive expected match.\n\texpected: '%s'\n\t actual: '%s'", test.name, expected, resultData)
return
}
} else {
if len(results) > 0 {
t.Errorf("%s: received a match for '%v' when one wasn't wanted", test.name, test.data)
return
}
}
})
}
}
func TestDenoDeploy_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
secret := testSecrets.MustGetField("DENODEPLOY")
inactiveSecret := testSecrets.MustGetField("DENODEPLOY_INACTIVE")
type args struct {
ctx context.Context
data []byte
verify bool
}
tests := []struct {
name string
s Scanner
args args
want []detectors.Result
wantErr bool
wantVerificationErr bool
}{
{
name: "found, verified",
s: Scanner{},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a denodeploy secret %s within", secret)),
verify: true,
},
want: []detectors.Result{
{
DetectorType: detectorspb.DetectorType_DenoDeploy,
Verified: true,
},
},
wantErr: false,
wantVerificationErr: false,
},
{
name: "found, unverified",
s: Scanner{},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a denodeploy secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
verify: true,
},
want: []detectors.Result{
{
DetectorType: detectorspb.DetectorType_DenoDeploy,
Verified: false,
},
},
wantErr: false,
wantVerificationErr: false,
},
{
name: "not found",
s: Scanner{},
args: args{
ctx: context.Background(),
data: []byte("You cannot find the secret within"),
verify: true,
},
want: nil,
wantErr: false,
wantVerificationErr: false,
},
{
name: "found, would be verified if not for timeout",
s: Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a denodeploy secret %s within", secret)),
verify: true,
},
want: []detectors.Result{
{
DetectorType: detectorspb.DetectorType_DenoDeploy,
Verified: false,
},
},
wantErr: false,
wantVerificationErr: true,
},
{
name: "found, verified but unexpected api surface",
s: Scanner{client: common.ConstantResponseHttpClient(404, "")},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a denodeploy secret %s within", secret)),
verify: true,
},
want: []detectors.Result{
{
DetectorType: detectorspb.DetectorType_DenoDeploy,
Verified: false,
},
},
wantErr: false,
wantVerificationErr: true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
if (err != nil) != tt.wantErr {
t.Errorf("Denodeploy.FromData() error = %v, wantErr %v", err, tt.wantErr)
return
}
for i := range got {
if len(got[i].Raw) == 0 {
t.Fatalf("no raw secret present: \n %+v", got[i])
}
if (got[i].VerificationError() != nil) != tt.wantVerificationErr {
t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError())
}
}
ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "verificationError", "ExtraData")
if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {
t.Errorf("Denodeploy.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
}
})
}
}
func BenchmarkFromData(benchmark *testing.B) {
ctx := context.Background()
s := Scanner{}
for name, data := range detectors.MustGetBenchmarkData() {
benchmark.Run(name, func(b *testing.B) {
b.ResetTimer()
for n := 0; n < b.N; n++ {
_, err := s.FromData(ctx, false, data)
if err != nil {
b.Fatal(err)
}
}
})
}
}
+2 -40
View File
@@ -8,8 +8,6 @@ import (
"strings"
"unicode"
"errors"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/source_metadatapb"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
@@ -61,50 +59,14 @@ type Result struct {
// This field should only be populated if the verification process itself failed in a way that provides no
// information about the verification status of the candidate secret, such as if the verification request timed out.
verificationError error
}
// SetVerificationError is the only way to set a verification error. Any sensetive values should be passed-in as secrets to be redacted.
func (r *Result) SetVerificationError(err error, secrets ...string) {
if err != nil {
r.verificationError = redactSecrets(err, secrets...)
}
}
// Public accessors for the fields could also be provided if needed.
func (r *Result) VerificationError() error {
return r.verificationError
}
// redactSecrets replaces all instances of the given secrets with [REDACTED] in the error message.
func redactSecrets(err error, secrets ...string) error {
lastErr := unwrapToLast(err)
errStr := lastErr.Error()
for _, secret := range secrets {
errStr = strings.Replace(errStr, secret, "[REDACTED]", -1)
}
return errors.New(errStr)
}
// unwrapToLast returns the last error in the chain of errors.
// This is added to exclude non-essential details (like URLs) for brevity and security.
// Also helps us optimize performance in redaction and enhance log clarity.
func unwrapToLast(err error) error {
for {
unwrapped := errors.Unwrap(err)
if unwrapped == nil {
// We've reached the last error in the chain
return err
}
err = unwrapped
}
VerificationError error
}
type ResultWithMetadata struct {
// SourceMetadata contains source-specific contextual information.
SourceMetadata *source_metadatapb.MetaData
// SourceID is the ID of the source that the API uses to map secrets to specific sources.
SourceID sources.SourceID
SourceID int64
// SourceType is the type of Source.
SourceType sourcespb.SourceType
// SourceName is the name of the Source.
+3 -3
View File
@@ -12,15 +12,15 @@ func TestPrefixRegex(t *testing.T) {
}{
{
keywords: []string{"securitytrails"},
expected: `(?i)(?:securitytrails)(?:.|[\n\r]){0,40}`,
expected: `(?i)(?:securitytrails).|(?:[\n\r]){0,40}`,
},
{
keywords: []string{"zipbooks"},
expected: `(?i)(?:zipbooks)(?:.|[\n\r]){0,40}`,
expected: `(?i)(?:zipbooks).|(?:[\n\r]){0,40}`,
},
{
keywords: []string{"wrike"},
expected: `(?i)(?:wrike)(?:.|[\n\r]){0,40}`,
expected: `(?i)(?:wrike).|(?:[\n\r]){0,40}`,
},
}
for _, tt := range tests {
-162
View File
@@ -1,162 +0,0 @@
//go:build detectors
// +build detectors
package eventbrite
import (
"context"
"fmt"
"testing"
"time"
"github.com/google/go-cmp/cmp"
"github.com/google/go-cmp/cmp/cmpopts"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
)
func TestEventbrite_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
secret := testSecrets.MustGetField("EVENTBRITE")
inactiveSecret := testSecrets.MustGetField("EVENTBRITE_INACTIVE")
type args struct {
ctx context.Context
data []byte
verify bool
}
tests := []struct {
name string
s Scanner
args args
want []detectors.Result
wantErr bool
wantVerificationErr bool
}{
{
name: "found, verified",
s: Scanner{},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a eventbrite secret %s within", secret)),
verify: true,
},
want: []detectors.Result{
{
DetectorType: detectorspb.DetectorType_Eventbrite,
Verified: true,
},
},
wantErr: false,
wantVerificationErr: false,
},
{
name: "found, unverified",
s: Scanner{},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a eventbrite secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
verify: true,
},
want: []detectors.Result{
{
DetectorType: detectorspb.DetectorType_Eventbrite,
Verified: false,
},
},
wantErr: false,
wantVerificationErr: false,
},
{
name: "not found",
s: Scanner{},
args: args{
ctx: context.Background(),
data: []byte("You cannot find the secret within"),
verify: true,
},
want: nil,
wantErr: false,
wantVerificationErr: false,
},
{
name: "found, would be verified if not for timeout",
s: Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a eventbrite secret %s within", secret)),
verify: true,
},
want: []detectors.Result{
{
DetectorType: detectorspb.DetectorType_Eventbrite,
Verified: false,
},
},
wantErr: false,
wantVerificationErr: true,
},
{
name: "found, verified but unexpected api surface",
s: Scanner{client: common.ConstantResponseHttpClient(404, "")},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a eventbrite secret %s within", secret)),
verify: true,
},
want: []detectors.Result{
{
DetectorType: detectorspb.DetectorType_Eventbrite,
Verified: false,
},
},
wantErr: false,
wantVerificationErr: true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
if (err != nil) != tt.wantErr {
t.Errorf("Eventbrite.FromData() error = %v, wantErr %v", err, tt.wantErr)
return
}
for i := range got {
if len(got[i].Raw) == 0 {
t.Fatalf("no raw secret present: \n %+v", got[i])
}
if (got[i].VerificationError() != nil) != tt.wantVerificationErr {
t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError())
}
}
ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "verificationError")
if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {
t.Errorf("Eventbrite.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
}
})
}
}
func BenchmarkFromData(benchmark *testing.B) {
ctx := context.Background()
s := Scanner{}
for name, data := range detectors.MustGetBenchmarkData() {
benchmark.Run(name, func(b *testing.B) {
b.ResetTimer()
for n := 0; n < b.N; n++ {
_, err := s.FromData(ctx, false, data)
if err != nil {
b.Fatal(err)
}
}
})
}
}
+1 -2
View File
@@ -59,8 +59,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
if verify {
// thanks https://stackoverflow.com/questions/15621471/validate-a-facebook-app-id-and-app-secret
// https://stackoverflow.com/questions/24401241/how-to-get-a-facebook-access-token-using-appid-and-app-secret-without-any-login
req, err := http.NewRequestWithContext(ctx, "GET", fmt.Sprintf("https://graph.facebook.com/me?access_token=%s|%s", apiIdRes, apiSecretRes), nil)
req, err := http.NewRequestWithContext(ctx, "GET", fmt.Sprintf("https://graph.facebook.com/%s?fields=roles&access_token=%s|%s", apiIdRes, apiIdRes, apiSecretRes), nil)
if err != nil {
continue
}
@@ -1,7 +1,9 @@
package metabase
package fakejson
import (
"context"
"fmt"
"io"
"net/http"
"regexp"
"strings"
@@ -20,23 +22,20 @@ var (
client = common.SaneHttpClient()
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"metabase"}) + `\b([a-zA-Z0-9-]{36})\b`)
baseURL = regexp.MustCompile(detectors.PrefixRegex([]string{"metabase"}) + `\b(https?:\/\/[-a-zA-Z0-9@:%._\+~#=]{7,256})\b`)
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"fakejson"}) + `\b([a-zA-Z0-9]{22})\b`)
)
// Keywords are used for efficiently pre-filtering chunks.
// Use identifiers in the secret preferably, or the provider name.
func (s Scanner) Keywords() []string {
return []string{"metabase"}
return []string{"fakejson"}
}
// FromData will find and optionally verify Metabase secrets in a given set of bytes.
// FromData will find and optionally verify FakeJSON secrets in a given set of bytes.
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
dataStr := string(data)
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
urlMatches := baseURL.FindAllStringSubmatch(dataStr, -1)
for _, match := range matches {
if len(match) != 2 {
@@ -44,29 +43,31 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
}
resMatch := strings.TrimSpace(match[1])
for _, urlMatch := range urlMatches {
if len(urlMatch) != 2 {
s1 := detectors.Result{
DetectorType: detectorspb.DetectorType_FakeJSON,
Raw: []byte(resMatch),
}
if verify {
payload := strings.NewReader(fmt.Sprintf(`{"token":"%s","data":{"user_name":"nameFirst","user_email":"internetEmail","_repeat":3}}`, resMatch))
req, err := http.NewRequestWithContext(ctx, "POST", "https://app.fakejson.com/q", payload)
if err != nil {
continue
}
resURLMatch := strings.TrimSpace(urlMatch[1])
s1 := detectors.Result{
DetectorType: detectorspb.DetectorType_Metabase,
Raw: []byte(resMatch),
RawV2: []byte(resMatch + resURLMatch),
}
if verify {
req, err := http.NewRequestWithContext(ctx, "GET", resURLMatch+"/api/user/current", nil)
if err != nil {
continue
}
req.Header.Add("X-Metabase-Session", resMatch)
res, err := client.Do(req)
req.Header.Add("Content-Type", "application/json")
res, err := client.Do(req)
if err == nil {
bodyBytes, err := io.ReadAll(res.Body)
if err == nil {
bodyString := string(bodyBytes)
validResponse := strings.Contains(bodyString, `"user_email":`)
defer res.Body.Close()
if res.StatusCode >= 200 && res.StatusCode < 300 {
s1.Verified = true
if validResponse {
s1.Verified = true
} else {
s1.Verified = false
}
} else {
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
@@ -75,14 +76,14 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
}
}
}
results = append(results, s1)
}
results = append(results, s1)
}
return results, nil
}
func (s Scanner) Type() detectorspb.DetectorType {
return detectorspb.DetectorType_Metabase
return detectorspb.DetectorType_FakeJSON
}
+120
View File
@@ -0,0 +1,120 @@
//go:build detectors
// +build detectors
package fakejson
import (
"context"
"fmt"
"testing"
"time"
"github.com/kylelemons/godebug/pretty"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
)
func TestFakeJSON_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
secret := testSecrets.MustGetField("FAKEJSON")
inactiveSecret := testSecrets.MustGetField("FAKEJSON_INACTIVE")
type args struct {
ctx context.Context
data []byte
verify bool
}
tests := []struct {
name string
s Scanner
args args
want []detectors.Result
wantErr bool
}{
{
name: "found, verified",
s: Scanner{},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a fakejson secret %s within", secret)),
verify: true,
},
want: []detectors.Result{
{
DetectorType: detectorspb.DetectorType_FakeJSON,
Verified: true,
},
},
wantErr: false,
},
{
name: "found, unverified",
s: Scanner{},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a fakejson secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
verify: true,
},
want: []detectors.Result{
{
DetectorType: detectorspb.DetectorType_FakeJSON,
Verified: false,
},
},
wantErr: false,
},
{
name: "not found",
s: Scanner{},
args: args{
ctx: context.Background(),
data: []byte("You cannot find the secret within"),
verify: true,
},
want: nil,
wantErr: false,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
s := Scanner{}
got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
if (err != nil) != tt.wantErr {
t.Errorf("FakeJSON.FromData() error = %v, wantErr %v", err, tt.wantErr)
return
}
for i := range got {
if len(got[i].Raw) == 0 {
t.Fatalf("no raw secret present: \n %+v", got[i])
}
got[i].Raw = nil
}
if diff := pretty.Compare(got, tt.want); diff != "" {
t.Errorf("FakeJSON.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
}
})
}
}
func BenchmarkFromData(benchmark *testing.B) {
ctx := context.Background()
s := Scanner{}
for name, data := range detectors.MustGetBenchmarkData() {
benchmark.Run(name, func(b *testing.B) {
b.ResetTimer()
for n := 0; n < b.N; n++ {
_, err := s.FromData(ctx, false, data)
if err != nil {
b.Fatal(err)
}
}
})
}
}
+21 -52
View File
@@ -2,10 +2,8 @@ package detectors
import (
_ "embed"
"math"
"strings"
"unicode"
"unicode/utf8"
)
var DefaultFalsePositives = []FalsePositive{"example", "xxxxxx", "aaaaaa", "abcde", "00000", "sample", "www"}
@@ -22,9 +20,9 @@ var wordList []byte
var programmingBookWords []byte
type Wordlists struct {
wordList map[string]struct{}
badList map[string]struct{}
programmingBookWords map[string]struct{}
wordList []string
badList []string
programmingBookWords []string
}
var FalsePositiveWordlists = Wordlists{
@@ -37,29 +35,36 @@ var FalsePositiveWordlists = Wordlists{
// Currently that includes: No number, english word in key, or matches common example pattens.
// Only the secret key material should be passed into this function
func IsKnownFalsePositive(match string, falsePositives []FalsePositive, wordCheck bool) bool {
if !utf8.ValidString(match) {
return true
}
lower := strings.ToLower(match)
for _, fp := range falsePositives {
if strings.Contains(lower, string(fp)) {
if strings.Contains(strings.ToLower(match), string(fp)) {
return true
}
}
if wordCheck {
// check against common substring badlist
if _, ok := FalsePositiveWordlists.badList[lower]; ok {
if hasDictWord(FalsePositiveWordlists.badList, match) {
return true
}
// check for dictionary word substrings
if _, ok := FalsePositiveWordlists.wordList[lower]; ok {
if hasDictWord(FalsePositiveWordlists.wordList, match) {
return true
}
// check for programming book token substrings
if _, ok := FalsePositiveWordlists.programmingBookWords[lower]; ok {
if hasDictWord(FalsePositiveWordlists.programmingBookWords, match) {
return true
}
}
return false
}
func hasDictWord(wordList []string, token string) bool {
lower := strings.ToLower(token)
for _, word := range wordList {
if strings.Contains(lower, word) {
return true
}
}
@@ -76,48 +81,12 @@ func HasDigit(key string) bool {
return false
}
func bytesToCleanWordList(data []byte) map[string]struct{} {
words := make(map[string]struct{})
func bytesToCleanWordList(data []byte) []string {
words := []string{}
for _, word := range strings.Split(string(data), "\n") {
if strings.TrimSpace(word) != "" {
words[strings.TrimSpace(strings.ToLower(word))] = struct{}{}
words = append(words, strings.TrimSpace(strings.ToLower(word)))
}
}
return words
}
func StringShannonEntropy(input string) float64 {
chars := make(map[rune]float64)
inverseTotal := 1 / float64(len(input)) // precompute the inverse
for _, char := range input {
chars[char]++
}
entropy := 0.0
for _, count := range chars {
probability := count * inverseTotal
entropy += probability * math.Log2(probability)
}
return -entropy
}
// FilterResultsWithEntropy filters out determinately unverified results that have a shannon entropy below the given value.
func FilterResultsWithEntropy(results []Result, entropy float64) []Result {
var filteredResults []Result
for _, result := range results {
if !result.Verified && result.VerificationError() == nil {
if result.RawV2 != nil {
if StringShannonEntropy(string(result.RawV2)) >= entropy {
filteredResults = append(filteredResults, result)
}
} else {
if StringShannonEntropy(string(result.Raw)) >= entropy {
filteredResults = append(filteredResults, result)
}
}
}
}
return filteredResults
}
+1 -58
View File
@@ -3,10 +3,7 @@
package detectors
import (
_ "embed"
"testing"
)
import "testing"
func TestIsFalsePositive(t *testing.T) {
type args struct {
@@ -43,57 +40,3 @@ func TestIsFalsePositive(t *testing.T) {
})
}
}
func TestStringShannonEntropy(t *testing.T) {
type args struct {
input string
}
tests := []struct {
name string
args args
want float64
}{
{
name: "entropy 1",
args: args{
input: "aaaaaaaaaaaaaaaaaaaaaaaaaaaa",
},
want: 0,
},
{
name: "entropy 2",
args: args{
input: "aaaaaaaaaaaaaaaaaaaaaaaaaaab",
},
want: 0.22228483068568816,
},
{
name: "entropy 3",
args: args{
input: "aaaaaaaaaaaaaaaaaaaaaaaaaaabaaaaaaaaaaaaaaaaaaaaaaaaaaab",
},
want: 0.22228483068568816,
},
{
name: "empty",
args: args{
input: "",
},
want: 0.0,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if got := StringShannonEntropy(tt.args.input); got != tt.want {
t.Errorf("StringShannonEntropy() = %v, want %v", got, tt.want)
}
})
}
}
func BenchmarkDefaultIsKnownFalsePositive(b *testing.B) {
for i := 0; i < b.N; i++ {
// Use a string that won't be found in any dictionary for the worst case check.
IsKnownFalsePositive("aoeuaoeuaoeuaoeuaoeuaoeu", DefaultFalsePositives, true)
}
}
@@ -2,7 +2,6 @@ package figmapersonalaccesstoken
import (
"context"
"fmt"
"net/http"
"regexp"
"strings"
@@ -12,18 +11,13 @@ import (
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
)
type Scanner struct {
client *http.Client
}
type Scanner struct{}
// Ensure the Scanner satisfies the interface at compile time.
var _ detectors.Detector = (*Scanner)(nil)
var _ detectors.Versioner = (*Scanner)(nil)
func (Scanner) Version() int { return 1 }
var (
defaultClient = common.SaneHttpClient()
client = common.SaneHttpClient()
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"figma"}) + `\b([0-9]{6}-[0-9a-z]{8}-[0-9a-z]{4}-[0-9a-z]{4}-[0-9a-z]{4}-[0-9a-z]{12})\b`)
)
@@ -52,11 +46,6 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
}
if verify {
client := s.client
if client == nil {
client = defaultClient
}
req, err := http.NewRequestWithContext(ctx, "GET", "https://api.figma.com/v1/me", nil)
if err != nil {
continue
@@ -67,18 +56,14 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
defer res.Body.Close()
if res.StatusCode >= 200 && res.StatusCode < 300 {
s1.Verified = true
} else if res.StatusCode != 403 {
err = fmt.Errorf("unexpected HTTP response status %d", res.StatusCode)
s1.SetVerificationError(err, resMatch)
} else {
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
continue
}
}
} else {
s1.SetVerificationError(err, resMatch)
}
}
if !s1.Verified && detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
continue
}
results = append(results, s1)
}
@@ -9,8 +9,8 @@ import (
"testing"
"time"
"github.com/google/go-cmp/cmp"
"github.com/google/go-cmp/cmp/cmpopts"
"github.com/kylelemons/godebug/pretty"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
@@ -32,12 +32,11 @@ func TestFigmaPersonalAccessToken_FromChunk(t *testing.T) {
verify bool
}
tests := []struct {
name string
s Scanner
args args
want []detectors.Result
wantErr bool
wantVerificationErr bool
name string
s Scanner
args args
want []detectors.Result
wantErr bool
}{
{
name: "found, verified",
@@ -71,40 +70,6 @@ func TestFigmaPersonalAccessToken_FromChunk(t *testing.T) {
},
wantErr: false,
},
{
name: "found, verified but unexpected api surface",
s: Scanner{client: common.ConstantResponseHttpClient(404, "")},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a figmapersonalaccesstoken secret %s within", secret)),
verify: true,
},
want: []detectors.Result{
{
DetectorType: detectorspb.DetectorType_FigmaPersonalAccessToken,
Verified: false,
},
},
wantErr: false,
wantVerificationErr: true,
},
{
name: "found, would be verified if not for timeout",
s: Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a figmapersonalaccesstoken secret %s within", secret)),
verify: true,
},
want: []detectors.Result{
{
DetectorType: detectorspb.DetectorType_FigmaPersonalAccessToken,
Verified: false,
},
},
wantErr: false,
wantVerificationErr: true,
},
{
name: "not found",
s: Scanner{},
@@ -119,7 +84,8 @@ func TestFigmaPersonalAccessToken_FromChunk(t *testing.T) {
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
s := Scanner{}
got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
if (err != nil) != tt.wantErr {
t.Errorf("FigmaPersonalAccessToken.FromData() error = %v, wantErr %v", err, tt.wantErr)
return
@@ -128,12 +94,9 @@ func TestFigmaPersonalAccessToken_FromChunk(t *testing.T) {
if len(got[i].Raw) == 0 {
t.Fatalf("no raw secret present: \n %+v", got[i])
}
if (got[i].VerificationError() != nil) != tt.wantVerificationErr {
t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError())
}
got[i].Raw = nil
}
ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "verificationError")
if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {
if diff := pretty.Compare(got, tt.want); diff != "" {
t.Errorf("FigmaPersonalAccessToken.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
}
})
@@ -1,157 +0,0 @@
//go:build detectors
// +build detectors
package figmapersonalaccesstoken
import (
"context"
"fmt"
"testing"
"time"
"github.com/google/go-cmp/cmp"
"github.com/google/go-cmp/cmp/cmpopts"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
)
func TestFigmaPersonalAccessToken_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
secret := testSecrets.MustGetField("FIGMAPERSONALACCESSTOKEN_V2_TOKEN")
inactiveSecret := testSecrets.MustGetField("FIGMAPERSONALACCESSTOKEN_V2_INACTIVE")
type args struct {
ctx context.Context
data []byte
verify bool
}
tests := []struct {
name string
s Scanner
args args
want []detectors.Result
wantErr bool
wantVerificationErr bool
}{
{
name: "found, verified",
s: Scanner{},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a figmapersonalaccesstoken secret %s within", secret)),
verify: true,
},
want: []detectors.Result{
{
DetectorType: detectorspb.DetectorType_FigmaPersonalAccessToken,
Verified: true,
},
},
wantErr: false,
},
{
name: "found, unverified",
s: Scanner{},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a figmapersonalaccesstoken secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
verify: true,
},
want: []detectors.Result{
{
DetectorType: detectorspb.DetectorType_FigmaPersonalAccessToken,
Verified: false,
},
},
wantErr: false,
},
{
name: "found, verified but unexpected api surface",
s: Scanner{client: common.ConstantResponseHttpClient(404, "")},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a figmapersonalaccesstoken secret %s within", secret)),
verify: true,
},
want: []detectors.Result{
{
DetectorType: detectorspb.DetectorType_FigmaPersonalAccessToken,
Verified: false,
},
},
wantErr: false,
wantVerificationErr: true,
},
{
name: "found, would be verified if not for timeout",
s: Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a figmapersonalaccesstoken secret %s within", secret)),
verify: true,
},
want: []detectors.Result{
{
DetectorType: detectorspb.DetectorType_FigmaPersonalAccessToken,
Verified: false,
},
},
wantErr: false,
wantVerificationErr: true,
},
{
name: "not found",
s: Scanner{},
args: args{
ctx: context.Background(),
data: []byte("You cannot find the secret within"),
verify: true,
},
want: nil,
wantErr: false,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
if (err != nil) != tt.wantErr {
t.Errorf("FigmaPersonalAccessToken.FromData() error = %v, wantErr %v", err, tt.wantErr)
return
}
for i := range got {
if len(got[i].Raw) == 0 {
t.Fatalf("no raw secret present: \n %+v", got[i])
}
if (got[i].VerificationError() != nil) != tt.wantVerificationErr {
t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError())
}
}
ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "verificationError")
if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {
t.Errorf("FigmaPersonalAccessToken.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
}
})
}
}
func BenchmarkFromData(benchmark *testing.B) {
ctx := context.Background()
s := Scanner{}
for name, data := range detectors.MustGetBenchmarkData() {
benchmark.Run(name, func(b *testing.B) {
b.ResetTimer()
for n := 0; n < b.N; n++ {
_, err := s.FromData(ctx, false, data)
if err != nil {
b.Fatal(err)
}
}
})
}
}

Some files were not shown because too many files have changed in this diff Show More