Compare commits
2 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 070cacfc49 | |||
| 6082e0c458 |
@@ -0,0 +1,15 @@
|
||||
# To get started with Dependabot version updates, you'll need to specify which
|
||||
# package ecosystems to update and where the package manifests are located.
|
||||
# Please see the documentation for all configuration options:
|
||||
# https://help.github.com/github/administering-a-repository/configuration-options-for-dependency-updates
|
||||
|
||||
version: 2
|
||||
updates:
|
||||
- package-ecosystem: "gomod" # See documentation for possible values
|
||||
directory: "/" # Location of package manifests
|
||||
schedule:
|
||||
interval: "monthly"
|
||||
- package-ecosystem: "github-actions"
|
||||
directory: "/"
|
||||
schedule:
|
||||
interval: "monthly"
|
||||
@@ -1,8 +0,0 @@
|
||||
{
|
||||
"$schema": "https://docs.renovatebot.com/renovate-schema.json",
|
||||
"extends": [
|
||||
"config:base"
|
||||
],
|
||||
"prConcurrentLimit": 3,
|
||||
"prHourlyLimit": 2
|
||||
}
|
||||
+3
-3
@@ -8,9 +8,9 @@ RUN --mount=type=cache,target=/go/pkg/mod \
|
||||
--mount=type=cache,target=/root/.cache/go-build \
|
||||
GOOS=${TARGETOS} GOARCH=${TARGETARCH} go build -o trufflehog .
|
||||
|
||||
FROM alpine:3.18
|
||||
RUN apk add --no-cache bash git openssh-client ca-certificates rpm2cpio \
|
||||
&& rm -rf /var/cache/apk/* && update-ca-certificates
|
||||
FROM alpine:3.15
|
||||
RUN apk add --no-cache bash git openssh-client ca-certificates \
|
||||
&& update-ca-certificates
|
||||
COPY --from=builder /build/trufflehog /usr/bin/trufflehog
|
||||
COPY entrypoint.sh /etc/entrypoint.sh
|
||||
RUN chmod +x /etc/entrypoint.sh
|
||||
|
||||
@@ -1,7 +1,8 @@
|
||||
FROM alpine:3.18
|
||||
FROM alpine:3.15
|
||||
|
||||
RUN apk add --no-cache bash git openssh-client ca-certificates rpm2cpio \
|
||||
&& rm -rf /var/cache/apk/* && update-ca-certificates
|
||||
RUN apk add --no-cache bash git openssh-client ca-certificates \
|
||||
&& rm -rf /var/cache/apk/* && \
|
||||
update-ca-certificates
|
||||
WORKDIR /usr/bin/
|
||||
COPY trufflehog .
|
||||
COPY entrypoint.sh /etc/entrypoint.sh
|
||||
|
||||
@@ -52,10 +52,10 @@ run-debug:
|
||||
CGO_ENABLED=0 go run . git file://. --json --debug
|
||||
|
||||
protos:
|
||||
docker run --rm -u "$(shell id -u)" -v "$(shell dirname $(realpath $(firstword $(MAKEFILE_LIST))))":/pwd "${PROTOS_IMAGE}" bash -c "cd /pwd; /pwd/scripts/gen_proto.sh"
|
||||
docker run -u "$(shell id -u)" -v "$(shell dirname $(realpath $(firstword $(MAKEFILE_LIST))))":/pwd "${PROTOS_IMAGE}" bash -c "cd /pwd; /pwd/scripts/gen_proto.sh"
|
||||
|
||||
protos-windows:
|
||||
docker run --rm -v "$(shell cygpath -w $(shell pwd))":/pwd "${PROTOS_IMAGE}" bash -c "cd /pwd; ./scripts/gen_proto.sh"
|
||||
docker run -v "$(shell cygpath -w $(shell pwd))":/pwd "${PROTOS_IMAGE}" bash -c "cd /pwd; ./scripts/gen_proto.sh"
|
||||
|
||||
release-protos-image:
|
||||
docker buildx build --push --platform=linux/amd64,linux/arm64 \
|
||||
|
||||
@@ -164,7 +164,7 @@ trufflehog docker --image trufflesecurity/secrets --only-verified
|
||||
|
||||
+ All I see is `🐷🔑🐷 TruffleHog. Unearth your secrets. 🐷🔑🐷` and the program exits, what gives?
|
||||
+ That means no secrets were detected
|
||||
+ Why is the scan taking a long time when I scan a GitHub org
|
||||
+ Why is the scan is taking a long time when I scan a GitHub org
|
||||
+ Unauthenticated GitHub scans have rate limits. To improve your rate limits, include the `--token` flag with a personal access token
|
||||
+ It says a private key was verified, what does that mean?
|
||||
+ Check out our Driftwood blog post to learn how to do this, in short we've confirmed the key can be used live for SSH or SSL [Blog post](https://trufflesecurity.com/blog/driftwood-know-if-private-keys-are-sensitive/)
|
||||
@@ -219,7 +219,7 @@ Flags:
|
||||
--concurrency=10 Number of concurrent workers.
|
||||
--no-verification Don't verify the results.
|
||||
--only-verified Only output verified results.
|
||||
--filter-unverified Only output first unverified result per chunk per detector if there is more than one result.
|
||||
--filter-unverified Only output first unverified result per chunk per detector if there are more than one results.
|
||||
--config=CONFIG Path to configuration file.
|
||||
--print-avg-detector-time Print the average time spent on each detector.
|
||||
--no-update Don't check for updates.
|
||||
|
||||
@@ -2,17 +2,21 @@ module github.com/trufflesecurity/trufflehog/v3
|
||||
|
||||
go 1.21
|
||||
|
||||
replace github.com/jpillora/overseer => github.com/trufflesecurity/overseer v1.2.7
|
||||
replace github.com/jpillora/overseer => github.com/trufflesecurity/overseer v1.1.7-custom5
|
||||
|
||||
// go wants to pull in the latest but after v10.2.5 has a breaking API change
|
||||
replace github.com/couchbase/gocbcore/v10 => github.com/couchbase/gocbcore/v10 v10.2.4
|
||||
|
||||
require (
|
||||
cloud.google.com/go/secretmanager v1.11.2
|
||||
cloud.google.com/go/secretmanager v1.11.1
|
||||
cloud.google.com/go/storage v1.33.0
|
||||
github.com/Azure/go-autorest/autorest/azure/auth v0.5.12
|
||||
github.com/Azure/go-autorest/autorest/azure/auth v0.5.11
|
||||
github.com/AzureAD/microsoft-authentication-library-for-go v1.2.0
|
||||
github.com/BobuSumisu/aho-corasick v1.0.3
|
||||
github.com/TheZeroSlave/zapsentry v1.17.0
|
||||
github.com/aws/aws-sdk-go v1.45.19
|
||||
github.com/aymanbagabas/go-osc52 v1.2.2
|
||||
github.com/aymanbagabas/go-osc52 v1.2.1
|
||||
github.com/bill-rich/disk-buffer-reader v0.1.7
|
||||
github.com/bill-rich/go-syslog v0.0.0-20220413021637-49edb52a574c
|
||||
github.com/bitfinexcom/bitfinex-api-go v0.0.0-20210608095005-9e0b26f200fb
|
||||
github.com/bradleyfalzon/ghinstallation/v2 v2.7.0
|
||||
@@ -20,14 +24,14 @@ require (
|
||||
github.com/charmbracelet/bubbletea v0.24.2
|
||||
github.com/charmbracelet/glamour v0.6.0
|
||||
github.com/charmbracelet/lipgloss v0.7.1
|
||||
github.com/couchbase/gocb/v2 v2.6.5
|
||||
github.com/couchbase/gocb/v2 v2.6.3
|
||||
github.com/crewjam/rfc5424 v0.1.0
|
||||
github.com/denisenkom/go-mssqldb v0.12.3
|
||||
github.com/envoyproxy/protoc-gen-validate v1.0.2
|
||||
github.com/fatih/color v1.15.0
|
||||
github.com/felixge/fgprof v0.9.3
|
||||
github.com/getsentry/sentry-go v0.24.1
|
||||
github.com/go-errors/errors v1.5.1
|
||||
github.com/go-errors/errors v1.4.2
|
||||
github.com/go-git/go-git/v5 v5.8.1
|
||||
github.com/go-ldap/ldap/v3 v3.4.6
|
||||
github.com/go-logr/logr v1.2.4
|
||||
@@ -36,10 +40,9 @@ require (
|
||||
github.com/go-sql-driver/mysql v1.7.1
|
||||
github.com/gobwas/glob v0.2.3
|
||||
github.com/golang-jwt/jwt v3.2.2+incompatible
|
||||
github.com/google/go-cmp v0.6.0
|
||||
github.com/google/go-cmp v0.5.9
|
||||
github.com/google/go-containerregistry v0.15.2
|
||||
github.com/google/go-github/v42 v42.0.0
|
||||
github.com/google/go-github/v56 v56.0.0
|
||||
github.com/googleapis/gax-go/v2 v2.12.0
|
||||
github.com/h2non/filetype v1.1.3
|
||||
github.com/hashicorp/go-retryablehttp v0.7.4
|
||||
@@ -49,11 +52,11 @@ require (
|
||||
github.com/joho/godotenv v1.5.1
|
||||
github.com/jpillora/overseer v1.1.6
|
||||
github.com/kylelemons/godebug v1.1.0
|
||||
github.com/launchdarkly/go-server-sdk/v6 v6.1.1
|
||||
github.com/launchdarkly/go-server-sdk/v6 v6.1.0
|
||||
github.com/lib/pq v1.10.9
|
||||
github.com/lrstanley/bubblezone v0.0.0-20230911164824-e3824f1adde9
|
||||
github.com/lrstanley/bubblezone v0.0.0-20221222153816-e95291e2243e
|
||||
github.com/marusama/semaphore/v2 v2.5.0
|
||||
github.com/mattn/go-isatty v0.0.20
|
||||
github.com/mattn/go-isatty v0.0.19
|
||||
github.com/mattn/go-sqlite3 v1.14.17
|
||||
github.com/mholt/archiver/v4 v4.0.0-alpha.8
|
||||
github.com/muesli/reflow v0.3.0
|
||||
@@ -63,18 +66,17 @@ require (
|
||||
github.com/prometheus/client_golang v1.16.0
|
||||
github.com/rabbitmq/amqp091-go v1.8.1
|
||||
github.com/sergi/go-diff v1.3.1
|
||||
github.com/snowflakedb/gosnowflake v1.6.25
|
||||
github.com/snowflakedb/gosnowflake v1.6.23
|
||||
github.com/stretchr/testify v1.8.4
|
||||
github.com/tailscale/depaware v0.0.0-20210622194025-720c4b409502
|
||||
github.com/trufflesecurity/disk-buffer-reader v0.1.9
|
||||
github.com/xanzy/go-gitlab v0.92.3
|
||||
go.mongodb.org/mongo-driver v1.12.1
|
||||
go.uber.org/mock v0.2.0
|
||||
go.uber.org/zap v1.24.0
|
||||
golang.org/x/crypto v0.14.0
|
||||
golang.org/x/exp v0.0.0-20231006140011-7918f672742d
|
||||
golang.org/x/oauth2 v0.13.0
|
||||
golang.org/x/sync v0.4.0
|
||||
golang.org/x/exp v0.0.0-20230206171751-46f607a40771
|
||||
golang.org/x/oauth2 v0.12.0
|
||||
golang.org/x/sync v0.3.0
|
||||
golang.org/x/text v0.13.0
|
||||
google.golang.org/api v0.132.0
|
||||
google.golang.org/protobuf v1.31.0
|
||||
@@ -139,7 +141,7 @@ require (
|
||||
github.com/connesc/cipherio v0.2.1 // indirect
|
||||
github.com/containerd/console v1.0.4-0.20230313162750-1ae8d489ac81 // indirect
|
||||
github.com/containerd/stargz-snapshotter/estargz v0.14.3 // indirect
|
||||
github.com/couchbase/gocbcore/v10 v10.2.9 // indirect
|
||||
github.com/couchbase/gocbcore/v10 v10.2.4 // indirect
|
||||
github.com/danieljoos/wincred v1.1.2 // indirect
|
||||
github.com/davecgh/go-spew v1.1.1 // indirect
|
||||
github.com/dimchansky/utfbom v1.1.1 // indirect
|
||||
@@ -192,7 +194,7 @@ require (
|
||||
github.com/launchdarkly/eventsource v1.6.2 // indirect
|
||||
github.com/launchdarkly/go-jsonstream/v3 v3.0.0 // indirect
|
||||
github.com/launchdarkly/go-sdk-common/v3 v3.0.1 // indirect
|
||||
github.com/launchdarkly/go-sdk-events/v2 v2.0.2 // indirect
|
||||
github.com/launchdarkly/go-sdk-events/v2 v2.0.1 // indirect
|
||||
github.com/launchdarkly/go-semver v1.0.2 // indirect
|
||||
github.com/launchdarkly/go-server-sdk-evaluation/v2 v2.0.2 // indirect
|
||||
github.com/lucasb-eyer/go-colorful v1.2.0 // indirect
|
||||
@@ -245,12 +247,12 @@ require (
|
||||
go.uber.org/atomic v1.7.0 // indirect
|
||||
go.uber.org/multierr v1.6.0 // indirect
|
||||
go4.org v0.0.0-20200411211856-f5505b9728dd // indirect
|
||||
golang.org/x/mod v0.13.0 // indirect
|
||||
golang.org/x/mod v0.12.0 // indirect
|
||||
golang.org/x/net v0.17.0 // indirect
|
||||
golang.org/x/sys v0.13.0 // indirect
|
||||
golang.org/x/term v0.13.0 // indirect
|
||||
golang.org/x/time v0.3.0 // indirect
|
||||
golang.org/x/tools v0.14.0 // indirect
|
||||
golang.org/x/tools v0.12.0 // indirect
|
||||
golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect
|
||||
google.golang.org/appengine v1.6.7 // indirect
|
||||
google.golang.org/genproto v0.0.0-20230706204954-ccb25ca9f130 // indirect
|
||||
|
||||
@@ -22,8 +22,6 @@ cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2k
|
||||
cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw=
|
||||
cloud.google.com/go/secretmanager v1.11.1 h1:cLTCwAjFh9fKvU6F13Y4L9vPcx9yiWPyWXE4+zkuEQs=
|
||||
cloud.google.com/go/secretmanager v1.11.1/go.mod h1:znq9JlXgTNdBeQk9TBW/FnR/W4uChEKGeqQWAJ8SXFw=
|
||||
cloud.google.com/go/secretmanager v1.11.2 h1:52Z78hH8NBWIqbvIG0wi0EoTaAmSx99KIOAmDXIlX0M=
|
||||
cloud.google.com/go/secretmanager v1.11.2/go.mod h1:MQm4t3deoSub7+WNwiC4/tRYgDBHJgJPvswqQVB1Vss=
|
||||
cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw=
|
||||
cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos=
|
||||
cloud.google.com/go/storage v1.33.0 h1:PVrDOkIC8qQVa1P3SXGpQvfuJhN2LHOoyZvWs8D2X5M=
|
||||
@@ -54,8 +52,6 @@ github.com/Azure/go-autorest/autorest/adal v0.9.18 h1:kLnPsRjzZZUF3K5REu/Kc+qMQr
|
||||
github.com/Azure/go-autorest/autorest/adal v0.9.18/go.mod h1:XVVeme+LZwABT8K5Lc3hA4nAe8LDBVle26gTrguhhPQ=
|
||||
github.com/Azure/go-autorest/autorest/azure/auth v0.5.11 h1:P6bYXFoao05z5uhOQzbC3Qd8JqF3jUoocoTeIxkp2cA=
|
||||
github.com/Azure/go-autorest/autorest/azure/auth v0.5.11/go.mod h1:84w/uV8E37feW2NCJ08uT9VBfjfUHpgLVnG2InYD6cg=
|
||||
github.com/Azure/go-autorest/autorest/azure/auth v0.5.12 h1:wkAZRgT/pn8HhFyzfe9UnqOjJYqlembgCTi72Bm/xKk=
|
||||
github.com/Azure/go-autorest/autorest/azure/auth v0.5.12/go.mod h1:84w/uV8E37feW2NCJ08uT9VBfjfUHpgLVnG2InYD6cg=
|
||||
github.com/Azure/go-autorest/autorest/azure/cli v0.4.5 h1:0W/yGmFdTIT77fvdlGZ0LMISoLHFJ7Tx4U0yeB+uFs4=
|
||||
github.com/Azure/go-autorest/autorest/azure/cli v0.4.5/go.mod h1:ADQAXrkgm7acgWVUNamOgh8YNrv4p27l3Wc55oVfpzg=
|
||||
github.com/Azure/go-autorest/autorest/date v0.3.0 h1:7gUk1U5M/CQbp9WoqinNzJar+8KY+LPI6wiWrP/myHw=
|
||||
@@ -151,8 +147,6 @@ github.com/aws/smithy-go v1.13.5/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J
|
||||
github.com/aymanbagabas/go-osc52 v1.0.3/go.mod h1:zT8H+Rk4VSabYN90pWyugflM3ZhpTZNC7cASDfUCdT4=
|
||||
github.com/aymanbagabas/go-osc52 v1.2.1 h1:q2sWUyDcozPLcLabEMd+a+7Ea2DitxZVN9hTxab9L4E=
|
||||
github.com/aymanbagabas/go-osc52 v1.2.1/go.mod h1:zT8H+Rk4VSabYN90pWyugflM3ZhpTZNC7cASDfUCdT4=
|
||||
github.com/aymanbagabas/go-osc52 v1.2.2 h1:NT7wkhEhPTcKnBCdPi9djmyy9L3JOL4+3SsfJyqptCo=
|
||||
github.com/aymanbagabas/go-osc52 v1.2.2/go.mod h1:zT8H+Rk4VSabYN90pWyugflM3ZhpTZNC7cASDfUCdT4=
|
||||
github.com/aymanbagabas/go-osc52/v2 v2.0.1 h1:HwpRHbFMcZLEVr42D4p7XBqjyuxQH5SMiErDT4WkJ2k=
|
||||
github.com/aymanbagabas/go-osc52/v2 v2.0.1/go.mod h1:uYgXzlJ7ZpABp8OJ+exZzJJhRNQ2ASbcXHWsFqH8hp8=
|
||||
github.com/aymerick/douceur v0.2.0 h1:Mv+mAeH1Q+n9Fr+oyamOlAkUNPWPlA8PPGR0QAaYuPk=
|
||||
@@ -161,6 +155,8 @@ github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLj
|
||||
github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA=
|
||||
github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
|
||||
github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
|
||||
github.com/bill-rich/disk-buffer-reader v0.1.7 h1:oPgDjsbeqErOrFJeWID/51LhIOGjU3cqN1Tj3V6RiwE=
|
||||
github.com/bill-rich/disk-buffer-reader v0.1.7/go.mod h1:VVzzsK1Ac2AnpOfp/5r9JlIFaFkZ9uSf7zisZayCt0Y=
|
||||
github.com/bill-rich/go-syslog v0.0.0-20220413021637-49edb52a574c h1:tSME5FDS02qQll3JYodI6RZR/g4EKOHApGv1wMZT+Z0=
|
||||
github.com/bill-rich/go-syslog v0.0.0-20220413021637-49edb52a574c/go.mod h1:+sCc6hztur+oZCLOsNk6wCCy+GLrnSNHSRmTnnL+8iQ=
|
||||
github.com/bitfinexcom/bitfinex-api-go v0.0.0-20210608095005-9e0b26f200fb h1:9v7Bzlg+1EBYi2IYcUmOwHReBEfqBbYIj3ZCi9cIe1Q=
|
||||
@@ -208,12 +204,8 @@ github.com/containerd/stargz-snapshotter/estargz v0.14.3 h1:OqlDCK3ZVUO6C3B/5FSk
|
||||
github.com/containerd/stargz-snapshotter/estargz v0.14.3/go.mod h1:KY//uOCIkSuNAHhJogcZtrNHdKrA99/FCCRjE3HD36o=
|
||||
github.com/couchbase/gocb/v2 v2.6.3 h1:5RsMo+RRfK0mVxHLAfpBz3/tHlgXZb1WBNItLk9Ab+c=
|
||||
github.com/couchbase/gocb/v2 v2.6.3/go.mod h1:yF5F6BHTZ/ZowhEuZbySbXrlI4rHd1TIhm5azOaMbJU=
|
||||
github.com/couchbase/gocb/v2 v2.6.5 h1:xaZu29o8UJEV1ZQ3n2s9jcRCUHz/JsQ6+y6JBnVsy5A=
|
||||
github.com/couchbase/gocb/v2 v2.6.5/go.mod h1:0vFM09y+VPhnXeNrIb8tS0wKHGpJvjJBrJnriWEiwGs=
|
||||
github.com/couchbase/gocbcore/v10 v10.2.4 h1:TDTQ1mSBUw9eajuV71ZDJcLRbmfGcsvZV9SMeUDAVGc=
|
||||
github.com/couchbase/gocbcore/v10 v10.2.4/go.mod h1:lYQIIk+tzoMcwtwU5GzPbDdqEkwkH3isI2rkSpfL0oM=
|
||||
github.com/couchbase/gocbcore/v10 v10.2.9 h1:zph/+ceu3JtZEDKhJMTRc6lGrahq+mnlQY/1dSepJuE=
|
||||
github.com/couchbase/gocbcore/v10 v10.2.9/go.mod h1:lYQIIk+tzoMcwtwU5GzPbDdqEkwkH3isI2rkSpfL0oM=
|
||||
github.com/couchbaselabs/gocaves/client v0.0.0-20230307083111-cc3960c624b1/go.mod h1:AVekAZwIY2stsJOMWLAS/0uA/+qdp7pjO8EHnl61QkY=
|
||||
github.com/couchbaselabs/gocaves/client v0.0.0-20230404095311-05e3ba4f0259 h1:2TXy68EGEzIMHOx9UvczR5ApVecwCfQZ0LjkmwMI6g4=
|
||||
github.com/couchbaselabs/gocaves/client v0.0.0-20230404095311-05e3ba4f0259/go.mod h1:AVekAZwIY2stsJOMWLAS/0uA/+qdp7pjO8EHnl61QkY=
|
||||
@@ -279,8 +271,6 @@ github.com/go-asn1-ber/asn1-ber v1.5.5 h1:MNHlNMBDgEKD4TcKr36vQN68BA00aDfjIt3/bD
|
||||
github.com/go-asn1-ber/asn1-ber v1.5.5/go.mod h1:hEBeB/ic+5LoWskz+yKT7vGhhPYkProFKoKdwZRWMe0=
|
||||
github.com/go-errors/errors v1.4.2 h1:J6MZopCL4uSllY1OfXM374weqZFFItUbrImctkmUxIA=
|
||||
github.com/go-errors/errors v1.4.2/go.mod h1:sIVyrIiJhuEF+Pj9Ebtd6P/rEYROXFi3BopGUQ5a5Og=
|
||||
github.com/go-errors/errors v1.5.1 h1:ZwEMSLRCapFLflTpT7NKaAc7ukJ8ZPEjzlxt8rPN8bk=
|
||||
github.com/go-errors/errors v1.5.1/go.mod h1:sIVyrIiJhuEF+Pj9Ebtd6P/rEYROXFi3BopGUQ5a5Og=
|
||||
github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 h1:+zs/tPmkDkHx3U66DAb0lQFJrpS6731Oaa12ikc+DiI=
|
||||
github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376/go.mod h1:an3vInlBmSxCcxctByoQdvwPiA7DTK7jaaFDBTtu0ic=
|
||||
github.com/go-git/go-billy/v5 v5.4.1 h1:Uwp5tDRkPr+l/TnbHOQzp+tmJfLceOlbVucgpTz8ix4=
|
||||
@@ -373,15 +363,12 @@ github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/
|
||||
github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
|
||||
github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
|
||||
github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
|
||||
github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
|
||||
github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
|
||||
github.com/google/go-containerregistry v0.15.2 h1:MMkSh+tjSdnmJZO7ljvEqV1DjfekB6VUEAZgy3a+TQE=
|
||||
github.com/google/go-containerregistry v0.15.2/go.mod h1:wWK+LnOv4jXMM23IT/F1wdYftGWGr47Is8CG+pmHK1Q=
|
||||
github.com/google/go-github/v42 v42.0.0 h1:YNT0FwjPrEysRkLIiKuEfSvBPCGKphW5aS5PxwaoLec=
|
||||
github.com/google/go-github/v42 v42.0.0/go.mod h1:jgg/jvyI0YlDOM1/ps6XYh04HNQ3vKf0CVko62/EhRg=
|
||||
github.com/google/go-github/v55 v55.0.0 h1:4pp/1tNMB9X/LuAhs5i0KQAE40NmiR/y6prLNb9x9cg=
|
||||
github.com/google/go-github/v55 v55.0.0/go.mod h1:JLahOTA1DnXzhxEymmFF5PP2tSS9JVNj68mSZNDwskA=
|
||||
github.com/google/go-github/v56 v56.0.0/go.mod h1:D8cdcX98YWJvi7TLo7zM4/h8ZTx6u6fwGEkCdisopo0=
|
||||
github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8=
|
||||
github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU=
|
||||
github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPgecwXBIDzw5no=
|
||||
@@ -488,16 +475,12 @@ github.com/launchdarkly/go-sdk-common/v3 v3.0.1 h1:rVdLusAIViduNvyjNKy06RA+SPwk0
|
||||
github.com/launchdarkly/go-sdk-common/v3 v3.0.1/go.mod h1:H/zISoCNhviHTTqqBjIKQy2YgSHT8ioL1FtgBKpiEGg=
|
||||
github.com/launchdarkly/go-sdk-events/v2 v2.0.1 h1:vnUN2Y7og/5wtOCcCZW7wYpmZcS++GAyclasc7gaTIY=
|
||||
github.com/launchdarkly/go-sdk-events/v2 v2.0.1/go.mod h1:Msqbl6brgFO83RUxmLaJAUx2sYG+WKULcy+Vf3+tKww=
|
||||
github.com/launchdarkly/go-sdk-events/v2 v2.0.2 h1:Ngp050AGYUyN3YRxX7zfu110NETHEPEOVys1ey2Hyvw=
|
||||
github.com/launchdarkly/go-sdk-events/v2 v2.0.2/go.mod h1:Msqbl6brgFO83RUxmLaJAUx2sYG+WKULcy+Vf3+tKww=
|
||||
github.com/launchdarkly/go-semver v1.0.2 h1:sYVRnuKyvxlmQCnCUyDkAhtmzSFRoX6rG2Xa21Mhg+w=
|
||||
github.com/launchdarkly/go-semver v1.0.2/go.mod h1:xFmMwXba5Mb+3h72Z+VeSs9ahCvKo2QFUTHRNHVqR28=
|
||||
github.com/launchdarkly/go-server-sdk-evaluation/v2 v2.0.2 h1:PAM0GvE0nIUBeOkjdiymIEKI+8FFLJ+fEsWTupW1yGU=
|
||||
github.com/launchdarkly/go-server-sdk-evaluation/v2 v2.0.2/go.mod h1:Mztipcz+7ZMatXVun3k/IfPa8IOgUnAqiZawtFh2MRg=
|
||||
github.com/launchdarkly/go-server-sdk/v6 v6.1.0 h1:pp/VBIon5JNbtSw7ih7I5MXN9mXHfo6T5xikKVy52dI=
|
||||
github.com/launchdarkly/go-server-sdk/v6 v6.1.0/go.mod h1:3TfS6vNsNksPT4LGeGuGKtT3zTjTbbOsCUK3nrj1neA=
|
||||
github.com/launchdarkly/go-server-sdk/v6 v6.1.1 h1:L57BC9Fy4IJG0zRZoYBkoxko6QTAC2OF8QzTVUqn2p8=
|
||||
github.com/launchdarkly/go-server-sdk/v6 v6.1.1/go.mod h1:67QqqyeRNyXx7S/fZiSZesRvaePCyxhzsnY+4fhK55U=
|
||||
github.com/launchdarkly/go-test-helpers/v2 v2.2.0 h1:L3kGILP/6ewikhzhdNkHy1b5y4zs50LueWenVF0sBbs=
|
||||
github.com/launchdarkly/go-test-helpers/v2 v2.2.0/go.mod h1:L7+th5govYp5oKU9iN7To5PgznBuIjBPn+ejqKR0avw=
|
||||
github.com/launchdarkly/go-test-helpers/v3 v3.0.2 h1:rh0085g1rVJM5qIukdaQ8z1XTWZztbJ49vRZuveqiuU=
|
||||
@@ -506,8 +489,6 @@ github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw=
|
||||
github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
|
||||
github.com/lrstanley/bubblezone v0.0.0-20221222153816-e95291e2243e h1:XoxHx8K6ZKoMtjzWOMDuM69LCdjDDsTOtTfWGrT/fns=
|
||||
github.com/lrstanley/bubblezone v0.0.0-20221222153816-e95291e2243e/go.mod h1:v5lEwWaguF1o2MW/ucO0ZIA/IZymdBYJJ+2cMRLE7LU=
|
||||
github.com/lrstanley/bubblezone v0.0.0-20230911164824-e3824f1adde9 h1:+7bxeCzFs4bfFPAnIZrjNmRt/MCffIy7aw2mPc9mxkU=
|
||||
github.com/lrstanley/bubblezone v0.0.0-20230911164824-e3824f1adde9/go.mod h1:v5lEwWaguF1o2MW/ucO0ZIA/IZymdBYJJ+2cMRLE7LU=
|
||||
github.com/lucasb-eyer/go-colorful v1.2.0 h1:1nnpGOrhyZZuNyfu1QjKiUICQ74+3FNCN69Aj6K7nkY=
|
||||
github.com/lucasb-eyer/go-colorful v1.2.0/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i7ruzyGqttikkLy0=
|
||||
github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0=
|
||||
@@ -521,8 +502,6 @@ github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovk
|
||||
github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
|
||||
github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APPA=
|
||||
github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
|
||||
github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
|
||||
github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
|
||||
github.com/mattn/go-localereader v0.0.1 h1:ygSAOl7ZXTx4RdPYinUpg6W99U8jWvWi9Ye2JC/oIi4=
|
||||
github.com/mattn/go-localereader v0.0.1/go.mod h1:8fBrzywKY7BI3czFoHkuzRoWE9C+EiG4R1k4Cjx5p88=
|
||||
github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI=
|
||||
@@ -639,8 +618,6 @@ github.com/smartystreets/gunit v1.1.3 h1:32x+htJCu3aMswhPw3teoJ+PnWPONqdNgaGs6Qt
|
||||
github.com/smartystreets/gunit v1.1.3/go.mod h1:EH5qMBab2UclzXUcpR8b93eHsIlp9u+pDQIRp5DZNzQ=
|
||||
github.com/snowflakedb/gosnowflake v1.6.23 h1:uO+zMTXJcSHzOm6ks5To8ergNjt5Dy6cr5QtStpRFT8=
|
||||
github.com/snowflakedb/gosnowflake v1.6.23/go.mod h1:KfO4F7bk+aXPUIvBqYxvPhxLlu2/w4TtSC8Rw/yr5Mg=
|
||||
github.com/snowflakedb/gosnowflake v1.6.25 h1:o5zUmxTOo0Eo9AdkEj8blCeiMuILrQJ+rjUMAeZhcRE=
|
||||
github.com/snowflakedb/gosnowflake v1.6.25/go.mod h1:KfO4F7bk+aXPUIvBqYxvPhxLlu2/w4TtSC8Rw/yr5Mg=
|
||||
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||
github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
|
||||
github.com/stretchr/objx v0.5.0 h1:1zr/of2m5FGMsad5YfcqgdqdWrIhu+EBEJRhR1U7z/c=
|
||||
@@ -662,12 +639,8 @@ github.com/tailscale/depaware v0.0.0-20210622194025-720c4b409502 h1:34icjjmqJ2HP
|
||||
github.com/tailscale/depaware v0.0.0-20210622194025-720c4b409502/go.mod h1:p9lPsd+cx33L3H9nNoecRRxPssFKUwwI50I3pZ0yT+8=
|
||||
github.com/therootcompany/xz v1.0.1 h1:CmOtsn1CbtmyYiusbfmhmkpAAETj0wBIH6kCYaX+xzw=
|
||||
github.com/therootcompany/xz v1.0.1/go.mod h1:3K3UH1yCKgBneZYhuQUvJ9HPD19UEXEI0BWbMn8qNMY=
|
||||
github.com/trufflesecurity/disk-buffer-reader v0.1.9 h1:1dji/Cx5aPL5DlsqI1B86xFMo6OfbBEhkQhXxgbX9WA=
|
||||
github.com/trufflesecurity/disk-buffer-reader v0.1.9/go.mod h1:uYwTCdxzV0o+qaeBMxflOsq4eu2WjrE46qGR2e80O9Y=
|
||||
github.com/trufflesecurity/overseer v1.1.7-custom5 h1:xu+Fg6fkSRifUPzUCl7N8HmobJ6WGOkIApGnM7mJS6w=
|
||||
github.com/trufflesecurity/overseer v1.1.7-custom5/go.mod h1:nT9w37AiO1Nop2VhVhNfzAFaPjthvxgpDV3XKsxYkcI=
|
||||
github.com/trufflesecurity/overseer v1.2.7 h1:DEiu2wrL/0f+XqshIT9C4/0nj7SHZvLmFeE4jELh4Oo=
|
||||
github.com/trufflesecurity/overseer v1.2.7/go.mod h1:nT9w37AiO1Nop2VhVhNfzAFaPjthvxgpDV3XKsxYkcI=
|
||||
github.com/ulikunitz/xz v0.5.6/go.mod h1:2bypXElzHzzJZwzH67Y6wb67pO62Rzfn7BSiF4ABRW8=
|
||||
github.com/ulikunitz/xz v0.5.10 h1:t92gobL9l3HE202wg3rlk19F6X+JOxl9BBrCCMYEYd8=
|
||||
github.com/ulikunitz/xz v0.5.10/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14=
|
||||
@@ -749,8 +722,6 @@ golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u0
|
||||
golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM=
|
||||
golang.org/x/exp v0.0.0-20230206171751-46f607a40771 h1:xP7rWLUr1e1n2xkK5YB4LI0hPEy3LJC6Wk+D4pGlOJg=
|
||||
golang.org/x/exp v0.0.0-20230206171751-46f607a40771/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc=
|
||||
golang.org/x/exp v0.0.0-20231006140011-7918f672742d h1:jtJma62tbqLibJ5sFQz8bKtEM8rJBtfilJ2qTU199MI=
|
||||
golang.org/x/exp v0.0.0-20231006140011-7918f672742d/go.mod h1:ldy0pHrwJyGW56pPQzzkH36rKxoZW1tw7ZJpeKx+hdo=
|
||||
golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
|
||||
golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
|
||||
golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
|
||||
@@ -775,8 +746,6 @@ golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91
|
||||
golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
|
||||
golang.org/x/mod v0.12.0 h1:rmsUpXtvNzj340zd98LZ4KntptpfRHwpFOHG188oHXc=
|
||||
golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
|
||||
golang.org/x/mod v0.13.0 h1:I/DsJXRlw/8l/0c24sM9yb0T4z9liZTduXvdAWYiysY=
|
||||
golang.org/x/mod v0.13.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
|
||||
golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
@@ -816,8 +785,6 @@ golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4Iltr
|
||||
golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
|
||||
golang.org/x/oauth2 v0.12.0 h1:smVPGxink+n1ZI5pkQa8y6fZT0RW0MgCO5bFpepy4B4=
|
||||
golang.org/x/oauth2 v0.12.0/go.mod h1:A74bZ3aGXgCY0qaIC9Ahg6Lglin4AMAco8cIv9baba4=
|
||||
golang.org/x/oauth2 v0.13.0 h1:jDDenyj+WgFtmV3zYVoi8aE2BwtXFLWOA67ZfNWftiY=
|
||||
golang.org/x/oauth2 v0.13.0/go.mod h1:/JMhi4ZRXAf4HG9LiNmxvk+45+96RUlVThiH8FzNBn0=
|
||||
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
@@ -830,8 +797,6 @@ golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJ
|
||||
golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.3.0 h1:ftCYgMx6zT/asHUrPw8BLLscYtGznsLAnjq5RH9P66E=
|
||||
golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y=
|
||||
golang.org/x/sync v0.4.0 h1:zxkM55ReGkDlKSM+Fu41A+zmbZuaPVbGMzvvdUPznYQ=
|
||||
golang.org/x/sync v0.4.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y=
|
||||
golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
@@ -942,8 +907,6 @@ golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc
|
||||
golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
|
||||
golang.org/x/tools v0.12.0 h1:YW6HUoUmYBpwSgyaGaZq1fHjrBjX1rlpZ54T6mu2kss=
|
||||
golang.org/x/tools v0.12.0/go.mod h1:Sc0INKfu04TlqNoRA1hgpFZbhYXHPr4V5DzpSBTPqQM=
|
||||
golang.org/x/tools v0.14.0 h1:jvNa2pY0M4r62jkRQ6RwEZZyPcymeL9XZMLBbV7U2nc=
|
||||
golang.org/x/tools v0.14.0/go.mod h1:uYBEerGOWcJyEORxN+Ek8+TT266gXkNlHdJBwexUsBg=
|
||||
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
package glob
|
||||
package common
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
package glob
|
||||
package common
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
@@ -4,7 +4,6 @@ import (
|
||||
"bytes"
|
||||
"encoding/base64"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
|
||||
)
|
||||
|
||||
@@ -26,8 +25,7 @@ func init() {
|
||||
}
|
||||
}
|
||||
|
||||
func (d *Base64) FromChunk(chunk *sources.Chunk) *DecodableChunk {
|
||||
decodableChunk := &DecodableChunk{Chunk: chunk, DecoderType: detectorspb.DecoderType_BASE64}
|
||||
func (d *Base64) FromChunk(chunk *sources.Chunk) *sources.Chunk {
|
||||
encodedSubstrings := getSubstringsOfCharacterSet(chunk.Data, 20, b64CharsetMapping, b64EndChars)
|
||||
decodedSubstrings := make(map[string][]byte)
|
||||
|
||||
@@ -63,7 +61,7 @@ func (d *Base64) FromChunk(chunk *sources.Chunk) *DecodableChunk {
|
||||
}
|
||||
result.Write(chunk.Data[start:])
|
||||
chunk.Data = result.Bytes()
|
||||
return decodableChunk
|
||||
return chunk
|
||||
}
|
||||
|
||||
return nil
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
package decoders
|
||||
|
||||
import (
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
|
||||
)
|
||||
|
||||
@@ -14,15 +13,8 @@ func DefaultDecoders() []Decoder {
|
||||
}
|
||||
}
|
||||
|
||||
// DecodableChunk is a chunk that includes the type of decoder used.
|
||||
// This allows us to avoid a type assertion on each decoder.
|
||||
type DecodableChunk struct {
|
||||
*sources.Chunk
|
||||
DecoderType detectorspb.DecoderType
|
||||
}
|
||||
|
||||
type Decoder interface {
|
||||
FromChunk(chunk *sources.Chunk) *DecodableChunk
|
||||
FromChunk(chunk *sources.Chunk) *sources.Chunk
|
||||
}
|
||||
|
||||
// Fuzz is an entrypoint for go-fuzz, which is an AFL-style fuzzing tool.
|
||||
|
||||
@@ -5,24 +5,22 @@ import (
|
||||
"encoding/binary"
|
||||
"unicode/utf8"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
|
||||
)
|
||||
|
||||
type UTF16 struct{}
|
||||
|
||||
func (d *UTF16) FromChunk(chunk *sources.Chunk) *DecodableChunk {
|
||||
func (d *UTF16) FromChunk(chunk *sources.Chunk) *sources.Chunk {
|
||||
if chunk == nil || len(chunk.Data) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
decodableChunk := &DecodableChunk{Chunk: chunk, DecoderType: detectorspb.DecoderType_UTF16}
|
||||
if utf16Data, err := utf16ToUTF8(chunk.Data); err == nil {
|
||||
if len(utf16Data) == 0 {
|
||||
return nil
|
||||
}
|
||||
chunk.Data = utf16Data
|
||||
return decodableChunk
|
||||
return chunk
|
||||
}
|
||||
|
||||
return nil
|
||||
|
||||
@@ -4,25 +4,22 @@ import (
|
||||
"bytes"
|
||||
"unicode/utf8"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
|
||||
)
|
||||
|
||||
type UTF8 struct{}
|
||||
|
||||
func (d *UTF8) FromChunk(chunk *sources.Chunk) *DecodableChunk {
|
||||
func (d *UTF8) FromChunk(chunk *sources.Chunk) *sources.Chunk {
|
||||
if chunk == nil || len(chunk.Data) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
decodableChunk := &DecodableChunk{Chunk: chunk, DecoderType: detectorspb.DecoderType_PLAIN}
|
||||
|
||||
if !utf8.Valid(chunk.Data) {
|
||||
chunk.Data = extractSubstrings(chunk.Data)
|
||||
return decodableChunk
|
||||
return chunk
|
||||
}
|
||||
|
||||
return decodableChunk
|
||||
return chunk
|
||||
}
|
||||
|
||||
// extractSubstrings performs similarly to the strings binutil,
|
||||
|
||||
@@ -1,8 +1,9 @@
|
||||
package privacy
|
||||
package baseapiio
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"strings"
|
||||
@@ -12,26 +13,25 @@ import (
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
|
||||
)
|
||||
|
||||
type Scanner struct {
|
||||
client *http.Client
|
||||
}
|
||||
type Scanner struct{}
|
||||
|
||||
// Ensure the Scanner satisfies the interface at compile time.
|
||||
var _ detectors.Detector = (*Scanner)(nil)
|
||||
|
||||
var (
|
||||
defaultClient = common.SaneHttpClient()
|
||||
client = common.SaneHttpClient()
|
||||
|
||||
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
|
||||
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"privacy"}) + `\b([a-z0-9]{8}-[a-z0-9]{4}-[a-z0-9]{4}-[a-z0-9]{4}-[a-z0-9]{12})\b`)
|
||||
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"baseapi", "base-api"}) + `\b([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})\b`)
|
||||
)
|
||||
|
||||
// Keywords are used for efficiently pre-filtering chunks.
|
||||
// Use identifiers in the secret preferably, or the provider name.
|
||||
func (s Scanner) Keywords() []string {
|
||||
return []string{"privacy"}
|
||||
return []string{"baseapi", "base-api"}
|
||||
}
|
||||
|
||||
// FromData will find and optionally verify Privacy secrets in a given set of bytes.
|
||||
// FromData will find and optionally verify BaseApiIO secrets in a given set of bytes.
|
||||
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
|
||||
dataStr := string(data)
|
||||
|
||||
@@ -44,38 +44,34 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
resMatch := strings.TrimSpace(match[1])
|
||||
|
||||
s1 := detectors.Result{
|
||||
DetectorType: detectorspb.DetectorType_Privacy,
|
||||
DetectorType: detectorspb.DetectorType_BaseApiIO,
|
||||
Raw: []byte(resMatch),
|
||||
}
|
||||
|
||||
if verify {
|
||||
client := s.client
|
||||
if client == nil {
|
||||
client = defaultClient
|
||||
}
|
||||
req, err := http.NewRequestWithContext(ctx, "GET", "https://api.privacy.com/v1/card?page=1&page_size=50", nil)
|
||||
req, err := http.NewRequestWithContext(ctx, "GET", "https://api.base-api.io/v1/forms?page=1&per_page=10", nil)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
req.Header.Set("Authorization", "api-key "+resMatch)
|
||||
req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", resMatch))
|
||||
res, err := client.Do(req)
|
||||
if err == nil {
|
||||
defer res.Body.Close()
|
||||
if res.StatusCode >= 200 && res.StatusCode < 300 {
|
||||
s1.Verified = true
|
||||
} else if res.StatusCode == 401 {
|
||||
// The secret is determinately not verified (nothing to do)
|
||||
} else {
|
||||
s1.VerificationError = fmt.Errorf("unexpected HTTP response status %d", res.StatusCode)
|
||||
bodyBytes, err := io.ReadAll(res.Body)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
} else {
|
||||
s1.VerificationError = err
|
||||
}
|
||||
}
|
||||
body := string(bodyBytes)
|
||||
|
||||
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
|
||||
if !s1.Verified && detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
|
||||
continue
|
||||
if strings.Contains(body, "items") {
|
||||
s1.Verified = true
|
||||
} else {
|
||||
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
results = append(results, s1)
|
||||
@@ -85,5 +81,5 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
}
|
||||
|
||||
func (s Scanner) Type() detectorspb.DetectorType {
|
||||
return detectorspb.DetectorType_Privacy
|
||||
return detectorspb.DetectorType_BaseApiIO
|
||||
}
|
||||
+23
-27
@@ -1,7 +1,7 @@
|
||||
//go:build detectors
|
||||
// +build detectors
|
||||
|
||||
package instamojo
|
||||
package baseapiio
|
||||
|
||||
import (
|
||||
"context"
|
||||
@@ -16,16 +16,15 @@ import (
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
|
||||
)
|
||||
|
||||
func TestInstamojo_FromChunk(t *testing.T) {
|
||||
func TestBaseApiIO_FromChunk(t *testing.T) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
|
||||
defer cancel()
|
||||
testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")
|
||||
testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")
|
||||
if err != nil {
|
||||
t.Fatalf("could not get test secrets from GCP: %s", err)
|
||||
}
|
||||
id := testSecrets.MustGetField("INSTAMOJO_CLIENT_ID")
|
||||
secret := testSecrets.MustGetField("INSTAMOJO_SECRET")
|
||||
inactiveSecret := testSecrets.MustGetField("INSTAMOJO_INACTIVE")
|
||||
secret := testSecrets.MustGetField("BASEAPIIO")
|
||||
inactiveSecret := testSecrets.MustGetField("BASEAPIIO_INACTIVE")
|
||||
|
||||
type args struct {
|
||||
ctx context.Context
|
||||
@@ -33,46 +32,43 @@ func TestInstamojo_FromChunk(t *testing.T) {
|
||||
verify bool
|
||||
}
|
||||
tests := []struct {
|
||||
name string
|
||||
s Scanner
|
||||
args args
|
||||
want []detectors.Result
|
||||
wantErr bool
|
||||
wantVerificationErr bool
|
||||
name string
|
||||
s Scanner
|
||||
args args
|
||||
want []detectors.Result
|
||||
wantErr bool
|
||||
}{
|
||||
{
|
||||
name: "found, verified",
|
||||
s: Scanner{},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a instamojo secret %s within id %s", secret, id)),
|
||||
data: []byte(fmt.Sprintf("You can find a baseapiio secret %s within", secret)),
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_Instamojo,
|
||||
DetectorType: detectorspb.DetectorType_BaseApiIO,
|
||||
Verified: true,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
wantVerificationErr: false,
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "found, unverified",
|
||||
s: Scanner{},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a instamojo secret %s within but not valid, within id %s", inactiveSecret, id)), // the secret would satisfy the regex but not pass validation
|
||||
data: []byte(fmt.Sprintf("You can find a baseapiio secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_Instamojo,
|
||||
VerificationError: fmt.Errorf("unexpected HTTP response status 401"),
|
||||
DetectorType: detectorspb.DetectorType_BaseApiIO,
|
||||
Verified: false,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
wantVerificationErr: true,
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "not found",
|
||||
@@ -82,16 +78,16 @@ func TestInstamojo_FromChunk(t *testing.T) {
|
||||
data: []byte("You cannot find the secret within"),
|
||||
verify: true,
|
||||
},
|
||||
want: nil,
|
||||
wantErr: false,
|
||||
wantVerificationErr: true,
|
||||
want: nil,
|
||||
wantErr: false,
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
|
||||
s := Scanner{}
|
||||
got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
|
||||
if (err != nil) != tt.wantErr {
|
||||
t.Errorf("Instamojo.FromData() error = %v, wantErr %v", err, tt.wantErr)
|
||||
t.Errorf("BaseApiIO.FromData() error = %v, wantErr %v", err, tt.wantErr)
|
||||
return
|
||||
}
|
||||
for i := range got {
|
||||
@@ -101,7 +97,7 @@ func TestInstamojo_FromChunk(t *testing.T) {
|
||||
got[i].Raw = nil
|
||||
}
|
||||
if diff := pretty.Compare(got, tt.want); diff != "" {
|
||||
t.Errorf("Instamojo.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
|
||||
t.Errorf("BaseApiIO.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
|
||||
}
|
||||
})
|
||||
}
|
||||
@@ -1,4 +1,4 @@
|
||||
package ip2location
|
||||
package blablabus
|
||||
|
||||
import (
|
||||
"context"
|
||||
@@ -12,24 +12,25 @@ import (
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
|
||||
)
|
||||
|
||||
type Scanner struct {}
|
||||
type Scanner struct{}
|
||||
|
||||
// Ensure the Scanner satisfies the interface at compile time.
|
||||
var _ detectors.Detector = (*Scanner)(nil)
|
||||
|
||||
var (
|
||||
client = common.SaneHttpClient()
|
||||
|
||||
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
|
||||
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"ip2location"}) + `\b([0-9A-Z]{32})\b`)
|
||||
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"blablabus"}) + `\b([0-9A-Za-z]{22})\b`)
|
||||
)
|
||||
|
||||
// Keywords are used for efficiently pre-filtering chunks.
|
||||
// Use identifiers in the secret preferably, or the provider name.
|
||||
func (s Scanner) Keywords() []string {
|
||||
return []string{"ip2location"}
|
||||
return []string{"blablabus"}
|
||||
}
|
||||
|
||||
// FromData will find and optionally verify Ip2location secrets in a given set of bytes.
|
||||
// FromData will find and optionally verify Blablabus secrets in a given set of bytes.
|
||||
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
|
||||
dataStr := string(data)
|
||||
|
||||
@@ -42,33 +43,30 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
resMatch := strings.TrimSpace(match[1])
|
||||
|
||||
s1 := detectors.Result{
|
||||
DetectorType: detectorspb.DetectorType_Ip2location,
|
||||
DetectorType: detectorspb.DetectorType_Blablabus,
|
||||
Raw: []byte(resMatch),
|
||||
}
|
||||
|
||||
if verify {
|
||||
req, err := http.NewRequestWithContext(ctx, "GET", "https://api.ip2location.io/?key=" + resMatch, nil)
|
||||
|
||||
req, err := http.NewRequestWithContext(ctx, "GET", "https://api.idbus.com/v3/stops", nil)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
req.Header.Add("Content-Type", "application/json")
|
||||
req.Header.Add("Authorization", fmt.Sprintf("Token %s", resMatch))
|
||||
res, err := client.Do(req)
|
||||
if err == nil {
|
||||
defer res.Body.Close()
|
||||
if res.StatusCode >= 200 && res.StatusCode < 300 {
|
||||
s1.Verified = true
|
||||
} else if res.StatusCode == 401 {
|
||||
// The secret is determinately not verified (nothing to do)
|
||||
} else {
|
||||
s1.VerificationError = fmt.Errorf("unexpected HTTP response status %d", res.StatusCode)
|
||||
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
|
||||
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
|
||||
continue
|
||||
}
|
||||
}
|
||||
} else {
|
||||
s1.VerificationError = err
|
||||
}
|
||||
}
|
||||
if !s1.Verified && detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
|
||||
continue
|
||||
}
|
||||
|
||||
results = append(results, s1)
|
||||
}
|
||||
@@ -77,8 +75,5 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
}
|
||||
|
||||
func (s Scanner) Type() detectorspb.DetectorType {
|
||||
return detectorspb.DetectorType_Ip2location
|
||||
return detectorspb.DetectorType_Blablabus
|
||||
}
|
||||
|
||||
|
||||
|
||||
+25
-36
@@ -1,7 +1,7 @@
|
||||
//go:build detectors
|
||||
// +build detectors
|
||||
|
||||
package portainer
|
||||
package blablabus
|
||||
|
||||
import (
|
||||
"context"
|
||||
@@ -9,25 +9,22 @@ import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/google/go-cmp/cmp"
|
||||
"github.com/google/go-cmp/cmp/cmpopts"
|
||||
|
||||
"github.com/kylelemons/godebug/pretty"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
|
||||
)
|
||||
|
||||
func TestPortainer_FromChunk(t *testing.T) {
|
||||
func TestBlablabus_FromChunk(t *testing.T) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
|
||||
defer cancel()
|
||||
testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")
|
||||
testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")
|
||||
if err != nil {
|
||||
t.Fatalf("could not get test secrets from GCP: %s", err)
|
||||
}
|
||||
secret := testSecrets.MustGetField("PORTAINER")
|
||||
endpoint := testSecrets.MustGetField("PORTAINER_ENDPOINT")
|
||||
inactiveSecret := testSecrets.MustGetField("PORTAINER_INACTIVE")
|
||||
secret := testSecrets.MustGetField("BLABLABUS")
|
||||
inactiveSecret := testSecrets.MustGetField("BLABLABUS_INACTIVE")
|
||||
|
||||
type args struct {
|
||||
ctx context.Context
|
||||
@@ -35,48 +32,43 @@ func TestPortainer_FromChunk(t *testing.T) {
|
||||
verify bool
|
||||
}
|
||||
tests := []struct {
|
||||
name string
|
||||
s Scanner
|
||||
args args
|
||||
want []detectors.Result
|
||||
wantErr bool
|
||||
wantVerificationErr bool
|
||||
name string
|
||||
s Scanner
|
||||
args args
|
||||
want []detectors.Result
|
||||
wantErr bool
|
||||
}{
|
||||
{
|
||||
name: "found, verified",
|
||||
s: Scanner{},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a portainer secret %s for portainer url %s", secret, endpoint)),
|
||||
data: []byte(fmt.Sprintf("You can find a blablabus secret %s within", secret)),
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_Portainer,
|
||||
DetectorType: detectorspb.DetectorType_Blablabus,
|
||||
Verified: true,
|
||||
RawV2: []byte(secret + endpoint),
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
wantVerificationErr: false,
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "found, unverified",
|
||||
s: Scanner{},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a portainer secret %s for portainer %s within but not valid", inactiveSecret, endpoint)), // the secret would satisfy the regex but not pass validation
|
||||
data: []byte(fmt.Sprintf("You can find a blablabus secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_Portainer,
|
||||
DetectorType: detectorspb.DetectorType_Blablabus,
|
||||
Verified: false,
|
||||
RawV2: []byte(inactiveSecret + endpoint),
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
wantVerificationErr: false,
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "not found",
|
||||
@@ -86,29 +78,26 @@ func TestPortainer_FromChunk(t *testing.T) {
|
||||
data: []byte("You cannot find the secret within"),
|
||||
verify: true,
|
||||
},
|
||||
want: nil,
|
||||
wantErr: false,
|
||||
wantVerificationErr: false,
|
||||
want: nil,
|
||||
wantErr: false,
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
|
||||
s := Scanner{}
|
||||
got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
|
||||
if (err != nil) != tt.wantErr {
|
||||
t.Errorf("Portainer.FromData() error = %v, wantErr %v", err, tt.wantErr)
|
||||
t.Errorf("Blablabus.FromData() error = %v, wantErr %v", err, tt.wantErr)
|
||||
return
|
||||
}
|
||||
for i := range got {
|
||||
if len(got[i].Raw) == 0 {
|
||||
t.Fatalf("no raw secret present: \n %+v", got[i])
|
||||
}
|
||||
if (got[i].VerificationError != nil) != tt.wantVerificationErr {
|
||||
t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError)
|
||||
}
|
||||
got[i].Raw = nil
|
||||
}
|
||||
ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "VerificationError")
|
||||
if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {
|
||||
t.Errorf("Portainer.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
|
||||
if diff := pretty.Compare(got, tt.want); diff != "" {
|
||||
t.Errorf("Blablabus.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
|
||||
}
|
||||
})
|
||||
}
|
||||
@@ -2,7 +2,6 @@ package cloudsmith
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"strings"
|
||||
@@ -30,10 +29,6 @@ func (s Scanner) Keywords() []string {
|
||||
return []string{"cloudsmith"}
|
||||
}
|
||||
|
||||
type response struct {
|
||||
Authenticated bool `json:"authenticated"`
|
||||
}
|
||||
|
||||
// FromData will find and optionally verify Cloudsmith secrets in a given set of bytes.
|
||||
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
|
||||
dataStr := string(data)
|
||||
@@ -62,14 +57,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
if err == nil {
|
||||
defer res.Body.Close()
|
||||
if res.StatusCode >= 200 && res.StatusCode < 300 {
|
||||
var r response
|
||||
if err := json.NewDecoder(res.Body).Decode(&r); err != nil {
|
||||
s1.VerificationError = err
|
||||
continue
|
||||
}
|
||||
if r.Authenticated {
|
||||
s1.Verified = true
|
||||
}
|
||||
s1.Verified = true
|
||||
} else {
|
||||
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
|
||||
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
|
||||
|
||||
@@ -2,7 +2,6 @@ package codeclimate
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"regexp"
|
||||
@@ -31,12 +30,6 @@ func (s Scanner) Keywords() []string {
|
||||
return []string{"codeclimate"}
|
||||
}
|
||||
|
||||
type response struct {
|
||||
Data struct {
|
||||
Id string `json:"id"`
|
||||
} `json:"data"`
|
||||
}
|
||||
|
||||
// FromData will find and optionally verify Codeclimate secrets in a given set of bytes.
|
||||
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
|
||||
dataStr := string(data)
|
||||
@@ -65,14 +58,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
if err == nil {
|
||||
defer res.Body.Close()
|
||||
if res.StatusCode >= 200 && res.StatusCode < 300 {
|
||||
var r response
|
||||
if err := json.NewDecoder(res.Body).Decode(&r); err != nil {
|
||||
s1.VerificationError = err
|
||||
continue
|
||||
}
|
||||
if r.Data.Id != "" {
|
||||
s1.Verified = true
|
||||
}
|
||||
s1.Verified = true
|
||||
} else {
|
||||
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key
|
||||
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
|
||||
|
||||
+14
-26
@@ -1,4 +1,4 @@
|
||||
package vagrantcloudpersonaltoken
|
||||
package datafire
|
||||
|
||||
import (
|
||||
"context"
|
||||
@@ -12,26 +12,25 @@ import (
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
|
||||
)
|
||||
|
||||
type Scanner struct {
|
||||
client *http.Client
|
||||
}
|
||||
type Scanner struct{}
|
||||
|
||||
// Ensure the Scanner satisfies the interface at compile time.
|
||||
var _ detectors.Detector = (*Scanner)(nil)
|
||||
|
||||
var (
|
||||
defaultClient = common.SaneHttpClient()
|
||||
client = common.SaneHttpClient()
|
||||
|
||||
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
|
||||
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"vagrant"}) + `\b([A-Za-z0-9]{14}.atlasv1.[A-Za-z0-9]{67})\b`)
|
||||
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"datafire"}) + `\b([a-z0-9\S]{175,190})\b`)
|
||||
)
|
||||
|
||||
// Keywords are used for efficiently pre-filtering chunks.
|
||||
// Use identifiers in the secret preferably, or the provider name.
|
||||
func (s Scanner) Keywords() []string {
|
||||
return []string{"vagrant"}
|
||||
return []string{"datafire"}
|
||||
}
|
||||
|
||||
// FromData will find and optionally verify Vagrantcloudpersonaltoken secrets in a given set of bytes.
|
||||
// FromData will find and optionally verify DataFire secrets in a given set of bytes.
|
||||
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
|
||||
dataStr := string(data)
|
||||
|
||||
@@ -44,17 +43,12 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
resMatch := strings.TrimSpace(match[1])
|
||||
|
||||
s1 := detectors.Result{
|
||||
DetectorType: detectorspb.DetectorType_VagrantCloudPersonalToken,
|
||||
DetectorType: detectorspb.DetectorType_DataFire,
|
||||
Raw: []byte(resMatch),
|
||||
}
|
||||
|
||||
if verify {
|
||||
client := s.client
|
||||
if client == nil {
|
||||
client = defaultClient
|
||||
}
|
||||
|
||||
req, err := http.NewRequestWithContext(ctx, "GET", "https://app.vagrantup.com/api/v2/authenticate", nil)
|
||||
req, err := http.NewRequestWithContext(ctx, "GET", "https://api.datafire.io/projects/", nil)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
@@ -64,21 +58,15 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
defer res.Body.Close()
|
||||
if res.StatusCode >= 200 && res.StatusCode < 300 {
|
||||
s1.Verified = true
|
||||
} else if res.StatusCode == 401 {
|
||||
// The secret is determinately not verified (nothing to do)
|
||||
} else {
|
||||
s1.VerificationError = fmt.Errorf("unexpected HTTP response status %d", res.StatusCode)
|
||||
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
|
||||
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
|
||||
continue
|
||||
}
|
||||
}
|
||||
} else {
|
||||
s1.VerificationError = err
|
||||
}
|
||||
}
|
||||
|
||||
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
|
||||
if !s1.Verified && detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
|
||||
continue
|
||||
}
|
||||
|
||||
results = append(results, s1)
|
||||
}
|
||||
|
||||
@@ -86,5 +74,5 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
}
|
||||
|
||||
func (s Scanner) Type() detectorspb.DetectorType {
|
||||
return detectorspb.DetectorType_VagrantCloudPersonalToken
|
||||
return detectorspb.DetectorType_DataFire
|
||||
}
|
||||
+25
-33
@@ -1,7 +1,7 @@
|
||||
//go:build detectors
|
||||
// +build detectors
|
||||
|
||||
package ip2location
|
||||
package datafire
|
||||
|
||||
import (
|
||||
"context"
|
||||
@@ -9,24 +9,22 @@ import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/google/go-cmp/cmp"
|
||||
"github.com/google/go-cmp/cmp/cmpopts"
|
||||
|
||||
"github.com/kylelemons/godebug/pretty"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
|
||||
)
|
||||
|
||||
func TestIp2location_FromChunk(t *testing.T) {
|
||||
func TestDataFire_FromChunk(t *testing.T) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
|
||||
defer cancel()
|
||||
testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")
|
||||
testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")
|
||||
if err != nil {
|
||||
t.Fatalf("could not get test secrets from GCP: %s", err)
|
||||
}
|
||||
secret := testSecrets.MustGetField("IP2LOCATION")
|
||||
inactiveSecret := testSecrets.MustGetField("IP2LOCATION_INACTIVE")
|
||||
secret := testSecrets.MustGetField("DATAFIRE")
|
||||
inactiveSecret := testSecrets.MustGetField("DATAFIRE_INACTIVE")
|
||||
|
||||
type args struct {
|
||||
ctx context.Context
|
||||
@@ -34,46 +32,43 @@ func TestIp2location_FromChunk(t *testing.T) {
|
||||
verify bool
|
||||
}
|
||||
tests := []struct {
|
||||
name string
|
||||
s Scanner
|
||||
args args
|
||||
want []detectors.Result
|
||||
wantErr bool
|
||||
wantVerificationErr bool
|
||||
name string
|
||||
s Scanner
|
||||
args args
|
||||
want []detectors.Result
|
||||
wantErr bool
|
||||
}{
|
||||
{
|
||||
name: "found, verified",
|
||||
s: Scanner{},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a ip2location secret %s within", secret)),
|
||||
data: []byte(fmt.Sprintf("You can find a datafire secret %s within", secret)),
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_Ip2location,
|
||||
DetectorType: detectorspb.DetectorType_DataFire,
|
||||
Verified: true,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
wantVerificationErr: false,
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "found, unverified",
|
||||
s: Scanner{},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a ip2location secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
|
||||
data: []byte(fmt.Sprintf("You can find a datafire secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_Ip2location,
|
||||
DetectorType: detectorspb.DetectorType_DataFire,
|
||||
Verified: false,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
wantVerificationErr: false,
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "not found",
|
||||
@@ -83,29 +78,26 @@ func TestIp2location_FromChunk(t *testing.T) {
|
||||
data: []byte("You cannot find the secret within"),
|
||||
verify: true,
|
||||
},
|
||||
want: nil,
|
||||
wantErr: false,
|
||||
wantVerificationErr: false,
|
||||
want: nil,
|
||||
wantErr: false,
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
|
||||
s := Scanner{}
|
||||
got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
|
||||
if (err != nil) != tt.wantErr {
|
||||
t.Errorf("Ip2location.FromData() error = %v, wantErr %v", err, tt.wantErr)
|
||||
t.Errorf("DataFire.FromData() error = %v, wantErr %v", err, tt.wantErr)
|
||||
return
|
||||
}
|
||||
for i := range got {
|
||||
if len(got[i].Raw) == 0 {
|
||||
t.Fatalf("no raw secret present: \n %+v", got[i])
|
||||
}
|
||||
if (got[i].VerificationError != nil) != tt.wantVerificationErr {
|
||||
t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError)
|
||||
}
|
||||
got[i].Raw = nil
|
||||
}
|
||||
ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "VerificationError")
|
||||
if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {
|
||||
t.Errorf("Ip2location.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
|
||||
if diff := pretty.Compare(got, tt.want); diff != "" {
|
||||
t.Errorf("DataFire.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
|
||||
}
|
||||
})
|
||||
}
|
||||
@@ -59,8 +59,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
|
||||
if verify {
|
||||
// thanks https://stackoverflow.com/questions/15621471/validate-a-facebook-app-id-and-app-secret
|
||||
// https://stackoverflow.com/questions/24401241/how-to-get-a-facebook-access-token-using-appid-and-app-secret-without-any-login
|
||||
req, err := http.NewRequestWithContext(ctx, "GET", fmt.Sprintf("https://graph.facebook.com/me?access_token=%s|%s", apiIdRes, apiSecretRes), nil)
|
||||
req, err := http.NewRequestWithContext(ctx, "GET", fmt.Sprintf("https://graph.facebook.com/%s?fields=roles&access_token=%s|%s", apiIdRes, apiIdRes, apiSecretRes), nil)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
|
||||
@@ -1,8 +1,7 @@
|
||||
package ipinfo
|
||||
package glitterlyapi
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"strings"
|
||||
@@ -12,26 +11,25 @@ import (
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
|
||||
)
|
||||
|
||||
type Scanner struct {
|
||||
client *http.Client
|
||||
}
|
||||
type Scanner struct{}
|
||||
|
||||
// Ensure the Scanner satisfies the interface at compile time.
|
||||
var _ detectors.Detector = (*Scanner)(nil)
|
||||
|
||||
var (
|
||||
defaultClient = common.SaneHttpClient()
|
||||
client = common.SaneHttpClient()
|
||||
|
||||
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
|
||||
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"ipinfo"}) + `\b([a-f0-9]{14})\b`)
|
||||
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"glitterlyapi"}) + `\b([0-9a-z-]{36})\b`)
|
||||
)
|
||||
|
||||
// Keywords are used for efficiently pre-filtering chunks.
|
||||
// Use identifiers in the secret preferably, or the provider name.
|
||||
func (s Scanner) Keywords() []string {
|
||||
return []string{"ipinfo"}
|
||||
return []string{"glitterlyapi"}
|
||||
}
|
||||
|
||||
// FromData will find and optionally verify Ipinfo secrets in a given set of bytes.
|
||||
// FromData will find and optionally verify GlitterlyAPI secrets in a given set of bytes.
|
||||
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
|
||||
dataStr := string(data)
|
||||
|
||||
@@ -44,40 +42,31 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
resMatch := strings.TrimSpace(match[1])
|
||||
|
||||
s1 := detectors.Result{
|
||||
DetectorType: detectorspb.DetectorType_IPInfo,
|
||||
DetectorType: detectorspb.DetectorType_GlitterlyAPI,
|
||||
Raw: []byte(resMatch),
|
||||
}
|
||||
|
||||
if verify {
|
||||
client := s.client
|
||||
if client == nil {
|
||||
client = defaultClient
|
||||
}
|
||||
req, err := http.NewRequestWithContext(ctx, "GET", "https://ipinfo.io/json?token="+resMatch, nil)
|
||||
req, err := http.NewRequestWithContext(ctx, "GET", "https://www.glitterlyapi.com/auth", nil)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
req.Header.Add("Content-Type", "application/json")
|
||||
req.Header.Add("x-api-key", resMatch)
|
||||
res, err := client.Do(req)
|
||||
if err == nil {
|
||||
fmt.Println(res.Status, resMatch)
|
||||
defer res.Body.Close()
|
||||
if res.StatusCode >= 200 && res.StatusCode < 300 {
|
||||
s1.Verified = true
|
||||
} else if res.StatusCode == 403 {
|
||||
// The secret is determinately not verified (nothing to do)
|
||||
} else {
|
||||
s1.VerificationError = fmt.Errorf("unexpected HTTP response status %d", res.StatusCode)
|
||||
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
|
||||
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
|
||||
continue
|
||||
}
|
||||
}
|
||||
} else {
|
||||
s1.VerificationError = err
|
||||
}
|
||||
}
|
||||
|
||||
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
|
||||
if !s1.Verified && detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
|
||||
continue
|
||||
}
|
||||
|
||||
results = append(results, s1)
|
||||
}
|
||||
|
||||
@@ -85,5 +74,5 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
}
|
||||
|
||||
func (s Scanner) Type() detectorspb.DetectorType {
|
||||
return detectorspb.DetectorType_IPInfo
|
||||
return detectorspb.DetectorType_GlitterlyAPI
|
||||
}
|
||||
+25
-33
@@ -1,7 +1,7 @@
|
||||
//go:build detectors
|
||||
// +build detectors
|
||||
|
||||
package vagrantcloudpersonaltoken
|
||||
package glitterlyapi
|
||||
|
||||
import (
|
||||
"context"
|
||||
@@ -9,24 +9,22 @@ import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/google/go-cmp/cmp"
|
||||
"github.com/google/go-cmp/cmp/cmpopts"
|
||||
|
||||
"github.com/kylelemons/godebug/pretty"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
|
||||
)
|
||||
|
||||
func TestVagrantcloudpersonaltoken_FromChunk(t *testing.T) {
|
||||
func TestGlitterlyAPI_FromChunk(t *testing.T) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
|
||||
defer cancel()
|
||||
testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")
|
||||
testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
|
||||
if err != nil {
|
||||
t.Fatalf("could not get test secrets from GCP: %s", err)
|
||||
}
|
||||
secret := testSecrets.MustGetField("VAGRANTCLOUDPERSONALTOKEN")
|
||||
inactiveSecret := testSecrets.MustGetField("VAGRANTCLOUDPERSONALTOKEN_INACTIVE")
|
||||
secret := testSecrets.MustGetField("GLITTERLYAPI")
|
||||
inactiveSecret := testSecrets.MustGetField("GLITTERLYAPI_INACTIVE")
|
||||
|
||||
type args struct {
|
||||
ctx context.Context
|
||||
@@ -34,46 +32,43 @@ func TestVagrantcloudpersonaltoken_FromChunk(t *testing.T) {
|
||||
verify bool
|
||||
}
|
||||
tests := []struct {
|
||||
name string
|
||||
s Scanner
|
||||
args args
|
||||
want []detectors.Result
|
||||
wantErr bool
|
||||
wantVerificationErr bool
|
||||
name string
|
||||
s Scanner
|
||||
args args
|
||||
want []detectors.Result
|
||||
wantErr bool
|
||||
}{
|
||||
{
|
||||
name: "found, verified",
|
||||
s: Scanner{},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a vagrantcloudpersonaltoken secret %s within", secret)),
|
||||
data: []byte(fmt.Sprintf("You can find a glitterlyapi secret %s within", secret)),
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_VagrantCloudPersonalToken,
|
||||
DetectorType: detectorspb.DetectorType_GlitterlyAPI,
|
||||
Verified: true,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
wantVerificationErr: false,
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "found, unverified",
|
||||
s: Scanner{},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a vagrantcloudpersonaltoken secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
|
||||
data: []byte(fmt.Sprintf("You can find a glitterlyapi secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_VagrantCloudPersonalToken,
|
||||
DetectorType: detectorspb.DetectorType_GlitterlyAPI,
|
||||
Verified: false,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
wantVerificationErr: false,
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "not found",
|
||||
@@ -83,29 +78,26 @@ func TestVagrantcloudpersonaltoken_FromChunk(t *testing.T) {
|
||||
data: []byte("You cannot find the secret within"),
|
||||
verify: true,
|
||||
},
|
||||
want: nil,
|
||||
wantErr: false,
|
||||
wantVerificationErr: false,
|
||||
want: nil,
|
||||
wantErr: false,
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
|
||||
s := Scanner{}
|
||||
got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
|
||||
if (err != nil) != tt.wantErr {
|
||||
t.Errorf("Vagrantcloudpersonaltoken.FromData() error = %v, wantErr %v", err, tt.wantErr)
|
||||
t.Errorf("GlitterlyAPI.FromData() error = %v, wantErr %v", err, tt.wantErr)
|
||||
return
|
||||
}
|
||||
for i := range got {
|
||||
if len(got[i].Raw) == 0 {
|
||||
t.Fatalf("no raw secret present: \n %+v", got[i])
|
||||
}
|
||||
if (got[i].VerificationError != nil) != tt.wantVerificationErr {
|
||||
t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError)
|
||||
}
|
||||
got[i].Raw = nil
|
||||
}
|
||||
ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "VerificationError")
|
||||
if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {
|
||||
t.Errorf("Vagrantcloudpersonaltoken.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
|
||||
if diff := pretty.Compare(got, tt.want); diff != "" {
|
||||
t.Errorf("GlitterlyAPI.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
|
||||
}
|
||||
})
|
||||
}
|
||||
@@ -1,105 +0,0 @@
|
||||
package instamojo
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
|
||||
)
|
||||
|
||||
type Scanner struct {
|
||||
client *http.Client
|
||||
}
|
||||
|
||||
// Ensure the Scanner satisfies the interface at compile time.
|
||||
var _ detectors.Detector = (*Scanner)(nil)
|
||||
|
||||
var (
|
||||
defaultClient = common.SaneHttpClient()
|
||||
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
|
||||
//KeyPat is client_id
|
||||
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"instamojo"}) + `\b([0-9a-zA-Z]{40})\b`)
|
||||
//Secretpat is Client_secret
|
||||
secretPat = regexp.MustCompile(detectors.PrefixRegex([]string{"instamojo"}) + `\b([0-9a-zA-Z]{128})\b`)
|
||||
)
|
||||
|
||||
// Keywords are used for efficiently pre-filtering chunks.
|
||||
// Use identifiers in the secret preferably, or the provider name.
|
||||
func (s Scanner) Keywords() []string {
|
||||
return []string{"instamojo"}
|
||||
}
|
||||
|
||||
// FromData will find and optionally verify Instamojo secrets in a given set of bytes.
|
||||
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
|
||||
dataStr := string(data)
|
||||
|
||||
secretMatches := secretPat.FindAllStringSubmatch(dataStr, -1)
|
||||
clientIdmatches := keyPat.FindAllStringSubmatch(dataStr, -1)
|
||||
|
||||
for _, match := range secretMatches {
|
||||
if len(match) != 2 {
|
||||
continue
|
||||
}
|
||||
resSecret := strings.TrimSpace(match[1])
|
||||
|
||||
for _, clientIdMatch := range clientIdmatches {
|
||||
if len(clientIdMatch) != 2 {
|
||||
continue
|
||||
}
|
||||
resClientId := strings.TrimSpace(clientIdMatch[1])
|
||||
|
||||
s1 := detectors.Result{
|
||||
DetectorType: detectorspb.DetectorType_Instamojo,
|
||||
Raw: []byte(resClientId),
|
||||
}
|
||||
|
||||
if verify {
|
||||
client := s.client
|
||||
if client == nil {
|
||||
client = defaultClient
|
||||
}
|
||||
payload := strings.NewReader("grant_type=client_credentials&client_id=" + resClientId + "&client_secret=" + resSecret)
|
||||
|
||||
req, err := http.NewRequestWithContext(ctx, "POST", "https://api.instamojo.com/oauth2/token/", payload)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
req.Header.Add("Content-Type", "application/x-www-form-urlencoded")
|
||||
res, err := client.Do(req)
|
||||
if err == nil {
|
||||
defer res.Body.Close()
|
||||
bodyBytes, err := io.ReadAll(res.Body)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
body := string(bodyBytes)
|
||||
if (res.StatusCode >= 200 && res.StatusCode < 300) && strings.Contains(body, "access_token") {
|
||||
s1.Verified = true
|
||||
} else {
|
||||
s1.VerificationError = fmt.Errorf("unexpected HTTP response status %d", res.StatusCode)
|
||||
}
|
||||
} else {
|
||||
s1.VerificationError = err
|
||||
}
|
||||
}
|
||||
|
||||
if !s1.Verified && detectors.IsKnownFalsePositive(string(s1.Raw), detectors.DefaultFalsePositives, true) {
|
||||
continue
|
||||
}
|
||||
|
||||
results = append(results, s1)
|
||||
}
|
||||
}
|
||||
|
||||
return results, nil
|
||||
}
|
||||
|
||||
func (s Scanner) Type() detectorspb.DetectorType {
|
||||
return detectorspb.DetectorType_Instamojo
|
||||
}
|
||||
@@ -1,161 +0,0 @@
|
||||
//go:build detectors
|
||||
// +build detectors
|
||||
|
||||
package ipinfo
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"github.com/google/go-cmp/cmp"
|
||||
"github.com/google/go-cmp/cmp/cmpopts"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
|
||||
)
|
||||
|
||||
func TestIpinfo_FromChunk(t *testing.T) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
|
||||
defer cancel()
|
||||
testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")
|
||||
if err != nil {
|
||||
t.Fatalf("could not get test secrets from GCP: %s", err)
|
||||
}
|
||||
secret := testSecrets.MustGetField("IPINFO")
|
||||
inactiveSecret := testSecrets.MustGetField("IPINFO_INACTIVE")
|
||||
|
||||
type args struct {
|
||||
ctx context.Context
|
||||
data []byte
|
||||
verify bool
|
||||
}
|
||||
tests := []struct {
|
||||
name string
|
||||
s Scanner
|
||||
args args
|
||||
want []detectors.Result
|
||||
wantErr bool
|
||||
wantVerificationErr bool
|
||||
}{
|
||||
{
|
||||
name: "found, verified",
|
||||
s: Scanner{},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a ipinfo secret %s within", secret)),
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_IPInfo,
|
||||
Verified: true,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
wantVerificationErr: false,
|
||||
},
|
||||
{
|
||||
name: "found, unverified",
|
||||
s: Scanner{},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a ipinfo secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_IPInfo,
|
||||
Verified: false,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
wantVerificationErr: false,
|
||||
},
|
||||
{
|
||||
name: "not found",
|
||||
s: Scanner{},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte("You cannot find the secret within"),
|
||||
verify: true,
|
||||
},
|
||||
want: nil,
|
||||
wantErr: false,
|
||||
wantVerificationErr: false,
|
||||
},
|
||||
{
|
||||
name: "found, would be verified if not for timeout",
|
||||
s: Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a ipinfo secret %s within", secret)),
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_IPInfo,
|
||||
Verified: false,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
wantVerificationErr: true,
|
||||
},
|
||||
{
|
||||
name: "found, verified but unexpected api surface",
|
||||
s: Scanner{client: common.ConstantResponseHttpClient(404, "")},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a ipinfo secret %s within", secret)),
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_IPInfo,
|
||||
Verified: false,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
wantVerificationErr: true,
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
|
||||
if (err != nil) != tt.wantErr {
|
||||
t.Errorf("Ipinfo.FromData() error = %v, wantErr %v", err, tt.wantErr)
|
||||
return
|
||||
}
|
||||
for i := range got {
|
||||
if len(got[i].Raw) == 0 {
|
||||
t.Fatalf("no raw secret present: \n %+v", got[i])
|
||||
}
|
||||
if (got[i].VerificationError != nil) != tt.wantVerificationErr {
|
||||
t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError)
|
||||
}
|
||||
}
|
||||
ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "VerificationError")
|
||||
if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {
|
||||
t.Errorf("Ipinfo.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkFromData(benchmark *testing.B) {
|
||||
ctx := context.Background()
|
||||
s := Scanner{}
|
||||
for name, data := range detectors.MustGetBenchmarkData() {
|
||||
benchmark.Run(name, func(b *testing.B) {
|
||||
b.ResetTimer()
|
||||
for n := 0; n < b.N; n++ {
|
||||
_, err := s.FromData(ctx, false, data)
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -1,101 +0,0 @@
|
||||
package loggly
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
|
||||
)
|
||||
|
||||
type Scanner struct {
|
||||
client *http.Client
|
||||
}
|
||||
|
||||
// Ensure the Scanner satisfies the interface at compile time.
|
||||
var _ detectors.Detector = (*Scanner)(nil)
|
||||
|
||||
var (
|
||||
defaultClient = common.SaneHttpClient()
|
||||
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
|
||||
domainPat = regexp.MustCompile(`\b([a-zA-Z0-9-]+\.loggly\.com)\b`)
|
||||
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"loggly"}) + `\b([a-z0-9]{8}-[a-z0-9]{4}-[a-z0-9]{4}-[a-z0-9]{4}-[a-z0-9]{12})\b`)
|
||||
)
|
||||
|
||||
// Keywords are used for efficiently pre-filtering chunks.
|
||||
// Use identifiers in the secret preferably, or the provider name.
|
||||
func (s Scanner) Keywords() []string {
|
||||
return []string{"loggly"}
|
||||
}
|
||||
|
||||
// FromData will find and optionally verify Loggly secrets in a given set of bytes.
|
||||
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
|
||||
dataStr := string(data)
|
||||
|
||||
keyMatches := keyPat.FindAllStringSubmatch(dataStr, -1)
|
||||
domainMatches := domainPat.FindAllStringSubmatch(dataStr, -1)
|
||||
|
||||
for _, match := range keyMatches {
|
||||
if len(match) != 2 {
|
||||
continue
|
||||
}
|
||||
key := strings.TrimSpace(match[1])
|
||||
|
||||
for _, domainMatch := range domainMatches {
|
||||
if len(domainMatch) != 2 {
|
||||
continue
|
||||
}
|
||||
|
||||
domainRes := strings.TrimSpace(domainMatch[1])
|
||||
|
||||
s1 := detectors.Result{
|
||||
DetectorType: detectorspb.DetectorType_Loggly,
|
||||
Raw: []byte(key),
|
||||
RawV2: []byte(fmt.Sprintf("%s:%s", domainRes, key)),
|
||||
|
||||
}
|
||||
|
||||
if verify {
|
||||
client := s.client
|
||||
if client == nil {
|
||||
client = defaultClient
|
||||
}
|
||||
req, err := http.NewRequestWithContext(ctx, "GET", "https://"+domainRes+"/apiv2/customer", nil)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", key))
|
||||
res, err := client.Do(req)
|
||||
if err == nil {
|
||||
defer res.Body.Close()
|
||||
if res.StatusCode >= 200 && res.StatusCode < 300 {
|
||||
s1.Verified = true
|
||||
} else if res.StatusCode == 401 {
|
||||
// The secret is determinately not verified (nothing to do)
|
||||
} else {
|
||||
s1.VerificationError = fmt.Errorf("unexpected HTTP response status %d", res.StatusCode)
|
||||
}
|
||||
} else {
|
||||
s1.VerificationError = err
|
||||
}
|
||||
}
|
||||
|
||||
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
|
||||
if !s1.Verified && detectors.IsKnownFalsePositive(key, detectors.DefaultFalsePositives, true) {
|
||||
continue
|
||||
}
|
||||
|
||||
results = append(results, s1)
|
||||
}
|
||||
}
|
||||
|
||||
return results, nil
|
||||
}
|
||||
|
||||
func (s Scanner) Type() detectorspb.DetectorType {
|
||||
return detectorspb.DetectorType_Loggly
|
||||
}
|
||||
@@ -1,162 +0,0 @@
|
||||
//go:build detectors
|
||||
// +build detectors
|
||||
|
||||
package loggly
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"github.com/google/go-cmp/cmp"
|
||||
"github.com/google/go-cmp/cmp/cmpopts"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
|
||||
)
|
||||
|
||||
func TestLoggly_FromChunk(t *testing.T) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
|
||||
defer cancel()
|
||||
testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors4")
|
||||
if err != nil {
|
||||
t.Fatalf("could not get test secrets from GCP: %s", err)
|
||||
}
|
||||
secret := testSecrets.MustGetField("LOGGLY")
|
||||
inactiveSecret := testSecrets.MustGetField("LOGGLY_INACTIVE")
|
||||
domain := testSecrets.MustGetField("LOGGLY_DOMAIN")
|
||||
|
||||
type args struct {
|
||||
ctx context.Context
|
||||
data []byte
|
||||
verify bool
|
||||
}
|
||||
tests := []struct {
|
||||
name string
|
||||
s Scanner
|
||||
args args
|
||||
want []detectors.Result
|
||||
wantErr bool
|
||||
wantVerificationErr bool
|
||||
}{
|
||||
{
|
||||
name: "found, verified",
|
||||
s: Scanner{},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a loggly secret %s within loggly Domain %s", secret, domain)),
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_Loggly,
|
||||
Verified: true,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
wantVerificationErr: false,
|
||||
},
|
||||
{
|
||||
name: "found, unverified",
|
||||
s: Scanner{},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a loggly secret %s within loggly Domain %s but not valid", inactiveSecret, domain)), // the secret would satisfy the regex but not pass validation
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_Loggly,
|
||||
Verified: false,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
wantVerificationErr: false,
|
||||
},
|
||||
{
|
||||
name: "not found",
|
||||
s: Scanner{},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte("You cannot find the secret within"),
|
||||
verify: true,
|
||||
},
|
||||
want: nil,
|
||||
wantErr: false,
|
||||
wantVerificationErr: false,
|
||||
},
|
||||
{
|
||||
name: "found, would be verified if not for timeout",
|
||||
s: Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a loggly secret %s within %s", secret, domain)),
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_Loggly,
|
||||
Verified: false,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
wantVerificationErr: true,
|
||||
},
|
||||
{
|
||||
name: "found, verified but unexpected api surface",
|
||||
s: Scanner{client: common.ConstantResponseHttpClient(404, "")},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a loggly secret %s within %s", secret, domain)),
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_Loggly,
|
||||
Verified: false,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
wantVerificationErr: true,
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
|
||||
if (err != nil) != tt.wantErr {
|
||||
t.Errorf("Loggly.FromData() error = %v, wantErr %v", err, tt.wantErr)
|
||||
return
|
||||
}
|
||||
for i := range got {
|
||||
if len(got[i].Raw) == 0 {
|
||||
t.Fatalf("no raw secret present: \n %+v", got[i])
|
||||
}
|
||||
if (got[i].VerificationError != nil) != tt.wantVerificationErr {
|
||||
t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError)
|
||||
}
|
||||
}
|
||||
ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "RawV2", "VerificationError")
|
||||
if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {
|
||||
t.Errorf("Loggly.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkFromData(benchmark *testing.B) {
|
||||
ctx := context.Background()
|
||||
s := Scanner{}
|
||||
for name, data := range detectors.MustGetBenchmarkData() {
|
||||
benchmark.Run(name, func(b *testing.B) {
|
||||
b.ResetTimer()
|
||||
for n := 0; n < b.N; n++ {
|
||||
_, err := s.FromData(ctx, false, data)
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -3,7 +3,6 @@ package meaningcloud
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"io"
|
||||
"mime/multipart"
|
||||
"net/http"
|
||||
@@ -33,10 +32,6 @@ func (s Scanner) Keywords() []string {
|
||||
return []string{"meaningcloud"}
|
||||
}
|
||||
|
||||
type response struct {
|
||||
DeepTime float64 `json:"deepTime"`
|
||||
}
|
||||
|
||||
// FromData will find and optionally verify MeaningCloud secrets in a given set of bytes.
|
||||
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
|
||||
dataStr := string(data)
|
||||
@@ -83,14 +78,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
if err == nil {
|
||||
defer res.Body.Close()
|
||||
if res.StatusCode >= 200 && res.StatusCode < 300 {
|
||||
var r response
|
||||
if err := json.NewDecoder(res.Body).Decode(&r); err != nil {
|
||||
s1.VerificationError = err
|
||||
continue
|
||||
}
|
||||
if r.DeepTime > 0 {
|
||||
s1.Verified = true
|
||||
}
|
||||
s1.Verified = true
|
||||
} else {
|
||||
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
|
||||
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
|
||||
|
||||
@@ -20,7 +20,7 @@ var (
|
||||
client = common.SaneHttpClient()
|
||||
|
||||
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
|
||||
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"monday"}) + `\b(eyJ[A-Za-z0-9-_]{15,100}\.eyJ[A-Za-z0-9-_]{100,300}\.[A-Za-z0-9-_]{25,100})\b`)
|
||||
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"monday"}) + `\b(ey[a-zA-Z0-9_.]{210,225})\b`)
|
||||
)
|
||||
|
||||
// Keywords are used for efficiently pre-filtering chunks.
|
||||
|
||||
@@ -19,7 +19,7 @@ import (
|
||||
func TestMonday_FromChunk(t *testing.T) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
|
||||
defer cancel()
|
||||
testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")
|
||||
testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
|
||||
if err != nil {
|
||||
t.Fatalf("could not get test secrets from GCP: %s", err)
|
||||
}
|
||||
|
||||
@@ -48,16 +48,11 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
|
||||
if verify {
|
||||
req, err := http.NewRequestWithContext(ctx, "GET", "https://api.newrelic.com/v2/users.json", nil)
|
||||
reqEU, errEU := http.NewRequestWithContext(ctx, "GET", "https://api.eu.newrelic.com/v2/users.json", nil)
|
||||
if err != nil || errEU != nil {
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
req.Header.Add("X-Api-Key", resMatch)
|
||||
reqEU.Header.Add("X-Api-Key", resMatch)
|
||||
|
||||
res, err := client.Do(req)
|
||||
resEU, errEU := client.Do(reqEU)
|
||||
|
||||
if err == nil {
|
||||
defer res.Body.Close()
|
||||
if res.StatusCode >= 200 && res.StatusCode < 300 {
|
||||
@@ -68,16 +63,6 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
continue
|
||||
}
|
||||
}
|
||||
} else if errEU == nil {
|
||||
defer resEU.Body.Close()
|
||||
if resEU.StatusCode >= 200 && resEU.StatusCode < 300 {
|
||||
s1.Verified = true
|
||||
} else {
|
||||
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
|
||||
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, false) {
|
||||
continue
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -1,109 +0,0 @@
|
||||
package openvpn
|
||||
|
||||
import (
|
||||
"context"
|
||||
b64 "encoding/base64"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
|
||||
)
|
||||
|
||||
type Scanner struct {
|
||||
client *http.Client
|
||||
}
|
||||
|
||||
// Ensure the Scanner satisfies the interface at compile time.
|
||||
var _ detectors.Detector = (*Scanner)(nil)
|
||||
|
||||
var (
|
||||
defaultClient = common.SaneHttpClient()
|
||||
|
||||
clientIDPat = regexp.MustCompile(detectors.PrefixRegex([]string{"openvpn"}) + `\b([A-Za-z0-9-]{3,40}\.[A-Za-z0-9-]{3,40})\b`)
|
||||
clientSecretPat = regexp.MustCompile(`\b([a-zA-Z0-9_-]{64,})\b`)
|
||||
domainPat = regexp.MustCompile(`\b(https?://[A-Za-z0-9-]+\.api\.openvpn\.com)\b`)
|
||||
|
||||
)
|
||||
|
||||
// Keywords are used for efficiently pre-filtering chunks.
|
||||
// Use identifiers in the secret preferably, or the provider name.
|
||||
func (s Scanner) Keywords() []string {
|
||||
return []string{"openvpn"}
|
||||
}
|
||||
|
||||
// FromData will find and optionally verify Openvpn secrets in a given set of bytes.
|
||||
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
|
||||
dataStr := string(data)
|
||||
|
||||
domainMatches := domainPat.FindAllStringSubmatch(dataStr, -1)
|
||||
clientIdMatches := clientIDPat.FindAllStringSubmatch(dataStr, -1)
|
||||
clientSecretMatches := clientSecretPat.FindAllStringSubmatch(dataStr, -1)
|
||||
|
||||
for _, clientIdMatch := range clientIdMatches {
|
||||
clientIDRes := strings.TrimSpace(clientIdMatch[1])
|
||||
for _, clientSecretMatch := range clientSecretMatches {
|
||||
clientSecretRes := strings.TrimSpace(clientSecretMatch[1])
|
||||
for _, domainMatch := range domainMatches {
|
||||
domainRes := strings.TrimSpace(domainMatch[1])
|
||||
|
||||
s1 := detectors.Result{
|
||||
DetectorType: detectorspb.DetectorType_OpenVpn,
|
||||
Raw: []byte(clientSecretRes),
|
||||
RawV2: []byte(clientIDRes + clientSecretRes),
|
||||
}
|
||||
|
||||
if verify {
|
||||
client := s.client
|
||||
if client == nil {
|
||||
client = defaultClient
|
||||
}
|
||||
|
||||
payload := strings.NewReader("grant_type=client_credentials")
|
||||
// OpenVPN API is in beta, We'll have to update the API endpoint once
|
||||
// Docs: https://openvpn.net/cloud-docs/developer/creating-api-credentials.html
|
||||
req, err := http.NewRequestWithContext(ctx, "POST", domainRes + "/api/beta/oauth/token", payload)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
|
||||
data := fmt.Sprintf("%s:%s", clientIDRes, clientSecretRes)
|
||||
sEnc := b64.StdEncoding.EncodeToString([]byte(data))
|
||||
|
||||
req.Header.Add("Authorization", fmt.Sprintf("Basic %s", sEnc))
|
||||
req.Header.Add("content-type", "application/x-www-form-urlencoded")
|
||||
res, err := client.Do(req)
|
||||
if err == nil {
|
||||
defer res.Body.Close()
|
||||
if res.StatusCode >= 200 && res.StatusCode < 300 {
|
||||
s1.Verified = true
|
||||
} else if res.StatusCode == 401 {
|
||||
// The secret is determinately not verified (nothing to do)
|
||||
} else {
|
||||
s1.VerificationError = fmt.Errorf("unexpected HTTP response status %d", res.StatusCode)
|
||||
}
|
||||
} else {
|
||||
s1.VerificationError = err
|
||||
}
|
||||
}
|
||||
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
|
||||
if detectors.IsKnownFalsePositive(clientSecretRes, detectors.DefaultFalsePositives, true) {
|
||||
continue
|
||||
}
|
||||
results = append(results, s1)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
}
|
||||
|
||||
return results, nil
|
||||
}
|
||||
|
||||
func (s Scanner) Type() detectorspb.DetectorType {
|
||||
return detectorspb.DetectorType_OpenVpn
|
||||
}
|
||||
@@ -1,133 +0,0 @@
|
||||
//go:build detectors
|
||||
// +build detectors
|
||||
|
||||
package openvpn
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/google/go-cmp/cmp"
|
||||
"github.com/google/go-cmp/cmp/cmpopts"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
|
||||
)
|
||||
|
||||
func TestOpenvpn_FromChunk(t *testing.T) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
|
||||
defer cancel()
|
||||
testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")
|
||||
if err != nil {
|
||||
t.Fatalf("could not get test secrets from GCP: %s", err)
|
||||
}
|
||||
|
||||
domain := testSecrets.MustGetField("OPENVPN_DOMAIN")
|
||||
clientId := testSecrets.MustGetField("OPENVPN_CLIENT_ID")
|
||||
clientSecret := testSecrets.MustGetField("OPENVPN_CLIENT_SECRET")
|
||||
inactiveClientSecret := testSecrets.MustGetField("OPENVPN_CLIENT_SECRET_INACTIVE")
|
||||
|
||||
type args struct {
|
||||
ctx context.Context
|
||||
data []byte
|
||||
verify bool
|
||||
}
|
||||
tests := []struct {
|
||||
name string
|
||||
s Scanner
|
||||
args args
|
||||
want []detectors.Result
|
||||
wantErr bool
|
||||
wantVerificationErr bool
|
||||
}{
|
||||
{
|
||||
name: "found, verified",
|
||||
s: Scanner{},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a openvpn secret %s openvpn clientId %s and domain %s within", clientSecret, clientId, domain)),
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_OpenVpn,
|
||||
Verified: true,
|
||||
RawV2: []byte(clientId + clientSecret),
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
wantVerificationErr: false,
|
||||
},
|
||||
{
|
||||
name: "found, unverified",
|
||||
s: Scanner{},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a openvpn secret %s openvpn clientId %s and domain %s within but not valid", inactiveClientSecret, clientId, domain)), // the secret would satisfy the regex but not pass validation
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_OpenVpn,
|
||||
Verified: false,
|
||||
RawV2: []byte(clientId + inactiveClientSecret),
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
wantVerificationErr: false,
|
||||
},
|
||||
{
|
||||
name: "not found",
|
||||
s: Scanner{},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte("You cannot find the secret within"),
|
||||
verify: true,
|
||||
},
|
||||
want: nil,
|
||||
wantErr: false,
|
||||
wantVerificationErr: false,
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
|
||||
if (err != nil) != tt.wantErr {
|
||||
t.Errorf("Openvpn.FromData() error = %v, wantErr %v", err, tt.wantErr)
|
||||
return
|
||||
}
|
||||
for i := range got {
|
||||
if len(got[i].Raw) == 0 {
|
||||
t.Fatalf("no raw secret present: \n %+v", got[i])
|
||||
}
|
||||
if (got[i].VerificationError != nil) != tt.wantVerificationErr {
|
||||
t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError)
|
||||
}
|
||||
}
|
||||
ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "VerificationError")
|
||||
if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {
|
||||
t.Errorf("Openvpn.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkFromData(benchmark *testing.B) {
|
||||
ctx := context.Background()
|
||||
s := Scanner{}
|
||||
for name, data := range detectors.MustGetBenchmarkData() {
|
||||
benchmark.Run(name, func(b *testing.B) {
|
||||
b.ResetTimer()
|
||||
for n := 0; n < b.N; n++ {
|
||||
_, err := s.FromData(ctx, false, data)
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -1,98 +0,0 @@
|
||||
package portainer
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
|
||||
)
|
||||
|
||||
type Scanner struct {
|
||||
client *http.Client
|
||||
}
|
||||
|
||||
// Ensure the Scanner satisfies the interface at compile time.
|
||||
var _ detectors.Detector = (*Scanner)(nil)
|
||||
|
||||
var (
|
||||
defaultClient = common.SaneHttpClient()
|
||||
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
|
||||
endpointPat = regexp.MustCompile(detectors.PrefixRegex([]string{"portainer"}) + `\b(https?:\/\/\S+(:[0-9]{4,5})?)\b`)
|
||||
tokenPat = regexp.MustCompile(detectors.PrefixRegex([]string{"portainer"}) + `\b(eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9\.[0-9A-Za-z]{50,310}\.[0-9A-Z-a-z\-_]{43})\b`)
|
||||
)
|
||||
|
||||
// Keywords are used for efficiently pre-filtering chunks.
|
||||
// Use identifiers in the secret preferably, or the provider name.
|
||||
func (s Scanner) Keywords() []string {
|
||||
return []string{"portainer"}
|
||||
}
|
||||
|
||||
// FromData will find and optionally verify Portainer secrets in a given set of bytes.
|
||||
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
|
||||
dataStr := string(data)
|
||||
|
||||
matches := tokenPat.FindAllStringSubmatch(dataStr, -1)
|
||||
endpointMatches := endpointPat.FindAllStringSubmatch(dataStr, -1)
|
||||
|
||||
for _, match := range matches {
|
||||
if len(match) != 2 {
|
||||
continue
|
||||
}
|
||||
resMatch := strings.TrimSpace(match[1])
|
||||
|
||||
for _, endpointMatch := range endpointMatches {
|
||||
resEndpointMatch := strings.TrimSpace(endpointMatch[1])
|
||||
|
||||
s1 := detectors.Result{
|
||||
DetectorType: detectorspb.DetectorType_Portainer,
|
||||
Raw: []byte(resMatch),
|
||||
RawV2: []byte(resMatch + resEndpointMatch),
|
||||
}
|
||||
|
||||
if verify {
|
||||
client := s.client
|
||||
if client == nil {
|
||||
client = defaultClient
|
||||
}
|
||||
req, err := http.NewRequestWithContext(ctx, "GET", resEndpointMatch + "/api/endpoints", nil)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", resMatch))
|
||||
res, err := client.Do(req)
|
||||
if err == nil {
|
||||
defer res.Body.Close()
|
||||
if res.StatusCode >= 200 && res.StatusCode < 300 {
|
||||
s1.Verified = true
|
||||
} else if res.StatusCode == 401 || res.StatusCode == 403 {
|
||||
// The secret is determinately not verified (nothing to do)
|
||||
} else {
|
||||
s1.VerificationError = fmt.Errorf("unexpected HTTP response status %d", res.StatusCode)
|
||||
}
|
||||
} else {
|
||||
s1.VerificationError = err
|
||||
}
|
||||
}
|
||||
|
||||
if !s1.Verified && detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
|
||||
continue
|
||||
}
|
||||
|
||||
if len(endpointMatches) > 0 {
|
||||
results = append(results, s1)
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
return results, nil
|
||||
}
|
||||
|
||||
func (s Scanner) Type() detectorspb.DetectorType {
|
||||
return detectorspb.DetectorType_Portainer
|
||||
}
|
||||
@@ -1,99 +0,0 @@
|
||||
package portainertoken
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
|
||||
)
|
||||
|
||||
type Scanner struct {
|
||||
client *http.Client
|
||||
}
|
||||
|
||||
// Ensure the Scanner satisfies the interface at compile time.
|
||||
var _ detectors.Detector = (*Scanner)(nil)
|
||||
|
||||
var (
|
||||
defaultClient = common.SaneHttpClient()
|
||||
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
|
||||
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"portainertoken"}) + `\b(ptr_[A-Za-z0-9\/_\-+=]{20,60})`)
|
||||
endpointPat = regexp.MustCompile(detectors.PrefixRegex([]string{"portainer"}) + `\b(https?:\/\/\S+(:[0-9]{4,5})?)\b`)
|
||||
)
|
||||
|
||||
// Keywords are used for efficiently pre-filtering chunks.
|
||||
// Use identifiers in the secret preferably, or the provider name.
|
||||
func (s Scanner) Keywords() []string {
|
||||
return []string{"portainertoken"}
|
||||
}
|
||||
|
||||
// FromData will find and optionally verify Portainertoken secrets in a given set of bytes.
|
||||
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
|
||||
dataStr := string(data)
|
||||
|
||||
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
|
||||
endpointMatches := endpointPat.FindAllStringSubmatch(dataStr, -1)
|
||||
|
||||
for _, match := range matches {
|
||||
if len(match) != 2 {
|
||||
continue
|
||||
}
|
||||
resMatch := strings.TrimSpace(match[1])
|
||||
|
||||
for _, endpointMatch := range endpointMatches {
|
||||
resEndpointMatch := strings.TrimSpace(endpointMatch[1])
|
||||
|
||||
s1 := detectors.Result{
|
||||
DetectorType: detectorspb.DetectorType_PortainerToken,
|
||||
Raw: []byte(resMatch),
|
||||
RawV2: []byte(resMatch + resEndpointMatch),
|
||||
}
|
||||
|
||||
if verify {
|
||||
client := s.client
|
||||
if client == nil {
|
||||
client = defaultClient
|
||||
}
|
||||
req, err := http.NewRequestWithContext(ctx, "GET", resEndpointMatch + "/api/stacks", nil)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
|
||||
req.Header.Add("X-API-Key", resMatch)
|
||||
|
||||
res, err := client.Do(req)
|
||||
|
||||
if err == nil {
|
||||
defer res.Body.Close()
|
||||
if res.StatusCode >= 200 && res.StatusCode < 300 {
|
||||
s1.Verified = true
|
||||
} else if res.StatusCode == 401 {
|
||||
// The secret is determinately not verified (nothing to do)
|
||||
} else {
|
||||
s1.VerificationError = fmt.Errorf("unexpected HTTP response status %d", res.StatusCode)
|
||||
}
|
||||
} else {
|
||||
s1.VerificationError = err
|
||||
}
|
||||
}
|
||||
|
||||
if !s1.Verified && detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
|
||||
continue
|
||||
}
|
||||
|
||||
if len(endpointMatches) > 0 {
|
||||
results = append(results, s1)
|
||||
}
|
||||
}
|
||||
}
|
||||
return results, nil
|
||||
}
|
||||
|
||||
func (s Scanner) Type() detectorspb.DetectorType {
|
||||
return detectorspb.DetectorType_PortainerToken
|
||||
}
|
||||
@@ -1,131 +0,0 @@
|
||||
//go:build detectors
|
||||
// +build detectors
|
||||
|
||||
package portainertoken
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/google/go-cmp/cmp"
|
||||
"github.com/google/go-cmp/cmp/cmpopts"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
|
||||
)
|
||||
|
||||
func TestPortainertoken_FromChunk(t *testing.T) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
|
||||
defer cancel()
|
||||
testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")
|
||||
if err != nil {
|
||||
t.Fatalf("could not get test secrets from GCP: %s", err)
|
||||
}
|
||||
secret := testSecrets.MustGetField("PORTAINERTOKEN")
|
||||
inactiveSecret := testSecrets.MustGetField("PORTAINERTOKEN_INACTIVE")
|
||||
endpoint := testSecrets.MustGetField("PORTAINER_ENDPOINT")
|
||||
|
||||
type args struct {
|
||||
ctx context.Context
|
||||
data []byte
|
||||
verify bool
|
||||
}
|
||||
tests := []struct {
|
||||
name string
|
||||
s Scanner
|
||||
args args
|
||||
want []detectors.Result
|
||||
wantErr bool
|
||||
wantVerificationErr bool
|
||||
}{
|
||||
{
|
||||
name: "found, verified",
|
||||
s: Scanner{},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a portainertoken secret %s within for portainer url %s", secret, endpoint)),
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_PortainerToken,
|
||||
Verified: true,
|
||||
RawV2: []byte(secret + endpoint),
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
wantVerificationErr: false,
|
||||
},
|
||||
{
|
||||
name: "found, unverified",
|
||||
s: Scanner{},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a portainertoken secret %s within but not valid for portainer url %s", inactiveSecret, endpoint)), // the secret would satisfy the regex but not pass validation
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_PortainerToken,
|
||||
Verified: false,
|
||||
RawV2: []byte(inactiveSecret + endpoint),
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
wantVerificationErr: false,
|
||||
},
|
||||
{
|
||||
name: "not found",
|
||||
s: Scanner{},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte("You cannot find the secret within"),
|
||||
verify: true,
|
||||
},
|
||||
want: nil,
|
||||
wantErr: false,
|
||||
wantVerificationErr: false,
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
|
||||
if (err != nil) != tt.wantErr {
|
||||
t.Errorf("Portainertoken.FromData() error = %v, wantErr %v", err, tt.wantErr)
|
||||
return
|
||||
}
|
||||
for i := range got {
|
||||
if len(got[i].Raw) == 0 {
|
||||
t.Fatalf("no raw secret present: \n %+v", got[i])
|
||||
}
|
||||
if (got[i].VerificationError != nil) != tt.wantVerificationErr {
|
||||
t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError)
|
||||
}
|
||||
}
|
||||
ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "VerificationError")
|
||||
if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {
|
||||
t.Errorf("Portainertoken.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkFromData(benchmark *testing.B) {
|
||||
ctx := context.Background()
|
||||
s := Scanner{}
|
||||
for name, data := range detectors.MustGetBenchmarkData() {
|
||||
benchmark.Run(name, func(b *testing.B) {
|
||||
b.ResetTimer()
|
||||
for n := 0; n < b.N; n++ {
|
||||
_, err := s.FromData(ctx, false, data)
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -1,161 +0,0 @@
|
||||
//go:build detectors
|
||||
// +build detectors
|
||||
|
||||
package privacy
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"github.com/google/go-cmp/cmp"
|
||||
"github.com/google/go-cmp/cmp/cmpopts"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
|
||||
)
|
||||
|
||||
func TestPrivacy_FromChunk(t *testing.T) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
|
||||
defer cancel()
|
||||
testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")
|
||||
if err != nil {
|
||||
t.Fatalf("could not get test secrets from GCP: %s", err)
|
||||
}
|
||||
secret := testSecrets.MustGetField("PRIVACY")
|
||||
inactiveSecret := testSecrets.MustGetField("PRIVACY_INACTIVE")
|
||||
|
||||
type args struct {
|
||||
ctx context.Context
|
||||
data []byte
|
||||
verify bool
|
||||
}
|
||||
tests := []struct {
|
||||
name string
|
||||
s Scanner
|
||||
args args
|
||||
want []detectors.Result
|
||||
wantErr bool
|
||||
wantVerificationErr bool
|
||||
}{
|
||||
{
|
||||
name: "found, verified",
|
||||
s: Scanner{},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a privacy secret %s within", secret)),
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_Privacy,
|
||||
Verified: true,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
wantVerificationErr: false,
|
||||
},
|
||||
{
|
||||
name: "found, unverified",
|
||||
s: Scanner{},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a privacy secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_Privacy,
|
||||
Verified: false,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
wantVerificationErr: false,
|
||||
},
|
||||
{
|
||||
name: "not found",
|
||||
s: Scanner{},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte("You cannot find the secret within"),
|
||||
verify: true,
|
||||
},
|
||||
want: nil,
|
||||
wantErr: false,
|
||||
wantVerificationErr: false,
|
||||
},
|
||||
{
|
||||
name: "found, would be verified if not for timeout",
|
||||
s: Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a privacy secret %s within", secret)),
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_Privacy,
|
||||
Verified: false,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
wantVerificationErr: true,
|
||||
},
|
||||
{
|
||||
name: "found, verified but unexpected api surface",
|
||||
s: Scanner{client: common.ConstantResponseHttpClient(404, "")},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a privacy secret %s within", secret)),
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_Privacy,
|
||||
Verified: false,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
wantVerificationErr: true,
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
|
||||
if (err != nil) != tt.wantErr {
|
||||
t.Errorf("Privacy.FromData() error = %v, wantErr %v", err, tt.wantErr)
|
||||
return
|
||||
}
|
||||
for i := range got {
|
||||
if len(got[i].Raw) == 0 {
|
||||
t.Fatalf("no raw secret present: \n %+v", got[i])
|
||||
}
|
||||
if (got[i].VerificationError != nil) != tt.wantVerificationErr {
|
||||
t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError)
|
||||
}
|
||||
}
|
||||
ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "VerificationError")
|
||||
if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {
|
||||
t.Errorf("Privacy.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkFromData(benchmark *testing.B) {
|
||||
ctx := context.Background()
|
||||
s := Scanner{}
|
||||
for name, data := range detectors.MustGetBenchmarkData() {
|
||||
benchmark.Run(name, func(b *testing.B) {
|
||||
b.ResetTimer()
|
||||
for n := 0; n < b.N; n++ {
|
||||
_, err := s.FromData(ctx, false, data)
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,79 @@
|
||||
package quickmetrics
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
|
||||
)
|
||||
|
||||
type Scanner struct{}
|
||||
|
||||
// Ensure the Scanner satisfies the interface at compile time.
|
||||
var _ detectors.Detector = (*Scanner)(nil)
|
||||
|
||||
var (
|
||||
client = common.SaneHttpClient()
|
||||
|
||||
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
|
||||
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"quickmetrics"}) + `\b([a-zA-Z0-9_-]{22})\b`)
|
||||
)
|
||||
|
||||
// Keywords are used for efficiently pre-filtering chunks.
|
||||
// Use identifiers in the secret preferably, or the provider name.
|
||||
func (s Scanner) Keywords() []string {
|
||||
return []string{"quickmetrics"}
|
||||
}
|
||||
|
||||
// FromData will find and optionally verify QuickMetrics secrets in a given set of bytes.
|
||||
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
|
||||
dataStr := string(data)
|
||||
|
||||
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
|
||||
|
||||
for _, match := range matches {
|
||||
if len(match) != 2 {
|
||||
continue
|
||||
}
|
||||
resMatch := strings.TrimSpace(match[1])
|
||||
|
||||
s1 := detectors.Result{
|
||||
DetectorType: detectorspb.DetectorType_QuickMetrics,
|
||||
Raw: []byte(resMatch),
|
||||
}
|
||||
|
||||
if verify {
|
||||
payload := strings.NewReader(`[{"name":"api.response.time","dimension":null,"values":[[1568319841,12.4],[1568319856,9.3],[1568319860,234],[1568319863,3.2]]},{"name":"click.color","dimension":"green","values":[[1568319841,1],[1568319856,1],[1568319860,1],[1568319863,1]]}]`)
|
||||
req, err := http.NewRequestWithContext(ctx, "POST", "https://qckm.io/list", payload)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
req.Header.Add("Content-Type", "application/json")
|
||||
req.Header.Add("x-qm-key", resMatch)
|
||||
res, err := client.Do(req)
|
||||
if err == nil {
|
||||
defer res.Body.Close()
|
||||
if res.StatusCode >= 200 && res.StatusCode < 300 {
|
||||
s1.Verified = true
|
||||
} else {
|
||||
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
|
||||
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
|
||||
continue
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
results = append(results, s1)
|
||||
}
|
||||
|
||||
return results, nil
|
||||
}
|
||||
|
||||
func (s Scanner) Type() detectorspb.DetectorType {
|
||||
return detectorspb.DetectorType_QuickMetrics
|
||||
}
|
||||
@@ -0,0 +1,120 @@
|
||||
//go:build detectors
|
||||
// +build detectors
|
||||
|
||||
package quickmetrics
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/kylelemons/godebug/pretty"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
|
||||
)
|
||||
|
||||
func TestQuickMetrics_FromChunk(t *testing.T) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
|
||||
defer cancel()
|
||||
testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")
|
||||
if err != nil {
|
||||
t.Fatalf("could not get test secrets from GCP: %s", err)
|
||||
}
|
||||
secret := testSecrets.MustGetField("QUICKMETRICS")
|
||||
inactiveSecret := testSecrets.MustGetField("QUICKMETRICS_INACTIVE")
|
||||
|
||||
type args struct {
|
||||
ctx context.Context
|
||||
data []byte
|
||||
verify bool
|
||||
}
|
||||
tests := []struct {
|
||||
name string
|
||||
s Scanner
|
||||
args args
|
||||
want []detectors.Result
|
||||
wantErr bool
|
||||
}{
|
||||
{
|
||||
name: "found, verified",
|
||||
s: Scanner{},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a quickmetrics secret %s within", secret)),
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_QuickMetrics,
|
||||
Verified: true,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "found, unverified",
|
||||
s: Scanner{},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a quickmetrics secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_QuickMetrics,
|
||||
Verified: false,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "not found",
|
||||
s: Scanner{},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte("You cannot find the secret within"),
|
||||
verify: true,
|
||||
},
|
||||
want: nil,
|
||||
wantErr: false,
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
s := Scanner{}
|
||||
got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
|
||||
if (err != nil) != tt.wantErr {
|
||||
t.Errorf("QuickMetrics.FromData() error = %v, wantErr %v", err, tt.wantErr)
|
||||
return
|
||||
}
|
||||
for i := range got {
|
||||
if len(got[i].Raw) == 0 {
|
||||
t.Fatalf("no raw secret present: \n %+v", got[i])
|
||||
}
|
||||
got[i].Raw = nil
|
||||
}
|
||||
if diff := pretty.Compare(got, tt.want); diff != "" {
|
||||
t.Errorf("QuickMetrics.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkFromData(benchmark *testing.B) {
|
||||
ctx := context.Background()
|
||||
s := Scanner{}
|
||||
for name, data := range detectors.MustGetBenchmarkData() {
|
||||
benchmark.Run(name, func(b *testing.B) {
|
||||
b.ResetTimer()
|
||||
for n := 0; n < b.N; n++ {
|
||||
_, err := s.FromData(ctx, false, data)
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -2,7 +2,6 @@ package screenshotapi
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"strings"
|
||||
@@ -31,10 +30,6 @@ func (s Scanner) Keywords() []string {
|
||||
return []string{"screenshotapi"}
|
||||
}
|
||||
|
||||
type response struct {
|
||||
Screenshot string `json:"screenshot"`
|
||||
}
|
||||
|
||||
// FromData will find and optionally verify ScreenshotAPI secrets in a given set of bytes.
|
||||
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
|
||||
dataStr := string(data)
|
||||
@@ -55,19 +50,14 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
if verify {
|
||||
timeout := 10 * time.Second
|
||||
client.Timeout = timeout
|
||||
req, err := http.NewRequestWithContext(ctx, "GET", "https://shot.screenshotapi.net/screenshot?token="+resMatch+"&url=https://google.com&width=1920&height=1080", nil)
|
||||
req, err := http.NewRequestWithContext(ctx, "GET", "https://shot.screenshotapi.net/screenshot?token="+resMatch+"&url=https://google.com&width=1920&height=1080&output=image", nil)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
res, err := client.Do(req)
|
||||
if err == nil {
|
||||
defer res.Body.Close()
|
||||
var r response
|
||||
if err := json.NewDecoder(res.Body).Decode(&r); err != nil {
|
||||
s1.VerificationError = err
|
||||
continue
|
||||
}
|
||||
if res.StatusCode >= 200 && res.StatusCode < 300 && r.Screenshot != "" {
|
||||
if res.StatusCode >= 200 && res.StatusCode < 300 {
|
||||
s1.Verified = true
|
||||
} else {
|
||||
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
|
||||
|
||||
@@ -47,7 +47,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
}
|
||||
|
||||
if verify {
|
||||
req, err := http.NewRequestWithContext(ctx, "GET", "https://api.supernotes.app/v1/user", nil)
|
||||
req, err := http.NewRequestWithContext(ctx, "GET", "https://api.supernotes.app/v1/user/", nil)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
|
||||
@@ -1,111 +0,0 @@
|
||||
package engine
|
||||
|
||||
import (
|
||||
"strings"
|
||||
|
||||
ahocorasick "github.com/BobuSumisu/aho-corasick"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/context"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
|
||||
)
|
||||
|
||||
// detectorKey is used to identify a detector in the keywordsToDetectors map.
|
||||
// Multiple detectors can have the same detector type but different versions.
|
||||
// This allows us to identify a detector by its type and version.
|
||||
type detectorKey struct {
|
||||
detectorType detectorspb.DetectorType
|
||||
version int
|
||||
}
|
||||
|
||||
// DetectorInfo is used to store a detector and whether it should be verified.
|
||||
type DetectorInfo struct {
|
||||
detectors.Detector
|
||||
ShouldVerify bool
|
||||
}
|
||||
|
||||
// AhoCorasickCore encapsulates the operations and data structures used for keyword matching via the
|
||||
// Aho-Corasick algorithm. It is responsible for constructing and managing the trie for efficient
|
||||
// substring searches, as well as mapping keywords to their associated detectors for rapid lookups.
|
||||
type AhoCorasickCore struct {
|
||||
// prefilter is a ahocorasick struct used for doing efficient string
|
||||
// matching given a set of words. (keywords from the rules in the config)
|
||||
prefilter ahocorasick.Trie
|
||||
// Maps for efficient lookups during detection.
|
||||
detectorTypeToDetectorInfo map[detectorKey]DetectorInfo
|
||||
detectors map[bool][]detectors.Detector
|
||||
keywordsToDetectors map[string][]detectorKey
|
||||
}
|
||||
|
||||
// NewAhoCorasickCore allocates and initializes a new instance of AhoCorasickCore.
|
||||
// It creates an empty keyword-to-detectors map for future string matching operations.
|
||||
// The map detectorTypeToDetectorInfo is pre-allocated based on the size of detectors
|
||||
// provided, for efficient storage and lookup of detector information.
|
||||
func NewAhoCorasickCore(detectors map[bool][]detectors.Detector) *AhoCorasickCore {
|
||||
return &AhoCorasickCore{
|
||||
keywordsToDetectors: make(map[string][]detectorKey),
|
||||
detectors: detectors,
|
||||
detectorTypeToDetectorInfo: make(map[detectorKey]DetectorInfo, len(detectors[true])+len(detectors[false])),
|
||||
}
|
||||
}
|
||||
|
||||
// Setup initializes the internal state of AhoCorasickCore to prepare it for keyword matching.
|
||||
// This involves pre-filtering setup and lookup optimization, critical for the engine's performance.
|
||||
func (ac *AhoCorasickCore) Setup(ctx context.Context) {
|
||||
// Prepare maps for fast detector lookups, instead of scanning through an array of detectors for every chunk.
|
||||
var keywords []string
|
||||
for verify, detectorsSet := range ac.detectors {
|
||||
for _, d := range detectorsSet {
|
||||
key := createDetectorKey(d)
|
||||
ac.detectorTypeToDetectorInfo[key] = DetectorInfo{Detector: d, ShouldVerify: verify}
|
||||
keywords = ac.extractAndMapKeywords(d, key, keywords)
|
||||
}
|
||||
}
|
||||
|
||||
// Use the Ahocorasick algorithm to create a trie structure for efficient keyword matching.
|
||||
// This ensures that we can rapidly match against a vast set of keywords without individually comparing each one.
|
||||
ac.prefilter = *ahocorasick.NewTrieBuilder().AddStrings(keywords).Build()
|
||||
ctx.Logger().V(4).Info("AhoCorasickCore Setup complete")
|
||||
}
|
||||
|
||||
// createDetectorKey creates a unique key for each detector. This key based on type and version,
|
||||
// it ensures faster lookups and reduces redundancy in our main detector store.
|
||||
func createDetectorKey(d detectors.Detector) detectorKey {
|
||||
detectorType := d.Type()
|
||||
var version int
|
||||
if v, ok := d.(detectors.Versioner); ok {
|
||||
version = v.Version()
|
||||
}
|
||||
return detectorKey{detectorType: detectorType, version: version}
|
||||
}
|
||||
|
||||
// extractAndMapKeywords captures keywords associated with each detector and maps them.
|
||||
// This allows us to quickly determine which detectors are relevant based on the presence of certain keywords.
|
||||
func (ac *AhoCorasickCore) extractAndMapKeywords(d detectors.Detector, key detectorKey, keywords []string) []string {
|
||||
for _, kw := range d.Keywords() {
|
||||
kwLower := strings.ToLower(kw)
|
||||
keywords = append(keywords, kwLower)
|
||||
ac.keywordsToDetectors[kwLower] = append(ac.keywordsToDetectors[kwLower], key)
|
||||
}
|
||||
return keywords
|
||||
}
|
||||
|
||||
// MatchString performs a string match using the Aho-Corasick algorithm, returning an array of matches.
|
||||
// Designed for internal use within the AhoCorasickCore component.
|
||||
func (ac *AhoCorasickCore) MatchString(input string) []*ahocorasick.Match {
|
||||
return ac.prefilter.MatchString(strings.ToLower(input))
|
||||
}
|
||||
|
||||
// PopulateDetectorsByMatch populates the given detectorMap based on the Aho-Corasick match results.
|
||||
// This method is designed to reuse the same map for performance optimization,
|
||||
// reducing the need for repeated allocations within each detector worker in the engine.
|
||||
func (ac *AhoCorasickCore) PopulateDetectorsByMatch(match *ahocorasick.Match, detectors map[detectorspb.DetectorType]DetectorInfo) bool {
|
||||
matchedKeys, ok := ac.keywordsToDetectors[match.MatchString()]
|
||||
if !ok {
|
||||
return false
|
||||
}
|
||||
for _, key := range matchedKeys {
|
||||
detectors[key.detectorType] = ac.detectorTypeToDetectorInfo[key]
|
||||
}
|
||||
return true
|
||||
}
|
||||
+10
-8
@@ -4,8 +4,6 @@ import (
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/anthropic"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/envoyapikey"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/huggingface"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/ip2location"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/openvpn"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/ramp"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/salesforce"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/snowflake"
|
||||
@@ -76,6 +74,7 @@ import (
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/azure"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/bannerbear"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/baremetrics"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/baseapiio"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/beamer"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/beebole"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/besnappy"
|
||||
@@ -86,6 +85,7 @@ import (
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/bitfinex"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/bitlyaccesstoken"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/bitmex"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/blablabus"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/blazemeter"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/blitapp"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/blocknative"
|
||||
@@ -192,6 +192,7 @@ import (
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/databox"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/databrickstoken"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/datadogtoken"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/datafire"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/datagov"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/deepai"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/deepgram"
|
||||
@@ -304,6 +305,7 @@ import (
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/gitlabv2"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/gitter"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/glassnode"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/glitterlyapi"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/gocanvas"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/gocardless"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/goodday"
|
||||
@@ -386,7 +388,6 @@ import (
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/livestorm"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/loadmill"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/locationiq"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/loggly"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/loginradius"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/lokalisetoken"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/loyverse"
|
||||
@@ -532,6 +533,7 @@ import (
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/qase"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/qualaroo"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/qubole"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/quickmetrics"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/rapidapi"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/rawg"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/razorpay"
|
||||
@@ -712,7 +714,6 @@ import (
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/urlscan"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/userflow"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/userstack"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/vagrantcloudpersonaltoken"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/vatlayer"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/vbout"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/vercel"
|
||||
@@ -1061,6 +1062,7 @@ func DefaultDetectors() []detectors.Detector {
|
||||
deepgram.Scanner{},
|
||||
brandfetch.Scanner{},
|
||||
typeform.Scanner{},
|
||||
blablabus.Scanner{},
|
||||
fxmarket.Scanner{},
|
||||
ipapi.Scanner{},
|
||||
clearbit.Scanner{},
|
||||
@@ -1186,6 +1188,7 @@ func DefaultDetectors() []detectors.Detector {
|
||||
restpackscreenshotapi.Scanner{},
|
||||
shortcut.Scanner{},
|
||||
// nasdaqdatalink.Scanner{},
|
||||
quickmetrics.Scanner{},
|
||||
neutrinoapi.Scanner{},
|
||||
bitmex.Scanner{},
|
||||
deepai.Scanner{},
|
||||
@@ -1211,6 +1214,7 @@ func DefaultDetectors() []detectors.Detector {
|
||||
scraperbox.Scanner{},
|
||||
passbase.Scanner{},
|
||||
ticketmaster.Scanner{},
|
||||
datafire.Scanner{},
|
||||
iexcloud.Scanner{},
|
||||
partnerstack.Scanner{},
|
||||
qubole.Scanner{},
|
||||
@@ -1244,6 +1248,7 @@ func DefaultDetectors() []detectors.Detector {
|
||||
scrapeowl.Scanner{},
|
||||
rebrandly.Scanner{},
|
||||
dandelion.Scanner{},
|
||||
baseapiio.Scanner{},
|
||||
purestake.Scanner{},
|
||||
carboninterface.Scanner{},
|
||||
signaturit.Scanner{},
|
||||
@@ -1398,6 +1403,7 @@ func DefaultDetectors() []detectors.Detector {
|
||||
convier.Scanner{},
|
||||
loadmill.Scanner{},
|
||||
magicbell.Scanner{},
|
||||
glitterlyapi.Scanner{},
|
||||
apitemplate.Scanner{},
|
||||
knapsackpro.Scanner{},
|
||||
twitter.Scanner{},
|
||||
@@ -1545,15 +1551,11 @@ func DefaultDetectors() []detectors.Detector {
|
||||
salesforce.Scanner{},
|
||||
sourcegraph.Scanner{},
|
||||
tailscale.Scanner{},
|
||||
loggly.Scanner{},
|
||||
web3storage.Scanner{},
|
||||
&ramp.Scanner{},
|
||||
&anthropic.Scanner{},
|
||||
&sourcegraphcody.Scanner{},
|
||||
voiceflow.Scanner{},
|
||||
ip2location.Scanner{},
|
||||
vagrantcloudpersonaltoken.Scanner{},
|
||||
openvpn.Scanner{},
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
+111
-99
@@ -3,11 +3,14 @@ package engine
|
||||
import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
"reflect"
|
||||
"runtime"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
ahocorasick "github.com/BobuSumisu/aho-corasick"
|
||||
lru "github.com/hashicorp/golang-lru"
|
||||
"google.golang.org/protobuf/proto"
|
||||
|
||||
@@ -63,8 +66,9 @@ type Engine struct {
|
||||
onlyVerified bool
|
||||
printAvgDetectorTime bool
|
||||
|
||||
// ahoCorasickHandler manages the Aho-Corasick trie and related keyword lookups.
|
||||
ahoCorasickCore *AhoCorasickCore
|
||||
// prefilter is a ahocorasick struct used for doing efficient string
|
||||
// matching given a set of words (keywords from the rules in the config)
|
||||
prefilter ahocorasick.Trie
|
||||
|
||||
// Engine synchronization primitives.
|
||||
sourceManager *sources.SourceManager
|
||||
@@ -89,10 +93,9 @@ type Engine struct {
|
||||
dedupeCache *lru.Cache
|
||||
}
|
||||
|
||||
// Option is used to configure the engine during initialization using functional options.
|
||||
type Option func(*Engine)
|
||||
type EngineOption func(*Engine)
|
||||
|
||||
func WithConcurrency(concurrency uint8) Option {
|
||||
func WithConcurrency(concurrency uint8) EngineOption {
|
||||
return func(e *Engine) {
|
||||
e.concurrency = concurrency
|
||||
}
|
||||
@@ -100,7 +103,7 @@ func WithConcurrency(concurrency uint8) Option {
|
||||
|
||||
const ignoreTag = "trufflehog:ignore"
|
||||
|
||||
func WithDetectors(verify bool, d ...detectors.Detector) Option {
|
||||
func WithDetectors(verify bool, d ...detectors.Detector) EngineOption {
|
||||
return func(e *Engine) {
|
||||
if e.detectors == nil {
|
||||
e.detectors = make(map[bool][]detectors.Detector)
|
||||
@@ -113,7 +116,7 @@ func WithDetectors(verify bool, d ...detectors.Detector) Option {
|
||||
}
|
||||
}
|
||||
|
||||
func WithDecoders(decoders ...decoders.Decoder) Option {
|
||||
func WithDecoders(decoders ...decoders.Decoder) EngineOption {
|
||||
return func(e *Engine) {
|
||||
e.decoders = decoders
|
||||
}
|
||||
@@ -121,14 +124,14 @@ func WithDecoders(decoders ...decoders.Decoder) Option {
|
||||
|
||||
// WithFilterUnverified sets the filterUnverified flag on the engine. If set to
|
||||
// true, the engine will only return the first unverified result for a chunk for a detector.
|
||||
func WithFilterUnverified(filter bool) Option {
|
||||
func WithFilterUnverified(filter bool) EngineOption {
|
||||
return func(e *Engine) {
|
||||
e.filterUnverified = filter
|
||||
}
|
||||
}
|
||||
|
||||
// WithFilterEntropy filters out unverified results using Shannon entropy.
|
||||
func WithFilterEntropy(entropy float64) Option {
|
||||
func WithFilterEntropy(entropy float64) EngineOption {
|
||||
return func(e *Engine) {
|
||||
if entropy > 0 {
|
||||
e.filterEntropy = &entropy
|
||||
@@ -138,7 +141,7 @@ func WithFilterEntropy(entropy float64) Option {
|
||||
|
||||
// WithOnlyVerified sets the onlyVerified flag on the engine. If set to true,
|
||||
// the engine will only print verified results.
|
||||
func WithOnlyVerified(onlyVerified bool) Option {
|
||||
func WithOnlyVerified(onlyVerified bool) EngineOption {
|
||||
return func(e *Engine) {
|
||||
e.onlyVerified = onlyVerified
|
||||
}
|
||||
@@ -150,7 +153,7 @@ func WithOnlyVerified(onlyVerified bool) Option {
|
||||
// the engine is configured to print the results.
|
||||
// Calculating the average time taken by each detector is an expensive operation
|
||||
// and should be avoided unless specified by the user.
|
||||
func WithPrintAvgDetectorTime(printAvgDetectorTime bool) Option {
|
||||
func WithPrintAvgDetectorTime(printAvgDetectorTime bool) EngineOption {
|
||||
return func(e *Engine) {
|
||||
e.printAvgDetectorTime = printAvgDetectorTime
|
||||
}
|
||||
@@ -160,7 +163,7 @@ func WithPrintAvgDetectorTime(printAvgDetectorTime bool) Option {
|
||||
// the filterFunc returns true, the detector will be included for scanning.
|
||||
// This option applies to the existing list of detectors configured, so the
|
||||
// order this option appears matters. All filtering happens before scanning.
|
||||
func WithFilterDetectors(filterFunc func(detectors.Detector) bool) Option {
|
||||
func WithFilterDetectors(filterFunc func(detectors.Detector) bool) EngineOption {
|
||||
return func(e *Engine) {
|
||||
// If no detectors are configured, do nothing.
|
||||
if e.detectors == nil {
|
||||
@@ -172,7 +175,7 @@ func WithFilterDetectors(filterFunc func(detectors.Detector) bool) Option {
|
||||
}
|
||||
|
||||
// WithPrinter sets the Printer on the engine.
|
||||
func WithPrinter(printer Printer) Option {
|
||||
func WithPrinter(printer Printer) EngineOption {
|
||||
return func(e *Engine) {
|
||||
e.printer = printer
|
||||
}
|
||||
@@ -266,60 +269,32 @@ func (e *Engine) DetectorAvgTime() map[string][]time.Duration {
|
||||
return avgTime
|
||||
}
|
||||
|
||||
// Start initializes and activates the engine's processing pipeline.
|
||||
// It sets up various default configurations, prepares lookup structures for
|
||||
// detectors, conducts basic sanity checks, and kickstarts all necessary workers.
|
||||
// Once started, the engine begins processing input data to identify secrets.
|
||||
func Start(ctx context.Context, options ...Option) (*Engine, error) {
|
||||
e := &Engine{}
|
||||
|
||||
if err := e.initialize(ctx, options...); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
e.setDefaults(ctx)
|
||||
ctx.Logger().V(4).Info("setting up aho-corasick core")
|
||||
e.ahoCorasickCore.Setup(ctx)
|
||||
e.sanityChecks(ctx)
|
||||
e.startWorkers(ctx)
|
||||
|
||||
return e, nil
|
||||
}
|
||||
|
||||
const defaultChannelBuffer = 1
|
||||
|
||||
// initialize prepares the engine's internal structures. The LRU cache optimizes
|
||||
// deduplication efforts, allowing the engine to quickly check if a chunk has
|
||||
// been processed before, thereby saving computational overhead.
|
||||
func (e *Engine) initialize(ctx context.Context, options ...Option) error {
|
||||
// TODO (ahrav): Determine the optimal cache size.
|
||||
const cacheSize = 512 // number of entries in the LRU cache
|
||||
// Start the engine with options.
|
||||
func Start(ctx context.Context, options ...EngineOption) (*Engine, error) {
|
||||
const (
|
||||
defaultChannelBuffer = 1
|
||||
// TODO (ahrav): Determine the optimal cache size.
|
||||
cacheSize = 512 // number of entries in the LRU cache
|
||||
)
|
||||
|
||||
cache, err := lru.New(cacheSize)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to initialize LRU cache: %w", err)
|
||||
return nil, fmt.Errorf("failed to initialize LRU cache: %w", err)
|
||||
}
|
||||
|
||||
// Channels are used for communication between different parts of the engine,
|
||||
// ensuring that data flows smoothly without race conditions.
|
||||
e.detectableChunksChan = make(chan detectableChunk, defaultChannelBuffer)
|
||||
e.results = make(chan detectors.ResultWithMetadata, defaultChannelBuffer)
|
||||
e.dedupeCache = cache
|
||||
e.printer = new(output.PlainPrinter)
|
||||
e.metrics = runtimeMetrics{Metrics: Metrics{scanStartTime: time.Now()}}
|
||||
e := &Engine{
|
||||
detectableChunksChan: make(chan detectableChunk, defaultChannelBuffer),
|
||||
results: make(chan detectors.ResultWithMetadata, defaultChannelBuffer),
|
||||
dedupeCache: cache,
|
||||
printer: new(output.PlainPrinter), // default printer
|
||||
metrics: runtimeMetrics{Metrics: Metrics{scanStartTime: time.Now()}},
|
||||
}
|
||||
|
||||
for _, option := range options {
|
||||
option(e)
|
||||
}
|
||||
ctx.Logger().V(4).Info("engine initialized")
|
||||
e.ahoCorasickCore = NewAhoCorasickCore(e.detectors)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// setDefaults ensures that if specific engine properties aren't provided,
|
||||
// they're set to reasonable default values. It makes the engine robust to
|
||||
// incomplete configuration.
|
||||
func (e *Engine) setDefaults(ctx context.Context) {
|
||||
// Set defaults.
|
||||
if e.concurrency == 0 {
|
||||
numCPU := runtime.NumCPU()
|
||||
ctx.Logger().Info("No concurrency specified, defaulting to max", "cpu", numCPU)
|
||||
@@ -327,13 +302,12 @@ func (e *Engine) setDefaults(ctx context.Context) {
|
||||
}
|
||||
ctx.Logger().V(3).Info("engine started", "workers", e.concurrency)
|
||||
|
||||
// Create SourceManager.
|
||||
e.sourceManager = sources.NewManager(
|
||||
sources.WithConcurrentSources(int(e.concurrency)),
|
||||
sources.WithConcurrentUnits(int(e.concurrency)),
|
||||
sources.WithBufferedOutput(defaultChannelBuffer),
|
||||
)
|
||||
|
||||
// Default decoders handle common encoding formats.
|
||||
if len(e.decoders) == 0 {
|
||||
e.decoders = decoders.DefaultDecoders()
|
||||
}
|
||||
@@ -343,30 +317,46 @@ func (e *Engine) setDefaults(ctx context.Context) {
|
||||
e.detectors[true] = DefaultDetectors()
|
||||
e.detectors[false] = []detectors.Detector{}
|
||||
}
|
||||
ctx.Logger().V(4).Info("default engine options set")
|
||||
}
|
||||
|
||||
// Sanity check detectors for duplicate configuration. Only log in case
|
||||
// a detector has been configured in a way that isn't represented by
|
||||
// the DetectorID (type and version).
|
||||
func (e *Engine) sanityChecks(ctx context.Context) {
|
||||
dets := append(e.detectors[true], e.detectors[false]...)
|
||||
seenDetectors := make(map[config.DetectorID]struct{}, len(dets))
|
||||
for _, det := range dets {
|
||||
id := config.GetDetectorID(det)
|
||||
if _, ok := seenDetectors[id]; ok && id.ID != detectorspb.DetectorType_CustomRegex {
|
||||
ctx.Logger().Info("possible duplicate detector configured", "detector", id)
|
||||
// build ahocorasick prefilter for efficient string matching
|
||||
// on keywords
|
||||
keywords := []string{}
|
||||
for _, d := range e.detectors[false] {
|
||||
for _, kw := range d.Keywords() {
|
||||
keywords = append(keywords, strings.ToLower(kw))
|
||||
}
|
||||
}
|
||||
for _, d := range e.detectors[true] {
|
||||
for _, kw := range d.Keywords() {
|
||||
keywords = append(keywords, strings.ToLower(kw))
|
||||
}
|
||||
}
|
||||
e.prefilter = *ahocorasick.NewTrieBuilder().AddStrings(keywords).Build()
|
||||
|
||||
ctx.Logger().V(3).Info("loaded decoders", "count", len(e.decoders))
|
||||
ctx.Logger().V(3).Info("loaded detectors",
|
||||
"total", len(e.detectors[true])+len(e.detectors[false]),
|
||||
"verification_enabled", len(e.detectors[true]),
|
||||
"verification_disabled", len(e.detectors[false]),
|
||||
)
|
||||
|
||||
// Sanity check detectors for duplicate configuration. Only log in case
|
||||
// a detector has been configured in a way that isn't represented by
|
||||
// the DetectorID (type and version).
|
||||
{
|
||||
dets := append(e.detectors[true], e.detectors[false]...)
|
||||
seenDetectors := make(map[config.DetectorID]struct{}, len(dets))
|
||||
for _, det := range dets {
|
||||
id := config.GetDetectorID(det)
|
||||
if _, ok := seenDetectors[id]; ok && id.ID != detectorspb.DetectorType_CustomRegex {
|
||||
ctx.Logger().Info("possible duplicate detector configured", "detector", id)
|
||||
}
|
||||
seenDetectors[id] = struct{}{}
|
||||
}
|
||||
seenDetectors[id] = struct{}{}
|
||||
}
|
||||
}
|
||||
|
||||
// startWorkers initiates all necessary workers. Workers handle processing of
|
||||
// chunks concurrently. Separating the initialization of different types of
|
||||
// workers helps in scalability and makes it easier to diagnose issues.
|
||||
func (e *Engine) startWorkers(ctx context.Context) {
|
||||
// Scanner workers process input data and extract chunks for detectors.
|
||||
ctx.Logger().V(2).Info("starting scanner workers", "count", e.concurrency)
|
||||
// Run the Secret scanner workers and Notifier pipelines.
|
||||
for worker := uint64(0); worker < uint64(e.concurrency); worker++ {
|
||||
e.workersWg.Add(1)
|
||||
go func() {
|
||||
@@ -377,7 +367,6 @@ func (e *Engine) startWorkers(ctx context.Context) {
|
||||
}()
|
||||
}
|
||||
|
||||
// Detector workers apply keyword matching, regexes and API calls to detect secrets in chunks.
|
||||
const detectorWorkerMultiplier = 50
|
||||
ctx.Logger().V(2).Info("starting detector workers", "count", e.concurrency*detectorWorkerMultiplier)
|
||||
for worker := uint64(0); worker < uint64(e.concurrency*detectorWorkerMultiplier); worker++ {
|
||||
@@ -390,7 +379,6 @@ func (e *Engine) startWorkers(ctx context.Context) {
|
||||
}()
|
||||
}
|
||||
|
||||
// Notifier workers communicate detected issues to the user or any downstream systems.
|
||||
// We want 1/4th of the notifier workers as the number of scanner workers.
|
||||
const notifierWorkerRatio = 4
|
||||
maxNotifierWorkers := 1
|
||||
@@ -407,6 +395,8 @@ func (e *Engine) startWorkers(ctx context.Context) {
|
||||
e.notifyResults(ctx)
|
||||
}()
|
||||
}
|
||||
|
||||
return e, nil
|
||||
}
|
||||
|
||||
// Finish waits for running sources to complete and workers to finish scanning
|
||||
@@ -455,42 +445,64 @@ type detectableChunk struct {
|
||||
func (e *Engine) detectorWorker(ctx context.Context) {
|
||||
var wgDetect sync.WaitGroup
|
||||
|
||||
// Reuse the same map to avoid allocations.
|
||||
const avgDetectorsPerChunk = 2
|
||||
chunkSpecificDetectors := make(map[detectorspb.DetectorType]DetectorInfo, avgDetectorsPerChunk)
|
||||
for originalChunk := range e.ChunksChan() {
|
||||
for chunk := range sources.Chunker(originalChunk) {
|
||||
matchedKeywords := make(map[string]struct{})
|
||||
atomic.AddUint64(&e.metrics.BytesScanned, uint64(len(chunk.Data)))
|
||||
for _, decoder := range e.decoders {
|
||||
var decoderType detectorspb.DecoderType
|
||||
switch decoder.(type) {
|
||||
case *decoders.UTF8:
|
||||
decoderType = detectorspb.DecoderType_PLAIN
|
||||
case *decoders.Base64:
|
||||
decoderType = detectorspb.DecoderType_BASE64
|
||||
case *decoders.UTF16:
|
||||
decoderType = detectorspb.DecoderType_UTF16
|
||||
default:
|
||||
ctx.Logger().Info("unknown decoder type", "type", reflect.TypeOf(decoder).String())
|
||||
decoderType = detectorspb.DecoderType_UNKNOWN
|
||||
}
|
||||
|
||||
decoded := decoder.FromChunk(chunk)
|
||||
|
||||
if decoded == nil {
|
||||
ctx.Logger().V(4).Info("no decoder found for chunk", "chunk", chunk)
|
||||
continue
|
||||
}
|
||||
|
||||
for _, match := range e.ahoCorasickCore.MatchString(string(decoded.Chunk.Data)) {
|
||||
if !e.ahoCorasickCore.PopulateDetectorsByMatch(match, chunkSpecificDetectors) {
|
||||
continue
|
||||
}
|
||||
// build a map of all keywords that were matched in the chunk
|
||||
for _, m := range e.prefilter.MatchString(strings.ToLower(string(decoded.Data))) {
|
||||
matchedKeywords[strings.ToLower(m.MatchString())] = struct{}{}
|
||||
}
|
||||
|
||||
for k, detector := range chunkSpecificDetectors {
|
||||
decoded.Chunk.Verify = detector.ShouldVerify
|
||||
wgDetect.Add(1)
|
||||
e.detectableChunksChan <- detectableChunk{
|
||||
chunk: *decoded.Chunk,
|
||||
detector: detector,
|
||||
decoder: decoded.DecoderType,
|
||||
wgDoneFn: wgDetect.Done,
|
||||
for verify, detectorsSet := range e.detectors {
|
||||
for _, detector := range detectorsSet {
|
||||
chunkContainsKeyword := false
|
||||
for _, kw := range detector.Keywords() {
|
||||
if _, ok := matchedKeywords[strings.ToLower(kw)]; ok {
|
||||
chunkContainsKeyword = true
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
if !chunkContainsKeyword {
|
||||
continue
|
||||
}
|
||||
|
||||
decoded.Verify = verify
|
||||
wgDetect.Add(1)
|
||||
e.detectableChunksChan <- detectableChunk{
|
||||
chunk: *decoded,
|
||||
detector: detector,
|
||||
decoder: decoderType,
|
||||
wgDoneFn: wgDetect.Done,
|
||||
}
|
||||
}
|
||||
delete(chunkSpecificDetectors, k)
|
||||
}
|
||||
}
|
||||
}
|
||||
atomic.AddUint64(&e.metrics.ChunksScanned, 1)
|
||||
}
|
||||
wgDetect.Wait()
|
||||
ctx.Logger().V(4).Info("finished scanning chunks")
|
||||
}
|
||||
|
||||
func (e *Engine) detectChunks(ctx context.Context) {
|
||||
@@ -623,7 +635,7 @@ func FragmentLineOffset(chunk *sources.Chunk, result *detectors.Result) (int64,
|
||||
if !found {
|
||||
return 0, false
|
||||
}
|
||||
lineNumber := int64(bytes.Count(before, []byte("\n")))
|
||||
lineNumber := int64(bytes.Count(before, []byte("\n")) + 1)
|
||||
// If the line contains the ignore tag, we should ignore the result.
|
||||
endLine := bytes.Index(after, []byte("\n"))
|
||||
if endLine == -1 {
|
||||
|
||||
@@ -32,7 +32,7 @@ func TestFragmentLineOffset(t *testing.T) {
|
||||
result: &detectors.Result{
|
||||
Raw: []byte("secret here"),
|
||||
},
|
||||
expectedLine: 2,
|
||||
expectedLine: 3,
|
||||
ignore: true,
|
||||
},
|
||||
{
|
||||
@@ -43,7 +43,7 @@ func TestFragmentLineOffset(t *testing.T) {
|
||||
result: &detectors.Result{
|
||||
Raw: []byte("secret here"),
|
||||
},
|
||||
expectedLine: 2,
|
||||
expectedLine: 3,
|
||||
ignore: false,
|
||||
},
|
||||
{
|
||||
@@ -54,7 +54,7 @@ func TestFragmentLineOffset(t *testing.T) {
|
||||
result: &detectors.Result{
|
||||
Raw: []byte("secret here"),
|
||||
},
|
||||
expectedLine: 4,
|
||||
expectedLine: 5,
|
||||
ignore: false,
|
||||
},
|
||||
{
|
||||
@@ -65,7 +65,7 @@ func TestFragmentLineOffset(t *testing.T) {
|
||||
result: &detectors.Result{
|
||||
Raw: []byte("secret\nhere"),
|
||||
},
|
||||
expectedLine: 4,
|
||||
expectedLine: 5,
|
||||
ignore: false,
|
||||
},
|
||||
{
|
||||
@@ -76,7 +76,7 @@ func TestFragmentLineOffset(t *testing.T) {
|
||||
result: &detectors.Result{
|
||||
Raw: []byte("secret\nhere"),
|
||||
},
|
||||
expectedLine: 3,
|
||||
expectedLine: 4,
|
||||
ignore: true,
|
||||
},
|
||||
{
|
||||
@@ -87,7 +87,7 @@ func TestFragmentLineOffset(t *testing.T) {
|
||||
result: &detectors.Result{
|
||||
Raw: []byte("secret here"),
|
||||
},
|
||||
expectedLine: 3,
|
||||
expectedLine: 4,
|
||||
ignore: true,
|
||||
},
|
||||
}
|
||||
|
||||
@@ -2,7 +2,6 @@ package engine
|
||||
|
||||
import (
|
||||
"os"
|
||||
"runtime"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
@@ -121,10 +120,9 @@ func BenchmarkGitEngine(b *testing.B) {
|
||||
defer cancel()
|
||||
|
||||
e, err := Start(ctx,
|
||||
WithConcurrency(uint8(runtime.NumCPU())),
|
||||
WithConcurrency(1),
|
||||
WithDecoders(decoders.DefaultDecoders()...),
|
||||
WithDetectors(false, DefaultDetectors()...),
|
||||
WithPrinter(new(discardPrinter)),
|
||||
)
|
||||
assert.Nil(b, err)
|
||||
|
||||
@@ -135,7 +133,6 @@ func BenchmarkGitEngine(b *testing.B) {
|
||||
}
|
||||
}()
|
||||
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
// TODO: this is measuring the time it takes to initialize the source
|
||||
// and not to do the full scan
|
||||
|
||||
@@ -11,7 +11,7 @@ import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
diskbufferreader "github.com/trufflesecurity/disk-buffer-reader"
|
||||
diskbufferreader "github.com/bill-rich/disk-buffer-reader"
|
||||
"github.com/stretchr/testify/assert"
|
||||
|
||||
logContext "github.com/trufflesecurity/trufflehog/v3/pkg/context"
|
||||
|
||||
@@ -4,7 +4,7 @@ import (
|
||||
"context"
|
||||
"io"
|
||||
|
||||
diskbufferreader "github.com/trufflesecurity/disk-buffer-reader"
|
||||
diskbufferreader "github.com/bill-rich/disk-buffer-reader"
|
||||
|
||||
logContext "github.com/trufflesecurity/trufflehog/v3/pkg/context"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
|
||||
|
||||
+1011
-1053
File diff suppressed because it is too large
Load Diff
+482
-501
File diff suppressed because it is too large
Load Diff
@@ -121,8 +121,6 @@ func (m *LocalSource) validate(all bool) error {
|
||||
}
|
||||
}
|
||||
|
||||
// no validation rules for ScanPeriod
|
||||
|
||||
if len(errors) > 0 {
|
||||
return LocalSourceMultiError(errors)
|
||||
}
|
||||
|
||||
@@ -7,7 +7,7 @@ import (
|
||||
"testing"
|
||||
"testing/iotest"
|
||||
|
||||
diskbufferreader "github.com/trufflesecurity/disk-buffer-reader"
|
||||
diskbufferreader "github.com/bill-rich/disk-buffer-reader"
|
||||
"github.com/stretchr/testify/assert"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/context"
|
||||
|
||||
@@ -7,7 +7,7 @@ import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
|
||||
diskbufferreader "github.com/trufflesecurity/disk-buffer-reader"
|
||||
diskbufferreader "github.com/bill-rich/disk-buffer-reader"
|
||||
"github.com/go-errors/errors"
|
||||
"github.com/go-logr/logr"
|
||||
"google.golang.org/protobuf/proto"
|
||||
|
||||
@@ -10,7 +10,7 @@ import (
|
||||
"sync"
|
||||
|
||||
"cloud.google.com/go/storage"
|
||||
diskbufferreader "github.com/trufflesecurity/disk-buffer-reader"
|
||||
diskbufferreader "github.com/bill-rich/disk-buffer-reader"
|
||||
"github.com/go-errors/errors"
|
||||
"github.com/go-logr/logr"
|
||||
"golang.org/x/oauth2"
|
||||
|
||||
@@ -13,7 +13,7 @@ import (
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
diskbufferreader "github.com/trufflesecurity/disk-buffer-reader"
|
||||
diskbufferreader "github.com/bill-rich/disk-buffer-reader"
|
||||
"github.com/go-errors/errors"
|
||||
"github.com/go-git/go-git/v5"
|
||||
"github.com/go-git/go-git/v5/plumbing"
|
||||
|
||||
@@ -222,10 +222,7 @@ func (s *Source) Init(aCtx context.Context, name string, jobID sources.JobID, so
|
||||
}
|
||||
s.conn = &conn
|
||||
|
||||
s.filteredRepoCache = s.newFilteredRepoCache(memory.New(),
|
||||
append(s.conn.GetRepositories(), s.conn.GetIncludeRepos()...),
|
||||
s.conn.GetIgnoreRepos(),
|
||||
)
|
||||
s.filteredRepoCache = s.newFilteredRepoCache(memory.New(), s.conn.GetRepositories(), s.conn.GetIgnoreRepos())
|
||||
s.memberCache = make(map[string]struct{})
|
||||
|
||||
s.repoSizes = newRepoSize()
|
||||
@@ -511,12 +508,12 @@ func (s *Source) enumerateUnauthenticated(ctx context.Context, apiEndpoint strin
|
||||
|
||||
for _, org := range s.orgsCache.Keys() {
|
||||
if err := s.getReposByOrg(ctx, org); err != nil {
|
||||
s.log.Error(err, "error fetching repos for org")
|
||||
s.log.Error(err, "error fetching repos for org or user")
|
||||
}
|
||||
|
||||
// We probably don't need to do this, since getting repos by org makes more sense?
|
||||
if err := s.getReposByUser(ctx, org); err != nil {
|
||||
s.log.Error(err, "error fetching repos for user")
|
||||
s.log.Error(err, "error fetching repos for org or user")
|
||||
}
|
||||
|
||||
if s.conn.ScanUsers {
|
||||
@@ -1092,7 +1089,7 @@ func (s *Source) scanComments(ctx context.Context, repoPath string, chunksChan c
|
||||
}
|
||||
|
||||
func (s *Source) processGistComments(ctx context.Context, repoPath string, trimmedURL []string, repoURL *url.URL, chunksChan chan *sources.Chunk) error {
|
||||
ctx.Logger().V(2).Info("scanning github gist comments", "repository", repoPath)
|
||||
s.log.Info("scanning github gist comments", "repository", repoPath)
|
||||
// GitHub Gist URL.
|
||||
gistID, err := extractGistID(trimmedURL)
|
||||
if err != nil {
|
||||
@@ -1173,21 +1170,19 @@ func (s *Source) processRepoComments(ctx context.Context, repoPath string, trimm
|
||||
}
|
||||
|
||||
if s.includeIssueComments {
|
||||
ctx.Logger().V(2).Info("scanning github issues", "repository", repoInfo.repoPath)
|
||||
if err := s.processIssues(ctx, repoInfo, chunksChan); err != nil {
|
||||
if err := s.processIssueComments(ctx, repoInfo, chunksChan); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := s.processIssueComments(ctx, repoInfo, chunksChan); err != nil {
|
||||
if err := s.processIssues(ctx, repoInfo, chunksChan); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
if s.includePRComments {
|
||||
ctx.Logger().V(2).Info("scanning github pull requests", "repository", repoInfo.repoPath)
|
||||
if err := s.processPRs(ctx, repoInfo, chunksChan); err != nil {
|
||||
if err := s.processPRComments(ctx, repoInfo, chunksChan); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := s.processPRComments(ctx, repoInfo, chunksChan); err != nil {
|
||||
if err := s.processPRs(ctx, repoInfo, chunksChan); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
@@ -1197,6 +1192,8 @@ func (s *Source) processRepoComments(ctx context.Context, repoPath string, trimm
|
||||
}
|
||||
|
||||
func (s *Source) processIssues(ctx context.Context, info repoInfo, chunksChan chan *sources.Chunk) error {
|
||||
s.log.Info("scanning github issue descriptions", "repository", info.repoPath)
|
||||
|
||||
bodyTextsOpts := &github.IssueListByRepoOptions{
|
||||
Sort: sortType,
|
||||
Direction: directionType,
|
||||
@@ -1231,6 +1228,8 @@ func (s *Source) processIssues(ctx context.Context, info repoInfo, chunksChan ch
|
||||
}
|
||||
|
||||
func (s *Source) processIssueComments(ctx context.Context, info repoInfo, chunksChan chan *sources.Chunk) error {
|
||||
s.log.Info("scanning github issue comments", "repository", info.repoPath)
|
||||
|
||||
issueOpts := &github.IssueListCommentsOptions{
|
||||
Sort: &sortType,
|
||||
Direction: &directionType,
|
||||
@@ -1264,6 +1263,8 @@ func (s *Source) processIssueComments(ctx context.Context, info repoInfo, chunks
|
||||
}
|
||||
|
||||
func (s *Source) processPRs(ctx context.Context, info repoInfo, chunksChan chan *sources.Chunk) error {
|
||||
s.log.Info("scanning github pull request descriptions", "repository", info.repoPath)
|
||||
|
||||
prOpts := &github.PullRequestListOptions{
|
||||
Sort: sortType,
|
||||
Direction: directionType,
|
||||
@@ -1298,6 +1299,8 @@ func (s *Source) processPRs(ctx context.Context, info repoInfo, chunksChan chan
|
||||
}
|
||||
|
||||
func (s *Source) processPRComments(ctx context.Context, info repoInfo, chunksChan chan *sources.Chunk) error {
|
||||
s.log.Info("scanning github pull request comments", "repository", info.repoPath)
|
||||
|
||||
prOpts := &github.PullRequestListCommentsOptions{
|
||||
Sort: sortType,
|
||||
Direction: directionType,
|
||||
|
||||
@@ -14,7 +14,7 @@ import (
|
||||
"github.com/aws/aws-sdk-go/service/s3"
|
||||
"github.com/aws/aws-sdk-go/service/s3/s3manager"
|
||||
"github.com/aws/aws-sdk-go/service/sts"
|
||||
diskbufferreader "github.com/trufflesecurity/disk-buffer-reader"
|
||||
diskbufferreader "github.com/bill-rich/disk-buffer-reader"
|
||||
"github.com/go-errors/errors"
|
||||
"github.com/go-logr/logr"
|
||||
"golang.org/x/sync/errgroup"
|
||||
|
||||
+4
-13
@@ -361,7 +361,7 @@ enum DetectorType {
|
||||
Imagga = 349;
|
||||
SMSApi = 350;
|
||||
Distribusion = 351;
|
||||
Blablabus = 352 [deprecated = true];
|
||||
Blablabus = 352;
|
||||
WordsApi = 353;
|
||||
Currencylayer = 354;
|
||||
Html2Pdf = 355;
|
||||
@@ -588,7 +588,7 @@ enum DetectorType {
|
||||
Qubole = 577;
|
||||
CarbonInterface = 578;
|
||||
Intrinio = 579;
|
||||
QuickMetrics = 580 [deprecated = true];
|
||||
QuickMetrics = 580;
|
||||
ScrapeStack = 581;
|
||||
TechnicalAnalysisApi = 582;
|
||||
Urlscan = 583;
|
||||
@@ -628,7 +628,7 @@ enum DetectorType {
|
||||
Nylas = 617;
|
||||
Squareup = 618;
|
||||
Dandelion = 619;
|
||||
DataFire = 620 [deprecated = true];
|
||||
DataFire = 620;
|
||||
DeepAI = 621;
|
||||
MeaningCloud = 622;
|
||||
NeutrinoApi = 623;
|
||||
@@ -780,7 +780,7 @@ enum DetectorType {
|
||||
ConversionTools = 769;
|
||||
CraftMyPDF = 770;
|
||||
ExportSDK = 771;
|
||||
GlitterlyAPI = 772 [deprecated = true];
|
||||
GlitterlyAPI = 772;
|
||||
Hybiscus = 773;
|
||||
Miro = 774;
|
||||
Statuspage = 775;
|
||||
@@ -946,15 +946,6 @@ enum DetectorType {
|
||||
Klaviyo = 935;
|
||||
SourcegraphCody = 936;
|
||||
Voiceflow = 937;
|
||||
Privacy = 938;
|
||||
IPInfo = 939;
|
||||
Ip2location = 940;
|
||||
Instamojo = 941;
|
||||
Portainer = 942;
|
||||
PortainerToken = 943;
|
||||
Loggly = 944;
|
||||
OpenVpn = 945;
|
||||
VagrantCloudPersonalToken = 946;
|
||||
}
|
||||
|
||||
message Result {
|
||||
|
||||
+1
-8
@@ -49,16 +49,9 @@ enum SourceType {
|
||||
message LocalSource {
|
||||
string type = 1 ;
|
||||
string name = 2 ;
|
||||
// DEPRECATED: scan_interval is deprecated and can be removed when we no longer depend on the name.
|
||||
// Deprecating in favor of scan_period due to the fact that scan_interval is a duration
|
||||
// which is a fixed-length span of time represented as a count of seconds and fractions of seconds
|
||||
// at nanosecond resolution. Most of the time, we want to be able to specify a scan interval in
|
||||
// human-readable format (e.g. 45s, 30m, 12h, etc.) which is not possible with a duration.
|
||||
// https://protobuf.dev/reference/protobuf/google.protobuf/#duration
|
||||
google.protobuf.Duration scan_interval = 3 [deprecated = true];
|
||||
google.protobuf.Duration scan_interval = 3;
|
||||
bool verify = 4;
|
||||
google.protobuf.Any connection = 5;
|
||||
string scan_period = 6;
|
||||
}
|
||||
|
||||
message AzureStorage {
|
||||
|
||||
Reference in New Issue
Block a user