Compare commits

..

23 Commits

Author SHA1 Message Date
Ahrav Dutta 584ef1429b exported some methods and constructor since it will need to be be used by the enterprise pipeline as well 2023-10-22 19:18:41 -07:00
Ahrav Dutta f010f06edc address comments 2023-10-20 16:12:56 -07:00
Ahrav Dutta 60df4d6684 fix 2023-10-12 13:27:35 -07:00
ahrav 0a4a5f3ab4 Merge branch 'main' into filter-detectors-before-detecting 2023-10-12 13:26:40 -07:00
Ahrav Dutta 88c8f69532 update comments 2023-10-12 13:22:01 -07:00
Ahrav Dutta d144e9dc9b move structs to ahocorasickcore 2023-10-12 13:18:18 -07:00
Ahrav Dutta e43d95f1c5 update comments 2023-10-12 13:17:03 -07:00
Ahrav Dutta 0064b00c5f refactor ahocorasickcore into own file. 2023-10-12 13:10:38 -07:00
Ahrav Dutta ff3751fda2 remove old code. 2023-08-29 14:35:38 -07:00
Ahrav Dutta 5089383e71 delete the pool. 2023-08-29 14:32:06 -07:00
Ahrav Dutta 1311e7df40 use regular map. 2023-08-28 09:10:15 -07:00
Ahrav Dutta b90bb8cbe2 update comments. 2023-08-28 07:16:18 -07:00
Ahrav Dutta 621bb75f40 remove Engine prefix. 2023-08-28 07:01:09 -07:00
Ahrav Dutta 916a4f9ecf add comment. 2023-08-28 06:56:50 -07:00
Ahrav Dutta 11a0d1d31b Update benchmark. 2023-08-27 19:57:48 -07:00
Ahrav Dutta 04e4bc1dcf cleanup. 2023-08-27 16:55:39 -07:00
Ahrav Dutta 7dd3aa639d Break apart Start. 2023-08-27 16:42:22 -07:00
Ahrav Dutta 22564bc483 allow versioned detectors. 2023-08-27 16:23:24 -07:00
Ahrav Dutta 32bacdda7e use correct verify. 2023-08-27 15:47:28 -07:00
Ahrav Dutta 4c44a31d10 lint. 2023-08-27 11:36:46 -07:00
Ahrav Dutta 913a94f4b4 use sync.Map for concurrent access. 2023-08-27 09:54:28 -07:00
Ahrav Dutta 9f4623958d Optimize the engine to prevent iterating overing all detectors. 2023-08-26 21:29:12 -07:00
Ahrav Dutta f071a2ff14 pre filter detectors that include the keywords in the chunk. 2023-08-26 13:52:08 -07:00
77 changed files with 2061 additions and 4992 deletions
+15
View File
@@ -0,0 +1,15 @@
# To get started with Dependabot version updates, you'll need to specify which
# package ecosystems to update and where the package manifests are located.
# Please see the documentation for all configuration options:
# https://help.github.com/github/administering-a-repository/configuration-options-for-dependency-updates
version: 2
updates:
- package-ecosystem: "gomod" # See documentation for possible values
directory: "/" # Location of package manifests
schedule:
interval: "monthly"
- package-ecosystem: "github-actions"
directory: "/"
schedule:
interval: "monthly"
-8
View File
@@ -1,8 +0,0 @@
{
"$schema": "https://docs.renovatebot.com/renovate-schema.json",
"extends": [
"config:base"
],
"prConcurrentLimit": 3,
"prHourlyLimit": 2
}
+3 -3
View File
@@ -8,9 +8,9 @@ RUN --mount=type=cache,target=/go/pkg/mod \
--mount=type=cache,target=/root/.cache/go-build \
GOOS=${TARGETOS} GOARCH=${TARGETARCH} go build -o trufflehog .
FROM alpine:3.18
RUN apk add --no-cache bash git openssh-client ca-certificates rpm2cpio \
&& rm -rf /var/cache/apk/* && update-ca-certificates
FROM alpine:3.15
RUN apk add --no-cache bash git openssh-client ca-certificates \
&& update-ca-certificates
COPY --from=builder /build/trufflehog /usr/bin/trufflehog
COPY entrypoint.sh /etc/entrypoint.sh
RUN chmod +x /etc/entrypoint.sh
+4 -3
View File
@@ -1,7 +1,8 @@
FROM alpine:3.18
FROM alpine:3.15
RUN apk add --no-cache bash git openssh-client ca-certificates rpm2cpio \
&& rm -rf /var/cache/apk/* && update-ca-certificates
RUN apk add --no-cache bash git openssh-client ca-certificates \
&& rm -rf /var/cache/apk/* && \
update-ca-certificates
WORKDIR /usr/bin/
COPY trufflehog .
COPY entrypoint.sh /etc/entrypoint.sh
+2 -2
View File
@@ -52,10 +52,10 @@ run-debug:
CGO_ENABLED=0 go run . git file://. --json --debug
protos:
docker run --rm -u "$(shell id -u)" -v "$(shell dirname $(realpath $(firstword $(MAKEFILE_LIST))))":/pwd "${PROTOS_IMAGE}" bash -c "cd /pwd; /pwd/scripts/gen_proto.sh"
docker run -u "$(shell id -u)" -v "$(shell dirname $(realpath $(firstword $(MAKEFILE_LIST))))":/pwd "${PROTOS_IMAGE}" bash -c "cd /pwd; /pwd/scripts/gen_proto.sh"
protos-windows:
docker run --rm -v "$(shell cygpath -w $(shell pwd))":/pwd "${PROTOS_IMAGE}" bash -c "cd /pwd; ./scripts/gen_proto.sh"
docker run -v "$(shell cygpath -w $(shell pwd))":/pwd "${PROTOS_IMAGE}" bash -c "cd /pwd; ./scripts/gen_proto.sh"
release-protos-image:
docker buildx build --push --platform=linux/amd64,linux/arm64 \
+2 -2
View File
@@ -164,7 +164,7 @@ trufflehog docker --image trufflesecurity/secrets --only-verified
+ All I see is `🐷🔑🐷 TruffleHog. Unearth your secrets. 🐷🔑🐷` and the program exits, what gives?
+ That means no secrets were detected
+ Why is the scan taking a long time when I scan a GitHub org
+ Why is the scan is taking a long time when I scan a GitHub org
+ Unauthenticated GitHub scans have rate limits. To improve your rate limits, include the `--token` flag with a personal access token
+ It says a private key was verified, what does that mean?
+ Check out our Driftwood blog post to learn how to do this, in short we've confirmed the key can be used live for SSH or SSL [Blog post](https://trufflesecurity.com/blog/driftwood-know-if-private-keys-are-sensitive/)
@@ -219,7 +219,7 @@ Flags:
--concurrency=10 Number of concurrent workers.
--no-verification Don't verify the results.
--only-verified Only output verified results.
--filter-unverified Only output first unverified result per chunk per detector if there is more than one result.
--filter-unverified Only output first unverified result per chunk per detector if there are more than one results.
--config=CONFIG Path to configuration file.
--print-avg-detector-time Print the average time spent on each detector.
--no-update Don't check for updates.
+22 -22
View File
@@ -2,17 +2,21 @@ module github.com/trufflesecurity/trufflehog/v3
go 1.21
replace github.com/jpillora/overseer => github.com/trufflesecurity/overseer v1.2.7
replace github.com/jpillora/overseer => github.com/trufflesecurity/overseer v1.1.7-custom5
// go wants to pull in the latest but after v10.2.5 has a breaking API change
replace github.com/couchbase/gocbcore/v10 => github.com/couchbase/gocbcore/v10 v10.2.4
require (
cloud.google.com/go/secretmanager v1.11.2
cloud.google.com/go/secretmanager v1.11.1
cloud.google.com/go/storage v1.33.0
github.com/Azure/go-autorest/autorest/azure/auth v0.5.12
github.com/Azure/go-autorest/autorest/azure/auth v0.5.11
github.com/AzureAD/microsoft-authentication-library-for-go v1.2.0
github.com/BobuSumisu/aho-corasick v1.0.3
github.com/TheZeroSlave/zapsentry v1.17.0
github.com/aws/aws-sdk-go v1.45.19
github.com/aymanbagabas/go-osc52 v1.2.2
github.com/aymanbagabas/go-osc52 v1.2.1
github.com/bill-rich/disk-buffer-reader v0.1.7
github.com/bill-rich/go-syslog v0.0.0-20220413021637-49edb52a574c
github.com/bitfinexcom/bitfinex-api-go v0.0.0-20210608095005-9e0b26f200fb
github.com/bradleyfalzon/ghinstallation/v2 v2.7.0
@@ -20,14 +24,14 @@ require (
github.com/charmbracelet/bubbletea v0.24.2
github.com/charmbracelet/glamour v0.6.0
github.com/charmbracelet/lipgloss v0.7.1
github.com/couchbase/gocb/v2 v2.6.5
github.com/couchbase/gocb/v2 v2.6.3
github.com/crewjam/rfc5424 v0.1.0
github.com/denisenkom/go-mssqldb v0.12.3
github.com/envoyproxy/protoc-gen-validate v1.0.2
github.com/fatih/color v1.15.0
github.com/felixge/fgprof v0.9.3
github.com/getsentry/sentry-go v0.24.1
github.com/go-errors/errors v1.5.1
github.com/go-errors/errors v1.4.2
github.com/go-git/go-git/v5 v5.8.1
github.com/go-ldap/ldap/v3 v3.4.6
github.com/go-logr/logr v1.2.4
@@ -36,24 +40,22 @@ require (
github.com/go-sql-driver/mysql v1.7.1
github.com/gobwas/glob v0.2.3
github.com/golang-jwt/jwt v3.2.2+incompatible
github.com/google/go-cmp v0.6.0
github.com/google/go-cmp v0.5.9
github.com/google/go-containerregistry v0.15.2
github.com/google/go-github/v42 v42.0.0
github.com/google/go-github/v56 v56.0.0
github.com/googleapis/gax-go/v2 v2.12.0
github.com/h2non/filetype v1.1.3
github.com/hashicorp/go-retryablehttp v0.7.4
github.com/hashicorp/golang-lru v0.5.1
github.com/hashicorp/golang-lru/v2 v2.0.7
github.com/jlaffaye/ftp v0.2.0
github.com/joho/godotenv v1.5.1
github.com/jpillora/overseer v1.1.6
github.com/kylelemons/godebug v1.1.0
github.com/launchdarkly/go-server-sdk/v6 v6.1.1
github.com/launchdarkly/go-server-sdk/v6 v6.1.0
github.com/lib/pq v1.10.9
github.com/lrstanley/bubblezone v0.0.0-20230911164824-e3824f1adde9
github.com/lrstanley/bubblezone v0.0.0-20221222153816-e95291e2243e
github.com/marusama/semaphore/v2 v2.5.0
github.com/mattn/go-isatty v0.0.20
github.com/mattn/go-isatty v0.0.19
github.com/mattn/go-sqlite3 v1.14.17
github.com/mholt/archiver/v4 v4.0.0-alpha.8
github.com/muesli/reflow v0.3.0
@@ -63,24 +65,22 @@ require (
github.com/prometheus/client_golang v1.16.0
github.com/rabbitmq/amqp091-go v1.8.1
github.com/sergi/go-diff v1.3.1
github.com/snowflakedb/gosnowflake v1.6.25
github.com/snowflakedb/gosnowflake v1.6.23
github.com/stretchr/testify v1.8.4
github.com/tailscale/depaware v0.0.0-20210622194025-720c4b409502
github.com/trufflesecurity/disk-buffer-reader v0.1.9
github.com/xanzy/go-gitlab v0.92.3
go.mongodb.org/mongo-driver v1.12.1
go.uber.org/mock v0.2.0
go.uber.org/zap v1.24.0
golang.org/x/crypto v0.14.0
golang.org/x/exp v0.0.0-20231006140011-7918f672742d
golang.org/x/oauth2 v0.13.0
golang.org/x/sync v0.4.0
golang.org/x/exp v0.0.0-20230206171751-46f607a40771
golang.org/x/oauth2 v0.12.0
golang.org/x/sync v0.3.0
golang.org/x/text v0.13.0
google.golang.org/api v0.132.0
google.golang.org/protobuf v1.31.0
gopkg.in/alecthomas/kingpin.v2 v2.2.6
gopkg.in/h2non/gock.v1 v1.1.2
pgregory.net/rapid v1.1.0
sigs.k8s.io/yaml v1.3.0
)
@@ -139,7 +139,7 @@ require (
github.com/connesc/cipherio v0.2.1 // indirect
github.com/containerd/console v1.0.4-0.20230313162750-1ae8d489ac81 // indirect
github.com/containerd/stargz-snapshotter/estargz v0.14.3 // indirect
github.com/couchbase/gocbcore/v10 v10.2.9 // indirect
github.com/couchbase/gocbcore/v10 v10.2.4 // indirect
github.com/danieljoos/wincred v1.1.2 // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/dimchansky/utfbom v1.1.1 // indirect
@@ -192,7 +192,7 @@ require (
github.com/launchdarkly/eventsource v1.6.2 // indirect
github.com/launchdarkly/go-jsonstream/v3 v3.0.0 // indirect
github.com/launchdarkly/go-sdk-common/v3 v3.0.1 // indirect
github.com/launchdarkly/go-sdk-events/v2 v2.0.2 // indirect
github.com/launchdarkly/go-sdk-events/v2 v2.0.1 // indirect
github.com/launchdarkly/go-semver v1.0.2 // indirect
github.com/launchdarkly/go-server-sdk-evaluation/v2 v2.0.2 // indirect
github.com/lucasb-eyer/go-colorful v1.2.0 // indirect
@@ -245,12 +245,12 @@ require (
go.uber.org/atomic v1.7.0 // indirect
go.uber.org/multierr v1.6.0 // indirect
go4.org v0.0.0-20200411211856-f5505b9728dd // indirect
golang.org/x/mod v0.13.0 // indirect
golang.org/x/mod v0.12.0 // indirect
golang.org/x/net v0.17.0 // indirect
golang.org/x/sys v0.13.0 // indirect
golang.org/x/term v0.13.0 // indirect
golang.org/x/time v0.3.0 // indirect
golang.org/x/tools v0.14.0 // indirect
golang.org/x/tools v0.12.0 // indirect
golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect
google.golang.org/appengine v1.6.7 // indirect
google.golang.org/genproto v0.0.0-20230706204954-ccb25ca9f130 // indirect
+2 -43
View File
@@ -22,8 +22,6 @@ cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2k
cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw=
cloud.google.com/go/secretmanager v1.11.1 h1:cLTCwAjFh9fKvU6F13Y4L9vPcx9yiWPyWXE4+zkuEQs=
cloud.google.com/go/secretmanager v1.11.1/go.mod h1:znq9JlXgTNdBeQk9TBW/FnR/W4uChEKGeqQWAJ8SXFw=
cloud.google.com/go/secretmanager v1.11.2 h1:52Z78hH8NBWIqbvIG0wi0EoTaAmSx99KIOAmDXIlX0M=
cloud.google.com/go/secretmanager v1.11.2/go.mod h1:MQm4t3deoSub7+WNwiC4/tRYgDBHJgJPvswqQVB1Vss=
cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw=
cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos=
cloud.google.com/go/storage v1.33.0 h1:PVrDOkIC8qQVa1P3SXGpQvfuJhN2LHOoyZvWs8D2X5M=
@@ -54,8 +52,6 @@ github.com/Azure/go-autorest/autorest/adal v0.9.18 h1:kLnPsRjzZZUF3K5REu/Kc+qMQr
github.com/Azure/go-autorest/autorest/adal v0.9.18/go.mod h1:XVVeme+LZwABT8K5Lc3hA4nAe8LDBVle26gTrguhhPQ=
github.com/Azure/go-autorest/autorest/azure/auth v0.5.11 h1:P6bYXFoao05z5uhOQzbC3Qd8JqF3jUoocoTeIxkp2cA=
github.com/Azure/go-autorest/autorest/azure/auth v0.5.11/go.mod h1:84w/uV8E37feW2NCJ08uT9VBfjfUHpgLVnG2InYD6cg=
github.com/Azure/go-autorest/autorest/azure/auth v0.5.12 h1:wkAZRgT/pn8HhFyzfe9UnqOjJYqlembgCTi72Bm/xKk=
github.com/Azure/go-autorest/autorest/azure/auth v0.5.12/go.mod h1:84w/uV8E37feW2NCJ08uT9VBfjfUHpgLVnG2InYD6cg=
github.com/Azure/go-autorest/autorest/azure/cli v0.4.5 h1:0W/yGmFdTIT77fvdlGZ0LMISoLHFJ7Tx4U0yeB+uFs4=
github.com/Azure/go-autorest/autorest/azure/cli v0.4.5/go.mod h1:ADQAXrkgm7acgWVUNamOgh8YNrv4p27l3Wc55oVfpzg=
github.com/Azure/go-autorest/autorest/date v0.3.0 h1:7gUk1U5M/CQbp9WoqinNzJar+8KY+LPI6wiWrP/myHw=
@@ -151,8 +147,6 @@ github.com/aws/smithy-go v1.13.5/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J
github.com/aymanbagabas/go-osc52 v1.0.3/go.mod h1:zT8H+Rk4VSabYN90pWyugflM3ZhpTZNC7cASDfUCdT4=
github.com/aymanbagabas/go-osc52 v1.2.1 h1:q2sWUyDcozPLcLabEMd+a+7Ea2DitxZVN9hTxab9L4E=
github.com/aymanbagabas/go-osc52 v1.2.1/go.mod h1:zT8H+Rk4VSabYN90pWyugflM3ZhpTZNC7cASDfUCdT4=
github.com/aymanbagabas/go-osc52 v1.2.2 h1:NT7wkhEhPTcKnBCdPi9djmyy9L3JOL4+3SsfJyqptCo=
github.com/aymanbagabas/go-osc52 v1.2.2/go.mod h1:zT8H+Rk4VSabYN90pWyugflM3ZhpTZNC7cASDfUCdT4=
github.com/aymanbagabas/go-osc52/v2 v2.0.1 h1:HwpRHbFMcZLEVr42D4p7XBqjyuxQH5SMiErDT4WkJ2k=
github.com/aymanbagabas/go-osc52/v2 v2.0.1/go.mod h1:uYgXzlJ7ZpABp8OJ+exZzJJhRNQ2ASbcXHWsFqH8hp8=
github.com/aymerick/douceur v0.2.0 h1:Mv+mAeH1Q+n9Fr+oyamOlAkUNPWPlA8PPGR0QAaYuPk=
@@ -161,6 +155,8 @@ github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLj
github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA=
github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
github.com/bill-rich/disk-buffer-reader v0.1.7 h1:oPgDjsbeqErOrFJeWID/51LhIOGjU3cqN1Tj3V6RiwE=
github.com/bill-rich/disk-buffer-reader v0.1.7/go.mod h1:VVzzsK1Ac2AnpOfp/5r9JlIFaFkZ9uSf7zisZayCt0Y=
github.com/bill-rich/go-syslog v0.0.0-20220413021637-49edb52a574c h1:tSME5FDS02qQll3JYodI6RZR/g4EKOHApGv1wMZT+Z0=
github.com/bill-rich/go-syslog v0.0.0-20220413021637-49edb52a574c/go.mod h1:+sCc6hztur+oZCLOsNk6wCCy+GLrnSNHSRmTnnL+8iQ=
github.com/bitfinexcom/bitfinex-api-go v0.0.0-20210608095005-9e0b26f200fb h1:9v7Bzlg+1EBYi2IYcUmOwHReBEfqBbYIj3ZCi9cIe1Q=
@@ -208,12 +204,8 @@ github.com/containerd/stargz-snapshotter/estargz v0.14.3 h1:OqlDCK3ZVUO6C3B/5FSk
github.com/containerd/stargz-snapshotter/estargz v0.14.3/go.mod h1:KY//uOCIkSuNAHhJogcZtrNHdKrA99/FCCRjE3HD36o=
github.com/couchbase/gocb/v2 v2.6.3 h1:5RsMo+RRfK0mVxHLAfpBz3/tHlgXZb1WBNItLk9Ab+c=
github.com/couchbase/gocb/v2 v2.6.3/go.mod h1:yF5F6BHTZ/ZowhEuZbySbXrlI4rHd1TIhm5azOaMbJU=
github.com/couchbase/gocb/v2 v2.6.5 h1:xaZu29o8UJEV1ZQ3n2s9jcRCUHz/JsQ6+y6JBnVsy5A=
github.com/couchbase/gocb/v2 v2.6.5/go.mod h1:0vFM09y+VPhnXeNrIb8tS0wKHGpJvjJBrJnriWEiwGs=
github.com/couchbase/gocbcore/v10 v10.2.4 h1:TDTQ1mSBUw9eajuV71ZDJcLRbmfGcsvZV9SMeUDAVGc=
github.com/couchbase/gocbcore/v10 v10.2.4/go.mod h1:lYQIIk+tzoMcwtwU5GzPbDdqEkwkH3isI2rkSpfL0oM=
github.com/couchbase/gocbcore/v10 v10.2.9 h1:zph/+ceu3JtZEDKhJMTRc6lGrahq+mnlQY/1dSepJuE=
github.com/couchbase/gocbcore/v10 v10.2.9/go.mod h1:lYQIIk+tzoMcwtwU5GzPbDdqEkwkH3isI2rkSpfL0oM=
github.com/couchbaselabs/gocaves/client v0.0.0-20230307083111-cc3960c624b1/go.mod h1:AVekAZwIY2stsJOMWLAS/0uA/+qdp7pjO8EHnl61QkY=
github.com/couchbaselabs/gocaves/client v0.0.0-20230404095311-05e3ba4f0259 h1:2TXy68EGEzIMHOx9UvczR5ApVecwCfQZ0LjkmwMI6g4=
github.com/couchbaselabs/gocaves/client v0.0.0-20230404095311-05e3ba4f0259/go.mod h1:AVekAZwIY2stsJOMWLAS/0uA/+qdp7pjO8EHnl61QkY=
@@ -279,8 +271,6 @@ github.com/go-asn1-ber/asn1-ber v1.5.5 h1:MNHlNMBDgEKD4TcKr36vQN68BA00aDfjIt3/bD
github.com/go-asn1-ber/asn1-ber v1.5.5/go.mod h1:hEBeB/ic+5LoWskz+yKT7vGhhPYkProFKoKdwZRWMe0=
github.com/go-errors/errors v1.4.2 h1:J6MZopCL4uSllY1OfXM374weqZFFItUbrImctkmUxIA=
github.com/go-errors/errors v1.4.2/go.mod h1:sIVyrIiJhuEF+Pj9Ebtd6P/rEYROXFi3BopGUQ5a5Og=
github.com/go-errors/errors v1.5.1 h1:ZwEMSLRCapFLflTpT7NKaAc7ukJ8ZPEjzlxt8rPN8bk=
github.com/go-errors/errors v1.5.1/go.mod h1:sIVyrIiJhuEF+Pj9Ebtd6P/rEYROXFi3BopGUQ5a5Og=
github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 h1:+zs/tPmkDkHx3U66DAb0lQFJrpS6731Oaa12ikc+DiI=
github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376/go.mod h1:an3vInlBmSxCcxctByoQdvwPiA7DTK7jaaFDBTtu0ic=
github.com/go-git/go-billy/v5 v5.4.1 h1:Uwp5tDRkPr+l/TnbHOQzp+tmJfLceOlbVucgpTz8ix4=
@@ -373,15 +363,12 @@ github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/
github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
github.com/google/go-containerregistry v0.15.2 h1:MMkSh+tjSdnmJZO7ljvEqV1DjfekB6VUEAZgy3a+TQE=
github.com/google/go-containerregistry v0.15.2/go.mod h1:wWK+LnOv4jXMM23IT/F1wdYftGWGr47Is8CG+pmHK1Q=
github.com/google/go-github/v42 v42.0.0 h1:YNT0FwjPrEysRkLIiKuEfSvBPCGKphW5aS5PxwaoLec=
github.com/google/go-github/v42 v42.0.0/go.mod h1:jgg/jvyI0YlDOM1/ps6XYh04HNQ3vKf0CVko62/EhRg=
github.com/google/go-github/v55 v55.0.0 h1:4pp/1tNMB9X/LuAhs5i0KQAE40NmiR/y6prLNb9x9cg=
github.com/google/go-github/v55 v55.0.0/go.mod h1:JLahOTA1DnXzhxEymmFF5PP2tSS9JVNj68mSZNDwskA=
github.com/google/go-github/v56 v56.0.0/go.mod h1:D8cdcX98YWJvi7TLo7zM4/h8ZTx6u6fwGEkCdisopo0=
github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8=
github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU=
github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPgecwXBIDzw5no=
@@ -431,8 +418,6 @@ github.com/hashicorp/go-retryablehttp v0.7.4/go.mod h1:Jy/gPYAdjqffZ/yFGCFV2doI5
github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
github.com/hashicorp/golang-lru v0.5.1 h1:0hERBMJE1eitiLkihrMvRVBYAkpHzc/J3QdDN+dAcgU=
github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k=
github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM=
github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
github.com/ianlancetaylor/demangle v0.0.0-20210905161508-09a460cdf81d/go.mod h1:aYm2/VgdVmcIU8iMfdMvDMsRAQjcfZSKFby6HOFvi/w=
@@ -488,16 +473,12 @@ github.com/launchdarkly/go-sdk-common/v3 v3.0.1 h1:rVdLusAIViduNvyjNKy06RA+SPwk0
github.com/launchdarkly/go-sdk-common/v3 v3.0.1/go.mod h1:H/zISoCNhviHTTqqBjIKQy2YgSHT8ioL1FtgBKpiEGg=
github.com/launchdarkly/go-sdk-events/v2 v2.0.1 h1:vnUN2Y7og/5wtOCcCZW7wYpmZcS++GAyclasc7gaTIY=
github.com/launchdarkly/go-sdk-events/v2 v2.0.1/go.mod h1:Msqbl6brgFO83RUxmLaJAUx2sYG+WKULcy+Vf3+tKww=
github.com/launchdarkly/go-sdk-events/v2 v2.0.2 h1:Ngp050AGYUyN3YRxX7zfu110NETHEPEOVys1ey2Hyvw=
github.com/launchdarkly/go-sdk-events/v2 v2.0.2/go.mod h1:Msqbl6brgFO83RUxmLaJAUx2sYG+WKULcy+Vf3+tKww=
github.com/launchdarkly/go-semver v1.0.2 h1:sYVRnuKyvxlmQCnCUyDkAhtmzSFRoX6rG2Xa21Mhg+w=
github.com/launchdarkly/go-semver v1.0.2/go.mod h1:xFmMwXba5Mb+3h72Z+VeSs9ahCvKo2QFUTHRNHVqR28=
github.com/launchdarkly/go-server-sdk-evaluation/v2 v2.0.2 h1:PAM0GvE0nIUBeOkjdiymIEKI+8FFLJ+fEsWTupW1yGU=
github.com/launchdarkly/go-server-sdk-evaluation/v2 v2.0.2/go.mod h1:Mztipcz+7ZMatXVun3k/IfPa8IOgUnAqiZawtFh2MRg=
github.com/launchdarkly/go-server-sdk/v6 v6.1.0 h1:pp/VBIon5JNbtSw7ih7I5MXN9mXHfo6T5xikKVy52dI=
github.com/launchdarkly/go-server-sdk/v6 v6.1.0/go.mod h1:3TfS6vNsNksPT4LGeGuGKtT3zTjTbbOsCUK3nrj1neA=
github.com/launchdarkly/go-server-sdk/v6 v6.1.1 h1:L57BC9Fy4IJG0zRZoYBkoxko6QTAC2OF8QzTVUqn2p8=
github.com/launchdarkly/go-server-sdk/v6 v6.1.1/go.mod h1:67QqqyeRNyXx7S/fZiSZesRvaePCyxhzsnY+4fhK55U=
github.com/launchdarkly/go-test-helpers/v2 v2.2.0 h1:L3kGILP/6ewikhzhdNkHy1b5y4zs50LueWenVF0sBbs=
github.com/launchdarkly/go-test-helpers/v2 v2.2.0/go.mod h1:L7+th5govYp5oKU9iN7To5PgznBuIjBPn+ejqKR0avw=
github.com/launchdarkly/go-test-helpers/v3 v3.0.2 h1:rh0085g1rVJM5qIukdaQ8z1XTWZztbJ49vRZuveqiuU=
@@ -506,8 +487,6 @@ github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw=
github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
github.com/lrstanley/bubblezone v0.0.0-20221222153816-e95291e2243e h1:XoxHx8K6ZKoMtjzWOMDuM69LCdjDDsTOtTfWGrT/fns=
github.com/lrstanley/bubblezone v0.0.0-20221222153816-e95291e2243e/go.mod h1:v5lEwWaguF1o2MW/ucO0ZIA/IZymdBYJJ+2cMRLE7LU=
github.com/lrstanley/bubblezone v0.0.0-20230911164824-e3824f1adde9 h1:+7bxeCzFs4bfFPAnIZrjNmRt/MCffIy7aw2mPc9mxkU=
github.com/lrstanley/bubblezone v0.0.0-20230911164824-e3824f1adde9/go.mod h1:v5lEwWaguF1o2MW/ucO0ZIA/IZymdBYJJ+2cMRLE7LU=
github.com/lucasb-eyer/go-colorful v1.2.0 h1:1nnpGOrhyZZuNyfu1QjKiUICQ74+3FNCN69Aj6K7nkY=
github.com/lucasb-eyer/go-colorful v1.2.0/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i7ruzyGqttikkLy0=
github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0=
@@ -521,8 +500,6 @@ github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovk
github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APPA=
github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
github.com/mattn/go-localereader v0.0.1 h1:ygSAOl7ZXTx4RdPYinUpg6W99U8jWvWi9Ye2JC/oIi4=
github.com/mattn/go-localereader v0.0.1/go.mod h1:8fBrzywKY7BI3czFoHkuzRoWE9C+EiG4R1k4Cjx5p88=
github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI=
@@ -639,8 +616,6 @@ github.com/smartystreets/gunit v1.1.3 h1:32x+htJCu3aMswhPw3teoJ+PnWPONqdNgaGs6Qt
github.com/smartystreets/gunit v1.1.3/go.mod h1:EH5qMBab2UclzXUcpR8b93eHsIlp9u+pDQIRp5DZNzQ=
github.com/snowflakedb/gosnowflake v1.6.23 h1:uO+zMTXJcSHzOm6ks5To8ergNjt5Dy6cr5QtStpRFT8=
github.com/snowflakedb/gosnowflake v1.6.23/go.mod h1:KfO4F7bk+aXPUIvBqYxvPhxLlu2/w4TtSC8Rw/yr5Mg=
github.com/snowflakedb/gosnowflake v1.6.25 h1:o5zUmxTOo0Eo9AdkEj8blCeiMuILrQJ+rjUMAeZhcRE=
github.com/snowflakedb/gosnowflake v1.6.25/go.mod h1:KfO4F7bk+aXPUIvBqYxvPhxLlu2/w4TtSC8Rw/yr5Mg=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
github.com/stretchr/objx v0.5.0 h1:1zr/of2m5FGMsad5YfcqgdqdWrIhu+EBEJRhR1U7z/c=
@@ -662,12 +637,8 @@ github.com/tailscale/depaware v0.0.0-20210622194025-720c4b409502 h1:34icjjmqJ2HP
github.com/tailscale/depaware v0.0.0-20210622194025-720c4b409502/go.mod h1:p9lPsd+cx33L3H9nNoecRRxPssFKUwwI50I3pZ0yT+8=
github.com/therootcompany/xz v1.0.1 h1:CmOtsn1CbtmyYiusbfmhmkpAAETj0wBIH6kCYaX+xzw=
github.com/therootcompany/xz v1.0.1/go.mod h1:3K3UH1yCKgBneZYhuQUvJ9HPD19UEXEI0BWbMn8qNMY=
github.com/trufflesecurity/disk-buffer-reader v0.1.9 h1:1dji/Cx5aPL5DlsqI1B86xFMo6OfbBEhkQhXxgbX9WA=
github.com/trufflesecurity/disk-buffer-reader v0.1.9/go.mod h1:uYwTCdxzV0o+qaeBMxflOsq4eu2WjrE46qGR2e80O9Y=
github.com/trufflesecurity/overseer v1.1.7-custom5 h1:xu+Fg6fkSRifUPzUCl7N8HmobJ6WGOkIApGnM7mJS6w=
github.com/trufflesecurity/overseer v1.1.7-custom5/go.mod h1:nT9w37AiO1Nop2VhVhNfzAFaPjthvxgpDV3XKsxYkcI=
github.com/trufflesecurity/overseer v1.2.7 h1:DEiu2wrL/0f+XqshIT9C4/0nj7SHZvLmFeE4jELh4Oo=
github.com/trufflesecurity/overseer v1.2.7/go.mod h1:nT9w37AiO1Nop2VhVhNfzAFaPjthvxgpDV3XKsxYkcI=
github.com/ulikunitz/xz v0.5.6/go.mod h1:2bypXElzHzzJZwzH67Y6wb67pO62Rzfn7BSiF4ABRW8=
github.com/ulikunitz/xz v0.5.10 h1:t92gobL9l3HE202wg3rlk19F6X+JOxl9BBrCCMYEYd8=
github.com/ulikunitz/xz v0.5.10/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14=
@@ -749,8 +720,6 @@ golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u0
golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM=
golang.org/x/exp v0.0.0-20230206171751-46f607a40771 h1:xP7rWLUr1e1n2xkK5YB4LI0hPEy3LJC6Wk+D4pGlOJg=
golang.org/x/exp v0.0.0-20230206171751-46f607a40771/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc=
golang.org/x/exp v0.0.0-20231006140011-7918f672742d h1:jtJma62tbqLibJ5sFQz8bKtEM8rJBtfilJ2qTU199MI=
golang.org/x/exp v0.0.0-20231006140011-7918f672742d/go.mod h1:ldy0pHrwJyGW56pPQzzkH36rKxoZW1tw7ZJpeKx+hdo=
golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
@@ -775,8 +744,6 @@ golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91
golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
golang.org/x/mod v0.12.0 h1:rmsUpXtvNzj340zd98LZ4KntptpfRHwpFOHG188oHXc=
golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
golang.org/x/mod v0.13.0 h1:I/DsJXRlw/8l/0c24sM9yb0T4z9liZTduXvdAWYiysY=
golang.org/x/mod v0.13.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@@ -816,8 +783,6 @@ golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4Iltr
golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
golang.org/x/oauth2 v0.12.0 h1:smVPGxink+n1ZI5pkQa8y6fZT0RW0MgCO5bFpepy4B4=
golang.org/x/oauth2 v0.12.0/go.mod h1:A74bZ3aGXgCY0qaIC9Ahg6Lglin4AMAco8cIv9baba4=
golang.org/x/oauth2 v0.13.0 h1:jDDenyj+WgFtmV3zYVoi8aE2BwtXFLWOA67ZfNWftiY=
golang.org/x/oauth2 v0.13.0/go.mod h1:/JMhi4ZRXAf4HG9LiNmxvk+45+96RUlVThiH8FzNBn0=
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@@ -830,8 +795,6 @@ golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJ
golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.3.0 h1:ftCYgMx6zT/asHUrPw8BLLscYtGznsLAnjq5RH9P66E=
golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y=
golang.org/x/sync v0.4.0 h1:zxkM55ReGkDlKSM+Fu41A+zmbZuaPVbGMzvvdUPznYQ=
golang.org/x/sync v0.4.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y=
golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@@ -942,8 +905,6 @@ golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc
golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
golang.org/x/tools v0.12.0 h1:YW6HUoUmYBpwSgyaGaZq1fHjrBjX1rlpZ54T6mu2kss=
golang.org/x/tools v0.12.0/go.mod h1:Sc0INKfu04TlqNoRA1hgpFZbhYXHPr4V5DzpSBTPqQM=
golang.org/x/tools v0.14.0 h1:jvNa2pY0M4r62jkRQ6RwEZZyPcymeL9XZMLBbV7U2nc=
golang.org/x/tools v0.14.0/go.mod h1:uYBEerGOWcJyEORxN+Ek8+TT266gXkNlHdJBwexUsBg=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
@@ -1052,8 +1013,6 @@ honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWh
honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg=
pgregory.net/rapid v1.1.0 h1:CMa0sjHSru3puNx+J0MIAuiiEV4N0qj8/cMWGBBCsjw=
pgregory.net/rapid v1.1.0/go.mod h1:PY5XlDGj0+V1FCq0o192FdRhpKHGTRIWBgqjDBTrq04=
rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=
rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0=
rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA=
-3
View File
@@ -152,9 +152,6 @@ func init() {
cli.Version("trufflehog " + version.BuildVersion)
//Support -h for help
cli.HelpFlag.Short('h')
if len(os.Args) <= 1 && isatty.IsTerminal(os.Stdout.Fd()) {
args := tui.Run()
if len(args) == 0 {
-120
View File
@@ -1,120 +0,0 @@
package glob
import (
"fmt"
"github.com/gobwas/glob"
)
// Filter is a generic filter for excluding and including globs (limited
// regular expressions). Exclusion takes precedence if both include and exclude
// lists are provided.
type Filter struct {
exclude []glob.Glob
include []glob.Glob
}
type globFilterOpt func(*Filter) error
// WithExcludeGlobs adds exclude globs to the filter.
func WithExcludeGlobs(excludes ...string) globFilterOpt {
return func(f *Filter) error {
for _, exclude := range excludes {
g, err := glob.Compile(exclude)
if err != nil {
return fmt.Errorf("invalid exclude glob %q: %w", exclude, err)
}
f.exclude = append(f.exclude, g)
}
return nil
}
}
// WithIncludeGlobs adds include globs to the filter.
func WithIncludeGlobs(includes ...string) globFilterOpt {
return func(f *Filter) error {
for _, include := range includes {
g, err := glob.Compile(include)
if err != nil {
return fmt.Errorf("invalid include glob %q: %w", include, err)
}
f.include = append(f.include, g)
}
return nil
}
}
// NewGlobFilter creates a new Filter with the provided options.
func NewGlobFilter(opts ...globFilterOpt) (*Filter, error) {
filter := &Filter{}
for _, opt := range opts {
if err := opt(filter); err != nil {
return nil, err
}
}
return filter, nil
}
// ShouldInclude returns whether the object is in the include list or not in
// the exclude list (exclude taking precedence).
func (f *Filter) ShouldInclude(object string) bool {
if f == nil {
return true
}
exclude, include := len(f.exclude), len(f.include)
if exclude == 0 && include == 0 {
return true
} else if exclude > 0 && include == 0 {
return f.shouldIncludeFromExclude(object)
} else if exclude == 0 && include > 0 {
return f.shouldIncludeFromInclude(object)
} else {
if ok, err := f.shouldIncludeFromBoth(object); err == nil {
return ok
}
// Ambiguous case.
return false
}
}
// shouldIncludeFromExclude checks for explicitly excluded paths. This should
// only be called when the include list is empty.
func (f *Filter) shouldIncludeFromExclude(object string) bool {
for _, glob := range f.exclude {
if glob.Match(object) {
return false
}
}
return true
}
// shouldIncludeFromInclude checks for explicitly included paths. This should
// only be called when the exclude list is empty.
func (f *Filter) shouldIncludeFromInclude(object string) bool {
for _, glob := range f.include {
if glob.Match(object) {
return true
}
}
return false
}
// shouldIncludeFromBoth checks for either excluded or included paths. Exclusion
// takes precedence. If neither list contains the object, true is returned.
func (f *Filter) shouldIncludeFromBoth(object string) (bool, error) {
// Exclude takes precedence. If we find the object in the exclude list,
// we should not match.
for _, glob := range f.exclude {
if glob.Match(object) {
return false, nil
}
}
// If we find the object in the include list, we should match.
for _, glob := range f.include {
if glob.Match(object) {
return true, nil
}
}
// If we find it in neither, return an error to let the caller decide.
return false, fmt.Errorf("ambiguous match")
}
-138
View File
@@ -1,138 +0,0 @@
package glob
import (
"testing"
"github.com/stretchr/testify/assert"
"pgregory.net/rapid"
)
type globTest struct {
input string
shouldInclude bool
}
func testGlobs(t *testing.T, filter *Filter, tests ...globTest) {
for _, tt := range tests {
t.Run(tt.input, func(t *testing.T) {
// Invert because mentally it's easier to say whether an
// input should be included.
assert.Equal(t, tt.shouldInclude, filter.ShouldInclude(tt.input))
})
}
}
func TestGlobFilterExclude(t *testing.T) {
filter, err := NewGlobFilter(WithExcludeGlobs("foo", "bar*"))
assert.NoError(t, err)
testGlobs(t, filter,
globTest{"foo", false},
globTest{"bar", false},
globTest{"bara", false},
globTest{"barb", false},
globTest{"barbosa", false},
globTest{"foobar", true},
globTest{"food", true},
globTest{"anything else", true},
)
}
func TestGlobFilterInclude(t *testing.T) {
filter, err := NewGlobFilter(WithIncludeGlobs("foo", "bar*"))
assert.NoError(t, err)
testGlobs(t, filter,
globTest{"foo", true},
globTest{"bar", true},
globTest{"bara", true},
globTest{"barb", true},
globTest{"barbosa", true},
globTest{"foobar", false},
globTest{"food", false},
globTest{"anything else", false},
)
}
func TestGlobFilterEmpty(t *testing.T) {
filter, err := NewGlobFilter()
assert.NoError(t, err)
testGlobs(t, filter,
globTest{"foo", true},
globTest{"bar", true},
globTest{"bara", true},
globTest{"barb", true},
globTest{"barbosa", true},
globTest{"foobar", true},
globTest{"food", true},
globTest{"anything else", true},
)
}
func TestGlobFilterExcludeInclude(t *testing.T) {
filter, err := NewGlobFilter(WithExcludeGlobs("/foo/bar/**"), WithIncludeGlobs("/foo/**"))
assert.NoError(t, err)
testGlobs(t, filter,
globTest{"/foo/a", true},
globTest{"/foo/b", true},
globTest{"/foo/c/d/e", true},
globTest{"/foo/bar/a", false},
globTest{"/foo/bar/b", false},
globTest{"/foo/bar/c/d/e", false},
globTest{"/any/other/path", false},
)
}
func TestGlobFilterExcludePrecedence(t *testing.T) {
filter, err := NewGlobFilter(WithExcludeGlobs("foo"), WithIncludeGlobs("foo*"))
assert.NoError(t, err)
testGlobs(t, filter,
globTest{"foo", false},
globTest{"foobar", true},
)
}
func TestGlobErrorContainsGlob(t *testing.T) {
invalidGlob := "[this is invalid because it doesn't close the capture group"
_, err := NewGlobFilter(WithExcludeGlobs(invalidGlob))
assert.Error(t, err)
assert.Contains(t, err.Error(), invalidGlob)
}
// The filters in this test should be mutually exclusive because one includes
// and the other excludes the same glob.
func TestGlobInverse(t *testing.T) {
for _, glob := range []string{
"a",
"a*",
"a**",
"*a",
"**a",
"*",
} {
include, err := NewGlobFilter(WithIncludeGlobs(glob))
assert.NoError(t, err)
exclude, err := NewGlobFilter(WithExcludeGlobs(glob))
assert.NoError(t, err)
rapid.Check(t, func(t *rapid.T) {
input := rapid.String().Draw(t, "input")
a, b := include.ShouldInclude(input), exclude.ShouldInclude(input)
if a == b {
t.Fatalf("Filter(Include(%q)) == Filter(Exclude(%q)) == %v for input %q", glob, glob, a, input)
}
})
}
}
func TestGlobDefaultFilters(t *testing.T) {
for _, filter := range []*Filter{nil, {}} {
rapid.Check(t, func(t *rapid.T) {
if !filter.ShouldInclude(rapid.String().Draw(t, "input")) {
t.Fatalf("filter %#v did not include input", filter)
}
})
}
}
+2 -27
View File
@@ -2,7 +2,6 @@ package airtableapikey
import (
"context"
"encoding/json"
"fmt"
"net/http"
"regexp"
@@ -21,9 +20,8 @@ var _ detectors.Detector = (*Scanner)(nil)
var (
client = common.SaneHttpClient()
appPat = regexp.MustCompile(`(app[a-zA-Z0-9_-]{14})`) // could be part of url
keyPat = regexp.MustCompile(`\b(key[a-zA-Z0-9_-]{14})\b`)
personalPat = regexp.MustCompile(`(\bpat[[:alnum:]]{14}\.[[:alnum:]]{64}\b)`)
appPat = regexp.MustCompile(`(app[a-zA-Z0-9_-]{14})`) // could be part of url
keyPat = regexp.MustCompile(`\b(key[a-zA-Z0-9_-]{14})\b`)
)
// Keywords are used for efficiently pre-filtering chunks.
@@ -32,31 +30,17 @@ func (s Scanner) Keywords() []string {
return []string{"airtable"}
}
type response struct {
Error struct {
Type string `json:"type"`
Message string `json:"message"`
} `json:"error"`
}
// FromData will find and optionally verify AirtableApiKey secrets in a given set of bytes.
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
dataStr := string(data)
appMatches := appPat.FindAllStringSubmatch(dataStr, -1)
keyMatches := keyPat.FindAllStringSubmatch(dataStr, -1)
personalKeyMatches := personalPat.FindAllStringSubmatch(dataStr, -1)
if len(keyMatches) == 0 {
keyMatches = personalKeyMatches
}
for _, keyMatch := range keyMatches {
if len(keyMatch) != 2 {
continue
}
keyRes := strings.TrimSpace(keyMatch[1])
for _, appMatch := range appMatches {
@@ -83,15 +67,6 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
defer res.Body.Close()
if res.StatusCode >= 200 && res.StatusCode < 300 {
s1.Verified = true
} else if res.StatusCode == 403 {
var resp response
if err = json.NewDecoder(res.Body).Decode(&resp); err == nil {
// check if the error is due to invalid permissions or model not found
if resp.Error.Type == "INVALID_PERMISSIONS_OR_MODEL_NOT_FOUND" {
// The key is verified as it works, but the user must enumerate the tables or permissions for the key.
s1.Verified = true
}
}
} else {
if detectors.IsKnownFalsePositive(keyRes, detectors.DefaultFalsePositives, true) {
continue
@@ -9,10 +9,9 @@ import (
"testing"
"time"
"github.com/kylelemons/godebug/pretty"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/google/go-cmp/cmp"
"github.com/google/go-cmp/cmp/cmpopts"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
)
@@ -100,9 +99,8 @@ func TestAirtableApiKey_FromChunk(t *testing.T) {
}
got[i].Raw = nil
}
ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "RawV2", "VerificationError")
if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {
t.Errorf("Airtable.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
if diff := pretty.Compare(got, tt.want); diff != "" {
t.Errorf("AirtableApiKey.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
}
})
}
@@ -1,8 +1,9 @@
package privacy
package baseapiio
import (
"context"
"fmt"
"io"
"net/http"
"regexp"
"strings"
@@ -12,26 +13,25 @@ import (
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
)
type Scanner struct {
client *http.Client
}
type Scanner struct{}
// Ensure the Scanner satisfies the interface at compile time.
var _ detectors.Detector = (*Scanner)(nil)
var (
defaultClient = common.SaneHttpClient()
client = common.SaneHttpClient()
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"privacy"}) + `\b([a-z0-9]{8}-[a-z0-9]{4}-[a-z0-9]{4}-[a-z0-9]{4}-[a-z0-9]{12})\b`)
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"baseapi", "base-api"}) + `\b([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})\b`)
)
// Keywords are used for efficiently pre-filtering chunks.
// Use identifiers in the secret preferably, or the provider name.
func (s Scanner) Keywords() []string {
return []string{"privacy"}
return []string{"baseapi", "base-api"}
}
// FromData will find and optionally verify Privacy secrets in a given set of bytes.
// FromData will find and optionally verify BaseApiIO secrets in a given set of bytes.
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
dataStr := string(data)
@@ -44,38 +44,34 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
resMatch := strings.TrimSpace(match[1])
s1 := detectors.Result{
DetectorType: detectorspb.DetectorType_Privacy,
DetectorType: detectorspb.DetectorType_BaseApiIO,
Raw: []byte(resMatch),
}
if verify {
client := s.client
if client == nil {
client = defaultClient
}
req, err := http.NewRequestWithContext(ctx, "GET", "https://api.privacy.com/v1/card?page=1&page_size=50", nil)
req, err := http.NewRequestWithContext(ctx, "GET", "https://api.base-api.io/v1/forms?page=1&per_page=10", nil)
if err != nil {
continue
}
req.Header.Set("Authorization", "api-key "+resMatch)
req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", resMatch))
res, err := client.Do(req)
if err == nil {
defer res.Body.Close()
if res.StatusCode >= 200 && res.StatusCode < 300 {
s1.Verified = true
} else if res.StatusCode == 401 {
// The secret is determinately not verified (nothing to do)
} else {
s1.VerificationError = fmt.Errorf("unexpected HTTP response status %d", res.StatusCode)
bodyBytes, err := io.ReadAll(res.Body)
if err != nil {
continue
}
} else {
s1.VerificationError = err
}
}
body := string(bodyBytes)
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
if !s1.Verified && detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
continue
if strings.Contains(body, "items") {
s1.Verified = true
} else {
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
continue
}
}
}
}
results = append(results, s1)
@@ -85,5 +81,5 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
}
func (s Scanner) Type() detectorspb.DetectorType {
return detectorspb.DetectorType_Privacy
return detectorspb.DetectorType_BaseApiIO
}
@@ -1,7 +1,7 @@
//go:build detectors
// +build detectors
package instamojo
package baseapiio
import (
"context"
@@ -16,16 +16,15 @@ import (
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
)
func TestInstamojo_FromChunk(t *testing.T) {
func TestBaseApiIO_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")
testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
id := testSecrets.MustGetField("INSTAMOJO_CLIENT_ID")
secret := testSecrets.MustGetField("INSTAMOJO_SECRET")
inactiveSecret := testSecrets.MustGetField("INSTAMOJO_INACTIVE")
secret := testSecrets.MustGetField("BASEAPIIO")
inactiveSecret := testSecrets.MustGetField("BASEAPIIO_INACTIVE")
type args struct {
ctx context.Context
@@ -33,46 +32,43 @@ func TestInstamojo_FromChunk(t *testing.T) {
verify bool
}
tests := []struct {
name string
s Scanner
args args
want []detectors.Result
wantErr bool
wantVerificationErr bool
name string
s Scanner
args args
want []detectors.Result
wantErr bool
}{
{
name: "found, verified",
s: Scanner{},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a instamojo secret %s within id %s", secret, id)),
data: []byte(fmt.Sprintf("You can find a baseapiio secret %s within", secret)),
verify: true,
},
want: []detectors.Result{
{
DetectorType: detectorspb.DetectorType_Instamojo,
DetectorType: detectorspb.DetectorType_BaseApiIO,
Verified: true,
},
},
wantErr: false,
wantVerificationErr: false,
wantErr: false,
},
{
name: "found, unverified",
s: Scanner{},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a instamojo secret %s within but not valid, within id %s", inactiveSecret, id)), // the secret would satisfy the regex but not pass validation
data: []byte(fmt.Sprintf("You can find a baseapiio secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
verify: true,
},
want: []detectors.Result{
{
DetectorType: detectorspb.DetectorType_Instamojo,
VerificationError: fmt.Errorf("unexpected HTTP response status 401"),
DetectorType: detectorspb.DetectorType_BaseApiIO,
Verified: false,
},
},
wantErr: false,
wantVerificationErr: true,
wantErr: false,
},
{
name: "not found",
@@ -82,16 +78,16 @@ func TestInstamojo_FromChunk(t *testing.T) {
data: []byte("You cannot find the secret within"),
verify: true,
},
want: nil,
wantErr: false,
wantVerificationErr: true,
want: nil,
wantErr: false,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
s := Scanner{}
got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
if (err != nil) != tt.wantErr {
t.Errorf("Instamojo.FromData() error = %v, wantErr %v", err, tt.wantErr)
t.Errorf("BaseApiIO.FromData() error = %v, wantErr %v", err, tt.wantErr)
return
}
for i := range got {
@@ -101,7 +97,7 @@ func TestInstamojo_FromChunk(t *testing.T) {
got[i].Raw = nil
}
if diff := pretty.Compare(got, tt.want); diff != "" {
t.Errorf("Instamojo.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
t.Errorf("BaseApiIO.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
}
})
}
@@ -1,4 +1,4 @@
package ip2location
package blablabus
import (
"context"
@@ -12,24 +12,25 @@ import (
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
)
type Scanner struct {}
type Scanner struct{}
// Ensure the Scanner satisfies the interface at compile time.
var _ detectors.Detector = (*Scanner)(nil)
var (
client = common.SaneHttpClient()
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"ip2location"}) + `\b([0-9A-Z]{32})\b`)
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"blablabus"}) + `\b([0-9A-Za-z]{22})\b`)
)
// Keywords are used for efficiently pre-filtering chunks.
// Use identifiers in the secret preferably, or the provider name.
func (s Scanner) Keywords() []string {
return []string{"ip2location"}
return []string{"blablabus"}
}
// FromData will find and optionally verify Ip2location secrets in a given set of bytes.
// FromData will find and optionally verify Blablabus secrets in a given set of bytes.
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
dataStr := string(data)
@@ -42,33 +43,30 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
resMatch := strings.TrimSpace(match[1])
s1 := detectors.Result{
DetectorType: detectorspb.DetectorType_Ip2location,
DetectorType: detectorspb.DetectorType_Blablabus,
Raw: []byte(resMatch),
}
if verify {
req, err := http.NewRequestWithContext(ctx, "GET", "https://api.ip2location.io/?key=" + resMatch, nil)
req, err := http.NewRequestWithContext(ctx, "GET", "https://api.idbus.com/v3/stops", nil)
if err != nil {
continue
}
req.Header.Add("Content-Type", "application/json")
req.Header.Add("Authorization", fmt.Sprintf("Token %s", resMatch))
res, err := client.Do(req)
if err == nil {
defer res.Body.Close()
if res.StatusCode >= 200 && res.StatusCode < 300 {
s1.Verified = true
} else if res.StatusCode == 401 {
// The secret is determinately not verified (nothing to do)
} else {
s1.VerificationError = fmt.Errorf("unexpected HTTP response status %d", res.StatusCode)
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
continue
}
}
} else {
s1.VerificationError = err
}
}
if !s1.Verified && detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
continue
}
results = append(results, s1)
}
@@ -77,8 +75,5 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
}
func (s Scanner) Type() detectorspb.DetectorType {
return detectorspb.DetectorType_Ip2location
return detectorspb.DetectorType_Blablabus
}
@@ -1,7 +1,7 @@
//go:build detectors
// +build detectors
package portainer
package blablabus
import (
"context"
@@ -9,25 +9,22 @@ import (
"testing"
"time"
"github.com/google/go-cmp/cmp"
"github.com/google/go-cmp/cmp/cmpopts"
"github.com/kylelemons/godebug/pretty"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
)
func TestPortainer_FromChunk(t *testing.T) {
func TestBlablabus_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")
testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
secret := testSecrets.MustGetField("PORTAINER")
endpoint := testSecrets.MustGetField("PORTAINER_ENDPOINT")
inactiveSecret := testSecrets.MustGetField("PORTAINER_INACTIVE")
secret := testSecrets.MustGetField("BLABLABUS")
inactiveSecret := testSecrets.MustGetField("BLABLABUS_INACTIVE")
type args struct {
ctx context.Context
@@ -35,48 +32,43 @@ func TestPortainer_FromChunk(t *testing.T) {
verify bool
}
tests := []struct {
name string
s Scanner
args args
want []detectors.Result
wantErr bool
wantVerificationErr bool
name string
s Scanner
args args
want []detectors.Result
wantErr bool
}{
{
name: "found, verified",
s: Scanner{},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a portainer secret %s for portainer url %s", secret, endpoint)),
data: []byte(fmt.Sprintf("You can find a blablabus secret %s within", secret)),
verify: true,
},
want: []detectors.Result{
{
DetectorType: detectorspb.DetectorType_Portainer,
DetectorType: detectorspb.DetectorType_Blablabus,
Verified: true,
RawV2: []byte(secret + endpoint),
},
},
wantErr: false,
wantVerificationErr: false,
wantErr: false,
},
{
name: "found, unverified",
s: Scanner{},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a portainer secret %s for portainer %s within but not valid", inactiveSecret, endpoint)), // the secret would satisfy the regex but not pass validation
data: []byte(fmt.Sprintf("You can find a blablabus secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
verify: true,
},
want: []detectors.Result{
{
DetectorType: detectorspb.DetectorType_Portainer,
DetectorType: detectorspb.DetectorType_Blablabus,
Verified: false,
RawV2: []byte(inactiveSecret + endpoint),
},
},
wantErr: false,
wantVerificationErr: false,
wantErr: false,
},
{
name: "not found",
@@ -86,29 +78,26 @@ func TestPortainer_FromChunk(t *testing.T) {
data: []byte("You cannot find the secret within"),
verify: true,
},
want: nil,
wantErr: false,
wantVerificationErr: false,
want: nil,
wantErr: false,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
s := Scanner{}
got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
if (err != nil) != tt.wantErr {
t.Errorf("Portainer.FromData() error = %v, wantErr %v", err, tt.wantErr)
t.Errorf("Blablabus.FromData() error = %v, wantErr %v", err, tt.wantErr)
return
}
for i := range got {
if len(got[i].Raw) == 0 {
t.Fatalf("no raw secret present: \n %+v", got[i])
}
if (got[i].VerificationError != nil) != tt.wantVerificationErr {
t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError)
}
got[i].Raw = nil
}
ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "VerificationError")
if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {
t.Errorf("Portainer.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
if diff := pretty.Compare(got, tt.want); diff != "" {
t.Errorf("Blablabus.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
}
})
}
+1 -13
View File
@@ -2,7 +2,6 @@ package cloudsmith
import (
"context"
"encoding/json"
"net/http"
"regexp"
"strings"
@@ -30,10 +29,6 @@ func (s Scanner) Keywords() []string {
return []string{"cloudsmith"}
}
type response struct {
Authenticated bool `json:"authenticated"`
}
// FromData will find and optionally verify Cloudsmith secrets in a given set of bytes.
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
dataStr := string(data)
@@ -62,14 +57,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
if err == nil {
defer res.Body.Close()
if res.StatusCode >= 200 && res.StatusCode < 300 {
var r response
if err := json.NewDecoder(res.Body).Decode(&r); err != nil {
s1.VerificationError = err
continue
}
if r.Authenticated {
s1.Verified = true
}
s1.Verified = true
} else {
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
+1 -15
View File
@@ -2,7 +2,6 @@ package codeclimate
import (
"context"
"encoding/json"
"fmt"
"net/http"
"regexp"
@@ -31,12 +30,6 @@ func (s Scanner) Keywords() []string {
return []string{"codeclimate"}
}
type response struct {
Data struct {
Id string `json:"id"`
} `json:"data"`
}
// FromData will find and optionally verify Codeclimate secrets in a given set of bytes.
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
dataStr := string(data)
@@ -65,14 +58,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
if err == nil {
defer res.Body.Close()
if res.StatusCode >= 200 && res.StatusCode < 300 {
var r response
if err := json.NewDecoder(res.Body).Decode(&r); err != nil {
s1.VerificationError = err
continue
}
if r.Data.Id != "" {
s1.Verified = true
}
s1.Verified = true
} else {
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
@@ -1,4 +1,4 @@
package vagrantcloudpersonaltoken
package datafire
import (
"context"
@@ -12,26 +12,25 @@ import (
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
)
type Scanner struct {
client *http.Client
}
type Scanner struct{}
// Ensure the Scanner satisfies the interface at compile time.
var _ detectors.Detector = (*Scanner)(nil)
var (
defaultClient = common.SaneHttpClient()
client = common.SaneHttpClient()
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"vagrant"}) + `\b([A-Za-z0-9]{14}.atlasv1.[A-Za-z0-9]{67})\b`)
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"datafire"}) + `\b([a-z0-9\S]{175,190})\b`)
)
// Keywords are used for efficiently pre-filtering chunks.
// Use identifiers in the secret preferably, or the provider name.
func (s Scanner) Keywords() []string {
return []string{"vagrant"}
return []string{"datafire"}
}
// FromData will find and optionally verify Vagrantcloudpersonaltoken secrets in a given set of bytes.
// FromData will find and optionally verify DataFire secrets in a given set of bytes.
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
dataStr := string(data)
@@ -44,17 +43,12 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
resMatch := strings.TrimSpace(match[1])
s1 := detectors.Result{
DetectorType: detectorspb.DetectorType_VagrantCloudPersonalToken,
DetectorType: detectorspb.DetectorType_DataFire,
Raw: []byte(resMatch),
}
if verify {
client := s.client
if client == nil {
client = defaultClient
}
req, err := http.NewRequestWithContext(ctx, "GET", "https://app.vagrantup.com/api/v2/authenticate", nil)
req, err := http.NewRequestWithContext(ctx, "GET", "https://api.datafire.io/projects/", nil)
if err != nil {
continue
}
@@ -64,21 +58,15 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
defer res.Body.Close()
if res.StatusCode >= 200 && res.StatusCode < 300 {
s1.Verified = true
} else if res.StatusCode == 401 {
// The secret is determinately not verified (nothing to do)
} else {
s1.VerificationError = fmt.Errorf("unexpected HTTP response status %d", res.StatusCode)
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
continue
}
}
} else {
s1.VerificationError = err
}
}
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
if !s1.Verified && detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
continue
}
results = append(results, s1)
}
@@ -86,5 +74,5 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
}
func (s Scanner) Type() detectorspb.DetectorType {
return detectorspb.DetectorType_VagrantCloudPersonalToken
return detectorspb.DetectorType_DataFire
}
@@ -1,7 +1,7 @@
//go:build detectors
// +build detectors
package ip2location
package datafire
import (
"context"
@@ -9,24 +9,22 @@ import (
"testing"
"time"
"github.com/google/go-cmp/cmp"
"github.com/google/go-cmp/cmp/cmpopts"
"github.com/kylelemons/godebug/pretty"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
)
func TestIp2location_FromChunk(t *testing.T) {
func TestDataFire_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")
testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
secret := testSecrets.MustGetField("IP2LOCATION")
inactiveSecret := testSecrets.MustGetField("IP2LOCATION_INACTIVE")
secret := testSecrets.MustGetField("DATAFIRE")
inactiveSecret := testSecrets.MustGetField("DATAFIRE_INACTIVE")
type args struct {
ctx context.Context
@@ -34,46 +32,43 @@ func TestIp2location_FromChunk(t *testing.T) {
verify bool
}
tests := []struct {
name string
s Scanner
args args
want []detectors.Result
wantErr bool
wantVerificationErr bool
name string
s Scanner
args args
want []detectors.Result
wantErr bool
}{
{
name: "found, verified",
s: Scanner{},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a ip2location secret %s within", secret)),
data: []byte(fmt.Sprintf("You can find a datafire secret %s within", secret)),
verify: true,
},
want: []detectors.Result{
{
DetectorType: detectorspb.DetectorType_Ip2location,
DetectorType: detectorspb.DetectorType_DataFire,
Verified: true,
},
},
wantErr: false,
wantVerificationErr: false,
wantErr: false,
},
{
name: "found, unverified",
s: Scanner{},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a ip2location secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
data: []byte(fmt.Sprintf("You can find a datafire secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
verify: true,
},
want: []detectors.Result{
{
DetectorType: detectorspb.DetectorType_Ip2location,
DetectorType: detectorspb.DetectorType_DataFire,
Verified: false,
},
},
wantErr: false,
wantVerificationErr: false,
wantErr: false,
},
{
name: "not found",
@@ -83,29 +78,26 @@ func TestIp2location_FromChunk(t *testing.T) {
data: []byte("You cannot find the secret within"),
verify: true,
},
want: nil,
wantErr: false,
wantVerificationErr: false,
want: nil,
wantErr: false,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
s := Scanner{}
got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
if (err != nil) != tt.wantErr {
t.Errorf("Ip2location.FromData() error = %v, wantErr %v", err, tt.wantErr)
t.Errorf("DataFire.FromData() error = %v, wantErr %v", err, tt.wantErr)
return
}
for i := range got {
if len(got[i].Raw) == 0 {
t.Fatalf("no raw secret present: \n %+v", got[i])
}
if (got[i].VerificationError != nil) != tt.wantVerificationErr {
t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError)
}
got[i].Raw = nil
}
ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "VerificationError")
if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {
t.Errorf("Ip2location.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
if diff := pretty.Compare(got, tt.want); diff != "" {
t.Errorf("DataFire.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
}
})
}
+1 -2
View File
@@ -59,8 +59,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
if verify {
// thanks https://stackoverflow.com/questions/15621471/validate-a-facebook-app-id-and-app-secret
// https://stackoverflow.com/questions/24401241/how-to-get-a-facebook-access-token-using-appid-and-app-secret-without-any-login
req, err := http.NewRequestWithContext(ctx, "GET", fmt.Sprintf("https://graph.facebook.com/me?access_token=%s|%s", apiIdRes, apiSecretRes), nil)
req, err := http.NewRequestWithContext(ctx, "GET", fmt.Sprintf("https://graph.facebook.com/%s?fields=roles&access_token=%s|%s", apiIdRes, apiIdRes, apiSecretRes), nil)
if err != nil {
continue
}
-7
View File
@@ -74,13 +74,6 @@ func TestStringShannonEntropy(t *testing.T) {
},
want: 0.22228483068568816,
},
{
name: "empty",
args: args{
input: "",
},
want: 0.0,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
@@ -1,8 +1,7 @@
package ipinfo
package glitterlyapi
import (
"context"
"fmt"
"net/http"
"regexp"
"strings"
@@ -12,26 +11,25 @@ import (
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
)
type Scanner struct {
client *http.Client
}
type Scanner struct{}
// Ensure the Scanner satisfies the interface at compile time.
var _ detectors.Detector = (*Scanner)(nil)
var (
defaultClient = common.SaneHttpClient()
client = common.SaneHttpClient()
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"ipinfo"}) + `\b([a-f0-9]{14})\b`)
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"glitterlyapi"}) + `\b([0-9a-z-]{36})\b`)
)
// Keywords are used for efficiently pre-filtering chunks.
// Use identifiers in the secret preferably, or the provider name.
func (s Scanner) Keywords() []string {
return []string{"ipinfo"}
return []string{"glitterlyapi"}
}
// FromData will find and optionally verify Ipinfo secrets in a given set of bytes.
// FromData will find and optionally verify GlitterlyAPI secrets in a given set of bytes.
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
dataStr := string(data)
@@ -44,40 +42,31 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
resMatch := strings.TrimSpace(match[1])
s1 := detectors.Result{
DetectorType: detectorspb.DetectorType_IPInfo,
DetectorType: detectorspb.DetectorType_GlitterlyAPI,
Raw: []byte(resMatch),
}
if verify {
client := s.client
if client == nil {
client = defaultClient
}
req, err := http.NewRequestWithContext(ctx, "GET", "https://ipinfo.io/json?token="+resMatch, nil)
req, err := http.NewRequestWithContext(ctx, "GET", "https://www.glitterlyapi.com/auth", nil)
if err != nil {
continue
}
req.Header.Add("Content-Type", "application/json")
req.Header.Add("x-api-key", resMatch)
res, err := client.Do(req)
if err == nil {
fmt.Println(res.Status, resMatch)
defer res.Body.Close()
if res.StatusCode >= 200 && res.StatusCode < 300 {
s1.Verified = true
} else if res.StatusCode == 403 {
// The secret is determinately not verified (nothing to do)
} else {
s1.VerificationError = fmt.Errorf("unexpected HTTP response status %d", res.StatusCode)
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
continue
}
}
} else {
s1.VerificationError = err
}
}
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
if !s1.Verified && detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
continue
}
results = append(results, s1)
}
@@ -85,5 +74,5 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
}
func (s Scanner) Type() detectorspb.DetectorType {
return detectorspb.DetectorType_IPInfo
return detectorspb.DetectorType_GlitterlyAPI
}
@@ -1,7 +1,7 @@
//go:build detectors
// +build detectors
package vagrantcloudpersonaltoken
package glitterlyapi
import (
"context"
@@ -9,24 +9,22 @@ import (
"testing"
"time"
"github.com/google/go-cmp/cmp"
"github.com/google/go-cmp/cmp/cmpopts"
"github.com/kylelemons/godebug/pretty"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
)
func TestVagrantcloudpersonaltoken_FromChunk(t *testing.T) {
func TestGlitterlyAPI_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")
testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
secret := testSecrets.MustGetField("VAGRANTCLOUDPERSONALTOKEN")
inactiveSecret := testSecrets.MustGetField("VAGRANTCLOUDPERSONALTOKEN_INACTIVE")
secret := testSecrets.MustGetField("GLITTERLYAPI")
inactiveSecret := testSecrets.MustGetField("GLITTERLYAPI_INACTIVE")
type args struct {
ctx context.Context
@@ -34,46 +32,43 @@ func TestVagrantcloudpersonaltoken_FromChunk(t *testing.T) {
verify bool
}
tests := []struct {
name string
s Scanner
args args
want []detectors.Result
wantErr bool
wantVerificationErr bool
name string
s Scanner
args args
want []detectors.Result
wantErr bool
}{
{
name: "found, verified",
s: Scanner{},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a vagrantcloudpersonaltoken secret %s within", secret)),
data: []byte(fmt.Sprintf("You can find a glitterlyapi secret %s within", secret)),
verify: true,
},
want: []detectors.Result{
{
DetectorType: detectorspb.DetectorType_VagrantCloudPersonalToken,
DetectorType: detectorspb.DetectorType_GlitterlyAPI,
Verified: true,
},
},
wantErr: false,
wantVerificationErr: false,
wantErr: false,
},
{
name: "found, unverified",
s: Scanner{},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a vagrantcloudpersonaltoken secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
data: []byte(fmt.Sprintf("You can find a glitterlyapi secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
verify: true,
},
want: []detectors.Result{
{
DetectorType: detectorspb.DetectorType_VagrantCloudPersonalToken,
DetectorType: detectorspb.DetectorType_GlitterlyAPI,
Verified: false,
},
},
wantErr: false,
wantVerificationErr: false,
wantErr: false,
},
{
name: "not found",
@@ -83,29 +78,26 @@ func TestVagrantcloudpersonaltoken_FromChunk(t *testing.T) {
data: []byte("You cannot find the secret within"),
verify: true,
},
want: nil,
wantErr: false,
wantVerificationErr: false,
want: nil,
wantErr: false,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
s := Scanner{}
got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
if (err != nil) != tt.wantErr {
t.Errorf("Vagrantcloudpersonaltoken.FromData() error = %v, wantErr %v", err, tt.wantErr)
t.Errorf("GlitterlyAPI.FromData() error = %v, wantErr %v", err, tt.wantErr)
return
}
for i := range got {
if len(got[i].Raw) == 0 {
t.Fatalf("no raw secret present: \n %+v", got[i])
}
if (got[i].VerificationError != nil) != tt.wantVerificationErr {
t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError)
}
got[i].Raw = nil
}
ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "VerificationError")
if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {
t.Errorf("Vagrantcloudpersonaltoken.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
if diff := pretty.Compare(got, tt.want); diff != "" {
t.Errorf("GlitterlyAPI.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
}
})
}
-105
View File
@@ -1,105 +0,0 @@
package instamojo
import (
"context"
"fmt"
"io"
"net/http"
"regexp"
"strings"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
)
type Scanner struct {
client *http.Client
}
// Ensure the Scanner satisfies the interface at compile time.
var _ detectors.Detector = (*Scanner)(nil)
var (
defaultClient = common.SaneHttpClient()
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
//KeyPat is client_id
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"instamojo"}) + `\b([0-9a-zA-Z]{40})\b`)
//Secretpat is Client_secret
secretPat = regexp.MustCompile(detectors.PrefixRegex([]string{"instamojo"}) + `\b([0-9a-zA-Z]{128})\b`)
)
// Keywords are used for efficiently pre-filtering chunks.
// Use identifiers in the secret preferably, or the provider name.
func (s Scanner) Keywords() []string {
return []string{"instamojo"}
}
// FromData will find and optionally verify Instamojo secrets in a given set of bytes.
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
dataStr := string(data)
secretMatches := secretPat.FindAllStringSubmatch(dataStr, -1)
clientIdmatches := keyPat.FindAllStringSubmatch(dataStr, -1)
for _, match := range secretMatches {
if len(match) != 2 {
continue
}
resSecret := strings.TrimSpace(match[1])
for _, clientIdMatch := range clientIdmatches {
if len(clientIdMatch) != 2 {
continue
}
resClientId := strings.TrimSpace(clientIdMatch[1])
s1 := detectors.Result{
DetectorType: detectorspb.DetectorType_Instamojo,
Raw: []byte(resClientId),
}
if verify {
client := s.client
if client == nil {
client = defaultClient
}
payload := strings.NewReader("grant_type=client_credentials&client_id=" + resClientId + "&client_secret=" + resSecret)
req, err := http.NewRequestWithContext(ctx, "POST", "https://api.instamojo.com/oauth2/token/", payload)
if err != nil {
continue
}
req.Header.Add("Content-Type", "application/x-www-form-urlencoded")
res, err := client.Do(req)
if err == nil {
defer res.Body.Close()
bodyBytes, err := io.ReadAll(res.Body)
if err != nil {
continue
}
body := string(bodyBytes)
if (res.StatusCode >= 200 && res.StatusCode < 300) && strings.Contains(body, "access_token") {
s1.Verified = true
} else {
s1.VerificationError = fmt.Errorf("unexpected HTTP response status %d", res.StatusCode)
}
} else {
s1.VerificationError = err
}
}
if !s1.Verified && detectors.IsKnownFalsePositive(string(s1.Raw), detectors.DefaultFalsePositives, true) {
continue
}
results = append(results, s1)
}
}
return results, nil
}
func (s Scanner) Type() detectorspb.DetectorType {
return detectorspb.DetectorType_Instamojo
}
-161
View File
@@ -1,161 +0,0 @@
//go:build detectors
// +build detectors
package ipinfo
import (
"context"
"fmt"
"github.com/google/go-cmp/cmp"
"github.com/google/go-cmp/cmp/cmpopts"
"testing"
"time"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
)
func TestIpinfo_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
secret := testSecrets.MustGetField("IPINFO")
inactiveSecret := testSecrets.MustGetField("IPINFO_INACTIVE")
type args struct {
ctx context.Context
data []byte
verify bool
}
tests := []struct {
name string
s Scanner
args args
want []detectors.Result
wantErr bool
wantVerificationErr bool
}{
{
name: "found, verified",
s: Scanner{},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a ipinfo secret %s within", secret)),
verify: true,
},
want: []detectors.Result{
{
DetectorType: detectorspb.DetectorType_IPInfo,
Verified: true,
},
},
wantErr: false,
wantVerificationErr: false,
},
{
name: "found, unverified",
s: Scanner{},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a ipinfo secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
verify: true,
},
want: []detectors.Result{
{
DetectorType: detectorspb.DetectorType_IPInfo,
Verified: false,
},
},
wantErr: false,
wantVerificationErr: false,
},
{
name: "not found",
s: Scanner{},
args: args{
ctx: context.Background(),
data: []byte("You cannot find the secret within"),
verify: true,
},
want: nil,
wantErr: false,
wantVerificationErr: false,
},
{
name: "found, would be verified if not for timeout",
s: Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a ipinfo secret %s within", secret)),
verify: true,
},
want: []detectors.Result{
{
DetectorType: detectorspb.DetectorType_IPInfo,
Verified: false,
},
},
wantErr: false,
wantVerificationErr: true,
},
{
name: "found, verified but unexpected api surface",
s: Scanner{client: common.ConstantResponseHttpClient(404, "")},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a ipinfo secret %s within", secret)),
verify: true,
},
want: []detectors.Result{
{
DetectorType: detectorspb.DetectorType_IPInfo,
Verified: false,
},
},
wantErr: false,
wantVerificationErr: true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
if (err != nil) != tt.wantErr {
t.Errorf("Ipinfo.FromData() error = %v, wantErr %v", err, tt.wantErr)
return
}
for i := range got {
if len(got[i].Raw) == 0 {
t.Fatalf("no raw secret present: \n %+v", got[i])
}
if (got[i].VerificationError != nil) != tt.wantVerificationErr {
t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError)
}
}
ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "VerificationError")
if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {
t.Errorf("Ipinfo.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
}
})
}
}
func BenchmarkFromData(benchmark *testing.B) {
ctx := context.Background()
s := Scanner{}
for name, data := range detectors.MustGetBenchmarkData() {
benchmark.Run(name, func(b *testing.B) {
b.ResetTimer()
for n := 0; n < b.N; n++ {
_, err := s.FromData(ctx, false, data)
if err != nil {
b.Fatal(err)
}
}
})
}
}
-101
View File
@@ -1,101 +0,0 @@
package loggly
import (
"context"
"fmt"
"net/http"
"regexp"
"strings"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
)
type Scanner struct {
client *http.Client
}
// Ensure the Scanner satisfies the interface at compile time.
var _ detectors.Detector = (*Scanner)(nil)
var (
defaultClient = common.SaneHttpClient()
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
domainPat = regexp.MustCompile(`\b([a-zA-Z0-9-]+\.loggly\.com)\b`)
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"loggly"}) + `\b([a-z0-9]{8}-[a-z0-9]{4}-[a-z0-9]{4}-[a-z0-9]{4}-[a-z0-9]{12})\b`)
)
// Keywords are used for efficiently pre-filtering chunks.
// Use identifiers in the secret preferably, or the provider name.
func (s Scanner) Keywords() []string {
return []string{"loggly"}
}
// FromData will find and optionally verify Loggly secrets in a given set of bytes.
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
dataStr := string(data)
keyMatches := keyPat.FindAllStringSubmatch(dataStr, -1)
domainMatches := domainPat.FindAllStringSubmatch(dataStr, -1)
for _, match := range keyMatches {
if len(match) != 2 {
continue
}
key := strings.TrimSpace(match[1])
for _, domainMatch := range domainMatches {
if len(domainMatch) != 2 {
continue
}
domainRes := strings.TrimSpace(domainMatch[1])
s1 := detectors.Result{
DetectorType: detectorspb.DetectorType_Loggly,
Raw: []byte(key),
RawV2: []byte(fmt.Sprintf("%s:%s", domainRes, key)),
}
if verify {
client := s.client
if client == nil {
client = defaultClient
}
req, err := http.NewRequestWithContext(ctx, "GET", "https://"+domainRes+"/apiv2/customer", nil)
if err != nil {
continue
}
req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", key))
res, err := client.Do(req)
if err == nil {
defer res.Body.Close()
if res.StatusCode >= 200 && res.StatusCode < 300 {
s1.Verified = true
} else if res.StatusCode == 401 {
// The secret is determinately not verified (nothing to do)
} else {
s1.VerificationError = fmt.Errorf("unexpected HTTP response status %d", res.StatusCode)
}
} else {
s1.VerificationError = err
}
}
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
if !s1.Verified && detectors.IsKnownFalsePositive(key, detectors.DefaultFalsePositives, true) {
continue
}
results = append(results, s1)
}
}
return results, nil
}
func (s Scanner) Type() detectorspb.DetectorType {
return detectorspb.DetectorType_Loggly
}
-162
View File
@@ -1,162 +0,0 @@
//go:build detectors
// +build detectors
package loggly
import (
"context"
"fmt"
"github.com/google/go-cmp/cmp"
"github.com/google/go-cmp/cmp/cmpopts"
"testing"
"time"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
)
func TestLoggly_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors4")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
secret := testSecrets.MustGetField("LOGGLY")
inactiveSecret := testSecrets.MustGetField("LOGGLY_INACTIVE")
domain := testSecrets.MustGetField("LOGGLY_DOMAIN")
type args struct {
ctx context.Context
data []byte
verify bool
}
tests := []struct {
name string
s Scanner
args args
want []detectors.Result
wantErr bool
wantVerificationErr bool
}{
{
name: "found, verified",
s: Scanner{},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a loggly secret %s within loggly Domain %s", secret, domain)),
verify: true,
},
want: []detectors.Result{
{
DetectorType: detectorspb.DetectorType_Loggly,
Verified: true,
},
},
wantErr: false,
wantVerificationErr: false,
},
{
name: "found, unverified",
s: Scanner{},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a loggly secret %s within loggly Domain %s but not valid", inactiveSecret, domain)), // the secret would satisfy the regex but not pass validation
verify: true,
},
want: []detectors.Result{
{
DetectorType: detectorspb.DetectorType_Loggly,
Verified: false,
},
},
wantErr: false,
wantVerificationErr: false,
},
{
name: "not found",
s: Scanner{},
args: args{
ctx: context.Background(),
data: []byte("You cannot find the secret within"),
verify: true,
},
want: nil,
wantErr: false,
wantVerificationErr: false,
},
{
name: "found, would be verified if not for timeout",
s: Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a loggly secret %s within %s", secret, domain)),
verify: true,
},
want: []detectors.Result{
{
DetectorType: detectorspb.DetectorType_Loggly,
Verified: false,
},
},
wantErr: false,
wantVerificationErr: true,
},
{
name: "found, verified but unexpected api surface",
s: Scanner{client: common.ConstantResponseHttpClient(404, "")},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a loggly secret %s within %s", secret, domain)),
verify: true,
},
want: []detectors.Result{
{
DetectorType: detectorspb.DetectorType_Loggly,
Verified: false,
},
},
wantErr: false,
wantVerificationErr: true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
if (err != nil) != tt.wantErr {
t.Errorf("Loggly.FromData() error = %v, wantErr %v", err, tt.wantErr)
return
}
for i := range got {
if len(got[i].Raw) == 0 {
t.Fatalf("no raw secret present: \n %+v", got[i])
}
if (got[i].VerificationError != nil) != tt.wantVerificationErr {
t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError)
}
}
ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "RawV2", "VerificationError")
if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {
t.Errorf("Loggly.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
}
})
}
}
func BenchmarkFromData(benchmark *testing.B) {
ctx := context.Background()
s := Scanner{}
for name, data := range detectors.MustGetBenchmarkData() {
benchmark.Run(name, func(b *testing.B) {
b.ResetTimer()
for n := 0; n < b.N; n++ {
_, err := s.FromData(ctx, false, data)
if err != nil {
b.Fatal(err)
}
}
})
}
}
+1 -13
View File
@@ -3,7 +3,6 @@ package meaningcloud
import (
"bytes"
"context"
"encoding/json"
"io"
"mime/multipart"
"net/http"
@@ -33,10 +32,6 @@ func (s Scanner) Keywords() []string {
return []string{"meaningcloud"}
}
type response struct {
DeepTime float64 `json:"deepTime"`
}
// FromData will find and optionally verify MeaningCloud secrets in a given set of bytes.
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
dataStr := string(data)
@@ -83,14 +78,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
if err == nil {
defer res.Body.Close()
if res.StatusCode >= 200 && res.StatusCode < 300 {
var r response
if err := json.NewDecoder(res.Body).Decode(&r); err != nil {
s1.VerificationError = err
continue
}
if r.DeepTime > 0 {
s1.Verified = true
}
s1.Verified = true
} else {
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
+1 -1
View File
@@ -20,7 +20,7 @@ var (
client = common.SaneHttpClient()
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"monday"}) + `\b(eyJ[A-Za-z0-9-_]{15,100}\.eyJ[A-Za-z0-9-_]{100,300}\.[A-Za-z0-9-_]{25,100})\b`)
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"monday"}) + `\b(ey[a-zA-Z0-9_.]{210,225})\b`)
)
// Keywords are used for efficiently pre-filtering chunks.
+1 -1
View File
@@ -19,7 +19,7 @@ import (
func TestMonday_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")
testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
@@ -48,16 +48,11 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
if verify {
req, err := http.NewRequestWithContext(ctx, "GET", "https://api.newrelic.com/v2/users.json", nil)
reqEU, errEU := http.NewRequestWithContext(ctx, "GET", "https://api.eu.newrelic.com/v2/users.json", nil)
if err != nil || errEU != nil {
if err != nil {
continue
}
req.Header.Add("X-Api-Key", resMatch)
reqEU.Header.Add("X-Api-Key", resMatch)
res, err := client.Do(req)
resEU, errEU := client.Do(reqEU)
if err == nil {
defer res.Body.Close()
if res.StatusCode >= 200 && res.StatusCode < 300 {
@@ -68,16 +63,6 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
continue
}
}
} else if errEU == nil {
defer resEU.Body.Close()
if resEU.StatusCode >= 200 && resEU.StatusCode < 300 {
s1.Verified = true
} else {
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, false) {
continue
}
}
}
}
-109
View File
@@ -1,109 +0,0 @@
package openvpn
import (
"context"
b64 "encoding/base64"
"fmt"
"net/http"
"regexp"
"strings"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
)
type Scanner struct {
client *http.Client
}
// Ensure the Scanner satisfies the interface at compile time.
var _ detectors.Detector = (*Scanner)(nil)
var (
defaultClient = common.SaneHttpClient()
clientIDPat = regexp.MustCompile(detectors.PrefixRegex([]string{"openvpn"}) + `\b([A-Za-z0-9-]{3,40}\.[A-Za-z0-9-]{3,40})\b`)
clientSecretPat = regexp.MustCompile(`\b([a-zA-Z0-9_-]{64,})\b`)
domainPat = regexp.MustCompile(`\b(https?://[A-Za-z0-9-]+\.api\.openvpn\.com)\b`)
)
// Keywords are used for efficiently pre-filtering chunks.
// Use identifiers in the secret preferably, or the provider name.
func (s Scanner) Keywords() []string {
return []string{"openvpn"}
}
// FromData will find and optionally verify Openvpn secrets in a given set of bytes.
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
dataStr := string(data)
domainMatches := domainPat.FindAllStringSubmatch(dataStr, -1)
clientIdMatches := clientIDPat.FindAllStringSubmatch(dataStr, -1)
clientSecretMatches := clientSecretPat.FindAllStringSubmatch(dataStr, -1)
for _, clientIdMatch := range clientIdMatches {
clientIDRes := strings.TrimSpace(clientIdMatch[1])
for _, clientSecretMatch := range clientSecretMatches {
clientSecretRes := strings.TrimSpace(clientSecretMatch[1])
for _, domainMatch := range domainMatches {
domainRes := strings.TrimSpace(domainMatch[1])
s1 := detectors.Result{
DetectorType: detectorspb.DetectorType_OpenVpn,
Raw: []byte(clientSecretRes),
RawV2: []byte(clientIDRes + clientSecretRes),
}
if verify {
client := s.client
if client == nil {
client = defaultClient
}
payload := strings.NewReader("grant_type=client_credentials")
// OpenVPN API is in beta, We'll have to update the API endpoint once
// Docs: https://openvpn.net/cloud-docs/developer/creating-api-credentials.html
req, err := http.NewRequestWithContext(ctx, "POST", domainRes + "/api/beta/oauth/token", payload)
if err != nil {
continue
}
data := fmt.Sprintf("%s:%s", clientIDRes, clientSecretRes)
sEnc := b64.StdEncoding.EncodeToString([]byte(data))
req.Header.Add("Authorization", fmt.Sprintf("Basic %s", sEnc))
req.Header.Add("content-type", "application/x-www-form-urlencoded")
res, err := client.Do(req)
if err == nil {
defer res.Body.Close()
if res.StatusCode >= 200 && res.StatusCode < 300 {
s1.Verified = true
} else if res.StatusCode == 401 {
// The secret is determinately not verified (nothing to do)
} else {
s1.VerificationError = fmt.Errorf("unexpected HTTP response status %d", res.StatusCode)
}
} else {
s1.VerificationError = err
}
}
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
if detectors.IsKnownFalsePositive(clientSecretRes, detectors.DefaultFalsePositives, true) {
continue
}
results = append(results, s1)
}
}
}
return results, nil
}
func (s Scanner) Type() detectorspb.DetectorType {
return detectorspb.DetectorType_OpenVpn
}
-133
View File
@@ -1,133 +0,0 @@
//go:build detectors
// +build detectors
package openvpn
import (
"context"
"fmt"
"testing"
"time"
"github.com/google/go-cmp/cmp"
"github.com/google/go-cmp/cmp/cmpopts"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
)
func TestOpenvpn_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
domain := testSecrets.MustGetField("OPENVPN_DOMAIN")
clientId := testSecrets.MustGetField("OPENVPN_CLIENT_ID")
clientSecret := testSecrets.MustGetField("OPENVPN_CLIENT_SECRET")
inactiveClientSecret := testSecrets.MustGetField("OPENVPN_CLIENT_SECRET_INACTIVE")
type args struct {
ctx context.Context
data []byte
verify bool
}
tests := []struct {
name string
s Scanner
args args
want []detectors.Result
wantErr bool
wantVerificationErr bool
}{
{
name: "found, verified",
s: Scanner{},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a openvpn secret %s openvpn clientId %s and domain %s within", clientSecret, clientId, domain)),
verify: true,
},
want: []detectors.Result{
{
DetectorType: detectorspb.DetectorType_OpenVpn,
Verified: true,
RawV2: []byte(clientId + clientSecret),
},
},
wantErr: false,
wantVerificationErr: false,
},
{
name: "found, unverified",
s: Scanner{},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a openvpn secret %s openvpn clientId %s and domain %s within but not valid", inactiveClientSecret, clientId, domain)), // the secret would satisfy the regex but not pass validation
verify: true,
},
want: []detectors.Result{
{
DetectorType: detectorspb.DetectorType_OpenVpn,
Verified: false,
RawV2: []byte(clientId + inactiveClientSecret),
},
},
wantErr: false,
wantVerificationErr: false,
},
{
name: "not found",
s: Scanner{},
args: args{
ctx: context.Background(),
data: []byte("You cannot find the secret within"),
verify: true,
},
want: nil,
wantErr: false,
wantVerificationErr: false,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
if (err != nil) != tt.wantErr {
t.Errorf("Openvpn.FromData() error = %v, wantErr %v", err, tt.wantErr)
return
}
for i := range got {
if len(got[i].Raw) == 0 {
t.Fatalf("no raw secret present: \n %+v", got[i])
}
if (got[i].VerificationError != nil) != tt.wantVerificationErr {
t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError)
}
}
ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "VerificationError")
if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {
t.Errorf("Openvpn.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
}
})
}
}
func BenchmarkFromData(benchmark *testing.B) {
ctx := context.Background()
s := Scanner{}
for name, data := range detectors.MustGetBenchmarkData() {
benchmark.Run(name, func(b *testing.B) {
b.ResetTimer()
for n := 0; n < b.N; n++ {
_, err := s.FromData(ctx, false, data)
if err != nil {
b.Fatal(err)
}
}
})
}
}
@@ -24,7 +24,7 @@ const verifyURL = "https://api.pagerduty.com/users"
var (
defaultClient = common.SaneHttpClient()
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"pagerduty", "pager_duty", "pd_", "pd-"}) + `\b(u\+[a-zA-Z0-9_+-]{18})\b`)
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"pagerduty", "pager_duty", "pd_", "pd-"}) + `\b([a-zA-Z0-9_+-]{20})\b`)
)
// Keywords are used for efficiently pre-filtering chunks.
+20 -25
View File
@@ -2,7 +2,6 @@ package plaidkey
import (
"context"
"fmt"
"net/http"
"regexp"
"strings"
@@ -54,36 +53,32 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
DetectorType: detectorspb.DetectorType_PlaidKey,
Raw: []byte(resMatch),
}
environments := []string{"development", "production"}
if verify {
for _, env := range environments {
payload := strings.NewReader(`{"client_id":"` + idresMatch + `","secret":"` + resMatch + `","user":{"client_user_id":"60e3ee4019a2660010f8bc54","phone_number_verified_time":"0001-01-01T00:00:00Z","email_address_verified_time":"0001-01-01T00:00:00Z"},"client_name":"Plaid Test App","products":["auth","transactions"],"country_codes":["US"],"webhook":"https://webhook-uri.com","account_filters":{"depository":{"account_subtypes":["checking","savings"]}},"language":"en","link_customization_name":"default"}`)
req, err := http.NewRequestWithContext(ctx, "POST", "https://"+env+".plaid.com/link/token/create", payload)
if err != nil {
continue
}
req.Header.Add("Content-Type", "application/json")
res, err := client.Do(req)
if err == nil {
defer res.Body.Close()
if res.StatusCode >= 200 && res.StatusCode < 300 {
s1.Verified = true
s1.ExtraData = map[string]string{"environment": fmt.Sprintf("https://%s.plaid.com", env)}
} else {
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
continue
}
payload := strings.NewReader(`{"client_id":"` + idresMatch + `","secret":"` + resMatch + `","user":{"client_user_id":"60e3ee4019a2660010f8bc54","phone_number_verified_time":"0001-01-01T00:00:00Z","email_address_verified_time":"0001-01-01T00:00:00Z"},"client_name":"Plaid Test App","products":["auth","transactions"],"country_codes":["US"],"webhook":"https://webhook-uri.com","account_filters":{"depository":{"account_subtypes":["checking","savings"]}},"language":"en","link_customization_name":"default"}`)
req, err := http.NewRequestWithContext(ctx, "POST", "https://development.plaid.com/link/token/create", payload)
if err != nil {
continue
}
req.Header.Add("Content-Type", "application/json")
res, err := client.Do(req)
if err == nil {
defer res.Body.Close()
if res.StatusCode >= 200 && res.StatusCode < 300 {
s1.Verified = true
} else {
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
continue
}
}
}
results = append(results, s1)
// if the environment is dev, we don't need to check production
if s1.Verified {
break
}
}
results = append(results, s1)
}
}
return results, nil
-5
View File
@@ -26,8 +26,6 @@ func TestPlaidKey_FromChunk(t *testing.T) {
secret := testSecrets.MustGetField("PLAIDKEY_SECRET")
inactiveSecret := testSecrets.MustGetField("PLAIDKEY_SECRET_INACTIVE")
id := testSecrets.MustGetField("PLAIDKEY_CLIENTID")
// env := testSecrets.MustGetField("PLAIDKEY_ENVIRONMENT") // development or production
env := "development"
type args struct {
ctx context.Context
@@ -53,9 +51,6 @@ func TestPlaidKey_FromChunk(t *testing.T) {
{
DetectorType: detectorspb.DetectorType_PlaidKey,
Verified: true,
ExtraData: map[string]string{
"environment": fmt.Sprintf("https://%s.plaid.com", env),
},
},
},
wantErr: false,
-98
View File
@@ -1,98 +0,0 @@
package portainer
import (
"context"
"fmt"
"net/http"
"regexp"
"strings"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
)
type Scanner struct {
client *http.Client
}
// Ensure the Scanner satisfies the interface at compile time.
var _ detectors.Detector = (*Scanner)(nil)
var (
defaultClient = common.SaneHttpClient()
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
endpointPat = regexp.MustCompile(detectors.PrefixRegex([]string{"portainer"}) + `\b(https?:\/\/\S+(:[0-9]{4,5})?)\b`)
tokenPat = regexp.MustCompile(detectors.PrefixRegex([]string{"portainer"}) + `\b(eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9\.[0-9A-Za-z]{50,310}\.[0-9A-Z-a-z\-_]{43})\b`)
)
// Keywords are used for efficiently pre-filtering chunks.
// Use identifiers in the secret preferably, or the provider name.
func (s Scanner) Keywords() []string {
return []string{"portainer"}
}
// FromData will find and optionally verify Portainer secrets in a given set of bytes.
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
dataStr := string(data)
matches := tokenPat.FindAllStringSubmatch(dataStr, -1)
endpointMatches := endpointPat.FindAllStringSubmatch(dataStr, -1)
for _, match := range matches {
if len(match) != 2 {
continue
}
resMatch := strings.TrimSpace(match[1])
for _, endpointMatch := range endpointMatches {
resEndpointMatch := strings.TrimSpace(endpointMatch[1])
s1 := detectors.Result{
DetectorType: detectorspb.DetectorType_Portainer,
Raw: []byte(resMatch),
RawV2: []byte(resMatch + resEndpointMatch),
}
if verify {
client := s.client
if client == nil {
client = defaultClient
}
req, err := http.NewRequestWithContext(ctx, "GET", resEndpointMatch + "/api/endpoints", nil)
if err != nil {
continue
}
req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", resMatch))
res, err := client.Do(req)
if err == nil {
defer res.Body.Close()
if res.StatusCode >= 200 && res.StatusCode < 300 {
s1.Verified = true
} else if res.StatusCode == 401 || res.StatusCode == 403 {
// The secret is determinately not verified (nothing to do)
} else {
s1.VerificationError = fmt.Errorf("unexpected HTTP response status %d", res.StatusCode)
}
} else {
s1.VerificationError = err
}
}
if !s1.Verified && detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
continue
}
if len(endpointMatches) > 0 {
results = append(results, s1)
}
}
}
return results, nil
}
func (s Scanner) Type() detectorspb.DetectorType {
return detectorspb.DetectorType_Portainer
}
@@ -1,99 +0,0 @@
package portainertoken
import (
"context"
"fmt"
"net/http"
"regexp"
"strings"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
)
type Scanner struct {
client *http.Client
}
// Ensure the Scanner satisfies the interface at compile time.
var _ detectors.Detector = (*Scanner)(nil)
var (
defaultClient = common.SaneHttpClient()
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"portainertoken"}) + `\b(ptr_[A-Za-z0-9\/_\-+=]{20,60})`)
endpointPat = regexp.MustCompile(detectors.PrefixRegex([]string{"portainer"}) + `\b(https?:\/\/\S+(:[0-9]{4,5})?)\b`)
)
// Keywords are used for efficiently pre-filtering chunks.
// Use identifiers in the secret preferably, or the provider name.
func (s Scanner) Keywords() []string {
return []string{"portainertoken"}
}
// FromData will find and optionally verify Portainertoken secrets in a given set of bytes.
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
dataStr := string(data)
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
endpointMatches := endpointPat.FindAllStringSubmatch(dataStr, -1)
for _, match := range matches {
if len(match) != 2 {
continue
}
resMatch := strings.TrimSpace(match[1])
for _, endpointMatch := range endpointMatches {
resEndpointMatch := strings.TrimSpace(endpointMatch[1])
s1 := detectors.Result{
DetectorType: detectorspb.DetectorType_PortainerToken,
Raw: []byte(resMatch),
RawV2: []byte(resMatch + resEndpointMatch),
}
if verify {
client := s.client
if client == nil {
client = defaultClient
}
req, err := http.NewRequestWithContext(ctx, "GET", resEndpointMatch + "/api/stacks", nil)
if err != nil {
continue
}
req.Header.Add("X-API-Key", resMatch)
res, err := client.Do(req)
if err == nil {
defer res.Body.Close()
if res.StatusCode >= 200 && res.StatusCode < 300 {
s1.Verified = true
} else if res.StatusCode == 401 {
// The secret is determinately not verified (nothing to do)
} else {
s1.VerificationError = fmt.Errorf("unexpected HTTP response status %d", res.StatusCode)
}
} else {
s1.VerificationError = err
}
}
if !s1.Verified && detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
continue
}
if len(endpointMatches) > 0 {
results = append(results, s1)
}
}
}
return results, nil
}
func (s Scanner) Type() detectorspb.DetectorType {
return detectorspb.DetectorType_PortainerToken
}
@@ -1,131 +0,0 @@
//go:build detectors
// +build detectors
package portainertoken
import (
"context"
"fmt"
"testing"
"time"
"github.com/google/go-cmp/cmp"
"github.com/google/go-cmp/cmp/cmpopts"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
)
func TestPortainertoken_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
secret := testSecrets.MustGetField("PORTAINERTOKEN")
inactiveSecret := testSecrets.MustGetField("PORTAINERTOKEN_INACTIVE")
endpoint := testSecrets.MustGetField("PORTAINER_ENDPOINT")
type args struct {
ctx context.Context
data []byte
verify bool
}
tests := []struct {
name string
s Scanner
args args
want []detectors.Result
wantErr bool
wantVerificationErr bool
}{
{
name: "found, verified",
s: Scanner{},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a portainertoken secret %s within for portainer url %s", secret, endpoint)),
verify: true,
},
want: []detectors.Result{
{
DetectorType: detectorspb.DetectorType_PortainerToken,
Verified: true,
RawV2: []byte(secret + endpoint),
},
},
wantErr: false,
wantVerificationErr: false,
},
{
name: "found, unverified",
s: Scanner{},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a portainertoken secret %s within but not valid for portainer url %s", inactiveSecret, endpoint)), // the secret would satisfy the regex but not pass validation
verify: true,
},
want: []detectors.Result{
{
DetectorType: detectorspb.DetectorType_PortainerToken,
Verified: false,
RawV2: []byte(inactiveSecret + endpoint),
},
},
wantErr: false,
wantVerificationErr: false,
},
{
name: "not found",
s: Scanner{},
args: args{
ctx: context.Background(),
data: []byte("You cannot find the secret within"),
verify: true,
},
want: nil,
wantErr: false,
wantVerificationErr: false,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
if (err != nil) != tt.wantErr {
t.Errorf("Portainertoken.FromData() error = %v, wantErr %v", err, tt.wantErr)
return
}
for i := range got {
if len(got[i].Raw) == 0 {
t.Fatalf("no raw secret present: \n %+v", got[i])
}
if (got[i].VerificationError != nil) != tt.wantVerificationErr {
t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError)
}
}
ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "VerificationError")
if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {
t.Errorf("Portainertoken.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
}
})
}
}
func BenchmarkFromData(benchmark *testing.B) {
ctx := context.Background()
s := Scanner{}
for name, data := range detectors.MustGetBenchmarkData() {
benchmark.Run(name, func(b *testing.B) {
b.ResetTimer()
for n := 0; n < b.N; n++ {
_, err := s.FromData(ctx, false, data)
if err != nil {
b.Fatal(err)
}
}
})
}
}
+8 -21
View File
@@ -20,13 +20,13 @@ var (
client = common.SaneHttpClient()
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
keyPat = regexp.MustCompile(`\b(phx_[a-zA-Z0-9_]{43})\b`)
keyPat = regexp.MustCompile(`\b(phc_[a-zA-Z0-9_]{43})\b`)
)
// Keywords are used for efficiently pre-filtering chunks.
// Use identifiers in the secret preferably, or the provider name.
func (s Scanner) Keywords() []string {
return []string{"phx_"}
return []string{"phc_"}
}
// FromData will find and optionally verify AppPosthog secrets in a given set of bytes.
@@ -47,33 +47,20 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
}
if verify {
req, err := http.NewRequestWithContext(ctx, "GET", "https://app.posthog.com/api/event/?personal_api_key="+resMatch, nil)
reqEU, errEU := http.NewRequestWithContext(ctx, "GET", "https://eu.posthog.com/api/event/?personal_api_key="+resMatch, nil)
if err != nil || errEU != nil {
payload := strings.NewReader(` {
"api_key": "` + resMatch + `",
"distinct_id": "1234"
}`)
req, err := http.NewRequestWithContext(ctx, "POST", "https://app.posthog.com/decide/", payload)
if err != nil {
continue
}
req.Header.Add("Content-Type", "application/json")
reqEU.Header.Add("Content-Type", "application/json")
res, err := client.Do(req)
if err == nil {
defer res.Body.Close()
if res.StatusCode >= 200 && res.StatusCode < 300 {
s1.Verified = true
} else if res.StatusCode == 401 {
//Try EU Endpoint only if other one fails.
res, err := client.Do(reqEU)
if err == nil {
defer res.Body.Close()
if res.StatusCode >= 200 && res.StatusCode < 300 {
s1.Verified = true
} else {
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
continue
}
}
}
} else {
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
-161
View File
@@ -1,161 +0,0 @@
//go:build detectors
// +build detectors
package privacy
import (
"context"
"fmt"
"github.com/google/go-cmp/cmp"
"github.com/google/go-cmp/cmp/cmpopts"
"testing"
"time"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
)
func TestPrivacy_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
secret := testSecrets.MustGetField("PRIVACY")
inactiveSecret := testSecrets.MustGetField("PRIVACY_INACTIVE")
type args struct {
ctx context.Context
data []byte
verify bool
}
tests := []struct {
name string
s Scanner
args args
want []detectors.Result
wantErr bool
wantVerificationErr bool
}{
{
name: "found, verified",
s: Scanner{},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a privacy secret %s within", secret)),
verify: true,
},
want: []detectors.Result{
{
DetectorType: detectorspb.DetectorType_Privacy,
Verified: true,
},
},
wantErr: false,
wantVerificationErr: false,
},
{
name: "found, unverified",
s: Scanner{},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a privacy secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
verify: true,
},
want: []detectors.Result{
{
DetectorType: detectorspb.DetectorType_Privacy,
Verified: false,
},
},
wantErr: false,
wantVerificationErr: false,
},
{
name: "not found",
s: Scanner{},
args: args{
ctx: context.Background(),
data: []byte("You cannot find the secret within"),
verify: true,
},
want: nil,
wantErr: false,
wantVerificationErr: false,
},
{
name: "found, would be verified if not for timeout",
s: Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a privacy secret %s within", secret)),
verify: true,
},
want: []detectors.Result{
{
DetectorType: detectorspb.DetectorType_Privacy,
Verified: false,
},
},
wantErr: false,
wantVerificationErr: true,
},
{
name: "found, verified but unexpected api surface",
s: Scanner{client: common.ConstantResponseHttpClient(404, "")},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a privacy secret %s within", secret)),
verify: true,
},
want: []detectors.Result{
{
DetectorType: detectorspb.DetectorType_Privacy,
Verified: false,
},
},
wantErr: false,
wantVerificationErr: true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
if (err != nil) != tt.wantErr {
t.Errorf("Privacy.FromData() error = %v, wantErr %v", err, tt.wantErr)
return
}
for i := range got {
if len(got[i].Raw) == 0 {
t.Fatalf("no raw secret present: \n %+v", got[i])
}
if (got[i].VerificationError != nil) != tt.wantVerificationErr {
t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError)
}
}
ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "VerificationError")
if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {
t.Errorf("Privacy.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
}
})
}
}
func BenchmarkFromData(benchmark *testing.B) {
ctx := context.Background()
s := Scanner{}
for name, data := range detectors.MustGetBenchmarkData() {
benchmark.Run(name, func(b *testing.B) {
b.ResetTimer()
for n := 0; n < b.N; n++ {
_, err := s.FromData(ctx, false, data)
if err != nil {
b.Fatal(err)
}
}
})
}
}
@@ -0,0 +1,79 @@
package quickmetrics
import (
"context"
"net/http"
"regexp"
"strings"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
)
type Scanner struct{}
// Ensure the Scanner satisfies the interface at compile time.
var _ detectors.Detector = (*Scanner)(nil)
var (
client = common.SaneHttpClient()
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"quickmetrics"}) + `\b([a-zA-Z0-9_-]{22})\b`)
)
// Keywords are used for efficiently pre-filtering chunks.
// Use identifiers in the secret preferably, or the provider name.
func (s Scanner) Keywords() []string {
return []string{"quickmetrics"}
}
// FromData will find and optionally verify QuickMetrics secrets in a given set of bytes.
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
dataStr := string(data)
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
for _, match := range matches {
if len(match) != 2 {
continue
}
resMatch := strings.TrimSpace(match[1])
s1 := detectors.Result{
DetectorType: detectorspb.DetectorType_QuickMetrics,
Raw: []byte(resMatch),
}
if verify {
payload := strings.NewReader(`[{"name":"api.response.time","dimension":null,"values":[[1568319841,12.4],[1568319856,9.3],[1568319860,234],[1568319863,3.2]]},{"name":"click.color","dimension":"green","values":[[1568319841,1],[1568319856,1],[1568319860,1],[1568319863,1]]}]`)
req, err := http.NewRequestWithContext(ctx, "POST", "https://qckm.io/list", payload)
if err != nil {
continue
}
req.Header.Add("Content-Type", "application/json")
req.Header.Add("x-qm-key", resMatch)
res, err := client.Do(req)
if err == nil {
defer res.Body.Close()
if res.StatusCode >= 200 && res.StatusCode < 300 {
s1.Verified = true
} else {
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
continue
}
}
}
}
results = append(results, s1)
}
return results, nil
}
func (s Scanner) Type() detectorspb.DetectorType {
return detectorspb.DetectorType_QuickMetrics
}
@@ -0,0 +1,120 @@
//go:build detectors
// +build detectors
package quickmetrics
import (
"context"
"fmt"
"testing"
"time"
"github.com/kylelemons/godebug/pretty"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
)
func TestQuickMetrics_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
secret := testSecrets.MustGetField("QUICKMETRICS")
inactiveSecret := testSecrets.MustGetField("QUICKMETRICS_INACTIVE")
type args struct {
ctx context.Context
data []byte
verify bool
}
tests := []struct {
name string
s Scanner
args args
want []detectors.Result
wantErr bool
}{
{
name: "found, verified",
s: Scanner{},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a quickmetrics secret %s within", secret)),
verify: true,
},
want: []detectors.Result{
{
DetectorType: detectorspb.DetectorType_QuickMetrics,
Verified: true,
},
},
wantErr: false,
},
{
name: "found, unverified",
s: Scanner{},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a quickmetrics secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
verify: true,
},
want: []detectors.Result{
{
DetectorType: detectorspb.DetectorType_QuickMetrics,
Verified: false,
},
},
wantErr: false,
},
{
name: "not found",
s: Scanner{},
args: args{
ctx: context.Background(),
data: []byte("You cannot find the secret within"),
verify: true,
},
want: nil,
wantErr: false,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
s := Scanner{}
got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
if (err != nil) != tt.wantErr {
t.Errorf("QuickMetrics.FromData() error = %v, wantErr %v", err, tt.wantErr)
return
}
for i := range got {
if len(got[i].Raw) == 0 {
t.Fatalf("no raw secret present: \n %+v", got[i])
}
got[i].Raw = nil
}
if diff := pretty.Compare(got, tt.want); diff != "" {
t.Errorf("QuickMetrics.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
}
})
}
}
func BenchmarkFromData(benchmark *testing.B) {
ctx := context.Background()
s := Scanner{}
for name, data := range detectors.MustGetBenchmarkData() {
benchmark.Run(name, func(b *testing.B) {
b.ResetTimer()
for n := 0; n < b.N; n++ {
_, err := s.FromData(ctx, false, data)
if err != nil {
b.Fatal(err)
}
}
})
}
}
+2 -12
View File
@@ -2,7 +2,6 @@ package screenshotapi
import (
"context"
"encoding/json"
"net/http"
"regexp"
"strings"
@@ -31,10 +30,6 @@ func (s Scanner) Keywords() []string {
return []string{"screenshotapi"}
}
type response struct {
Screenshot string `json:"screenshot"`
}
// FromData will find and optionally verify ScreenshotAPI secrets in a given set of bytes.
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
dataStr := string(data)
@@ -55,19 +50,14 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
if verify {
timeout := 10 * time.Second
client.Timeout = timeout
req, err := http.NewRequestWithContext(ctx, "GET", "https://shot.screenshotapi.net/screenshot?token="+resMatch+"&url=https://google.com&width=1920&height=1080", nil)
req, err := http.NewRequestWithContext(ctx, "GET", "https://shot.screenshotapi.net/screenshot?token="+resMatch+"&url=https://google.com&width=1920&height=1080&output=image", nil)
if err != nil {
continue
}
res, err := client.Do(req)
if err == nil {
defer res.Body.Close()
var r response
if err := json.NewDecoder(res.Body).Decode(&r); err != nil {
s1.VerificationError = err
continue
}
if res.StatusCode >= 200 && res.StatusCode < 300 && r.Screenshot != "" {
if res.StatusCode >= 200 && res.StatusCode < 300 {
s1.Verified = true
} else {
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
@@ -1,91 +0,0 @@
package sourcegraphcody
import (
"context"
"fmt"
"net/http"
"regexp"
"strings"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
)
type Scanner struct {
client *http.Client
}
// Ensure the Scanner satisfies the interface at compile time.
var _ detectors.Detector = (*Scanner)(nil)
var (
defaultClient = common.SaneHttpClient()
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
keyPat = regexp.MustCompile(`\b(slk_[a-f0-9]{64})\b`)
)
// Keywords are used for efficiently pre-filtering chunks.
// Use identifiers in the secret preferably, or the provider name.
func (s Scanner) Keywords() []string {
return []string{"slk_"}
}
// FromData will find and optionally verify Sourcegraphcody secrets in a given set of bytes.
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
dataStr := string(data)
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
for _, match := range matches {
if len(match) != 2 {
continue
}
resMatch := strings.TrimSpace(match[1])
s1 := detectors.Result{
DetectorType: detectorspb.DetectorType_SourcegraphCody,
Raw: []byte(resMatch),
}
if verify {
client := s.client
if client == nil {
client = defaultClient
}
req, err := http.NewRequestWithContext(ctx, "GET", "https://cody-gateway.sourcegraph.com/v1/limits", nil)
req.Header.Add("Authorization", "Bearer "+resMatch)
if err != nil {
continue
}
res, err := client.Do(req)
if err == nil {
defer res.Body.Close()
if res.StatusCode >= 200 && res.StatusCode < 300 {
s1.Verified = true
} else if res.StatusCode == 403 || res.StatusCode == 401 {
// 403 when the gateway is not configured to accept the token,
// 401 when the token is invalid.
// The secret is determinately not verified (nothing to do)
} else {
s1.VerificationError = fmt.Errorf("unexpected HTTP response status %d", res.StatusCode)
}
} else {
s1.VerificationError = err
}
}
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
if !s1.Verified && detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
continue
}
results = append(results, s1)
}
return results, nil
}
func (s Scanner) Type() detectorspb.DetectorType {
return detectorspb.DetectorType_SourcegraphCody
}
@@ -1,162 +0,0 @@
//go:build detectors
// +build detectors
package sourcegraphcody
import (
"context"
"fmt"
"testing"
"time"
"github.com/google/go-cmp/cmp"
"github.com/google/go-cmp/cmp/cmpopts"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
)
func TestSourcegraphcody_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
secret := testSecrets.MustGetField("SOURCEGRAPHCODY")
inactiveSecret := testSecrets.MustGetField("SOURCEGRAPHCODY_INACTIVE")
type args struct {
ctx context.Context
data []byte
verify bool
}
tests := []struct {
name string
s Scanner
args args
want []detectors.Result
wantErr bool
wantVerificationErr bool
}{
{
name: "found, verified",
s: Scanner{},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a sourcegraphcody secret %s within", secret)),
verify: true,
},
want: []detectors.Result{
{
DetectorType: detectorspb.DetectorType_SourcegraphCody,
Verified: true,
},
},
wantErr: false,
wantVerificationErr: false,
},
{
name: "found, unverified",
s: Scanner{},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a sourcegraphcody secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
verify: true,
},
want: []detectors.Result{
{
DetectorType: detectorspb.DetectorType_SourcegraphCody,
Verified: false,
},
},
wantErr: false,
wantVerificationErr: false,
},
{
name: "not found",
s: Scanner{},
args: args{
ctx: context.Background(),
data: []byte("You cannot find the secret within"),
verify: true,
},
want: nil,
wantErr: false,
wantVerificationErr: false,
},
{
name: "found, would be verified if not for timeout",
s: Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a sourcegraphcody secret %s within", secret)),
verify: true,
},
want: []detectors.Result{
{
DetectorType: detectorspb.DetectorType_SourcegraphCody,
Verified: false,
},
},
wantErr: false,
wantVerificationErr: true,
},
{
name: "found, verified but unexpected api surface",
s: Scanner{client: common.ConstantResponseHttpClient(404, "")},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a sourcegraphcody secret %s within", secret)),
verify: true,
},
want: []detectors.Result{
{
DetectorType: detectorspb.DetectorType_SourcegraphCody,
Verified: false,
},
},
wantErr: false,
wantVerificationErr: true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
if (err != nil) != tt.wantErr {
t.Errorf("Sourcegraphcody.FromData() error = %v, wantErr %v", err, tt.wantErr)
return
}
for i := range got {
if len(got[i].Raw) == 0 {
t.Fatalf("no raw secret present: \n %+v", got[i])
}
if (got[i].VerificationError != nil) != tt.wantVerificationErr {
t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError)
}
}
ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "VerificationError")
if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {
t.Errorf("Sourcegraphcody.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
}
})
}
}
func BenchmarkFromData(benchmark *testing.B) {
ctx := context.Background()
s := Scanner{}
for name, data := range detectors.MustGetBenchmarkData() {
benchmark.Run(name, func(b *testing.B) {
b.ResetTimer()
for n := 0; n < b.N; n++ {
_, err := s.FromData(ctx, false, data)
if err != nil {
b.Fatal(err)
}
}
})
}
}
+1 -1
View File
@@ -47,7 +47,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
}
if verify {
req, err := http.NewRequestWithContext(ctx, "GET", "https://api.supernotes.app/v1/user", nil)
req, err := http.NewRequestWithContext(ctx, "GET", "https://api.supernotes.app/v1/user/", nil)
if err != nil {
continue
}
+2 -1
View File
@@ -23,6 +23,7 @@ var (
identifierPat = regexp.MustCompile(`(?i)sid.{0,20}AC[0-9a-f]{32}`) // Should we have this? Seems restrictive.
sidPat = regexp.MustCompile(`\bAC[0-9a-f]{32}\b`)
keyPat = regexp.MustCompile(`\b[0-9a-f]{32}\b`)
client = common.SaneHttpClient()
)
// Keywords are used for efficiently pre-filtering chunks.
@@ -58,7 +59,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
}
if verify {
client := s.client
client = s.client
if client == nil {
client = defaultClient
}
+2 -2
View File
@@ -21,7 +21,7 @@ var (
client = common.SaneHttpClient()
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives
accessToken = regexp.MustCompile(detectors.PrefixRegex([]string{"twist"}) + `\b(?:oauth2:)?([0-9a-f]{40})\b`)
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"twist"}) + `\b([0-9a-f:]{40,47})\b`)
)
// Keywords are used for efficiently pre-filtering chunks.
@@ -34,7 +34,7 @@ func (s Scanner) Keywords() []string {
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
dataStr := string(data)
matches := accessToken.FindAllStringSubmatch(dataStr, -1)
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
for _, match := range matches {
if len(match) != 2 {
-99
View File
@@ -1,99 +0,0 @@
package voiceflow
import (
"bytes"
"context"
"fmt"
"io"
"net/http"
"regexp"
"strings"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
)
type Scanner struct {
client *http.Client
}
// Ensure the Scanner satisfies the interface at compile time.
var _ detectors.Detector = (*Scanner)(nil)
var (
defaultClient = common.SaneHttpClient()
// Reference: https://developer.voiceflow.com/reference/project#dialog-manager-api-keys
//
//TODO: This includes Workspace and Legacy Workspace API keys; I haven't validated whether these actually work.
// https://github.com/voiceflow/general-runtime/blob/master/tests/runtime/lib/DataAPI/utils.unit.ts
keyPat = regexp.MustCompile(`\b(VF\.(?:(?:DM|WS)\.)?[a-fA-F0-9]{24}\.[a-zA-Z0-9]{16})\b`)
)
// Keywords are used for efficiently pre-filtering chunks.
// Use identifiers in the secret preferably, or the provider name.
func (s Scanner) Keywords() []string {
return []string{"vf", "dm"}
}
// FromData will find and optionally verify Voiceflow secrets in a given set of bytes.
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
dataStr := string(data)
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
for _, match := range matches {
if len(match) != 2 {
continue
}
resMatch := strings.TrimSpace(match[1])
s1 := detectors.Result{
DetectorType: detectorspb.DetectorType_Voiceflow,
Raw: []byte(resMatch),
}
if verify {
client := s.client
if client == nil {
client = defaultClient
}
// Fetch the state for a random user.
payload := []byte(`{"question": "why is the sky blue?"}`)
req, err := http.NewRequestWithContext(ctx, "POST", "https://general-runtime.voiceflow.com/knowledge-base/query", bytes.NewBuffer(payload))
if err != nil {
continue
}
req.Header.Set("Accept", "application/json")
req.Header.Set("Authorization", resMatch)
req.Header.Set("Content-Type", "application/json")
res, err := client.Do(req)
if err == nil {
if res.StatusCode == http.StatusOK {
s1.Verified = true
} else if res.StatusCode == http.StatusUnauthorized {
// The secret is determinately not verified (nothing to do)
} else {
var buf bytes.Buffer
var bodyString string
_, err = io.Copy(&buf, res.Body)
if err == nil {
bodyString = buf.String()
}
s1.VerificationError = fmt.Errorf("unexpected HTTP response [status=%d, body=%s]", res.StatusCode, bodyString)
}
_ = res.Body.Close()
} else {
s1.VerificationError = err
}
}
results = append(results, s1)
}
return results, nil
}
func (s Scanner) Type() detectorspb.DetectorType {
return detectorspb.DetectorType_Voiceflow
}
-302
View File
@@ -1,302 +0,0 @@
//go:build detectors
// +build detectors
package voiceflow
import (
"context"
"fmt"
"github.com/google/go-cmp/cmp"
"github.com/google/go-cmp/cmp/cmpopts"
"testing"
"time"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
)
func TestVoiceflow_Pattern(t *testing.T) {
tests := []struct {
name string
data string
shouldMatch bool
match string
}{
// True positives
// https://github.com/funDAOmental/endlessquest/blob/5c008f7c6a7e58c45a88b72fef4b965c258d665c/Voiceflow/agent-api/index.js#L6
{
name: `valid_result1`,
data: `// z0MG IT'S NOT A SECRET (but we'll delete it)
const API_KEY = "VF.DM.6469b4e5909a470007b96250.k4ip0SMy84jWlCsF"; // it should look like this: VF.DM.XXXXXXX.XXXXXX... keep this a secret!`,
shouldMatch: true,
match: `VF.DM.6469b4e5909a470007b96250.k4ip0SMy84jWlCsF`,
},
// https://github.com/sherifButt/ll-site/blob/b98b268214324da42a84e996e4c03c242e122680/src/components/Chatbot.jsx#L14
{
name: `valid_result2`,
data: ` const runtime = useRuntime({
verify: { authorization: 'VF.DM.652da078cde70b0008e1c5df.zsIo23VTxNXKfb9f' },
session: { userID: 'user_123' },
});`,
shouldMatch: true,
match: `VF.DM.652da078cde70b0008e1c5df.zsIo23VTxNXKfb9f`,
},
// https://github.com/the-vv/Voiceflow-chatbot/blob/324db17693dd46387ea7a020e92c4e79b94306c6/src/app/chat/chat.component.ts#L27
{
name: `valid_result3`,
data: ` this.http.delete('https://general-runtime.voiceflow.com/state/user/TEST_USER', {
headers: {
Authorization: "VF.DM.652ecc210267ec00078fc726.ZFPdEwvU0d1jiIMq"
}
}).subscribe(res => {
this.loading = false;
this.doPrompt('', { action: { type: 'launch' } });
})`,
shouldMatch: true,
match: `VF.DM.652ecc210267ec00078fc726.ZFPdEwvU0d1jiIMq`,
},
// https://github.com/legionX7/Graduation-Project-API/blob/451431771d3fba1d8c634b8855274b414d7aed6d/mainAPI.py#L547
{
name: `valid_result4`,
data: `
API_KEY = 'VF.DM.646388eb1419c80007bbbaa4.XHOqETFO3cvTxlGl'
VERSION_ID = '646bc'`,
shouldMatch: true,
match: `VF.DM.646388eb1419c80007bbbaa4.XHOqETFO3cvTxlGl`,
},
// https://github.com/voiceflow/general-runtime/blob/master/tests/runtime/lib/DataAPI/utils.unit.ts
{
name: `valid_result5`,
data: ` it('extracts ID from a Dialog Manager API key', () => {
// eslint-disable-next-line no-secrets/no-secrets
const key = 'VF.DM.628d5d92faf688001bda7907.dmC8KKO1oX8JO5ai';
const result = utils.extractAPIKeyID(key);
expect(result).to.equal('628d5d92faf688001bda7907');
});`,
shouldMatch: true,
match: `VF.DM.628d5d92faf688001bda7907.dmC8KKO1oX8JO5ai`,
},
{
name: `valid_result6_legacy`,
data: ` it('extracts ID from a Workspace API key', () => {
// eslint-disable-next-line no-secrets/no-secrets
const key = 'VF.WS.62bcb0cca5184300066f5ac7.egnKyyzZksiS5iGa';
const result = utils.extractAPIKeyID(key);
expect(result).to.equal('62bcb0cca5184300066f5ac7');
});
`,
shouldMatch: true,
match: `VF.WS.62bcb0cca5184300066f5ac7.egnKyyzZksiS5iGa`,
},
{
name: `valid_result7_legacy`,
data: ` it('extracts ID from a Legacy Workspace API key', () => {
// eslint-disable-next-line no-secrets/no-secrets
const key = 'VF.62bcb0cca5184300066f5ac7.dmC8KKO1oX8JO5az';
const result = utils.extractAPIKeyID(key);
expect(result).to.equal('62bcb0cca5184300066f5ac7');
});`,
shouldMatch: true,
match: `VF.62bcb0cca5184300066f5ac7.dmC8KKO1oX8JO5az`,
},
// False positives
// https://github.com/ImperialCollegeLondon/voiceflow-integration-whatsapp/blob/0f3d6a5638b9acb4989d5bf8e77081cc78e9b976/README.md?plain=1#L155
{
name: `invalid_result1`,
data: "Now, paste it in your .env file for the **VF_PROJECT_API** variable<br>\n```VF_PROJECT_API='VF.DM.62xxxxxxxxxxxxxxxxxxxxxxx'```",
shouldMatch: false,
},
// https://github.com/voiceflow/api-examples/blob/c3d8ba9ee8eced7ec8d241973b1eb0284aaec212/rust/src/main.rs#L5
{
name: `invalid_result2`,
data: `const API_KEY: &str = "YOUR_API_KEY_HERE"; // it should look like this: VF.DM.XXXXXXX.XXXXXX... keep this a secret!`,
shouldMatch: false,
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
s := Scanner{}
results, err := s.FromData(context.Background(), true, []byte(test.data))
if err != nil {
t.Errorf("CoinbaseWaaS.FromData() error = %v", err)
return
}
if test.shouldMatch {
if len(results) == 0 {
t.Errorf("%s: did not receive a match for '%v' when one was expected", test.name, test.data)
return
}
expected := test.data
if test.match != "" {
expected = test.match
}
result := results[0]
if result.VerificationError != nil {
fmt.Printf("VerificationError: %v\n", result.VerificationError)
}
resultData := string(result.Raw)
if resultData != expected {
t.Errorf("%s: did not receive expected match.\n\texpected: '%s'\n\t actual: '%s'", test.name, expected, resultData)
return
}
} else {
if len(results) > 0 {
t.Errorf("%s: received a match for '%v' when one wasn't wanted", test.name, test.data)
return
}
}
})
}
}
func TestVoiceflow_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
secret := testSecrets.MustGetField("VOICEFLOW")
inactiveSecret := testSecrets.MustGetField("VOICEFLOW_INACTIVE")
type args struct {
ctx context.Context
data []byte
verify bool
}
tests := []struct {
name string
s Scanner
args args
want []detectors.Result
wantErr bool
wantVerificationErr bool
}{
{
name: "found, verified",
s: Scanner{},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a voiceflow secret %s within", secret)),
verify: true,
},
want: []detectors.Result{
{
DetectorType: detectorspb.DetectorType_Voiceflow,
Verified: true,
},
},
wantErr: false,
wantVerificationErr: false,
},
{
name: "found, unverified",
s: Scanner{},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a voiceflow secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
verify: true,
},
want: []detectors.Result{
{
DetectorType: detectorspb.DetectorType_Voiceflow,
Verified: false,
},
},
wantErr: false,
wantVerificationErr: false,
},
{
name: "not found",
s: Scanner{},
args: args{
ctx: context.Background(),
data: []byte("You cannot find the secret within"),
verify: true,
},
want: nil,
wantErr: false,
wantVerificationErr: false,
},
{
name: "found, would be verified if not for timeout",
s: Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a voiceflow secret %s within", secret)),
verify: true,
},
want: []detectors.Result{
{
DetectorType: detectorspb.DetectorType_Voiceflow,
Verified: false,
},
},
wantErr: false,
wantVerificationErr: true,
},
{
name: "found, verified but unexpected api surface",
s: Scanner{client: common.ConstantResponseHttpClient(404, "")},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a voiceflow secret %s within", secret)),
verify: true,
},
want: []detectors.Result{
{
DetectorType: detectorspb.DetectorType_Voiceflow,
Verified: false,
},
},
wantErr: false,
wantVerificationErr: true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
if (err != nil) != tt.wantErr {
t.Errorf("Voiceflow.FromData() error = %v, wantErr %v", err, tt.wantErr)
return
}
for i := range got {
if len(got[i].Raw) == 0 {
t.Fatalf("no raw secret present: \n %+v", got[i])
}
if (got[i].VerificationError != nil) != tt.wantVerificationErr {
t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError)
}
}
ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "VerificationError")
if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {
t.Errorf("Voiceflow.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
}
})
}
}
func BenchmarkFromData(benchmark *testing.B) {
ctx := context.Background()
s := Scanner{}
for name, data := range detectors.MustGetBenchmarkData() {
benchmark.Run(name, func(b *testing.B) {
b.ResetTimer()
for n := 0; n < b.N; n++ {
_, err := s.FromData(ctx, false, data)
if err != nil {
b.Fatal(err)
}
}
})
}
}
+8 -7
View File
@@ -18,10 +18,10 @@ type detectorKey struct {
version int
}
// DetectorInfo is used to store a detector and whether it should be verified.
type DetectorInfo struct {
// detectorInfo is used to store a detector and whether it should be verified.
type detectorInfo struct {
detectors.Detector
ShouldVerify bool
shouldVerify bool
}
// AhoCorasickCore encapsulates the operations and data structures used for keyword matching via the
@@ -32,7 +32,7 @@ type AhoCorasickCore struct {
// matching given a set of words. (keywords from the rules in the config)
prefilter ahocorasick.Trie
// Maps for efficient lookups during detection.
detectorTypeToDetectorInfo map[detectorKey]DetectorInfo
detectorTypeToDetectorInfo map[detectorKey]detectorInfo
detectors map[bool][]detectors.Detector
keywordsToDetectors map[string][]detectorKey
}
@@ -45,7 +45,7 @@ func NewAhoCorasickCore(detectors map[bool][]detectors.Detector) *AhoCorasickCor
return &AhoCorasickCore{
keywordsToDetectors: make(map[string][]detectorKey),
detectors: detectors,
detectorTypeToDetectorInfo: make(map[detectorKey]DetectorInfo, len(detectors[true])+len(detectors[false])),
detectorTypeToDetectorInfo: make(map[detectorKey]detectorInfo, len(detectors[true])+len(detectors[false])),
}
}
@@ -57,7 +57,7 @@ func (ac *AhoCorasickCore) Setup(ctx context.Context) {
for verify, detectorsSet := range ac.detectors {
for _, d := range detectorsSet {
key := createDetectorKey(d)
ac.detectorTypeToDetectorInfo[key] = DetectorInfo{Detector: d, ShouldVerify: verify}
ac.detectorTypeToDetectorInfo[key] = detectorInfo{Detector: d, shouldVerify: verify}
keywords = ac.extractAndMapKeywords(d, key, keywords)
}
}
@@ -65,6 +65,7 @@ func (ac *AhoCorasickCore) Setup(ctx context.Context) {
// Use the Ahocorasick algorithm to create a trie structure for efficient keyword matching.
// This ensures that we can rapidly match against a vast set of keywords without individually comparing each one.
ac.prefilter = *ahocorasick.NewTrieBuilder().AddStrings(keywords).Build()
ctx.Logger().V(4).Info("AhoCorasickCore lookups built")
ctx.Logger().V(4).Info("AhoCorasickCore Setup complete")
}
@@ -99,7 +100,7 @@ func (ac *AhoCorasickCore) MatchString(input string) []*ahocorasick.Match {
// PopulateDetectorsByMatch populates the given detectorMap based on the Aho-Corasick match results.
// This method is designed to reuse the same map for performance optimization,
// reducing the need for repeated allocations within each detector worker in the engine.
func (ac *AhoCorasickCore) PopulateDetectorsByMatch(match *ahocorasick.Match, detectors map[detectorspb.DetectorType]DetectorInfo) bool {
func (ac *AhoCorasickCore) PopulateDetectorsByMatch(match *ahocorasick.Match, detectors map[detectorspb.DetectorType]detectorInfo) bool {
matchedKeys, ok := ac.keywordsToDetectors[match.MatchString()]
if !ok {
return false
+10 -12
View File
@@ -4,16 +4,12 @@ import (
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/anthropic"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/envoyapikey"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/huggingface"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/ip2location"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/openvpn"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/ramp"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/salesforce"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/snowflake"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/sourcegraph"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/sourcegraphcody"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/tailscale"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/trufflehogenterprise"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/voiceflow"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/abbysale"
@@ -76,6 +72,7 @@ import (
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/azure"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/bannerbear"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/baremetrics"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/baseapiio"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/beamer"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/beebole"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/besnappy"
@@ -86,6 +83,7 @@ import (
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/bitfinex"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/bitlyaccesstoken"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/bitmex"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/blablabus"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/blazemeter"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/blitapp"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/blocknative"
@@ -192,6 +190,7 @@ import (
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/databox"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/databrickstoken"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/datadogtoken"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/datafire"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/datagov"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/deepai"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/deepgram"
@@ -304,6 +303,7 @@ import (
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/gitlabv2"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/gitter"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/glassnode"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/glitterlyapi"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/gocanvas"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/gocardless"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/goodday"
@@ -386,7 +386,6 @@ import (
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/livestorm"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/loadmill"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/locationiq"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/loggly"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/loginradius"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/lokalisetoken"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/loyverse"
@@ -532,6 +531,7 @@ import (
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/qase"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/qualaroo"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/qubole"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/quickmetrics"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/rapidapi"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/rawg"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/razorpay"
@@ -712,7 +712,6 @@ import (
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/urlscan"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/userflow"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/userstack"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/vagrantcloudpersonaltoken"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/vatlayer"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/vbout"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/vercel"
@@ -1061,6 +1060,7 @@ func DefaultDetectors() []detectors.Detector {
deepgram.Scanner{},
brandfetch.Scanner{},
typeform.Scanner{},
blablabus.Scanner{},
fxmarket.Scanner{},
ipapi.Scanner{},
clearbit.Scanner{},
@@ -1186,6 +1186,7 @@ func DefaultDetectors() []detectors.Detector {
restpackscreenshotapi.Scanner{},
shortcut.Scanner{},
// nasdaqdatalink.Scanner{},
quickmetrics.Scanner{},
neutrinoapi.Scanner{},
bitmex.Scanner{},
deepai.Scanner{},
@@ -1211,6 +1212,7 @@ func DefaultDetectors() []detectors.Detector {
scraperbox.Scanner{},
passbase.Scanner{},
ticketmaster.Scanner{},
datafire.Scanner{},
iexcloud.Scanner{},
partnerstack.Scanner{},
qubole.Scanner{},
@@ -1244,6 +1246,7 @@ func DefaultDetectors() []detectors.Detector {
scrapeowl.Scanner{},
rebrandly.Scanner{},
dandelion.Scanner{},
baseapiio.Scanner{},
purestake.Scanner{},
carboninterface.Scanner{},
signaturit.Scanner{},
@@ -1398,6 +1401,7 @@ func DefaultDetectors() []detectors.Detector {
convier.Scanner{},
loadmill.Scanner{},
magicbell.Scanner{},
glitterlyapi.Scanner{},
apitemplate.Scanner{},
knapsackpro.Scanner{},
twitter.Scanner{},
@@ -1545,15 +1549,9 @@ func DefaultDetectors() []detectors.Detector {
salesforce.Scanner{},
sourcegraph.Scanner{},
tailscale.Scanner{},
loggly.Scanner{},
web3storage.Scanner{},
&ramp.Scanner{},
&anthropic.Scanner{},
&sourcegraphcody.Scanner{},
voiceflow.Scanner{},
ip2location.Scanner{},
vagrantcloudpersonaltoken.Scanner{},
openvpn.Scanner{},
}
}
+2 -2
View File
@@ -457,7 +457,7 @@ func (e *Engine) detectorWorker(ctx context.Context) {
// Reuse the same map to avoid allocations.
const avgDetectorsPerChunk = 2
chunkSpecificDetectors := make(map[detectorspb.DetectorType]DetectorInfo, avgDetectorsPerChunk)
chunkSpecificDetectors := make(map[detectorspb.DetectorType]detectorInfo, avgDetectorsPerChunk)
for originalChunk := range e.ChunksChan() {
for chunk := range sources.Chunker(originalChunk) {
atomic.AddUint64(&e.metrics.BytesScanned, uint64(len(chunk.Data)))
@@ -475,7 +475,7 @@ func (e *Engine) detectorWorker(ctx context.Context) {
}
for k, detector := range chunkSpecificDetectors {
decoded.Chunk.Verify = detector.ShouldVerify
decoded.Chunk.Verify = detector.shouldVerify
wgDetect.Add(1)
e.detectableChunksChan <- detectableChunk{
chunk: *decoded.Chunk,
-5
View File
@@ -158,11 +158,6 @@ func UpdateLinkLineNumber(ctx context.Context, link string, newLine int64) strin
return link
}
if newLine <= 0 {
// Don't change the link if the line number is 0.
return link
}
switch determineProvider(link) {
case providerBitbucket:
// For Bitbucket, it doesn't support line links (based on the GenerateLink function).
-8
View File
@@ -251,14 +251,6 @@ func TestUpdateLinkLineNumber(t *testing.T) {
},
want: "https://onprem.customdomain.com/org/repo/commit/xyz123#L50",
},
{
name: "Don't include line when it's 0",
args: args{
link: "https://github.com/coinbase/cbpay-js/issues/181",
newLine: int64(0),
},
want: "https://github.com/coinbase/cbpay-js/issues/181",
},
{
name: "Invalid link",
args: args{
+1 -1
View File
@@ -11,7 +11,7 @@ import (
"testing"
"time"
diskbufferreader "github.com/trufflesecurity/disk-buffer-reader"
diskbufferreader "github.com/bill-rich/disk-buffer-reader"
"github.com/stretchr/testify/assert"
logContext "github.com/trufflesecurity/trufflehog/v3/pkg/context"
+1 -1
View File
@@ -4,7 +4,7 @@ import (
"context"
"io"
diskbufferreader "github.com/trufflesecurity/disk-buffer-reader"
diskbufferreader "github.com/bill-rich/disk-buffer-reader"
logContext "github.com/trufflesecurity/trufflehog/v3/pkg/context"
"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff
-2
View File
@@ -121,8 +121,6 @@ func (m *LocalSource) validate(all bool) error {
}
}
// no validation rules for ScanPeriod
if len(errors) > 0 {
return LocalSourceMultiError(errors)
}
+1 -1
View File
@@ -7,7 +7,7 @@ import (
"testing"
"testing/iotest"
diskbufferreader "github.com/trufflesecurity/disk-buffer-reader"
diskbufferreader "github.com/bill-rich/disk-buffer-reader"
"github.com/stretchr/testify/assert"
"github.com/trufflesecurity/trufflehog/v3/pkg/context"
+3 -2
View File
@@ -7,7 +7,7 @@ import (
"os"
"path/filepath"
diskbufferreader "github.com/trufflesecurity/disk-buffer-reader"
diskbufferreader "github.com/bill-rich/disk-buffer-reader"
"github.com/go-errors/errors"
"github.com/go-logr/logr"
"google.golang.org/protobuf/proto"
@@ -39,7 +39,8 @@ type Source struct {
// Ensure the Source satisfies the interfaces at compile time
var _ sources.Source = (*Source)(nil)
var _ sources.SourceUnitUnmarshaller = (*Source)(nil)
var _ sources.SourceUnitEnumChunker = (*Source)(nil)
var _ sources.SourceUnitEnumerator = (*Source)(nil)
var _ sources.SourceUnitChunker = (*Source)(nil)
// Type returns the type of source.
// It is used for matching source types in configuration and job input.
+10 -195
View File
@@ -15,7 +15,6 @@ import (
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/source_metadatapb"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
"github.com/trufflesecurity/trufflehog/v3/pkg/sourcestest"
)
func TestSource_Scan(t *testing.T) {
@@ -85,15 +84,23 @@ func TestSource_Scan(t *testing.T) {
}
func TestScanFile(t *testing.T) {
tmpfile, err := os.CreateTemp("", "example.txt")
if err != nil {
t.Fatal(err)
}
defer os.Remove(tmpfile.Name())
chunkSize := sources.ChunkSize
secretPart1 := "SECRET"
secretPart2 := "SPLIT"
// Split the secret into two parts and pad the rest of the chunk with A's.
data := strings.Repeat("A", chunkSize-len(secretPart1)) + secretPart1 + secretPart2 + strings.Repeat("A", chunkSize-len(secretPart2))
tmpfile, cleanup, err := createTempFile("", data)
_, err = tmpfile.Write([]byte(data))
assert.Nil(t, err)
err = tmpfile.Close()
assert.Nil(t, err)
defer cleanup()
source := &Source{}
chunksChan := make(chan *sources.Chunk, 2)
@@ -113,195 +120,3 @@ func TestScanFile(t *testing.T) {
assert.Contains(t, foundSecret, secretPart1+secretPart2)
}
func TestEnumerate(t *testing.T) {
t.Parallel()
ctx := context.Background()
// Setup the connection to test enumeration.
units := []string{
"/one", "/two", "/three",
"/path/to/dir/", "/path/to/another/dir/",
}
conn, err := anypb.New(&sourcespb.Filesystem{
Paths: units[0:3],
Directories: units[3:],
})
assert.NoError(t, err)
// Initialize the source.
s := Source{}
err = s.Init(ctx, "test enumerate", 0, 0, true, conn, 1)
assert.NoError(t, err)
reporter := sourcestest.TestReporter{}
err = s.Enumerate(ctx, &reporter)
assert.NoError(t, err)
assert.Equal(t, len(units), len(reporter.Units))
assert.Equal(t, 0, len(reporter.UnitErrs))
for _, unit := range reporter.Units {
assert.Contains(t, units, unit.SourceUnitID())
}
for _, unit := range units {
assert.Contains(t, reporter.Units, sources.CommonSourceUnit{ID: unit})
}
}
func TestChunkUnit(t *testing.T) {
t.Parallel()
ctx := context.Background()
// Setup test file to chunk.
fileContents := "TestChunkUnit"
tmpfile, cleanup, err := createTempFile("", fileContents)
assert.NoError(t, err)
defer cleanup()
tmpdir, cleanup, err := createTempDir("", "foo", "bar", "baz")
assert.NoError(t, err)
defer cleanup()
conn, err := anypb.New(&sourcespb.Filesystem{})
assert.NoError(t, err)
// Initialize the source.
s := Source{}
err = s.Init(ctx, "test chunk unit", 0, 0, true, conn, 1)
assert.NoError(t, err)
// Happy path single file.
reporter := sourcestest.TestReporter{}
err = s.ChunkUnit(ctx, sources.CommonSourceUnit{
ID: tmpfile.Name(),
}, &reporter)
assert.NoError(t, err)
// Happy path directory.
err = s.ChunkUnit(ctx, sources.CommonSourceUnit{
ID: tmpdir,
}, &reporter)
assert.NoError(t, err)
// Error path.
err = s.ChunkUnit(ctx, sources.CommonSourceUnit{
ID: "/file/not/found",
}, &reporter)
assert.NoError(t, err)
assert.Equal(t, 4, len(reporter.Chunks))
assert.Equal(t, 1, len(reporter.ChunkErrs))
dataFound := make(map[string]struct{}, 4)
for _, chunk := range reporter.Chunks {
dataFound[string(chunk.Data)] = struct{}{}
}
assert.Contains(t, dataFound, fileContents)
assert.Contains(t, dataFound, "foo")
assert.Contains(t, dataFound, "bar")
assert.Contains(t, dataFound, "baz")
}
func TestEnumerateReporterErr(t *testing.T) {
t.Parallel()
ctx := context.Background()
// Setup the connection to test enumeration.
units := []string{
"/one", "/two", "/three",
"/path/to/dir/", "/path/to/another/dir/",
}
conn, err := anypb.New(&sourcespb.Filesystem{
Paths: units[0:3],
Directories: units[3:],
})
assert.NoError(t, err)
// Initialize the source.
s := Source{}
err = s.Init(ctx, "test enumerate", 0, 0, true, conn, 1)
assert.NoError(t, err)
// Enumerate should always return an error if the reporter returns an
// error.
reporter := sourcestest.ErrReporter{}
err = s.Enumerate(ctx, &reporter)
assert.Error(t, err)
}
func TestChunkUnitReporterErr(t *testing.T) {
t.Parallel()
ctx := context.Background()
// Setup test file to chunk.
tmpfile, err := os.CreateTemp("", "example.txt")
if err != nil {
t.Fatal(err)
}
defer os.Remove(tmpfile.Name())
fileContents := []byte("TestChunkUnit")
_, err = tmpfile.Write(fileContents)
assert.NoError(t, err)
assert.NoError(t, tmpfile.Close())
conn, err := anypb.New(&sourcespb.Filesystem{})
assert.NoError(t, err)
// Initialize the source.
s := Source{}
err = s.Init(ctx, "test chunk unit", 0, 0, true, conn, 1)
assert.NoError(t, err)
// Happy path. ChunkUnit should always return an error if the reporter
// returns an error.
reporter := sourcestest.ErrReporter{}
err = s.ChunkUnit(ctx, sources.CommonSourceUnit{
ID: tmpfile.Name(),
}, &reporter)
assert.Error(t, err)
// Error path. ChunkUnit should always return an error if the reporter
// returns an error.
err = s.ChunkUnit(ctx, sources.CommonSourceUnit{
ID: "/file/not/found",
}, &reporter)
assert.Error(t, err)
}
// createTempFile is a helper function to create a temporary file in the given
// directory with the provided contents. If dir is "", the operating system's
// temp directory is used.
func createTempFile(dir string, contents string) (*os.File, func(), error) {
tmpfile, err := os.CreateTemp(dir, "trufflehogtest")
if err != nil {
return nil, nil, err
}
if _, err := tmpfile.Write([]byte(contents)); err != nil {
_ = os.Remove(tmpfile.Name())
return nil, nil, err
}
if err := tmpfile.Close(); err != nil {
_ = os.Remove(tmpfile.Name())
return nil, nil, err
}
return tmpfile, func() { _ = os.Remove(tmpfile.Name()) }, nil
}
// createTempDir is a helper function to create a temporary directory in the
// given directory with files containing the provided contents. If dir is "",
// the operating system's temp directory is used.
func createTempDir(dir string, contents ...string) (string, func(), error) {
tmpdir, err := os.MkdirTemp(dir, "trufflehogtest")
if err != nil {
return "", nil, err
}
for _, content := range contents {
if _, _, err := createTempFile(tmpdir, content); err != nil {
_ = os.RemoveAll(tmpdir)
return "", nil, err
}
}
return tmpdir, func() { _ = os.RemoveAll(tmpdir) }, nil
}
+1 -1
View File
@@ -10,7 +10,7 @@ import (
"sync"
"cloud.google.com/go/storage"
diskbufferreader "github.com/trufflesecurity/disk-buffer-reader"
diskbufferreader "github.com/bill-rich/disk-buffer-reader"
"github.com/go-errors/errors"
"github.com/go-logr/logr"
"golang.org/x/oauth2"
+1 -1
View File
@@ -13,7 +13,7 @@ import (
"sync/atomic"
"time"
diskbufferreader "github.com/trufflesecurity/disk-buffer-reader"
diskbufferreader "github.com/bill-rich/disk-buffer-reader"
"github.com/go-errors/errors"
"github.com/go-git/go-git/v5"
"github.com/go-git/go-git/v5/plumbing"
+56 -77
View File
@@ -222,10 +222,7 @@ func (s *Source) Init(aCtx context.Context, name string, jobID sources.JobID, so
}
s.conn = &conn
s.filteredRepoCache = s.newFilteredRepoCache(memory.New(),
append(s.conn.GetRepositories(), s.conn.GetIncludeRepos()...),
s.conn.GetIgnoreRepos(),
)
s.filteredRepoCache = s.newFilteredRepoCache(memory.New(), s.conn.GetRepositories(), s.conn.GetIgnoreRepos())
s.memberCache = make(map[string]struct{})
s.repoSizes = newRepoSize()
@@ -236,7 +233,7 @@ func (s *Source) Init(aCtx context.Context, name string, jobID sources.JobID, so
aCtx.Logger().Error(err, "invalid repository", "repo", repo)
continue
}
s.filteredRepoCache.Set(repo, r)
s.filteredRepoCache.Set(r, r)
}
s.includeIssueComments = s.conn.IncludeIssueComments
@@ -511,12 +508,12 @@ func (s *Source) enumerateUnauthenticated(ctx context.Context, apiEndpoint strin
for _, org := range s.orgsCache.Keys() {
if err := s.getReposByOrg(ctx, org); err != nil {
s.log.Error(err, "error fetching repos for org")
s.log.Error(err, "error fetching repos for org or user")
}
// We probably don't need to do this, since getting repos by org makes more sense?
if err := s.getReposByUser(ctx, org); err != nil {
s.log.Error(err, "error fetching repos for user")
s.log.Error(err, "error fetching repos for org or user")
}
if s.conn.ScanUsers {
@@ -828,51 +825,38 @@ func (s *Source) scan(ctx context.Context, installationClient *github.Client, ch
// Unauthenticated access to most github endpoints has a rate limit of 60 requests per hour.
// This will likely only be exhausted if many users/orgs are scanned without auth
func (s *Source) handleRateLimit(errIn error, res *github.Response) bool {
var (
knownWait = true
remaining = 0
retryAfter time.Duration
)
// GitHub has both primary (RateLimit) and secondary (AbuseRateLimit) errors.
var rateLimit *github.RateLimitError
var abuseLimit *github.AbuseRateLimitError
if errors.As(errIn, &rateLimit) {
// Do nothing
} else if errors.As(errIn, &abuseLimit) {
retryAfter = abuseLimit.GetRetryAfter()
} else {
limit, ok := errIn.(*github.RateLimitError)
if !ok {
return false
}
githubNumRateLimitEncountered.WithLabelValues(s.name).Inc()
// Parse retry information from response headers, unless a Retry-After value was already provided.
// https://docs.github.com/en/rest/overview/resources-in-the-rest-api#exceeding-the-rate-limit
if retryAfter <= 0 && res != nil {
var err error
remaining, err = strconv.Atoi(res.Header.Get("x-ratelimit-remaining"))
if res != nil {
knownWait := true
remaining, err := strconv.Atoi(res.Header.Get("x-ratelimit-remaining"))
if err != nil {
knownWait = false
}
resetTime, err := strconv.Atoi(res.Header.Get("x-ratelimit-reset"))
if err != nil || resetTime == 0 {
knownWait = false
} else if resetTime > 0 {
retryAfter = time.Duration(int64(resetTime)-time.Now().Unix()) * time.Second
}
if knownWait && remaining == 0 {
waitTime := int64(resetTime) - time.Now().Unix()
if waitTime > 0 {
duration := time.Duration(waitTime+1) * time.Second
s.log.V(2).Info("rate limited", "resumeTime", time.Now().Add(duration).String())
time.Sleep(duration)
githubSecondsSpentRateLimited.WithLabelValues(s.name).Add(duration.Seconds())
return true
}
}
}
resumeTime := time.Now().Add(retryAfter).String()
if knownWait && remaining == 0 && retryAfter > 0 {
s.log.V(2).Info("rate limited", "retry_after", retryAfter.String(), "resume_time", resumeTime)
} else {
// TODO: Use exponential backoff instead of static retry time.
retryAfter = time.Minute * 5
s.log.V(2).Error(errIn, "unexpected rate limit error", "retry_after", retryAfter.String(), "resume_time", resumeTime)
}
time.Sleep(retryAfter)
githubSecondsSpentRateLimited.WithLabelValues(s.name).Add(retryAfter.Seconds())
s.log.V(2).Info("handling rate limit (5 minutes retry)", "retry-after", limit.Message)
time.Sleep(time.Minute * 5)
return true
}
@@ -1092,7 +1076,7 @@ func (s *Source) scanComments(ctx context.Context, repoPath string, chunksChan c
}
func (s *Source) processGistComments(ctx context.Context, repoPath string, trimmedURL []string, repoURL *url.URL, chunksChan chan *sources.Chunk) error {
ctx.Logger().V(2).Info("scanning github gist comments", "repository", repoPath)
s.log.Info("scanning github gist comments", "repository", repoPath)
// GitHub Gist URL.
gistID, err := extractGistID(trimmedURL)
if err != nil {
@@ -1147,10 +1131,9 @@ var (
)
type repoInfo struct {
owner string
repo string
repoPath string
visibility source_metadatapb.Visibility
owner string
repo string
repoPath string
}
func (s *Source) processRepoComments(ctx context.Context, repoPath string, trimmedURL []string, repoURL *url.URL, chunksChan chan *sources.Chunk) error {
@@ -1161,33 +1144,22 @@ func (s *Source) processRepoComments(ctx context.Context, repoPath string, trimm
owner := trimmedURL[1]
repo := trimmedURL[2]
if !(s.includeIssueComments || s.includePRComments) {
return nil
}
repoInfo := repoInfo{
owner: owner,
repo: repo,
repoPath: repoPath,
visibility: s.visibilityOf(ctx, repoPath),
}
repoInfo := repoInfo{owner: owner, repo: repo, repoPath: repoPath}
if s.includeIssueComments {
ctx.Logger().V(2).Info("scanning github issues", "repository", repoInfo.repoPath)
if err := s.processIssues(ctx, repoInfo, chunksChan); err != nil {
if err := s.processIssueComments(ctx, repoInfo, chunksChan); err != nil {
return err
}
if err := s.processIssueComments(ctx, repoInfo, chunksChan); err != nil {
if err := s.processIssues(ctx, repoInfo, chunksChan); err != nil {
return err
}
}
if s.includePRComments {
ctx.Logger().V(2).Info("scanning github pull requests", "repository", repoInfo.repoPath)
if err := s.processPRs(ctx, repoInfo, chunksChan); err != nil {
if err := s.processPRComments(ctx, repoInfo, chunksChan); err != nil {
return err
}
if err := s.processPRComments(ctx, repoInfo, chunksChan); err != nil {
if err := s.processPRs(ctx, repoInfo, chunksChan); err != nil {
return err
}
}
@@ -1197,6 +1169,8 @@ func (s *Source) processRepoComments(ctx context.Context, repoPath string, trimm
}
func (s *Source) processIssues(ctx context.Context, info repoInfo, chunksChan chan *sources.Chunk) error {
s.log.Info("scanning github issue descriptions", "repository", info.repoPath)
bodyTextsOpts := &github.IssueListByRepoOptions{
Sort: sortType,
Direction: directionType,
@@ -1217,7 +1191,7 @@ func (s *Source) processIssues(ctx context.Context, info repoInfo, chunksChan ch
return err
}
if err = s.chunkIssues(ctx, info, issues, chunksChan); err != nil {
if err = s.chunkIssues(ctx, info.repo, info.repoPath, issues, chunksChan); err != nil {
return err
}
@@ -1231,6 +1205,8 @@ func (s *Source) processIssues(ctx context.Context, info repoInfo, chunksChan ch
}
func (s *Source) processIssueComments(ctx context.Context, info repoInfo, chunksChan chan *sources.Chunk) error {
s.log.Info("scanning github issue comments", "repository", info.repoPath)
issueOpts := &github.IssueListCommentsOptions{
Sort: &sortType,
Direction: &directionType,
@@ -1250,7 +1226,7 @@ func (s *Source) processIssueComments(ctx context.Context, info repoInfo, chunks
return err
}
if err = s.chunkIssueComments(ctx, info, issueComments, chunksChan); err != nil {
if err = s.chunkIssueComments(ctx, info.repo, info.repoPath, issueComments, chunksChan); err != nil {
return err
}
@@ -1264,6 +1240,8 @@ func (s *Source) processIssueComments(ctx context.Context, info repoInfo, chunks
}
func (s *Source) processPRs(ctx context.Context, info repoInfo, chunksChan chan *sources.Chunk) error {
s.log.Info("scanning github pull request descriptions", "repository", info.repoPath)
prOpts := &github.PullRequestListOptions{
Sort: sortType,
Direction: directionType,
@@ -1284,7 +1262,7 @@ func (s *Source) processPRs(ctx context.Context, info repoInfo, chunksChan chan
return err
}
if err = s.chunkPullRequests(ctx, info, prs, chunksChan); err != nil {
if err = s.chunkPullRequests(ctx, info.repo, prs, chunksChan); err != nil {
return err
}
@@ -1298,6 +1276,8 @@ func (s *Source) processPRs(ctx context.Context, info repoInfo, chunksChan chan
}
func (s *Source) processPRComments(ctx context.Context, info repoInfo, chunksChan chan *sources.Chunk) error {
s.log.Info("scanning github pull request comments", "repository", info.repoPath)
prOpts := &github.PullRequestListCommentsOptions{
Sort: sortType,
Direction: directionType,
@@ -1317,7 +1297,7 @@ func (s *Source) processPRComments(ctx context.Context, info repoInfo, chunksCha
return err
}
if err = s.chunkPullRequestComments(ctx, info, prComments, chunksChan); err != nil {
if err = s.chunkPullRequestComments(ctx, info.repo, prComments, chunksChan); err != nil {
return err
}
@@ -1330,7 +1310,7 @@ func (s *Source) processPRComments(ctx context.Context, info repoInfo, chunksCha
return nil
}
func (s *Source) chunkIssues(ctx context.Context, repoInfo repoInfo, issues []*github.Issue, chunksChan chan *sources.Chunk) error {
func (s *Source) chunkIssues(ctx context.Context, repo, repoPath string, issues []*github.Issue, chunksChan chan *sources.Chunk) error {
for _, issue := range issues {
// Skip pull requests since covered by processPRs.
@@ -1350,9 +1330,9 @@ func (s *Source) chunkIssues(ctx context.Context, repoInfo repoInfo, issues []*g
Link: sanitizer.UTF8(issue.GetHTMLURL()),
Username: sanitizer.UTF8(issue.GetUser().GetLogin()),
Email: sanitizer.UTF8(issue.GetUser().GetEmail()),
Repository: sanitizer.UTF8(repoInfo.repo),
Repository: sanitizer.UTF8(repo),
Timestamp: sanitizer.UTF8(issue.GetCreatedAt().String()),
Visibility: repoInfo.visibility,
Visibility: s.visibilityOf(ctx, repoPath),
},
},
},
@@ -1369,7 +1349,7 @@ func (s *Source) chunkIssues(ctx context.Context, repoInfo repoInfo, issues []*g
return nil
}
func (s *Source) chunkIssueComments(ctx context.Context, repoInfo repoInfo, comments []*github.IssueComment, chunksChan chan *sources.Chunk) error {
func (s *Source) chunkIssueComments(ctx context.Context, repo, repoPath string, comments []*github.IssueComment, chunksChan chan *sources.Chunk) error {
for _, comment := range comments {
// Create chunk and send it to the channel.
chunk := &sources.Chunk{
@@ -1383,9 +1363,9 @@ func (s *Source) chunkIssueComments(ctx context.Context, repoInfo repoInfo, comm
Link: sanitizer.UTF8(comment.GetHTMLURL()),
Username: sanitizer.UTF8(comment.GetUser().GetLogin()),
Email: sanitizer.UTF8(comment.GetUser().GetEmail()),
Repository: sanitizer.UTF8(repoInfo.repo),
Repository: sanitizer.UTF8(repo),
Timestamp: sanitizer.UTF8(comment.GetCreatedAt().String()),
Visibility: repoInfo.visibility,
Visibility: s.visibilityOf(ctx, repoPath),
},
},
},
@@ -1402,7 +1382,7 @@ func (s *Source) chunkIssueComments(ctx context.Context, repoInfo repoInfo, comm
return nil
}
func (s *Source) chunkPullRequestComments(ctx context.Context, repoInfo repoInfo, comments []*github.PullRequestComment, chunksChan chan *sources.Chunk) error {
func (s *Source) chunkPullRequestComments(ctx context.Context, repo string, comments []*github.PullRequestComment, chunksChan chan *sources.Chunk) error {
for _, comment := range comments {
// Create chunk and send it to the channel.
chunk := &sources.Chunk{
@@ -1416,9 +1396,8 @@ func (s *Source) chunkPullRequestComments(ctx context.Context, repoInfo repoInfo
Link: sanitizer.UTF8(comment.GetHTMLURL()),
Username: sanitizer.UTF8(comment.GetUser().GetLogin()),
Email: sanitizer.UTF8(comment.GetUser().GetEmail()),
Repository: sanitizer.UTF8(repoInfo.repo),
Repository: sanitizer.UTF8(repo),
Timestamp: sanitizer.UTF8(comment.GetCreatedAt().String()),
Visibility: repoInfo.visibility,
},
},
},
@@ -1435,7 +1414,7 @@ func (s *Source) chunkPullRequestComments(ctx context.Context, repoInfo repoInfo
return nil
}
func (s *Source) chunkPullRequests(ctx context.Context, repoInfo repoInfo, prs []*github.PullRequest, chunksChan chan *sources.Chunk) error {
func (s *Source) chunkPullRequests(ctx context.Context, repo string, prs []*github.PullRequest, chunksChan chan *sources.Chunk) error {
for _, pr := range prs {
// Create chunk and send it to the channel.
chunk := &sources.Chunk{
@@ -1449,9 +1428,8 @@ func (s *Source) chunkPullRequests(ctx context.Context, repoInfo repoInfo, prs [
Link: sanitizer.UTF8(pr.GetHTMLURL()),
Username: sanitizer.UTF8(pr.GetUser().GetLogin()),
Email: sanitizer.UTF8(pr.GetUser().GetEmail()),
Repository: sanitizer.UTF8(repoInfo.repo),
Repository: sanitizer.UTF8(repo),
Timestamp: sanitizer.UTF8(pr.GetCreatedAt().String()),
Visibility: repoInfo.visibility,
},
},
},
@@ -1484,7 +1462,8 @@ func (s *Source) chunkGistComments(ctx context.Context, gistUrl string, comments
Email: sanitizer.UTF8(comment.GetUser().GetEmail()),
Repository: sanitizer.UTF8(gistUrl),
Timestamp: sanitizer.UTF8(comment.GetCreatedAt().String()),
// TODO: Fetching this requires making an additional API call. We may want to include this in the future.
// Fetching this information requires making an additional API call.
// We may want to include this in the future.
// Visibility: s.visibilityOf(ctx, repoPath),
},
},
+4 -18
View File
@@ -89,14 +89,14 @@ func (f Fatal) Unwrap() error { return f.error }
// ChunkError is a custom error type for errors encountered during chunking of
// a specific unit.
type ChunkError struct {
Unit SourceUnit
Err error
unit SourceUnit
err error
}
func (f ChunkError) Error() string {
return fmt.Sprintf("error chunking unit %q: %s", f.Unit.SourceUnitID(), f.Err.Error())
return fmt.Sprintf("error chunking unit %q: %s", f.unit.SourceUnitID(), f.err.Error())
}
func (f ChunkError) Unwrap() error { return f.Err }
func (f ChunkError) Unwrap() error { return f.err }
// JobProgress aggregates information about a run of a Source.
type JobProgress struct {
@@ -351,17 +351,3 @@ func (m JobProgressMetrics) ElapsedTime() time.Duration {
}
return m.EndTime.Sub(m.StartTime)
}
// ErrorsFor returns all the errors for the given SourceUnit. If there are no
// errors, including the case that the unit has not been encountered, nil will
// be returned.
func (m JobProgressMetrics) ErrorsFor(unit SourceUnit) []error {
var errs []error
for _, err := range m.Errors {
var chunkErr ChunkError
if errors.As(err, &chunkErr) && chunkErr.Unit == unit {
errs = append(errs, err)
}
}
return errs
}
-218
View File
@@ -1,218 +0,0 @@
package sources
import (
"errors"
"fmt"
"strings"
"sync"
"time"
lru "github.com/hashicorp/golang-lru/v2"
"github.com/trufflesecurity/trufflehog/v3/pkg/context"
)
// UnitHook implements JobProgressHook for tracking the progress of each
// individual unit.
type UnitHook struct {
metrics *lru.Cache[string, *UnitMetrics]
mu sync.Mutex
NoopHook
}
type UnitHookOpt func(*UnitHook)
func WithUnitHookCache(cache *lru.Cache[string, *UnitMetrics]) UnitHookOpt {
return func(hook *UnitHook) { hook.metrics = cache }
}
func NewUnitHook(ctx context.Context, opts ...UnitHookOpt) *UnitHook {
// lru.NewWithEvict can only fail if the size is < 0.
cache, _ := lru.NewWithEvict(1024, func(key string, value *UnitMetrics) {
if value.handled {
return
}
ctx.Logger().Error(fmt.Errorf("eviction"), "dropping unit metric",
"id", key,
"metric", value,
)
})
hook := UnitHook{metrics: cache}
for _, opt := range opts {
opt(&hook)
}
return &hook
}
// id is a helper method to generate an ID for the given job and unit.
func (u *UnitHook) id(ref JobProgressRef, unit SourceUnit) string {
unitID := ""
if unit != nil {
unitID = unit.SourceUnitID()
}
return fmt.Sprintf("%d/%d/%s", ref.SourceID, ref.JobID, unitID)
}
func (u *UnitHook) StartUnitChunking(ref JobProgressRef, unit SourceUnit, start time.Time) {
id := u.id(ref, unit)
u.mu.Lock()
defer u.mu.Unlock()
u.metrics.Add(id, &UnitMetrics{
Unit: unit,
Parent: ref,
StartTime: start,
})
}
func (u *UnitHook) EndUnitChunking(ref JobProgressRef, unit SourceUnit, end time.Time) {
id := u.id(ref, unit)
u.mu.Lock()
defer u.mu.Unlock()
metrics, ok := u.metrics.Get(id)
if !ok {
return
}
metrics.EndTime = end
}
func (u *UnitHook) ReportChunk(ref JobProgressRef, unit SourceUnit, chunk *Chunk) {
id := u.id(ref, unit)
u.mu.Lock()
defer u.mu.Unlock()
metrics, ok := u.metrics.Get(id)
if !ok && unit != nil {
// The unit has been evicted.
return
} else if !ok && unit == nil {
// This is a chunk from a non-unit source.
metrics = &UnitMetrics{
Unit: nil,
Parent: ref,
StartTime: ref.Snapshot().StartTime,
}
u.metrics.Add(id, metrics)
}
metrics.TotalChunks++
metrics.TotalBytes += uint64(len(chunk.Data))
}
func (u *UnitHook) ReportError(ref JobProgressRef, err error) {
u.mu.Lock()
defer u.mu.Unlock()
// Always add the error to the nil unit if it exists.
if metrics, ok := u.metrics.Get(u.id(ref, nil)); ok {
metrics.Errors = append(metrics.Errors, err)
}
// Check if it's a ChunkError for a specific unit.
var chunkErr ChunkError
if !errors.As(err, &chunkErr) {
return
}
id := u.id(ref, chunkErr.Unit)
metrics, ok := u.metrics.Get(id)
if !ok {
return
}
metrics.Errors = append(metrics.Errors, err)
}
func (u *UnitHook) Finish(ref JobProgressRef) {
u.mu.Lock()
defer u.mu.Unlock()
// Clear out any metrics on this job. This covers the case for the
// source running without unit support.
prefix := u.id(ref, nil)
for _, id := range u.metrics.Keys() {
if !strings.HasPrefix(id, prefix) {
continue
}
metric, ok := u.metrics.Get(id)
if !ok {
continue
}
// If the unit is nil, the source does not support units.
// Use the overall job metrics instead.
if metric.Unit == nil {
snap := ref.Snapshot()
metric.StartTime = snap.StartTime
metric.EndTime = snap.EndTime
metric.Errors = snap.Errors
}
}
}
// UnitMetrics gets all the currently active or newly finished metrics for this
// job. If a unit returned from this method has finished, it will be removed
// from the cache and no longer returned in successive calls to UnitMetrics().
func (u *UnitHook) UnitMetrics() []UnitMetrics {
u.mu.Lock()
defer u.mu.Unlock()
output := make([]UnitMetrics, 0, u.metrics.Len())
for _, id := range u.metrics.Keys() {
metric, ok := u.metrics.Get(id)
if !ok {
continue
}
output = append(output, *metric)
if metric.IsFinished() {
metric.handled = true
u.metrics.Remove(id)
}
}
return output
}
type UnitMetrics struct {
Unit SourceUnit
Parent JobProgressRef
// Start and end time for chunking this unit.
StartTime time.Time
EndTime time.Time
// Total number of chunks produced from this unit.
TotalChunks uint64
// Total number of bytes produced from this unit.
TotalBytes uint64
// All errors encountered by this unit.
Errors []error
// Flag to mark that these metrics were intentionally evicted from
// the cache.
handled bool
}
func (u UnitMetrics) IsFinished() bool {
return !u.EndTime.IsZero()
}
// ElapsedTime is a convenience method that provides the elapsed time the job
// has been running. If it hasn't started yet, 0 is returned. If it has
// finished, the total time is returned.
func (u UnitMetrics) ElapsedTime() time.Duration {
if u.StartTime.IsZero() {
return 0
}
if u.EndTime.IsZero() {
return time.Since(u.StartTime)
}
return u.EndTime.Sub(u.StartTime)
}
// NoopHook implements JobProgressHook by doing nothing. This is useful for
// embedding in other structs to overwrite only the methods of the interface
// that you care about.
type NoopHook struct{}
func (NoopHook) Start(JobProgressRef, time.Time) {}
func (NoopHook) End(JobProgressRef, time.Time) {}
func (NoopHook) StartEnumerating(JobProgressRef, time.Time) {}
func (NoopHook) EndEnumerating(JobProgressRef, time.Time) {}
func (NoopHook) StartUnitChunking(JobProgressRef, SourceUnit, time.Time) {}
func (NoopHook) EndUnitChunking(JobProgressRef, SourceUnit, time.Time) {}
func (NoopHook) ReportError(JobProgressRef, error) {}
func (NoopHook) ReportUnit(JobProgressRef, SourceUnit) {}
func (NoopHook) ReportChunk(JobProgressRef, SourceUnit, *Chunk) {}
func (NoopHook) Finish(JobProgressRef) {}
-23
View File
@@ -126,26 +126,3 @@ func TestJobProgressElapsedTime(t *testing.T) {
metrics.EndTime = metrics.StartTime.Add(1 * time.Hour)
assert.Equal(t, metrics.ElapsedTime(), 1*time.Hour)
}
func TestJobProgressErrorsFor(t *testing.T) {
metrics := JobProgressMetrics{
Errors: []error{
Fatal{ChunkError{
Unit: CommonSourceUnit{ID: "foo"},
Err: fmt.Errorf("foo error"),
}},
ChunkError{
Unit: CommonSourceUnit{ID: "foo"},
Err: fmt.Errorf("foo again error"),
},
ChunkError{
Unit: CommonSourceUnit{ID: "bar"},
Err: fmt.Errorf("bar error"),
},
fmt.Errorf("hi there"),
},
}
assert.Equal(t, 2, len(metrics.ErrorsFor(CommonSourceUnit{ID: "foo"})))
assert.Equal(t, 1, len(metrics.ErrorsFor(CommonSourceUnit{ID: "bar"})))
assert.Equal(t, 0, len(metrics.ErrorsFor(CommonSourceUnit{ID: "baz"})))
}
+1 -1
View File
@@ -14,7 +14,7 @@ import (
"github.com/aws/aws-sdk-go/service/s3"
"github.com/aws/aws-sdk-go/service/s3/s3manager"
"github.com/aws/aws-sdk-go/service/sts"
diskbufferreader "github.com/trufflesecurity/disk-buffer-reader"
diskbufferreader "github.com/bill-rich/disk-buffer-reader"
"github.com/go-errors/errors"
"github.com/go-logr/logr"
"golang.org/x/sync/errgroup"
-115
View File
@@ -3,10 +3,8 @@ package sources
import (
"errors"
"fmt"
"sort"
"testing"
lru "github.com/hashicorp/golang-lru/v2"
"github.com/stretchr/testify/assert"
"google.golang.org/protobuf/types/known/anypb"
@@ -202,8 +200,6 @@ func (c *unitChunker) ChunkUnit(ctx context.Context, unit SourceUnit, rep ChunkR
if err := rep.ChunkErr(ctx, fmt.Errorf(step.err)); err != nil {
return err
}
}
if step.output == "" {
continue
}
if err := rep.ChunkOk(ctx, Chunk{Data: []byte(step.output)}); err != nil {
@@ -314,114 +310,3 @@ func TestSourceManagerAvailableCapacity(t *testing.T) {
<-ref.Done() // Wait for the job to finish.
assert.Equal(t, 1337, mgr.AvailableCapacity())
}
func TestSourceManagerUnitHook(t *testing.T) {
hook := NewUnitHook(context.TODO())
input := []unitChunk{
{unit: "one", output: "bar"},
{unit: "two", err: "oh no"},
{unit: "three", err: "not again"},
}
mgr := NewManager(
WithBufferedOutput(8),
WithSourceUnits(), WithConcurrentUnits(1),
WithReportHook(hook),
)
source, err := buildDummy(&unitChunker{input})
assert.NoError(t, err)
ref, err := mgr.Run(context.Background(), "dummy", source)
assert.NoError(t, err)
<-ref.Done()
metrics := hook.UnitMetrics()
assert.Equal(t, 3, len(metrics))
sort.Slice(metrics, func(i, j int) bool {
return metrics[i].EndTime.Before(metrics[j].EndTime)
})
m0, m1, m2 := metrics[0], metrics[1], metrics[2]
assert.Equal(t, "one", m0.Unit.SourceUnitID())
assert.Equal(t, uint64(1), m0.TotalChunks)
assert.Equal(t, uint64(3), m0.TotalBytes)
assert.NotZero(t, m0.StartTime)
assert.NotZero(t, m0.EndTime)
assert.NotZero(t, m0.ElapsedTime())
assert.Equal(t, 0, len(m0.Errors))
assert.Equal(t, "two", m1.Unit.SourceUnitID())
assert.Equal(t, uint64(0), m1.TotalChunks)
assert.Equal(t, uint64(0), m1.TotalBytes)
assert.NotZero(t, m1.StartTime)
assert.NotZero(t, m1.EndTime)
assert.NotZero(t, m1.ElapsedTime())
assert.Equal(t, 1, len(m1.Errors))
assert.Equal(t, "three", m2.Unit.SourceUnitID())
assert.Equal(t, uint64(0), m2.TotalChunks)
assert.Equal(t, uint64(0), m2.TotalBytes)
assert.NotZero(t, m2.StartTime)
assert.NotZero(t, m2.EndTime)
assert.NotZero(t, m2.ElapsedTime())
assert.Equal(t, 1, len(m2.Errors))
}
// TestSourceManagerUnitHookNoBlock tests that the UnitHook drops metrics if
// they aren't handled fast enough.
func TestSourceManagerUnitHookNoBlock(t *testing.T) {
var evictedKeys []string
cache, _ := lru.NewWithEvict(1, func(key string, _ *UnitMetrics) {
evictedKeys = append(evictedKeys, key)
})
hook := NewUnitHook(context.TODO(), WithUnitHookCache(cache))
input := []unitChunk{
{unit: "one", output: "bar"},
{unit: "two", err: "oh no"},
{unit: "three", err: "not again"},
}
mgr := NewManager(
WithBufferedOutput(8),
WithSourceUnits(), WithConcurrentUnits(1),
WithReportHook(hook),
)
source, err := buildDummy(&unitChunker{input})
assert.NoError(t, err)
ref, err := mgr.Run(context.Background(), "dummy", source)
assert.NoError(t, err)
<-ref.Done()
assert.Equal(t, 2, len(evictedKeys))
metrics := hook.UnitMetrics()
assert.Equal(t, 1, len(metrics))
assert.Equal(t, "three", metrics[0].Unit.SourceUnitID())
}
// TestSourceManagerUnitHookNoUnits tests whether the UnitHook works for
// sources that don't support units.
func TestSourceManagerUnitHookNoUnits(t *testing.T) {
hook := NewUnitHook(context.TODO())
mgr := NewManager(
WithBufferedOutput(8),
WithReportHook(hook),
)
source, err := buildDummy(&counterChunker{count: 5})
assert.NoError(t, err)
ref, err := mgr.Run(context.Background(), "dummy", source)
assert.NoError(t, err)
<-ref.Done()
metrics := hook.UnitMetrics()
assert.Equal(t, 1, len(metrics))
m := metrics[0]
assert.Equal(t, nil, m.Unit)
assert.Equal(t, uint64(5), m.TotalChunks)
assert.Equal(t, uint64(5), m.TotalBytes)
assert.NotZero(t, m.StartTime)
assert.NotZero(t, m.EndTime)
assert.NotZero(t, m.ElapsedTime())
assert.Equal(t, 0, len(m.Errors))
}
-61
View File
@@ -1,61 +0,0 @@
package sourcestest
import (
"fmt"
"github.com/trufflesecurity/trufflehog/v3/pkg/context"
"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
)
type reporter interface {
sources.UnitReporter
sources.ChunkReporter
}
var (
_ reporter = (*TestReporter)(nil)
_ reporter = (*ErrReporter)(nil)
)
// TestReporter is a helper struct that implements both UnitReporter and
// ChunkReporter by simply recording the values passed in the methods.
type TestReporter struct {
Units []sources.SourceUnit
UnitErrs []error
Chunks []sources.Chunk
ChunkErrs []error
}
func (t *TestReporter) UnitOk(_ context.Context, unit sources.SourceUnit) error {
t.Units = append(t.Units, unit)
return nil
}
func (t *TestReporter) UnitErr(_ context.Context, err error) error {
t.UnitErrs = append(t.UnitErrs, err)
return nil
}
func (t *TestReporter) ChunkOk(_ context.Context, chunk sources.Chunk) error {
t.Chunks = append(t.Chunks, chunk)
return nil
}
func (t *TestReporter) ChunkErr(_ context.Context, err error) error {
t.ChunkErrs = append(t.ChunkErrs, err)
return nil
}
// ErrReporter implements UnitReporter and ChunkReporter but always returns an
// error.
type ErrReporter struct{}
func (ErrReporter) UnitOk(context.Context, sources.SourceUnit) error {
return fmt.Errorf("ErrReporter: UnitOk error")
}
func (ErrReporter) UnitErr(context.Context, error) error {
return fmt.Errorf("ErrReporter: UnitErr error")
}
func (ErrReporter) ChunkOk(context.Context, sources.Chunk) error {
return fmt.Errorf("ErrReporter: ChunkOk error")
}
func (ErrReporter) ChunkErr(context.Context, error) error {
return fmt.Errorf("ErrReporter: ChunkErr error")
}
+4 -15
View File
@@ -361,7 +361,7 @@ enum DetectorType {
Imagga = 349;
SMSApi = 350;
Distribusion = 351;
Blablabus = 352 [deprecated = true];
Blablabus = 352;
WordsApi = 353;
Currencylayer = 354;
Html2Pdf = 355;
@@ -588,7 +588,7 @@ enum DetectorType {
Qubole = 577;
CarbonInterface = 578;
Intrinio = 579;
QuickMetrics = 580 [deprecated = true];
QuickMetrics = 580;
ScrapeStack = 581;
TechnicalAnalysisApi = 582;
Urlscan = 583;
@@ -628,7 +628,7 @@ enum DetectorType {
Nylas = 617;
Squareup = 618;
Dandelion = 619;
DataFire = 620 [deprecated = true];
DataFire = 620;
DeepAI = 621;
MeaningCloud = 622;
NeutrinoApi = 623;
@@ -780,7 +780,7 @@ enum DetectorType {
ConversionTools = 769;
CraftMyPDF = 770;
ExportSDK = 771;
GlitterlyAPI = 772 [deprecated = true];
GlitterlyAPI = 772;
Hybiscus = 773;
Miro = 774;
Statuspage = 775;
@@ -944,17 +944,6 @@ enum DetectorType {
Anthropic = 933;
Ramp = 934;
Klaviyo = 935;
SourcegraphCody = 936;
Voiceflow = 937;
Privacy = 938;
IPInfo = 939;
Ip2location = 940;
Instamojo = 941;
Portainer = 942;
PortainerToken = 943;
Loggly = 944;
OpenVpn = 945;
VagrantCloudPersonalToken = 946;
}
message Result {
+1 -8
View File
@@ -49,16 +49,9 @@ enum SourceType {
message LocalSource {
string type = 1 ;
string name = 2 ;
// DEPRECATED: scan_interval is deprecated and can be removed when we no longer depend on the name.
// Deprecating in favor of scan_period due to the fact that scan_interval is a duration
// which is a fixed-length span of time represented as a count of seconds and fractions of seconds
// at nanosecond resolution. Most of the time, we want to be able to specify a scan interval in
// human-readable format (e.g. 45s, 30m, 12h, etc.) which is not possible with a duration.
// https://protobuf.dev/reference/protobuf/google.protobuf/#duration
google.protobuf.Duration scan_interval = 3 [deprecated = true];
google.protobuf.Duration scan_interval = 3;
bool verify = 4;
google.protobuf.Any connection = 5;
string scan_period = 6;
}
message AzureStorage {