Compare commits
28 Commits
v3.6.9
...
gitparse-hack
| Author | SHA1 | Date | |
|---|---|---|---|
 Bill Rich
|
0a3679671a | ||
|
Bill Rich
|
3b49183caf | ||
|
Bill Rich
|
809b35a843 | ||
|
Bill Rich
|
6dd13a0b7c | ||
|
Bill Rich
|
bd1539d22a | ||
|
ahrav
|
2cab951ee5 | ||
|
ahrav
|
73f9d3f0a0 | ||
|
Miccah
|
2bd17052bb | ||
|
Bill Rich
|
4a93e49eea | ||
|
Bill Rich
|
a473b9aa99 | ||
|
ahrav
|
dcc102a81c | ||
|
ahrav
|
30ebe84e3e | ||
|
Dustin Decker
|
8826e369cf | ||
|
ahrav
|
dfb7e9a405 | ||
|
Dustin Decker
|
d8ed02643a | ||
|
Sergej
|
b081766d9b | ||
|
Max Thomson
|
b0a58fcf72 | ||
|
Bill Rich
|
7273dc9058 | ||
|
Dustin Decker
|
047e2b4607 | ||
|
dependabot[bot]
|
29f12d2d35 | ||
|
Jeff Wenzbauer
|
4a3bba78cc | ||
|
dependabot[bot]
|
dc516933f3 | ||
|
dependabot[bot]
|
ff8cf8fa09 | ||
|
Dustin Decker
|
5b7eff5c57 | ||
|
ahrav
|
21e1ff4a8a | ||
|
Adrian Gąsior
|
0752503872 | ||
|
Ariel Ril
|
4b0315c90a | ||
|
trufflesteeeve
|
176552b07a |
@@ -13,6 +13,8 @@ builds:
|
||||
dockers:
|
||||
- image_templates: ["trufflesecurity/{{ .ProjectName }}:{{ .Version }}-amd64"]
|
||||
dockerfile: Dockerfile.goreleaser
|
||||
extra_files:
|
||||
- entrypoint.sh
|
||||
use: buildx
|
||||
build_flag_templates:
|
||||
- --platform=linux/amd64
|
||||
@@ -26,6 +28,8 @@ dockers:
|
||||
- image_templates: ["trufflesecurity/{{ .ProjectName }}:{{ .Version }}-arm64v8"]
|
||||
goarch: arm64
|
||||
dockerfile: Dockerfile.goreleaser
|
||||
extra_files:
|
||||
- entrypoint.sh
|
||||
use: buildx
|
||||
build_flag_templates:
|
||||
- --platform=linux/arm64/v8
|
||||
@@ -38,6 +42,8 @@ dockers:
|
||||
- --label=org.opencontainers.image.licenses=AGPL-3.0
|
||||
- image_templates: ["ghcr.io/trufflesecurity/{{ .ProjectName }}:{{ .Version }}-amd64"]
|
||||
dockerfile: Dockerfile.goreleaser
|
||||
extra_files:
|
||||
- entrypoint.sh
|
||||
use: buildx
|
||||
build_flag_templates:
|
||||
- --platform=linux/amd64
|
||||
@@ -51,6 +57,8 @@ dockers:
|
||||
- image_templates: ["ghcr.io/trufflesecurity/{{ .ProjectName }}:{{ .Version }}-arm64v8"]
|
||||
goarch: arm64
|
||||
dockerfile: Dockerfile.goreleaser
|
||||
extra_files:
|
||||
- entrypoint.sh
|
||||
use: buildx
|
||||
build_flag_templates:
|
||||
- --platform=linux/arm64/v8
|
||||
|
||||
+12
-7
@@ -1,13 +1,18 @@
|
||||
FROM golang:bullseye as builder
|
||||
FROM --platform=${BUILDPLATFORM} golang:bullseye as builder
|
||||
|
||||
WORKDIR /build
|
||||
COPY go.mod go.sum ./
|
||||
RUN go mod download
|
||||
COPY . .
|
||||
RUN CGO_ENABLED=0 go build -a -o trufflehog main.go
|
||||
ENV CGO_ENABLED=0
|
||||
ARG TARGETOS TARGETARCH
|
||||
RUN --mount=type=cache,target=/go/pkg/mod \
|
||||
--mount=type=cache,target=/root/.cache/go-build \
|
||||
GOOS=${TARGETOS} GOARCH=${TARGETARCH} go build -o trufflehog .
|
||||
|
||||
FROM alpine:3.15
|
||||
RUN apk add --no-cache git
|
||||
COPY --from=builder /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt
|
||||
RUN apk add --no-cache git ca-certificates \
|
||||
&& rm -rf /var/cache/apk/* && \
|
||||
update-ca-certificates
|
||||
COPY --from=builder /build/trufflehog /usr/bin/trufflehog
|
||||
ENTRYPOINT ["/usr/bin/trufflehog"]
|
||||
COPY entrypoint.sh /etc/entrypoint.sh
|
||||
RUN chmod +x /etc/entrypoint.sh
|
||||
ENTRYPOINT ["/etc/entrypoint.sh"]
|
||||
|
||||
@@ -1,7 +1,10 @@
|
||||
FROM alpine:3.15
|
||||
|
||||
RUN apk add --no-cache git
|
||||
RUN apk add --no-cache git ca-certificates \
|
||||
&& rm -rf /var/cache/apk/* && \
|
||||
update-ca-certificates
|
||||
WORKDIR /usr/bin/
|
||||
COPY trufflehog .
|
||||
|
||||
ENTRYPOINT ["/usr/bin/trufflehog"]
|
||||
COPY entrypoint.sh /etc/entrypoint.sh
|
||||
RUN chmod +x /etc/entrypoint.sh
|
||||
ENTRYPOINT ["/etc/entrypoint.sh"]
|
||||
|
||||
@@ -28,13 +28,13 @@ test-failing:
|
||||
CGO_ENABLED=0 go test -timeout=5m $(shell go list ./... | grep -v /vendor/) | grep FAIL
|
||||
|
||||
test:
|
||||
CGO_ENABLED=0 go test -timeout=5m $(shell go list ./... | grep -v /vendor/)
|
||||
CGO_ENABLED=0 go test -timeout=5m $(shell go list ./... | grep -v /vendor/ | grep -v pkg/detectors)
|
||||
|
||||
test-race:
|
||||
CGO_ENABLED=1 go test -timeout=5m -race $(shell go list ./... | grep -v /vendor/)
|
||||
CGO_ENABLED=1 go test -timeout=5m -race $(shell go list ./... | grep -v /vendor/ | grep -v pkg/detectors)
|
||||
|
||||
test-detectors:
|
||||
CGO_ENABLED=0 go test -tags=detectors -timeout=5m $(shell go list ./...)
|
||||
CGO_ENABLED=0 go test -tags=detectors -timeout=5m $(shell go list ./... | grep pkg/detectors)
|
||||
|
||||
bench:
|
||||
CGO_ENABLED=0 go test $(shell go list ./pkg/secrets/... | grep -v /vendor/) -benchmem -run=xxx -bench .
|
||||
@@ -57,3 +57,6 @@ release-protos-image:
|
||||
|
||||
snifftest:
|
||||
./hack/snifftest/snifftest.sh
|
||||
|
||||
test-release:
|
||||
goreleaser release --rm-dist --skip-publish --snapshot
|
||||
@@ -6,14 +6,14 @@
|
||||
|
||||
---
|
||||
|
||||
<center>
|
||||
<div align="center">
|
||||
|
||||
[](https://github.com/trufflesecurity/trufflehog/actions)
|
||||
[](https://goreportcard.com/report/github.com/trufflesecurity/trufflehog/v3)
|
||||
[](/LICENSE)
|
||||
[](/pkg/detectors) <!-- Badge must be run from staging, see badges/shields#5967 -->
|
||||
[](/pkg/detectors)
|
||||
|
||||
</center>
|
||||
</div>
|
||||
|
||||
---
|
||||
|
||||
@@ -169,6 +169,8 @@ docker run -it -v "$PWD:/pwd" trufflesecurity/trufflehog:latest github --org=tru
|
||||
base:
|
||||
# Scan commits until here (usually dev branch).
|
||||
head: # optional
|
||||
# Extra args to be passed to the trufflehog cli.
|
||||
extra_args: # optional
|
||||
```
|
||||
|
||||
The TruffleHog OSS Github Action can be used to scan a range of commits for leaked credentials. The action will fail if
|
||||
@@ -192,6 +194,26 @@ jobs:
|
||||
path: ./
|
||||
base: ${{ github.event.repository.default_branch }}
|
||||
head: HEAD
|
||||
extra_args: --debug
|
||||
```
|
||||
|
||||
### Precommit Hook
|
||||
|
||||
Trufflehog can be used in a precommit hook to prevent credentials from leaking before they ever leave your computer.
|
||||
An example `.pre-commit-config.yaml` is provided (see [pre-commit.com](https://pre-commit.com/) for installation).
|
||||
|
||||
```yaml
|
||||
repos:
|
||||
- repo: local
|
||||
hooks:
|
||||
- id: trufflehog
|
||||
name: TruffleHog
|
||||
description: Detect secrets in your data.
|
||||
entry: bash -c 'trufflehog git file://. --only-verified --fail'
|
||||
# For running trufflehog in docker, use the following entry instead:
|
||||
# entry: bash -c 'docker run -v "$(pwd):/workdir" -i --rm trufflesecurity/trufflehog:latest git file:///workdir --only-verified --fail'
|
||||
language: system
|
||||
stages: ["commit", "push"]
|
||||
```
|
||||
|
||||
## Contributors
|
||||
|
||||
@@ -12,6 +12,10 @@ inputs:
|
||||
head:
|
||||
description: Scan commits until here (usually dev branch).
|
||||
required: false
|
||||
extra_args:
|
||||
default: ''
|
||||
description: Extra args to be passed to the trufflehog cli.
|
||||
required: false
|
||||
branding:
|
||||
icon: "shield"
|
||||
color: "green"
|
||||
@@ -26,3 +30,4 @@ runs:
|
||||
- --branch
|
||||
- ${{ inputs.head }}
|
||||
- --fail
|
||||
- ${{ inputs.extra_args }}
|
||||
|
||||
Regular → Executable
+4
-3
@@ -1,4 +1,5 @@
|
||||
#! /usr/bin/env bash
|
||||
#!/usr/bin/env ash
|
||||
|
||||
args=("$@")
|
||||
/usr/bin/trufflehog ${args[@]}
|
||||
# `$*` expands the `args` supplied in an `array` individually
|
||||
# or splits `args` in a string separated by whitespace.
|
||||
/usr/bin/trufflehog $*
|
||||
@@ -4,14 +4,18 @@ go 1.18
|
||||
|
||||
replace github.com/jpillora/overseer => github.com/trufflesecurity/overseer v1.1.7-custom5
|
||||
|
||||
replace github.com/zricethezav/gitleaks/v8 => github.com/trufflesecurity/gitleaks/v8 v8.6.1-custom8
|
||||
replace github.com/zricethezav/gitleaks/v8 => github.com/trufflesecurity/gitleaks/v8 v8.6.1-custom10
|
||||
|
||||
replace github.com/gitleaks/go-gitdiff => github.com/trufflesecurity/go-gitdiff v0.7.6-zombies2
|
||||
|
||||
replace github.com/bill-rich/gitparser => /home/hrich/go/src/github.com/bill-rich/gitparse
|
||||
|
||||
require (
|
||||
cloud.google.com/go/secretmanager v1.5.0
|
||||
github.com/Azure/go-autorest/autorest/azure/auth v0.5.11
|
||||
github.com/aws/aws-sdk-go v1.44.56
|
||||
github.com/aws/aws-sdk-go v1.44.61
|
||||
github.com/bill-rich/disk-buffer-reader v0.1.2
|
||||
github.com/bill-rich/gitparser v0.0.0-00010101000000-000000000000
|
||||
github.com/bill-rich/go-syslog v0.0.0-20220413021637-49edb52a574c
|
||||
github.com/bitfinexcom/bitfinex-api-go v0.0.0-20210608095005-9e0b26f200fb
|
||||
github.com/bradleyfalzon/ghinstallation/v2 v2.1.0
|
||||
@@ -19,7 +23,6 @@ require (
|
||||
github.com/envoyproxy/protoc-gen-validate v0.6.7
|
||||
github.com/fatih/color v1.13.0
|
||||
github.com/felixge/fgprof v0.9.2
|
||||
github.com/gitleaks/go-gitdiff v0.7.4
|
||||
github.com/go-errors/errors v1.4.2
|
||||
github.com/go-git/go-git/v5 v5.4.2
|
||||
github.com/golang-jwt/jwt v3.2.2+incompatible
|
||||
@@ -31,22 +34,23 @@ require (
|
||||
github.com/jpillora/overseer v1.1.6
|
||||
github.com/kylelemons/godebug v1.1.0
|
||||
github.com/mattn/go-colorable v0.1.12
|
||||
github.com/mholt/archiver/v4 v4.0.0-alpha.7
|
||||
github.com/paulbellamy/ratecounter v0.2.0
|
||||
github.com/pkg/errors v0.9.1
|
||||
github.com/razorpay/razorpay-go v0.0.0-20210728161131-0341409a6ab2
|
||||
github.com/rs/zerolog v1.27.0
|
||||
github.com/sergi/go-diff v1.2.0
|
||||
github.com/sirupsen/logrus v1.8.1
|
||||
github.com/sirupsen/logrus v1.9.0
|
||||
github.com/stretchr/testify v1.8.0
|
||||
github.com/tailscale/depaware v0.0.0-20210622194025-720c4b409502
|
||||
github.com/xanzy/go-gitlab v0.69.0
|
||||
github.com/zricethezav/gitleaks/v8 v8.5.2
|
||||
github.com/zricethezav/gitleaks/v8 v8.10.3
|
||||
golang.org/x/crypto v0.0.0-20211215165025-cf75a172585e
|
||||
golang.org/x/net v0.0.0-20220617184016-355a448f1bc9
|
||||
golang.org/x/oauth2 v0.0.0-20220608161450-d0670ef3b1eb
|
||||
golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f
|
||||
google.golang.org/genproto v0.0.0-20220617124728-180714bec0ad
|
||||
google.golang.org/protobuf v1.28.0
|
||||
google.golang.org/protobuf v1.28.1
|
||||
gopkg.in/alecthomas/kingpin.v2 v2.2.6
|
||||
gopkg.in/h2non/gock.v1 v1.1.2
|
||||
)
|
||||
@@ -67,19 +71,23 @@ require (
|
||||
github.com/acomagu/bufpipe v1.0.3 // indirect
|
||||
github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 // indirect
|
||||
github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 // indirect
|
||||
github.com/andybalholm/brotli v1.0.4 // indirect
|
||||
github.com/davecgh/go-spew v1.1.1 // indirect
|
||||
github.com/dimchansky/utfbom v1.1.1 // indirect
|
||||
github.com/dsnet/compress v0.0.1 // indirect
|
||||
github.com/emirpasic/gods v1.12.0 // indirect
|
||||
github.com/gitleaks/go-gitdiff v0.7.6 // indirect
|
||||
github.com/go-git/gcfg v1.5.0 // indirect
|
||||
github.com/go-git/go-billy/v5 v5.3.1 // indirect
|
||||
github.com/go-ole/go-ole v1.2.6 // indirect
|
||||
github.com/golang-jwt/jwt/v4 v4.4.1 // indirect
|
||||
github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e // indirect
|
||||
github.com/golang/protobuf v1.5.2 // indirect
|
||||
github.com/golang/snappy v0.0.4 // indirect
|
||||
github.com/google/go-cmp v0.5.8 // indirect
|
||||
github.com/google/go-github/v45 v45.2.0 // indirect
|
||||
github.com/google/go-querystring v1.1.0 // indirect
|
||||
github.com/google/pprof v0.0.0-20211214055906-6f57359322fd // indirect
|
||||
github.com/google/pprof v0.0.0-20220729232143-a41b82acbcb1 // indirect
|
||||
github.com/googleapis/enterprise-certificate-proxy v0.1.0 // indirect
|
||||
github.com/googleapis/gax-go/v2 v2.4.0 // indirect
|
||||
github.com/h2non/parth v0.0.0-20190131123155-b4df798d6542 // indirect
|
||||
@@ -89,15 +97,21 @@ require (
|
||||
github.com/jmespath/go-jmespath v0.4.0 // indirect
|
||||
github.com/jpillora/s3 v1.1.4 // indirect
|
||||
github.com/kevinburke/ssh_config v0.0.0-20201106050909-4977a11b4351 // indirect
|
||||
github.com/klauspost/compress v1.15.5 // indirect
|
||||
github.com/klauspost/pgzip v1.2.5 // indirect
|
||||
github.com/mattn/go-isatty v0.0.14 // indirect
|
||||
github.com/mitchellh/go-homedir v1.1.0 // indirect
|
||||
github.com/nwaples/rardecode/v2 v2.0.0-beta.2 // indirect
|
||||
github.com/pierrec/lz4/v4 v4.1.14 // indirect
|
||||
github.com/pkg/diff v0.0.0-20200914180035-5b29258ca4f7 // indirect
|
||||
github.com/pmezard/go-difflib v1.0.0 // indirect
|
||||
github.com/therootcompany/xz v1.0.1 // indirect
|
||||
github.com/ulikunitz/xz v0.5.10 // indirect
|
||||
github.com/xanzy/ssh-agent v0.3.0 // indirect
|
||||
github.com/yusufpapurcu/wmi v1.2.2 // indirect
|
||||
go.opencensus.io v0.23.0 // indirect
|
||||
golang.org/x/mod v0.5.0 // indirect
|
||||
golang.org/x/sys v0.0.0-20220615213510-4f61da869c0c // indirect
|
||||
golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab // indirect
|
||||
golang.org/x/text v0.3.7 // indirect
|
||||
golang.org/x/time v0.0.0-20220411224347-583f2d630306 // indirect
|
||||
golang.org/x/tools v0.1.7 // indirect
|
||||
|
||||
@@ -91,13 +91,17 @@ github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 h1:JYp7IbQjafo
|
||||
github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
|
||||
github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 h1:s6gZFSlWYmbqAuRjVTiNNhvNRfY2Wxp9nhfyel4rklc=
|
||||
github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137/go.mod h1:OMCwj8VM1Kc9e19TLln2VL61YJF0x1XFtfdL4JdbSyE=
|
||||
github.com/andybalholm/brotli v1.0.4 h1:V7DdXeJtZscaqfNuAdSRuRFzuiKlHSC/Zh3zl9qY3JY=
|
||||
github.com/andybalholm/brotli v1.0.4/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig=
|
||||
github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239 h1:kFOfPq6dUM1hTo4JG6LR5AXSUEsOjtdm0kw0FtQtMJA=
|
||||
github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c=
|
||||
github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY=
|
||||
github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio=
|
||||
github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs=
|
||||
github.com/aws/aws-sdk-go v1.44.56 h1:bT+lExwagH7djxb6InKUVkEKGPAj5aAPnV85/m1fKro=
|
||||
github.com/aws/aws-sdk-go v1.44.56/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo=
|
||||
github.com/aws/aws-sdk-go v1.44.61 h1:NcpLSS3Z0MiVQIYugx4I40vSIEEAXT0baO684ExNRco=
|
||||
github.com/aws/aws-sdk-go v1.44.61/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo=
|
||||
github.com/bill-rich/disk-buffer-reader v0.1.2 h1:pN9K5JoacTcNWp2SCd3n7mPouSwMP9ouTS66Qa+5IPY=
|
||||
github.com/bill-rich/disk-buffer-reader v0.1.2/go.mod h1:VVzzsK1Ac2AnpOfp/5r9JlIFaFkZ9uSf7zisZayCt0Y=
|
||||
github.com/bill-rich/go-syslog v0.0.0-20220413021637-49edb52a574c h1:tSME5FDS02qQll3JYodI6RZR/g4EKOHApGv1wMZT+Z0=
|
||||
github.com/bill-rich/go-syslog v0.0.0-20220413021637-49edb52a574c/go.mod h1:+sCc6hztur+oZCLOsNk6wCCy+GLrnSNHSRmTnnL+8iQ=
|
||||
github.com/bitfinexcom/bitfinex-api-go v0.0.0-20210608095005-9e0b26f200fb h1:9v7Bzlg+1EBYi2IYcUmOwHReBEfqBbYIj3ZCi9cIe1Q=
|
||||
@@ -129,6 +133,9 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c
|
||||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/dimchansky/utfbom v1.1.1 h1:vV6w1AhK4VMnhBno/TPVCoK9U/LP0PkLCS9tbxHdi/U=
|
||||
github.com/dimchansky/utfbom v1.1.1/go.mod h1:SxdoEBH5qIqFocHMyGOXVAybYJdr71b1Q/j0mACtrfE=
|
||||
github.com/dsnet/compress v0.0.1 h1:PlZu0n3Tuv04TzpfPbrnI0HW/YwodEXDS+oPKahKF0Q=
|
||||
github.com/dsnet/compress v0.0.1/go.mod h1:Aw8dCMJ7RioblQeTqt88akK31OvO8Dhf5JflhBbQEHo=
|
||||
github.com/dsnet/golib v0.0.0-20171103203638-1ea166775780/go.mod h1:Lj+Z9rebOhdfkVLjJ8T6VcRQv3SXugXy999NBtR9aFY=
|
||||
github.com/emirpasic/gods v1.12.0 h1:QAUIPSaCu4G+POclxeqb3F+WPpdKqFGlw36+yOzGlrg=
|
||||
github.com/emirpasic/gods v1.12.0/go.mod h1:YfzfFFoVP/catgzJb4IKIqXjX78Ha8FMSDh3ymbK86o=
|
||||
github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
|
||||
@@ -210,6 +217,8 @@ github.com/golang/protobuf v1.5.1/go.mod h1:DopwsBzvsk0Fs44TXzsVbJyPhcCPeIwnvohx
|
||||
github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw=
|
||||
github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
|
||||
github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
|
||||
github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM=
|
||||
github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
|
||||
github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
|
||||
github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
|
||||
github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
|
||||
@@ -251,8 +260,9 @@ github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLe
|
||||
github.com/google/pprof v0.0.0-20210601050228-01bbb1931b22/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
|
||||
github.com/google/pprof v0.0.0-20210609004039-a478d1d731e9/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
|
||||
github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
|
||||
github.com/google/pprof v0.0.0-20211214055906-6f57359322fd h1:1FjCyPC+syAzJ5/2S8fqdZK1R22vvA0J7JZKcuOIQ7Y=
|
||||
github.com/google/pprof v0.0.0-20211214055906-6f57359322fd/go.mod h1:KgnwoLYCZ8IQu3XUZ8Nc/bM9CCZFOyjUNOSygVozoDg=
|
||||
github.com/google/pprof v0.0.0-20220729232143-a41b82acbcb1 h1:8pyqKJvrJqUYaKS851Ule26pwWvey6IDMiczaBLDKLQ=
|
||||
github.com/google/pprof v0.0.0-20220729232143-a41b82acbcb1/go.mod h1:gSuNB+gJaOiQKLEZ+q+PK9Mq3SOzhRcw2GsGS/FhYDk=
|
||||
github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
|
||||
github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
||||
github.com/googleapis/enterprise-certificate-proxy v0.0.0-20220520183353-fd19c99a87aa/go.mod h1:17drOmN3MwGY7t0e+Ei9b45FFGA3fBs3x36SsCg1hq8=
|
||||
@@ -306,6 +316,12 @@ github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/X
|
||||
github.com/kevinburke/ssh_config v0.0.0-20201106050909-4977a11b4351 h1:DowS9hvgyYSX4TO5NpyC606/Z4SxnNYbT+WX27or6Ck=
|
||||
github.com/kevinburke/ssh_config v0.0.0-20201106050909-4977a11b4351/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM=
|
||||
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
|
||||
github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=
|
||||
github.com/klauspost/compress v1.15.5 h1:qyCLMz2JCrKADihKOh9FxnW3houKeNsp2h5OEz0QSEA=
|
||||
github.com/klauspost/compress v1.15.5/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU=
|
||||
github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek=
|
||||
github.com/klauspost/pgzip v1.2.5 h1:qnWYvvKqedOF2ulHpMG72XQol4ILEJ8k2wwRl/Km8oE=
|
||||
github.com/klauspost/pgzip v1.2.5/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs=
|
||||
github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
|
||||
github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg=
|
||||
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
|
||||
@@ -326,14 +342,20 @@ github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb
|
||||
github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
|
||||
github.com/mattn/go-isatty v0.0.14 h1:yVuAays6BHfxijgZPzw+3Zlu5yQgKGP2/hcQbHb7S9Y=
|
||||
github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94=
|
||||
github.com/mholt/archiver/v4 v4.0.0-alpha.7 h1:xzByj8G8tj0Oq7ZYYU4+ixL/CVb5ruWCm0EZQ1PjOkE=
|
||||
github.com/mholt/archiver/v4 v4.0.0-alpha.7/go.mod h1:Fs8qUkO74HHaidabihzYephJH8qmGD/nCP6tE5xC9BM=
|
||||
github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y=
|
||||
github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
|
||||
github.com/nbio/st v0.0.0-20140626010706-e9e8d9816f32 h1:W6apQkHrMkS0Muv8G/TipAy/FJl/rCYT0+EuS8+Z0z4=
|
||||
github.com/nbio/st v0.0.0-20140626010706-e9e8d9816f32/go.mod h1:9wM+0iRr9ahx58uYLpLIr5fm8diHn0JbqRycJi6w0Ms=
|
||||
github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
|
||||
github.com/nwaples/rardecode/v2 v2.0.0-beta.2 h1:e3mzJFJs4k83GXBEiTaQ5HgSc/kOK8q0rDaRO0MPaOk=
|
||||
github.com/nwaples/rardecode/v2 v2.0.0-beta.2/go.mod h1:yntwv/HfMc/Hbvtq9I19D1n58te3h6KsqCf3GxyfBGY=
|
||||
github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk=
|
||||
github.com/paulbellamy/ratecounter v0.2.0 h1:2L/RhJq+HA8gBQImDXtLPrDXK5qAj6ozWVK/zFXVJGs=
|
||||
github.com/paulbellamy/ratecounter v0.2.0/go.mod h1:Hfx1hDpSGoqxkVVpBi/IlYD7kChlfo5C6hzIHwPqfFE=
|
||||
github.com/pierrec/lz4/v4 v4.1.14 h1:+fL8AQEZtz/ijeNnpduH0bROTu0O3NZAlPjQxGn8LwE=
|
||||
github.com/pierrec/lz4/v4 v4.1.14/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
|
||||
github.com/pkg/diff v0.0.0-20200914180035-5b29258ca4f7 h1:+/+DxvQaYifJ+grD4klzrS5y+KJXldn/2YTl5JG+vZ8=
|
||||
github.com/pkg/diff v0.0.0-20200914180035-5b29258ca4f7/go.mod h1:zO8QMzTeZd5cpnIkz/Gn6iK0jDfGicM1nynOkkPIl28=
|
||||
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
|
||||
@@ -355,8 +377,8 @@ github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNX
|
||||
github.com/sergi/go-diff v1.2.0 h1:XU+rvMAioB0UC3q1MFrIQy4Vo5/4VsRDQQXHsEya6xQ=
|
||||
github.com/sergi/go-diff v1.2.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM=
|
||||
github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q=
|
||||
github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE=
|
||||
github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
|
||||
github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0=
|
||||
github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
|
||||
github.com/smartystreets/assertions v1.0.1 h1:voD4ITNjPL5jjBfgR/r8fPIIBrliWrWHeiJApdr3r4w=
|
||||
github.com/smartystreets/assertions v1.0.1/go.mod h1:kHHU4qYBaI3q23Pp3VPrmWhuIUrLW/7eUrw0BU5VaoM=
|
||||
github.com/smartystreets/gunit v1.1.3 h1:32x+htJCu3aMswhPw3teoJ+PnWPONqdNgaGs6Qt8ZaU=
|
||||
@@ -377,12 +399,17 @@ github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PK
|
||||
github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
|
||||
github.com/tailscale/depaware v0.0.0-20210622194025-720c4b409502 h1:34icjjmqJ2HPjrSuJYEkdZ+0ItmGQAQ75cRHIiftIyE=
|
||||
github.com/tailscale/depaware v0.0.0-20210622194025-720c4b409502/go.mod h1:p9lPsd+cx33L3H9nNoecRRxPssFKUwwI50I3pZ0yT+8=
|
||||
github.com/trufflesecurity/gitleaks/v8 v8.6.1-custom8 h1:L3i09KBNoAfNvE1Ue4XOVDOsihidbPGkz7r7a7YAnSU=
|
||||
github.com/trufflesecurity/gitleaks/v8 v8.6.1-custom8/go.mod h1:2iZpX4Epnmx7VK2atbIMEjHW9rivie5RRe0ZhPWUFvM=
|
||||
github.com/therootcompany/xz v1.0.1 h1:CmOtsn1CbtmyYiusbfmhmkpAAETj0wBIH6kCYaX+xzw=
|
||||
github.com/therootcompany/xz v1.0.1/go.mod h1:3K3UH1yCKgBneZYhuQUvJ9HPD19UEXEI0BWbMn8qNMY=
|
||||
github.com/trufflesecurity/gitleaks/v8 v8.6.1-custom10 h1:QuGZ5bJcQpVz+3sfvKKPDkQwdUueiBg0V+2eMHzkryo=
|
||||
github.com/trufflesecurity/gitleaks/v8 v8.6.1-custom10/go.mod h1:2iZpX4Epnmx7VK2atbIMEjHW9rivie5RRe0ZhPWUFvM=
|
||||
github.com/trufflesecurity/go-gitdiff v0.7.6-zombies2 h1:srCJzbE3b44+ZIPcgJSfvinHCOQlkMwVghtKf23un6o=
|
||||
github.com/trufflesecurity/go-gitdiff v0.7.6-zombies2/go.mod h1:pKz0X4YzCKZs30BL+weqBIG7mx0jl4tF1uXV9ZyNvrA=
|
||||
github.com/trufflesecurity/overseer v1.1.7-custom5 h1:xu+Fg6fkSRifUPzUCl7N8HmobJ6WGOkIApGnM7mJS6w=
|
||||
github.com/trufflesecurity/overseer v1.1.7-custom5/go.mod h1:nT9w37AiO1Nop2VhVhNfzAFaPjthvxgpDV3XKsxYkcI=
|
||||
github.com/ulikunitz/xz v0.5.6/go.mod h1:2bypXElzHzzJZwzH67Y6wb67pO62Rzfn7BSiF4ABRW8=
|
||||
github.com/ulikunitz/xz v0.5.10 h1:t92gobL9l3HE202wg3rlk19F6X+JOxl9BBrCCMYEYd8=
|
||||
github.com/ulikunitz/xz v0.5.10/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14=
|
||||
github.com/xanzy/go-gitlab v0.69.0 h1:sPci9xHzlX+lcJvPqNu3y3BQpePuR2R694Bal4AeyB8=
|
||||
github.com/xanzy/go-gitlab v0.69.0/go.mod h1:o4yExCtdaqlM8YGdDJWuZoBmfxBsmA9TPEjs9mx1UO4=
|
||||
github.com/xanzy/ssh-agent v0.3.0 h1:wUMzuKtKilRgBAD1sUb8gOwwRr2FGoBVumcjoOACClI=
|
||||
@@ -546,7 +573,6 @@ golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7w
|
||||
golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
@@ -601,8 +627,10 @@ golang.org/x/sys v0.0.0-20220502124256-b6088ccd6cba/go.mod h1:oPkhp1MJrh7nUepCBc
|
||||
golang.org/x/sys v0.0.0-20220503163025-988cb79eb6c6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220610221304-9f5ed59c137d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220615213510-4f61da869c0c h1:aFV+BgZ4svzjfabn8ERpuB4JI4N6/rdy1iusx77G3oU=
|
||||
golang.org/x/sys v0.0.0-20220615213510-4f61da869c0c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab h1:2QkjZIsXupsJbJIdSjjUOgWK3aEtzyuh2mPt3l/CkeU=
|
||||
golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
|
||||
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 h1:JGgROgKl9N8DuW20oFS5gxc+lE67/N3FcwmBPMe7ArY=
|
||||
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
|
||||
@@ -858,8 +886,9 @@ google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlba
|
||||
google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
|
||||
google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
|
||||
google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
|
||||
google.golang.org/protobuf v1.28.0 h1:w43yiav+6bVFTBQFZX0r7ipe9JQ1QsbMgHwbBziscLw=
|
||||
google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
|
||||
google.golang.org/protobuf v1.28.1 h1:d0NfwRgPtno5B1Wa6L2DAG+KivqkdutMf1UhdNx175w=
|
||||
google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
|
||||
gopkg.in/alecthomas/kingpin.v2 v2.2.6 h1:jMFz6MfLP0/4fUyZle81rXUoxOBFi19VUFKVDOQfozc=
|
||||
gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
|
||||
@@ -17,9 +17,11 @@ import (
|
||||
"github.com/gorilla/mux"
|
||||
"github.com/jpillora/overseer"
|
||||
"github.com/sirupsen/logrus"
|
||||
"gopkg.in/alecthomas/kingpin.v2"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/updater"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/version"
|
||||
"gopkg.in/alecthomas/kingpin.v2"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/decoders"
|
||||
@@ -189,36 +191,75 @@ func run(state overseer.State) {
|
||||
if remote {
|
||||
defer os.RemoveAll(repoPath)
|
||||
}
|
||||
err = e.ScanGit(ctx, repoPath, *gitScanBranch, *gitScanSinceCommit, *gitScanMaxDepth, filter)
|
||||
if err != nil {
|
||||
logrus.WithError(err).Fatal("Failed to scan git.")
|
||||
|
||||
config := sources.Config{
|
||||
RepoPath: repoPath,
|
||||
HeadRef: *gitScanBranch,
|
||||
BaseRef: *gitScanSinceCommit,
|
||||
MaxDepth: *gitScanMaxDepth,
|
||||
Filter: filter,
|
||||
}
|
||||
|
||||
if err = e.ScanGit(ctx, config); err != nil {
|
||||
logrus.WithError(err).Fatal("Failed to scan Git.")
|
||||
}
|
||||
case githubScan.FullCommand():
|
||||
if len(*githubScanOrgs) == 0 && len(*githubScanRepos) == 0 {
|
||||
log.Fatal("You must specify at least one organization or repository.")
|
||||
}
|
||||
err = e.ScanGitHub(ctx, *githubScanEndpoint, *githubScanRepos, *githubScanOrgs, *githubScanToken, *githubIncludeForks, filter, *concurrency, *githubIncludeMembers)
|
||||
if err != nil {
|
||||
logrus.WithError(err).Fatal("Failed to scan git.")
|
||||
|
||||
github := func(c sources.Config) {
|
||||
c.Endpoint = *githubScanEndpoint
|
||||
c.Repos = *githubScanRepos
|
||||
c.Orgs = *githubScanOrgs
|
||||
c.Token = *githubScanToken
|
||||
c.IncludeForks = *githubIncludeForks
|
||||
c.IncludeMembers = *githubIncludeMembers
|
||||
c.Concurrency = *concurrency
|
||||
}
|
||||
|
||||
if err = e.ScanGitHub(ctx, sources.NewConfig(github)); err != nil {
|
||||
logrus.WithError(err).Fatal("Failed to scan Github.")
|
||||
}
|
||||
case gitlabScan.FullCommand():
|
||||
err := e.ScanGitLab(ctx, *gitlabScanEndpoint, *gitlabScanToken, *gitlabScanRepos)
|
||||
if err != nil {
|
||||
gitlab := func(c sources.Config) {
|
||||
c.Endpoint = *gitlabScanEndpoint
|
||||
c.Token = *gitlabScanToken
|
||||
c.Repos = *gitlabScanRepos
|
||||
}
|
||||
|
||||
if err = e.ScanGitLab(ctx, sources.NewConfig(gitlab)); err != nil {
|
||||
logrus.WithError(err).Fatal("Failed to scan GitLab.")
|
||||
}
|
||||
case filesystemScan.FullCommand():
|
||||
err := e.ScanFileSystem(ctx, *filesystemDirectories)
|
||||
if err != nil {
|
||||
fs := func(c sources.Config) {
|
||||
c.Directories = *filesystemDirectories
|
||||
}
|
||||
|
||||
if err = e.ScanFileSystem(ctx, sources.NewConfig(fs)); err != nil {
|
||||
logrus.WithError(err).Fatal("Failed to scan filesystem")
|
||||
}
|
||||
case s3Scan.FullCommand():
|
||||
err := e.ScanS3(ctx, *s3ScanKey, *s3ScanSecret, *s3ScanCloudEnv, *s3ScanBuckets)
|
||||
if err != nil {
|
||||
s3 := func(c sources.Config) {
|
||||
c.Key = *s3ScanKey
|
||||
c.Secret = *s3ScanSecret
|
||||
c.Buckets = *s3ScanBuckets
|
||||
}
|
||||
|
||||
if err = e.ScanS3(ctx, sources.NewConfig(s3)); err != nil {
|
||||
logrus.WithError(err).Fatal("Failed to scan S3.")
|
||||
}
|
||||
case syslogScan.FullCommand():
|
||||
err := e.ScanSyslog(ctx, *syslogAddress, *syslogProtocol, *syslogTLSCert, *syslogTLSKey, *syslogFormat, *concurrency)
|
||||
if err != nil {
|
||||
syslog := func(c sources.Config) {
|
||||
c.Address = *syslogAddress
|
||||
c.Protocol = *syslogProtocol
|
||||
c.CertPath = *syslogTLSCert
|
||||
c.KeyPath = *syslogTLSKey
|
||||
c.Format = *syslogFormat
|
||||
c.Concurrency = *concurrency
|
||||
}
|
||||
|
||||
if err = e.ScanSyslog(ctx, sources.NewConfig(syslog)); err != nil {
|
||||
logrus.WithError(err).Fatal("Failed to scan syslog.")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,36 @@
|
||||
package common
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"errors"
|
||||
"io"
|
||||
|
||||
log "github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
const (
|
||||
ChunkSize = 10 * 1024
|
||||
PeekSize = 3 * 1024
|
||||
)
|
||||
|
||||
func ChunkReader(r io.Reader) chan []byte {
|
||||
chunkChan := make(chan []byte)
|
||||
go func() {
|
||||
defer close(chunkChan)
|
||||
reader := bufio.NewReaderSize(bufio.NewReader(r), ChunkSize)
|
||||
for {
|
||||
chunk := make([]byte, ChunkSize)
|
||||
n, err := reader.Read(chunk)
|
||||
if err != nil && !errors.Is(err, io.EOF) {
|
||||
log.WithError(err).Error("Error chunking reader.")
|
||||
break
|
||||
}
|
||||
peekData, _ := reader.Peek(PeekSize)
|
||||
chunkChan <- append(chunk[:n], peekData...)
|
||||
if errors.Is(err, io.EOF) {
|
||||
break
|
||||
}
|
||||
}
|
||||
}()
|
||||
return chunkChan
|
||||
}
|
||||
@@ -0,0 +1,75 @@
|
||||
package common
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
"errors"
|
||||
"io"
|
||||
"net/http"
|
||||
"testing"
|
||||
|
||||
diskbufferreader "github.com/bill-rich/disk-buffer-reader"
|
||||
)
|
||||
|
||||
func TestChunker(t *testing.T) {
|
||||
resp, err := http.Get("https://raw.githubusercontent.com/bill-rich/bad-secrets/master/FifteenMB.gz")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
reReader, err := diskbufferreader.New(resp.Body)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer reReader.Close()
|
||||
|
||||
baseChunkCount := 0
|
||||
|
||||
// Count chunks from looping using chunk size.
|
||||
for {
|
||||
baseChunkCount++
|
||||
tmpChunk := make([]byte, ChunkSize)
|
||||
_, err := reReader.Read(tmpChunk)
|
||||
if err != nil {
|
||||
if errors.Is(err, io.EOF) {
|
||||
break
|
||||
}
|
||||
t.Error(err)
|
||||
}
|
||||
}
|
||||
_ = reReader.Reset()
|
||||
|
||||
// Get the first two chunks for comparing later.
|
||||
baseChunkOne := make([]byte, ChunkSize)
|
||||
baseChunkTwo := make([]byte, ChunkSize)
|
||||
|
||||
baseReader := bufio.NewReaderSize(reReader, ChunkSize)
|
||||
_, _ = baseReader.Read(baseChunkOne)
|
||||
peek, _ := baseReader.Peek(PeekSize)
|
||||
baseChunkOne = append(baseChunkOne, peek...)
|
||||
_, _ = baseReader.Read(baseChunkTwo)
|
||||
peek, _ = baseReader.Peek(PeekSize)
|
||||
baseChunkTwo = append(baseChunkTwo, peek...)
|
||||
|
||||
// Reset the reader to the beginning and use ChunkReader.
|
||||
_ = reReader.Reset()
|
||||
|
||||
testChunkCount := 0
|
||||
for chunk := range ChunkReader(reReader) {
|
||||
testChunkCount++
|
||||
switch testChunkCount {
|
||||
case 1:
|
||||
if !bytes.Equal(baseChunkOne, chunk) {
|
||||
t.Errorf("First chunk did not match expected. Got: %d bytes, expected: %d bytes", len(chunk), len(baseChunkOne))
|
||||
}
|
||||
case 2:
|
||||
if !bytes.Equal(baseChunkTwo, chunk) {
|
||||
t.Errorf("Second chunk did not match expected. Got: %d bytes, expected: %d bytes", len(chunk), len(baseChunkTwo))
|
||||
}
|
||||
}
|
||||
}
|
||||
if testChunkCount != baseChunkCount {
|
||||
t.Errorf("Wrong number of chunks received. Got %d, expected: %d.", testChunkCount, baseChunkCount)
|
||||
}
|
||||
|
||||
}
|
||||
@@ -51,6 +51,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
s1 := detectors.Result{
|
||||
DetectorType: detectorspb.DetectorType_AdobeIO,
|
||||
Raw: []byte(resMatch),
|
||||
RawV2: []byte(resMatch + resIdMatch),
|
||||
}
|
||||
|
||||
if verify {
|
||||
|
||||
@@ -52,6 +52,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
s1 := detectors.Result{
|
||||
DetectorType: detectorspb.DetectorType_Adzuna,
|
||||
Raw: []byte(resMatch),
|
||||
RawV2: []byte(resMatch + resIdMatch),
|
||||
}
|
||||
|
||||
if verify {
|
||||
|
||||
@@ -52,6 +52,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
s1 := detectors.Result{
|
||||
DetectorType: detectorspb.DetectorType_Aeroworkflow,
|
||||
Raw: []byte(resMatch),
|
||||
RawV2: []byte(resMatch + resIdMatch),
|
||||
}
|
||||
|
||||
if verify {
|
||||
|
||||
@@ -53,6 +53,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
s1 := detectors.Result{
|
||||
DetectorType: detectorspb.DetectorType_Agora,
|
||||
Raw: []byte(resMatch),
|
||||
RawV2: []byte(resMatch + resSecret),
|
||||
}
|
||||
|
||||
if verify {
|
||||
|
||||
@@ -53,6 +53,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
s1 := detectors.Result{
|
||||
DetectorType: detectorspb.DetectorType_AirbrakeProjectKey,
|
||||
Raw: []byte(resMatch),
|
||||
RawV2: []byte(resMatch + resIdMatch),
|
||||
}
|
||||
|
||||
if verify {
|
||||
|
||||
@@ -53,6 +53,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
DetectorType: detectorspb.DetectorType_AirtableApiKey,
|
||||
Redacted: appRes,
|
||||
Raw: []byte(keyRes),
|
||||
RawV2: []byte(keyRes + appRes),
|
||||
}
|
||||
|
||||
if verify {
|
||||
|
||||
@@ -53,6 +53,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
s1 := detectors.Result{
|
||||
DetectorType: detectorspb.DetectorType_Alegra,
|
||||
Raw: []byte(tokenPatMatch),
|
||||
RawV2: []byte(tokenPatMatch + userPatMatch),
|
||||
}
|
||||
|
||||
if verify {
|
||||
|
||||
@@ -51,6 +51,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
s1 := detectors.Result{
|
||||
DetectorType: detectorspb.DetectorType_AlgoliaAdminKey,
|
||||
Raw: []byte(resMatch),
|
||||
RawV2: []byte(resMatch + resIdMatch),
|
||||
}
|
||||
|
||||
if verify {
|
||||
|
||||
@@ -84,6 +84,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
s1 := detectors.Result{
|
||||
DetectorType: detectorspb.DetectorType_Alibaba,
|
||||
Raw: []byte(resMatch),
|
||||
RawV2: []byte(resMatch + resIdMatch),
|
||||
}
|
||||
|
||||
if verify {
|
||||
@@ -103,7 +104,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
params.Add("Version", "2014-05-26")
|
||||
|
||||
stringToSign := buildStringToSign(req.Method, params.Encode())
|
||||
signature := GetSignature(stringToSign, resMatch+"&") //Get Signature HMAC SHA1
|
||||
signature := GetSignature(stringToSign, resMatch+"&") // Get Signature HMAC SHA1
|
||||
params.Add("Signature", signature)
|
||||
req.URL.RawQuery = params.Encode()
|
||||
|
||||
|
||||
@@ -52,6 +52,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
s1 := detectors.Result{
|
||||
DetectorType: detectorspb.DetectorType_Amadeus,
|
||||
Raw: []byte(resMatch),
|
||||
RawV2: []byte(resMatch + resSecret),
|
||||
}
|
||||
|
||||
if verify {
|
||||
|
||||
@@ -52,6 +52,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
s1 := detectors.Result{
|
||||
DetectorType: detectorspb.DetectorType_AmplitudeApiKey,
|
||||
Raw: []byte(resMatch),
|
||||
RawV2: []byte(resMatch + resSecretMatch),
|
||||
}
|
||||
|
||||
if verify {
|
||||
|
||||
@@ -52,6 +52,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
s1 := detectors.Result{
|
||||
DetectorType: detectorspb.DetectorType_Anypoint,
|
||||
Raw: []byte(resMatch),
|
||||
RawV2: []byte(resMatch + orgRes),
|
||||
}
|
||||
|
||||
if verify {
|
||||
|
||||
@@ -61,7 +61,9 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
|
||||
var result Response
|
||||
if errBody == nil {
|
||||
json.Unmarshal(body, &result)
|
||||
if err := json.Unmarshal(body, &result); err != nil {
|
||||
continue
|
||||
}
|
||||
|
||||
if res.StatusCode >= 200 && res.StatusCode < 300 && result.ReturnCode == 0 {
|
||||
s1.Verified = true
|
||||
|
||||
@@ -53,6 +53,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
s1 := detectors.Result{
|
||||
DetectorType: detectorspb.DetectorType_ApiDeck,
|
||||
Raw: []byte(resMatch),
|
||||
RawV2: []byte(resMatch + resIdMatch),
|
||||
}
|
||||
|
||||
if verify {
|
||||
|
||||
@@ -52,6 +52,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
s1 := detectors.Result{
|
||||
DetectorType: detectorspb.DetectorType_Apiflash,
|
||||
Raw: []byte(resMatch),
|
||||
RawV2: []byte(resMatch + resUrlMatch),
|
||||
}
|
||||
|
||||
if verify {
|
||||
|
||||
@@ -54,6 +54,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
s1 := detectors.Result{
|
||||
DetectorType: detectorspb.DetectorType_APIMatic,
|
||||
Raw: []byte(userPatMatch),
|
||||
RawV2: []byte(userPatMatch + passPatMatch),
|
||||
}
|
||||
if verify {
|
||||
timeout := 10 * time.Second
|
||||
|
||||
@@ -63,6 +63,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
s1 := detectors.Result{
|
||||
DetectorType: detectorspb.DetectorType_Appcues,
|
||||
Raw: []byte(resMatch),
|
||||
RawV2: []byte(resMatch + resUserMatch),
|
||||
}
|
||||
if verify {
|
||||
req, err := http.NewRequestWithContext(ctx, "GET", fmt.Sprintf("https://api.appcues.com/v2/accounts/%s/flows", resIdMatch), nil)
|
||||
|
||||
@@ -53,6 +53,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
s1 := detectors.Result{
|
||||
DetectorType: detectorspb.DetectorType_Apptivo,
|
||||
Raw: []byte(resMatch),
|
||||
RawV2: []byte(resMatch + resIdMatch),
|
||||
}
|
||||
|
||||
if verify {
|
||||
|
||||
@@ -54,6 +54,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
s1 := detectors.Result{
|
||||
DetectorType: detectorspb.DetectorType_ArtifactoryAccessToken,
|
||||
Raw: []byte(resMatch),
|
||||
RawV2: []byte(resMatch + resURLMatch),
|
||||
}
|
||||
|
||||
if verify {
|
||||
|
||||
@@ -53,6 +53,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
s1 := detectors.Result{
|
||||
DetectorType: detectorspb.DetectorType_Artsy,
|
||||
Raw: []byte(resMatch),
|
||||
RawV2: []byte(resMatch + resIdMatch),
|
||||
}
|
||||
|
||||
if verify {
|
||||
|
||||
@@ -58,6 +58,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
DetectorType: detectorspb.DetectorType_Auth0ManagementApiToken,
|
||||
Redacted: domainRes,
|
||||
Raw: []byte(managementApiTokenRes),
|
||||
RawV2: []byte(managementApiTokenRes + domainRes),
|
||||
}
|
||||
|
||||
if verify {
|
||||
|
||||
@@ -2,14 +2,11 @@ package auth0oauth
|
||||
|
||||
import (
|
||||
"context"
|
||||
// "fmt"
|
||||
// "log"
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"io/ioutil"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
@@ -65,6 +62,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
DetectorType: detectorspb.DetectorType_Auth0oauth,
|
||||
Redacted: clientIdRes,
|
||||
Raw: []byte(clientSecretRes),
|
||||
RawV2: []byte(clientIdRes + clientSecretRes),
|
||||
}
|
||||
|
||||
if verify {
|
||||
|
||||
@@ -52,6 +52,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
s1 := detectors.Result{
|
||||
DetectorType: detectorspb.DetectorType_Autodesk,
|
||||
Raw: []byte(resMatch),
|
||||
RawV2: []byte(resMatch + resSecret),
|
||||
}
|
||||
|
||||
if verify {
|
||||
|
||||
@@ -27,7 +27,7 @@ var (
|
||||
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
|
||||
// Key types are from this list https://docs.aws.amazon.com/IAM/latest/UserGuide/reference_identifiers.html#identifiers-unique-ids
|
||||
idPat = regexp.MustCompile(`\b((?:AKIA|ABIA|ACCA|ASIA)[0-9A-Z]{16})\b`)
|
||||
secretPat = regexp.MustCompile(`\b([A-Za-z0-9+/]{40})[ \r\n'"\x60]`)
|
||||
secretPat = regexp.MustCompile(`[^A-Za-z0-9+\/]{0,1}([A-Za-z0-9+\/]{40})[^A-Za-z0-9+\/]{0,1}`)
|
||||
// Hashes, like those for git, do technically match the secret pattern.
|
||||
// But they are extremely unlikely to be generated as an actual AWS secret.
|
||||
// So when we find them, if they're not verified, we should ignore the result.
|
||||
@@ -81,6 +81,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
DetectorType: detectorspb.DetectorType_AWS,
|
||||
Raw: []byte(resIDMatch),
|
||||
Redacted: resIDMatch,
|
||||
RawV2: []byte(resIDMatch + resSecretMatch),
|
||||
}
|
||||
|
||||
if verify {
|
||||
|
||||
@@ -144,6 +144,23 @@ func TestAWS_FromChunk(t *testing.T) {
|
||||
},
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "found, unverified, with leading +",
|
||||
s: Scanner{},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a aws secret %s within aws %s but not valid", "+HaNv9cTwheDKGJaws/+BMF2GgybQgBWdhcOOdfF", id)), // the secret would satisfy the regex but not pass validation
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_AWS,
|
||||
Verified: false,
|
||||
Redacted: "AKIAWARWQKZNHMZBLY4I",
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
|
||||
@@ -52,6 +52,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
s1 := detectors.Result{
|
||||
DetectorType: detectorspb.DetectorType_Aylien,
|
||||
Raw: []byte(resMatch),
|
||||
RawV2: []byte(resMatch + resIdMatch),
|
||||
}
|
||||
if verify {
|
||||
req, err := http.NewRequestWithContext(ctx, "GET", "https://api.aylien.com/news/stories", nil)
|
||||
|
||||
@@ -52,6 +52,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
s1 := detectors.Result{
|
||||
DetectorType: detectorspb.DetectorType_Billomat,
|
||||
Raw: []byte(resMatch),
|
||||
RawV2: []byte(resMatch + resId),
|
||||
}
|
||||
|
||||
if verify {
|
||||
|
||||
@@ -58,6 +58,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
s1 := detectors.Result{
|
||||
DetectorType: detectorspb.DetectorType_Bitmex,
|
||||
Raw: []byte(resSecretMatch),
|
||||
RawV2: []byte(resMatch + resSecretMatch),
|
||||
}
|
||||
|
||||
if verify {
|
||||
|
||||
@@ -52,6 +52,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
s1 := detectors.Result{
|
||||
DetectorType: detectorspb.DetectorType_BrowserStack,
|
||||
Raw: []byte(resMatch),
|
||||
RawV2: []byte(resMatch + resUserMatch),
|
||||
}
|
||||
|
||||
if verify {
|
||||
|
||||
@@ -2,11 +2,11 @@ package bulksms
|
||||
|
||||
import (
|
||||
"context"
|
||||
b64 "encoding/base64"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"strings"
|
||||
b64 "encoding/base64"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
@@ -21,10 +21,9 @@ var _ detectors.Detector = (*Scanner)(nil)
|
||||
var (
|
||||
client = common.SaneHttpClient()
|
||||
|
||||
//Make sure that your group is surrounded in boundry characters such as below to reduce false positives
|
||||
// Make sure that your group is surrounded in boundry characters such as below to reduce false positives
|
||||
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"bulksms"}) + `\b([a-fA-Z0-9*]{29})\b`)
|
||||
idPat = regexp.MustCompile(detectors.PrefixRegex([]string{"bulksms"}) + `\b([A-F0-9-]{37})\b`)
|
||||
|
||||
idPat = regexp.MustCompile(detectors.PrefixRegex([]string{"bulksms"}) + `\b([A-F0-9-]{37})\b`)
|
||||
)
|
||||
|
||||
// Keywords are used for efficiently pre-filtering chunks.
|
||||
@@ -54,10 +53,11 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
s1 := detectors.Result{
|
||||
DetectorType: detectorspb.DetectorType_Bulksms,
|
||||
Raw: []byte(resMatch),
|
||||
RawV2: []byte(resMatch + resIdMatch),
|
||||
}
|
||||
|
||||
|
||||
if verify {
|
||||
data := fmt.Sprintf("%s:%s", resIdMatch,resMatch)
|
||||
data := fmt.Sprintf("%s:%s", resIdMatch, resMatch)
|
||||
sEnc := b64.StdEncoding.EncodeToString([]byte(data))
|
||||
req, err := http.NewRequestWithContext(ctx, "GET", "https://api.bulksms.com/v1/messages", nil)
|
||||
if err != nil {
|
||||
@@ -70,17 +70,17 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
if res.StatusCode >= 200 && res.StatusCode < 300 {
|
||||
s1.Verified = true
|
||||
} else {
|
||||
//This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key
|
||||
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key
|
||||
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
|
||||
continue
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
results = append(results, s1)
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
||||
return detectors.CleanResults(results), nil
|
||||
|
||||
@@ -52,6 +52,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
s1 := detectors.Result{
|
||||
DetectorType: detectorspb.DetectorType_CaptainData,
|
||||
Raw: []byte(resMatch),
|
||||
RawV2: []byte(resProjIdMatch + resMatch),
|
||||
}
|
||||
|
||||
if verify {
|
||||
|
||||
@@ -54,6 +54,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
s1 := detectors.Result{
|
||||
DetectorType: detectorspb.DetectorType_Cashboard,
|
||||
Raw: []byte(resMatch),
|
||||
RawV2: []byte(resMatch + resUser),
|
||||
}
|
||||
|
||||
if verify {
|
||||
|
||||
@@ -63,6 +63,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
s1 := detectors.Result{
|
||||
DetectorType: detectorspb.DetectorType_Caspio,
|
||||
Raw: []byte(resMatch),
|
||||
RawV2: []byte(resMatch + resIdMatch + resDomainMatch),
|
||||
}
|
||||
|
||||
if verify {
|
||||
|
||||
@@ -51,6 +51,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
s1 := detectors.Result{
|
||||
DetectorType: detectorspb.DetectorType_Censys,
|
||||
Raw: []byte(tokenPatMatch),
|
||||
RawV2: []byte(tokenPatMatch + userPatMatch),
|
||||
}
|
||||
|
||||
if verify {
|
||||
|
||||
@@ -69,6 +69,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
s1 := detectors.Result{
|
||||
DetectorType: detectorspb.DetectorType_CexIO,
|
||||
Raw: []byte(resKeyMatch),
|
||||
RawV2: []byte(resUserIdMatch + resSecretMatch),
|
||||
}
|
||||
|
||||
if verify {
|
||||
@@ -102,7 +103,9 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
}
|
||||
|
||||
var responseObject Response
|
||||
json.Unmarshal(body, &responseObject)
|
||||
if err := json.Unmarshal(body, &responseObject); err != nil {
|
||||
continue
|
||||
}
|
||||
|
||||
if res.StatusCode >= 200 && res.StatusCode < 300 && validResponse {
|
||||
s1.Verified = true
|
||||
|
||||
@@ -20,7 +20,7 @@ var (
|
||||
client = common.SaneHttpClient()
|
||||
|
||||
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
|
||||
//Tokens starting with sk_test are used for the app's sandbox environment while tokens starting with sk only are for production environment
|
||||
// Tokens starting with sk_test are used for the app's sandbox environment while tokens starting with sk only are for production environment
|
||||
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"checkout"}) + `\b((sk_|sk_test_)[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12})\b`)
|
||||
idPat = regexp.MustCompile(detectors.PrefixRegex([]string{"checkout"}) + `\b(cus_[0-9a-zA-Z]{26})\b`)
|
||||
)
|
||||
@@ -53,10 +53,11 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
s1 := detectors.Result{
|
||||
DetectorType: detectorspb.DetectorType_Checkout,
|
||||
Raw: []byte(resMatch),
|
||||
RawV2: []byte(resMatch + resIdMatch),
|
||||
}
|
||||
|
||||
if verify {
|
||||
//Used the app's sandbox environment for this case since I can't create a live account.
|
||||
// Used the app's sandbox environment for this case since I can't create a live account.
|
||||
req, err := http.NewRequestWithContext(ctx, "GET", "https://api.sandbox.checkout.com/customers/"+resIdMatch, nil)
|
||||
if err != nil {
|
||||
continue
|
||||
|
||||
@@ -53,6 +53,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
s1 := detectors.Result{
|
||||
DetectorType: detectorspb.DetectorType_Checkvist,
|
||||
Raw: []byte(resMatch),
|
||||
RawV2: []byte(resMatch + resEmailMatch),
|
||||
}
|
||||
|
||||
if verify {
|
||||
|
||||
@@ -61,6 +61,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
s1 := detectors.Result{
|
||||
DetectorType: detectorspb.DetectorType_ClickHelp,
|
||||
Raw: []byte(resServer),
|
||||
RawV2: []byte(resServer + resEmail),
|
||||
}
|
||||
|
||||
if verify {
|
||||
|
||||
@@ -50,6 +50,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
s1 := detectors.Result{
|
||||
DetectorType: detectorspb.DetectorType_ClickSendsms,
|
||||
Raw: []byte(resMatch),
|
||||
RawV2: []byte(resMatch + resIdMatch),
|
||||
}
|
||||
|
||||
if verify {
|
||||
|
||||
@@ -52,6 +52,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
s1 := detectors.Result{
|
||||
DetectorType: detectorspb.DetectorType_ClockworkSMS,
|
||||
Raw: []byte(resMatch),
|
||||
RawV2: []byte(resMatch + tokenRes),
|
||||
}
|
||||
|
||||
if verify {
|
||||
|
||||
@@ -54,6 +54,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
s1 := detectors.Result{
|
||||
DetectorType: detectorspb.DetectorType_CloudElements,
|
||||
Raw: []byte(resMatch),
|
||||
RawV2: []byte(resMatch + resOrgMatch),
|
||||
}
|
||||
|
||||
if verify {
|
||||
|
||||
@@ -60,6 +60,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
DetectorType: detectorspb.DetectorType_CloudflareGlobalApiKey,
|
||||
Redacted: emailRes,
|
||||
Raw: []byte(apiKeyRes),
|
||||
RawV2: []byte(apiKeyRes + emailRes),
|
||||
}
|
||||
|
||||
if verify {
|
||||
|
||||
@@ -53,6 +53,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
s1 := detectors.Result{
|
||||
DetectorType: detectorspb.DetectorType_CompanyHub,
|
||||
Raw: []byte(resMatch),
|
||||
RawV2: []byte(resMatch + resIdMatch),
|
||||
}
|
||||
|
||||
if verify {
|
||||
|
||||
@@ -54,6 +54,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
s1 := detectors.Result{
|
||||
DetectorType: detectorspb.DetectorType_Confluent,
|
||||
Raw: []byte(resMatch),
|
||||
RawV2: []byte(resMatch + resSecret),
|
||||
}
|
||||
|
||||
if verify {
|
||||
|
||||
@@ -51,6 +51,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
s1 := detectors.Result{
|
||||
DetectorType: detectorspb.DetectorType_Copper,
|
||||
Raw: []byte(resMatch),
|
||||
RawV2: []byte(resMatch + resIdMatch),
|
||||
}
|
||||
|
||||
if verify {
|
||||
|
||||
@@ -51,6 +51,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
s1 := detectors.Result{
|
||||
DetectorType: detectorspb.DetectorType_CrossBrowserTesting,
|
||||
Raw: []byte(resMatch),
|
||||
RawV2: []byte(resMatch + resIdMatch),
|
||||
}
|
||||
|
||||
if verify {
|
||||
|
||||
@@ -52,6 +52,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
s1 := detectors.Result{
|
||||
DetectorType: detectorspb.DetectorType_CustomerGuru,
|
||||
Raw: []byte(resMatch),
|
||||
RawV2: []byte(resMatch + resIdMatch),
|
||||
}
|
||||
|
||||
if verify {
|
||||
|
||||
@@ -52,6 +52,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
s1 := detectors.Result{
|
||||
DetectorType: detectorspb.DetectorType_CustomerIO,
|
||||
Raw: []byte(resMatch),
|
||||
RawV2: []byte(resMatch + resIdMatch),
|
||||
}
|
||||
|
||||
if verify {
|
||||
|
||||
@@ -29,6 +29,9 @@ type Result struct {
|
||||
Verified bool
|
||||
// Raw contains the raw secret identifier data. Prefer IDs over secrets since it is used for deduping after hashing.
|
||||
Raw []byte
|
||||
// RawV2 contains the raw secret identifier that is a combination of both the ID and the secret.
|
||||
// This is used for secrets that are multi part and could have the same ID. Ex: AWS credentials
|
||||
RawV2 []byte
|
||||
// Redacted contains the redacted version of the raw secret identification data for display purposes.
|
||||
// A secret ID should be used if available.
|
||||
Redacted string
|
||||
@@ -96,9 +99,9 @@ func PrefixRegex(keywords []string) string {
|
||||
return pre + middle + post
|
||||
}
|
||||
|
||||
//KeyIsRandom is a Low cost check to make sure that 'keys' include a number to reduce FPs.
|
||||
//Golang doesnt support regex lookaheads, so must be done in separate calls.
|
||||
//TODO improve checks. Shannon entropy did not work well.
|
||||
// KeyIsRandom is a Low cost check to make sure that 'keys' include a number to reduce FPs.
|
||||
// Golang doesnt support regex lookaheads, so must be done in separate calls.
|
||||
// TODO improve checks. Shannon entropy did not work well.
|
||||
func KeyIsRandom(key string) bool {
|
||||
for _, ch := range key {
|
||||
if unicode.IsDigit(ch) {
|
||||
|
||||
@@ -51,6 +51,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
DetectorType: detectorspb.DetectorType_DiscordBotToken,
|
||||
Redacted: resId,
|
||||
Raw: []byte(resMatch),
|
||||
RawV2: []byte(resMatch + resId),
|
||||
}
|
||||
|
||||
if verify {
|
||||
|
||||
@@ -52,6 +52,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
s1 := detectors.Result{
|
||||
DetectorType: detectorspb.DetectorType_Dnscheck,
|
||||
Raw: []byte(resMatch),
|
||||
RawV2: []byte(resMatch + resIdMatch),
|
||||
}
|
||||
|
||||
if verify {
|
||||
|
||||
@@ -56,6 +56,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
s1 := detectors.Result{
|
||||
DetectorType: detectorspb.DetectorType_Dwolla,
|
||||
Raw: []byte(idMatch),
|
||||
RawV2: []byte(idMatch + secretMatch),
|
||||
}
|
||||
|
||||
if verify {
|
||||
|
||||
@@ -2,13 +2,12 @@ package easyinsight
|
||||
|
||||
import (
|
||||
"context"
|
||||
b64 "encoding/base64"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
b64 "encoding/base64"
|
||||
"net/http"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
|
||||
@@ -55,6 +54,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
s1 := detectors.Result{
|
||||
DetectorType: detectorspb.DetectorType_EasyInsight,
|
||||
Raw: []byte(resMatch),
|
||||
RawV2: []byte(resMatch + resIdMatch),
|
||||
}
|
||||
|
||||
if verify {
|
||||
|
||||
@@ -51,6 +51,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
s1 := detectors.Result{
|
||||
DetectorType: detectorspb.DetectorType_Edamam,
|
||||
Raw: []byte(resMatch),
|
||||
RawV2: []byte(resMatch + resId),
|
||||
}
|
||||
|
||||
if verify {
|
||||
|
||||
@@ -55,6 +55,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
s1 := detectors.Result{
|
||||
DetectorType: detectorspb.DetectorType_EightxEight,
|
||||
Raw: []byte(resMatch),
|
||||
RawV2: []byte(resMatch + resIdMatch),
|
||||
}
|
||||
|
||||
if verify {
|
||||
|
||||
@@ -54,6 +54,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
DetectorType: detectorspb.DetectorType_FacebookOAuth,
|
||||
Redacted: apiIdRes,
|
||||
Raw: []byte(apiSecretRes),
|
||||
RawV2: []byte(apiIdRes + apiSecretRes),
|
||||
}
|
||||
|
||||
if verify {
|
||||
|
||||
@@ -52,6 +52,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
s1 := detectors.Result{
|
||||
DetectorType: detectorspb.DetectorType_FacePlusPlus,
|
||||
Raw: []byte(resMatch),
|
||||
RawV2: []byte(resMatch + resSecret),
|
||||
}
|
||||
|
||||
if verify {
|
||||
|
||||
@@ -51,6 +51,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
s1 := detectors.Result{
|
||||
DetectorType: detectorspb.DetectorType_FourSquare,
|
||||
Raw: []byte(resMatch),
|
||||
RawV2: []byte(resMatch + resSecret),
|
||||
}
|
||||
|
||||
if verify {
|
||||
|
||||
@@ -28,7 +28,7 @@ var (
|
||||
client = common.SaneHttpClient()
|
||||
|
||||
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
|
||||
//Removed bounds since there are some cases where the start and end of the token is a special character
|
||||
// Removed bounds since there are some cases where the start and end of the token is a special character
|
||||
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"gengo"}) + `([ ]{0,1}[0-9a-zA-Z\[\]\-\(\)\{\}|_^@$=~]{64}[ \r\n]{1})`)
|
||||
secretPat = regexp.MustCompile(detectors.PrefixRegex([]string{"gengo"}) + `([ ]{0,1}[0-9a-zA-Z\[\]\-\(\)\{\}|_^@$=~]{64}[ \r\n]{1})`)
|
||||
)
|
||||
@@ -61,6 +61,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
s1 := detectors.Result{
|
||||
DetectorType: detectorspb.DetectorType_Gengo,
|
||||
Raw: []byte(resSecretMatch),
|
||||
RawV2: []byte(resMatch + resSecretMatch),
|
||||
}
|
||||
|
||||
if verify {
|
||||
@@ -80,7 +81,9 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
|
||||
if errBody == nil {
|
||||
var response Response
|
||||
json.Unmarshal(body, &response)
|
||||
if err := json.Unmarshal(body, &response); err != nil {
|
||||
continue
|
||||
}
|
||||
|
||||
if res.StatusCode >= 200 && res.StatusCode < 300 && response.OpStat == "ok" {
|
||||
s1.Verified = true
|
||||
|
||||
@@ -52,6 +52,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
s1 := detectors.Result{
|
||||
DetectorType: detectorspb.DetectorType_Geocodio,
|
||||
Raw: []byte(resMatch),
|
||||
RawV2: []byte(resMatch + resSearchMatch),
|
||||
}
|
||||
|
||||
if verify {
|
||||
|
||||
@@ -76,7 +76,9 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
|
||||
if errBody == nil {
|
||||
response := Response{}
|
||||
xml.Unmarshal(body, &response)
|
||||
if err := xml.Unmarshal(body, &response); err != nil {
|
||||
continue
|
||||
}
|
||||
|
||||
if res.StatusCode >= 200 && res.StatusCode < 300 && response.Error == nil {
|
||||
s1.Verified = true
|
||||
|
||||
@@ -2,6 +2,10 @@ package kraken
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/hmac"
|
||||
"crypto/sha256"
|
||||
"crypto/sha512"
|
||||
"encoding/base64"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
@@ -10,11 +14,6 @@ import (
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"crypto/hmac"
|
||||
"crypto/sha256"
|
||||
"crypto/sha512"
|
||||
"encoding/base64"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
|
||||
@@ -29,7 +28,7 @@ var (
|
||||
client = common.SaneHttpClient()
|
||||
|
||||
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
|
||||
//Bounds have been removed because there are some cases that tokens have trailing frontslash(/) or plus sign (+)
|
||||
// Bounds have been removed because there are some cases that tokens have trailing frontslash(/) or plus sign (+)
|
||||
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"kraken"}) + `\b([0-9A-Za-z\/\+=]{56}[ "'\r\n]{1})`)
|
||||
privKeyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"kraken"}) + `\b([0-9A-Za-z\/\+=]{86,88}[ "'\r\n]{1})`)
|
||||
)
|
||||
@@ -62,11 +61,12 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
s1 := detectors.Result{
|
||||
DetectorType: detectorspb.DetectorType_Kraken,
|
||||
Raw: []byte(resMatch),
|
||||
RawV2: []byte(resMatch + resPrivKeyMatch),
|
||||
}
|
||||
|
||||
if verify {
|
||||
|
||||
//Increasing 64-bit integer, for each request that is made with a particular API key.
|
||||
// Increasing 64-bit integer, for each request that is made with a particular API key.
|
||||
apiNonce := strconv.FormatInt(time.Now().Unix(), 10)
|
||||
|
||||
payload := url.Values{}
|
||||
@@ -106,7 +106,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
return detectors.CleanResults(results), nil
|
||||
}
|
||||
|
||||
//Code from https://docs.kraken.com/rest/#section/Authentication/Headers-and-Signature
|
||||
// Code from https://docs.kraken.com/rest/#section/Authentication/Headers-and-Signature
|
||||
func getKrakenSignature(url_path string, values url.Values, secret []byte) string {
|
||||
|
||||
sha := sha256.New()
|
||||
|
||||
@@ -65,6 +65,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
s1 := detectors.Result{
|
||||
DetectorType: detectorspb.DetectorType_KuCoin,
|
||||
Raw: []byte(resKeyMatch),
|
||||
RawV2: []byte(resKeyMatch + resPassphraseMatch),
|
||||
}
|
||||
|
||||
if verify {
|
||||
|
||||
@@ -53,6 +53,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
s1 := detectors.Result{
|
||||
DetectorType: detectorspb.DetectorType_MattermostPersonalToken,
|
||||
Raw: []byte(resMatch),
|
||||
RawV2: []byte(resMatch + serverRes),
|
||||
}
|
||||
|
||||
if verify {
|
||||
|
||||
@@ -52,6 +52,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
s1 := detectors.Result{
|
||||
DetectorType: detectorspb.DetectorType_Mux,
|
||||
Raw: []byte(resMatch),
|
||||
RawV2: []byte(resMatch + resSecretMatch),
|
||||
}
|
||||
|
||||
if verify {
|
||||
|
||||
@@ -51,6 +51,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
s1 := detectors.Result{
|
||||
DetectorType: detectorspb.DetectorType_NexmoApiKey,
|
||||
Raw: []byte(resMatch),
|
||||
RawV2: []byte(resMatch + resSecret),
|
||||
}
|
||||
|
||||
if verify {
|
||||
|
||||
@@ -52,6 +52,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
DetectorType: detectorspb.DetectorType_Plivo,
|
||||
Redacted: id,
|
||||
Raw: []byte(resMatch),
|
||||
RawV2: []byte(resMatch + id),
|
||||
}
|
||||
stringResMatch := fmt.Sprintf("%s:%s", id, resMatch)
|
||||
decodeSecret := b64.StdEncoding.EncodeToString([]byte(stringResMatch))
|
||||
|
||||
@@ -58,6 +58,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
s1 := detectors.Result{
|
||||
DetectorType: detectorspb.DetectorType_Poloniex,
|
||||
Raw: []byte(resSecretMatch),
|
||||
RawV2: []byte(resMatch + resSecretMatch),
|
||||
}
|
||||
|
||||
if verify {
|
||||
|
||||
@@ -72,6 +72,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
s1 := detectors.Result{
|
||||
DetectorType: detectorspb.DetectorType_PusherChannelKey,
|
||||
Raw: []byte(resappMatch),
|
||||
RawV2: []byte(resappMatch + reskeyMatch),
|
||||
}
|
||||
|
||||
if verify {
|
||||
|
||||
@@ -54,6 +54,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
s1 := detectors.Result{
|
||||
DetectorType: detectorspb.DetectorType_Roaring,
|
||||
Raw: []byte(resClient),
|
||||
RawV2: []byte(resClient + resSecret),
|
||||
}
|
||||
|
||||
if verify {
|
||||
|
||||
@@ -61,6 +61,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
s1 := detectors.Result{
|
||||
DetectorType: detectorspb.DetectorType_Rownd,
|
||||
Raw: []byte(keyMatch),
|
||||
RawV2: []byte(keyMatch + secretMatch),
|
||||
}
|
||||
|
||||
if verify {
|
||||
|
||||
@@ -62,6 +62,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
s1 := detectors.Result{
|
||||
DetectorType: detectorspb.DetectorType_SatismeterProjectkey,
|
||||
Raw: []byte(resMatch),
|
||||
RawV2: []byte(resMatch + resPassMatch),
|
||||
}
|
||||
|
||||
if verify {
|
||||
|
||||
@@ -69,7 +69,9 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
if err == nil {
|
||||
defer res.Body.Close()
|
||||
var authResponse authRes
|
||||
json.NewDecoder(res.Body).Decode(&authResponse)
|
||||
if err := json.NewDecoder(res.Body).Decode(&authResponse); err != nil {
|
||||
continue
|
||||
}
|
||||
s.Verified = authResponse.Ok
|
||||
}
|
||||
}
|
||||
|
||||
@@ -54,6 +54,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
s1 := detectors.Result{
|
||||
DetectorType: detectorspb.DetectorType_Smooch,
|
||||
Raw: []byte(resMatch),
|
||||
RawV2: []byte(resMatch + resSecret),
|
||||
}
|
||||
|
||||
if verify {
|
||||
|
||||
@@ -60,6 +60,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
s1 := detectors.Result{
|
||||
DetectorType: detectorspb.DetectorType_Strava,
|
||||
Raw: []byte(resId),
|
||||
RawV2: []byte(resId + resSecret),
|
||||
}
|
||||
|
||||
if verify {
|
||||
|
||||
@@ -54,6 +54,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
s1 := detectors.Result{
|
||||
DetectorType: detectorspb.DetectorType_Textmagic,
|
||||
Raw: []byte(resMatch),
|
||||
RawV2: []byte(resMatch + resUser),
|
||||
}
|
||||
|
||||
if verify {
|
||||
|
||||
@@ -53,6 +53,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
s1 := detectors.Result{
|
||||
DetectorType: detectorspb.DetectorType_Tru,
|
||||
Raw: []byte(resMatch),
|
||||
RawV2: []byte(resMatch + resSecret),
|
||||
}
|
||||
|
||||
if verify {
|
||||
|
||||
@@ -54,6 +54,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
s1 := detectors.Result{
|
||||
DetectorType: detectorspb.DetectorType_Vouchery,
|
||||
Raw: []byte(resMatch),
|
||||
RawV2: []byte(resMatch + subMatch),
|
||||
}
|
||||
|
||||
if verify {
|
||||
|
||||
@@ -6,15 +6,18 @@ import (
|
||||
|
||||
"github.com/go-errors/errors"
|
||||
"github.com/sirupsen/logrus"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/sources/filesystem"
|
||||
"google.golang.org/protobuf/proto"
|
||||
"google.golang.org/protobuf/types/known/anypb"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/sources/filesystem"
|
||||
)
|
||||
|
||||
func (e *Engine) ScanFileSystem(ctx context.Context, directories []string) error {
|
||||
// ScanFileSystem scans a given file system.
|
||||
func (e *Engine) ScanFileSystem(ctx context.Context, c sources.Config) error {
|
||||
connection := &sourcespb.Filesystem{
|
||||
Directories: directories,
|
||||
Directories: c.Directories,
|
||||
}
|
||||
var conn anypb.Any
|
||||
err := anypb.MarshalFrom(&conn, connection, proto.MarshalOptions{})
|
||||
|
||||
+29
-28
@@ -6,39 +6,40 @@ import (
|
||||
"runtime"
|
||||
|
||||
"github.com/go-errors/errors"
|
||||
"github.com/go-git/go-git/v5/plumbing/object"
|
||||
|
||||
gogit "github.com/go-git/go-git/v5"
|
||||
"github.com/go-git/go-git/v5/plumbing"
|
||||
"github.com/go-git/go-git/v5/plumbing/object"
|
||||
"github.com/sirupsen/logrus"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/source_metadatapb"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/sources/git"
|
||||
)
|
||||
|
||||
func (e *Engine) ScanGit(ctx context.Context, repoPath, headRef, baseRef string, maxDepth int, filter *common.Filter) error {
|
||||
// ScanGit scans any git source.
|
||||
func (e *Engine) ScanGit(ctx context.Context, c sources.Config) error {
|
||||
logOptions := &gogit.LogOptions{}
|
||||
opts := []git.ScanOption{
|
||||
git.ScanOptionFilter(filter),
|
||||
git.ScanOptionFilter(c.Filter),
|
||||
git.ScanOptionLogOptions(logOptions),
|
||||
}
|
||||
|
||||
repo, err := gogit.PlainOpenWithOptions(repoPath, &gogit.PlainOpenOptions{DetectDotGit: true})
|
||||
repo, err := gogit.PlainOpenWithOptions(c.RepoPath, &gogit.PlainOpenOptions{DetectDotGit: true})
|
||||
if err != nil {
|
||||
return fmt.Errorf("could open repo: %s: %w", repoPath, err)
|
||||
return fmt.Errorf("could open repo: %s: %w", c.RepoPath, err)
|
||||
}
|
||||
|
||||
var baseCommit *object.Commit
|
||||
if len(baseRef) > 0 {
|
||||
baseHash := plumbing.NewHash(baseRef)
|
||||
if !plumbing.IsHash(baseRef) {
|
||||
base, err := git.TryAdditionalBaseRefs(repo, baseRef)
|
||||
if len(c.BaseRef) > 0 {
|
||||
baseHash := plumbing.NewHash(c.BaseRef)
|
||||
if !plumbing.IsHash(c.BaseRef) {
|
||||
base, err := git.TryAdditionalBaseRefs(repo, c.BaseRef)
|
||||
if err != nil {
|
||||
return errors.WrapPrefix(err, "unable to resolve base ref", 0)
|
||||
} else {
|
||||
baseRef = base.String()
|
||||
baseCommit, _ = repo.CommitObject(plumbing.NewHash(baseRef))
|
||||
c.BaseRef = base.String()
|
||||
baseCommit, _ = repo.CommitObject(plumbing.NewHash(c.BaseRef))
|
||||
}
|
||||
} else {
|
||||
baseCommit, err = repo.CommitObject(baseHash)
|
||||
@@ -49,15 +50,15 @@ func (e *Engine) ScanGit(ctx context.Context, repoPath, headRef, baseRef string,
|
||||
}
|
||||
|
||||
var headCommit *object.Commit
|
||||
if len(headRef) > 0 {
|
||||
headHash := plumbing.NewHash(headRef)
|
||||
if !plumbing.IsHash(headRef) {
|
||||
head, err := git.TryAdditionalBaseRefs(repo, headRef)
|
||||
if len(c.HeadRef) > 0 {
|
||||
headHash := plumbing.NewHash(c.HeadRef)
|
||||
if !plumbing.IsHash(c.HeadRef) {
|
||||
head, err := git.TryAdditionalBaseRefs(repo, c.HeadRef)
|
||||
if err != nil {
|
||||
return errors.WrapPrefix(err, "unable to resolve head ref", 0)
|
||||
} else {
|
||||
headRef = head.String()
|
||||
headCommit, _ = repo.CommitObject(plumbing.NewHash(baseRef))
|
||||
c.HeadRef = head.String()
|
||||
headCommit, _ = repo.CommitObject(plumbing.NewHash(c.BaseRef))
|
||||
}
|
||||
} else {
|
||||
headCommit, err = repo.CommitObject(headHash)
|
||||
@@ -67,23 +68,23 @@ func (e *Engine) ScanGit(ctx context.Context, repoPath, headRef, baseRef string,
|
||||
}
|
||||
}
|
||||
|
||||
// If baseCommit is an ancestor of headCommit, update baseRef to be the common ancestor.
|
||||
// If baseCommit is an ancestor of headCommit, update c.BaseRef to be the common ancestor.
|
||||
if headCommit != nil && baseCommit != nil {
|
||||
mergeBase, err := headCommit.MergeBase(baseCommit)
|
||||
if err != nil || len(mergeBase) < 1 {
|
||||
return errors.WrapPrefix(err, "could not find common base between the given references", 0)
|
||||
}
|
||||
baseRef = mergeBase[0].Hash.String()
|
||||
c.BaseRef = mergeBase[0].Hash.String()
|
||||
}
|
||||
|
||||
if maxDepth != 0 {
|
||||
opts = append(opts, git.ScanOptionMaxDepth(int64(maxDepth)))
|
||||
if c.MaxDepth != 0 {
|
||||
opts = append(opts, git.ScanOptionMaxDepth(int64(c.MaxDepth)))
|
||||
}
|
||||
if baseRef != "" {
|
||||
opts = append(opts, git.ScanOptionBaseHash(baseRef))
|
||||
if c.BaseRef != "" {
|
||||
opts = append(opts, git.ScanOptionBaseHash(c.BaseRef))
|
||||
}
|
||||
if headRef != "" {
|
||||
opts = append(opts, git.ScanOptionHeadCommit(headRef))
|
||||
if c.HeadRef != "" {
|
||||
opts = append(opts, git.ScanOptionHeadCommit(c.HeadRef))
|
||||
}
|
||||
scanOptions := git.NewScanOptions(opts...)
|
||||
|
||||
@@ -106,7 +107,7 @@ func (e *Engine) ScanGit(ctx context.Context, repoPath, headRef, baseRef string,
|
||||
e.sourcesWg.Add(1)
|
||||
go func() {
|
||||
defer e.sourcesWg.Done()
|
||||
err := gitSource.ScanRepo(ctx, repo, repoPath, scanOptions, e.ChunksChan())
|
||||
err := gitSource.ScanRepo(ctx, repo, c.RepoPath, scanOptions, e.ChunksChan())
|
||||
if err != nil {
|
||||
logrus.WithError(err).Fatal("could not scan repo")
|
||||
}
|
||||
|
||||
+19
-3
@@ -8,6 +8,7 @@ import (
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/decoders"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/source_metadatapb"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/sources/git"
|
||||
)
|
||||
|
||||
@@ -37,7 +38,7 @@ func TestGitEngine(t *testing.T) {
|
||||
"all_secrets": {
|
||||
expected: map[string]expResult{
|
||||
"70001020fab32b1fcf2f1f0e5c66424eae649826": {"AKIAXYZDQCEN4B6JSJQI", 2},
|
||||
"90c75f884c65dc3638ca1610bd9844e668f213c2": {"AKIAILE3JG6KMS3HZGCA", 4},
|
||||
"84e9c75e388ae3e866e121087ea2dd45a71068f2": {"AKIAILE3JG6KMS3HZGCA", 4},
|
||||
"8afb0ecd4998b1179e428db5ebbcdc8221214432": {"369963c1434c377428ca8531fbc46c0c43d037a0", 3},
|
||||
"27fbead3bf883cdb7de9d7825ed401f28f9398f1": {"ffc7e0f9400fb6300167009e42d2f842cd7956e2", 7},
|
||||
},
|
||||
@@ -56,7 +57,16 @@ func TestGitEngine(t *testing.T) {
|
||||
WithDecoders(decoders.DefaultDecoders()...),
|
||||
WithDetectors(false, DefaultDetectors()...),
|
||||
)
|
||||
e.ScanGit(ctx, path, tTest.branch, tTest.base, tTest.maxDepth, tTest.filter)
|
||||
cfg := sources.Config{
|
||||
RepoPath: path,
|
||||
HeadRef: tTest.branch,
|
||||
BaseRef: tTest.base,
|
||||
MaxDepth: tTest.maxDepth,
|
||||
Filter: tTest.filter,
|
||||
}
|
||||
if err := e.ScanGit(ctx, cfg); err != nil {
|
||||
return
|
||||
}
|
||||
go e.Finish()
|
||||
resultCount := 0
|
||||
for result := range e.ResultsChan() {
|
||||
@@ -104,7 +114,13 @@ func BenchmarkGitEngine(b *testing.B) {
|
||||
for i := 0; i < b.N; i++ {
|
||||
// TODO: this is measuring the time it takes to initialize the source
|
||||
// and not to do the full scan
|
||||
e.ScanGit(ctx, path, "", "", 0, common.FilterEmpty())
|
||||
cfg := sources.Config{
|
||||
RepoPath: path,
|
||||
Filter: common.FilterEmpty(),
|
||||
}
|
||||
if err := e.ScanGit(ctx, cfg); err != nil {
|
||||
return
|
||||
}
|
||||
}
|
||||
e.Finish()
|
||||
}
|
||||
|
||||
+11
-10
@@ -7,34 +7,35 @@ import (
|
||||
"google.golang.org/protobuf/proto"
|
||||
"google.golang.org/protobuf/types/known/anypb"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/sources/github"
|
||||
)
|
||||
|
||||
func (e *Engine) ScanGitHub(ctx context.Context, endpoint string, repos, orgs []string, token string, includeForks bool, filter *common.Filter, concurrency int, includeMembers bool) error {
|
||||
// ScanGitHub scans Github with the provided options.
|
||||
func (e *Engine) ScanGitHub(ctx context.Context, c sources.Config) error {
|
||||
source := github.Source{}
|
||||
connection := sourcespb.GitHub{
|
||||
Endpoint: endpoint,
|
||||
Organizations: orgs,
|
||||
Repositories: repos,
|
||||
ScanUsers: includeMembers,
|
||||
Endpoint: c.Endpoint,
|
||||
Organizations: c.Orgs,
|
||||
Repositories: c.Repos,
|
||||
ScanUsers: c.IncludeMembers,
|
||||
}
|
||||
if len(token) > 0 {
|
||||
if len(c.Token) > 0 {
|
||||
connection.Credential = &sourcespb.GitHub_Token{
|
||||
Token: token,
|
||||
Token: c.Token,
|
||||
}
|
||||
} else {
|
||||
connection.Credential = &sourcespb.GitHub_Unauthenticated{}
|
||||
}
|
||||
connection.IncludeForks = includeForks
|
||||
connection.IncludeForks = c.IncludeForks
|
||||
var conn anypb.Any
|
||||
err := anypb.MarshalFrom(&conn, &connection, proto.MarshalOptions{})
|
||||
if err != nil {
|
||||
logrus.WithError(err).Error("failed to marshal github connection")
|
||||
return err
|
||||
}
|
||||
err = source.Init(ctx, "trufflehog - github", 0, 0, false, &conn, concurrency)
|
||||
err = source.Init(ctx, "trufflehog - github", 0, 0, false, &conn, c.Concurrency)
|
||||
if err != nil {
|
||||
logrus.WithError(err).Error("failed to initialize github source")
|
||||
return err
|
||||
|
||||
+12
-9
@@ -6,31 +6,34 @@ import (
|
||||
|
||||
"github.com/go-errors/errors"
|
||||
"github.com/sirupsen/logrus"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/sources/gitlab"
|
||||
"golang.org/x/net/context"
|
||||
"google.golang.org/protobuf/proto"
|
||||
"google.golang.org/protobuf/types/known/anypb"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/sources/gitlab"
|
||||
)
|
||||
|
||||
func (e *Engine) ScanGitLab(ctx context.Context, endpoint, token string, repositories []string) error {
|
||||
// ScanGitLab scans GitLab with the provided configuration.
|
||||
func (e *Engine) ScanGitLab(ctx context.Context, c sources.Config) error {
|
||||
connection := &sourcespb.GitLab{}
|
||||
|
||||
switch {
|
||||
case len(token) > 0:
|
||||
case len(c.Token) > 0:
|
||||
connection.Credential = &sourcespb.GitLab_Token{
|
||||
Token: token,
|
||||
Token: c.Token,
|
||||
}
|
||||
default:
|
||||
return fmt.Errorf("must provide token")
|
||||
}
|
||||
|
||||
if len(endpoint) > 0 {
|
||||
connection.Endpoint = endpoint
|
||||
if len(c.Endpoint) > 0 {
|
||||
connection.Endpoint = c.Endpoint
|
||||
}
|
||||
|
||||
if len(repositories) > 0 {
|
||||
connection.Repositories = repositories
|
||||
if len(c.Repos) > 0 {
|
||||
connection.Repositories = c.Repos
|
||||
}
|
||||
|
||||
var conn anypb.Any
|
||||
|
||||
+14
-11
@@ -7,33 +7,36 @@ import (
|
||||
|
||||
"github.com/go-errors/errors"
|
||||
"github.com/sirupsen/logrus"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/credentialspb"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/sources/s3"
|
||||
"google.golang.org/protobuf/proto"
|
||||
"google.golang.org/protobuf/types/known/anypb"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/credentialspb"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/sources/s3"
|
||||
)
|
||||
|
||||
func (e *Engine) ScanS3(ctx context.Context, key, secret string, cloudCred bool, buckets []string) error {
|
||||
// ScanS3 scans S3 buckets.
|
||||
func (e *Engine) ScanS3(ctx context.Context, c sources.Config) error {
|
||||
connection := &sourcespb.S3{
|
||||
Credential: &sourcespb.S3_Unauthenticated{},
|
||||
}
|
||||
if cloudCred {
|
||||
if len(key) > 0 || len(secret) > 0 {
|
||||
if c.CloudCred {
|
||||
if len(c.Key) > 0 || len(c.Secret) > 0 {
|
||||
return fmt.Errorf("cannot use cloud credentials and basic auth together")
|
||||
}
|
||||
connection.Credential = &sourcespb.S3_CloudEnvironment{}
|
||||
}
|
||||
if len(key) > 0 && len(secret) > 0 {
|
||||
if len(c.Key) > 0 && len(c.Secret) > 0 {
|
||||
connection.Credential = &sourcespb.S3_AccessKey{
|
||||
AccessKey: &credentialspb.KeySecret{
|
||||
Key: key,
|
||||
Secret: secret,
|
||||
Key: c.Key,
|
||||
Secret: c.Secret,
|
||||
},
|
||||
}
|
||||
}
|
||||
if len(buckets) > 0 {
|
||||
connection.Buckets = buckets
|
||||
if len(c.Buckets) > 0 {
|
||||
connection.Buckets = c.Buckets
|
||||
}
|
||||
var conn anypb.Any
|
||||
err := anypb.MarshalFrom(&conn, connection, proto.MarshalOptions{})
|
||||
|
||||
+10
-8
@@ -10,24 +10,26 @@ import (
|
||||
"google.golang.org/protobuf/types/known/anypb"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/sources/syslog"
|
||||
)
|
||||
|
||||
func (e *Engine) ScanSyslog(ctx context.Context, address, protocol, certPath, keyPath, format string, concurrency int) error {
|
||||
// ScanSyslog is a source that scans syslog files.
|
||||
func (e *Engine) ScanSyslog(ctx context.Context, c sources.Config) error {
|
||||
connection := &sourcespb.Syslog{
|
||||
Protocol: protocol,
|
||||
ListenAddress: address,
|
||||
Format: format,
|
||||
Protocol: c.Protocol,
|
||||
ListenAddress: c.Address,
|
||||
Format: c.Format,
|
||||
}
|
||||
|
||||
if certPath != "" && keyPath != "" {
|
||||
cert, err := os.ReadFile(certPath)
|
||||
if c.CertPath != "" && c.KeyPath != "" {
|
||||
cert, err := os.ReadFile(c.CertPath)
|
||||
if err != nil {
|
||||
return errors.WrapPrefix(err, "could not open TLS cert file", 0)
|
||||
}
|
||||
connection.TlsCert = string(cert)
|
||||
|
||||
key, err := os.ReadFile(keyPath)
|
||||
key, err := os.ReadFile(c.KeyPath)
|
||||
if err != nil {
|
||||
return errors.WrapPrefix(err, "could not open TLS key file", 0)
|
||||
}
|
||||
@@ -40,7 +42,7 @@ func (e *Engine) ScanSyslog(ctx context.Context, address, protocol, certPath, ke
|
||||
return errors.WrapPrefix(err, "error unmarshalling connection", 0)
|
||||
}
|
||||
source := syslog.Source{}
|
||||
err = source.Init(ctx, "trufflehog - syslog", 0, 0, false, &conn, concurrency)
|
||||
err = source.Init(ctx, "trufflehog - syslog", 0, 0, false, &conn, c.Concurrency)
|
||||
source.InjectConnection(connection)
|
||||
if err != nil {
|
||||
logrus.WithError(err).Error("failed to initialize syslog source")
|
||||
|
||||
@@ -0,0 +1,240 @@
|
||||
//package main
|
||||
package gitparse
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
"fmt"
|
||||
"log"
|
||||
"os/exec"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
// DateFormat is the standard date format for git.
|
||||
const DateFormat = "Mon Jan 02 15:04:05 2006 -0700"
|
||||
|
||||
// Commit contains commit header info and diffs.
|
||||
type Commit struct {
|
||||
Hash string
|
||||
Author string
|
||||
Date time.Time
|
||||
Message strings.Builder
|
||||
Diffs []Diff
|
||||
}
|
||||
|
||||
// Diff contains the info about a file diff in a commit.
|
||||
type Diff struct {
|
||||
PathA string
|
||||
PathB string
|
||||
LineStart int
|
||||
Content bytes.Buffer
|
||||
IsBinary bool
|
||||
}
|
||||
|
||||
/*
|
||||
func main() {
|
||||
source := "/home/hrich/go/src/github.com/hashicorp/terraform"
|
||||
start := time.Now()
|
||||
fileChan := BillParser(source)
|
||||
for range fileChan {
|
||||
}
|
||||
newTime := time.Now().Sub(start)
|
||||
|
||||
start = time.Now()
|
||||
errChan := make(chan error)
|
||||
fileChan2, _ := glgo.GitLog(source, glgo.LogOpts{}, errChan)
|
||||
for range fileChan2 {
|
||||
}
|
||||
oldTime := time.Now().Sub(start)
|
||||
|
||||
fmt.Printf("Gitleaks parser: %d seconds\nNew parser: %d seconds", oldTime.Seconds(), newTime.Seconds())
|
||||
|
||||
}
|
||||
*/
|
||||
|
||||
func BillParser(source string) chan Commit {
|
||||
cmd := exec.Command("git", "-C", source, "log", "-p", "-U0", "--full-history", "--all", "--diff-filter=AM", "--date=format:%a %b %d %H:%M:%S %Y %z")
|
||||
|
||||
stdOut, err := cmd.StdoutPipe()
|
||||
if err != nil {
|
||||
errs(err)
|
||||
}
|
||||
// TODO: Handle errors
|
||||
// stdErr, err := cmd.StderrPipe()
|
||||
// if err != nil {
|
||||
// errs(err)
|
||||
// }
|
||||
|
||||
err = cmd.Start()
|
||||
if err != nil {
|
||||
errs(err)
|
||||
}
|
||||
|
||||
outReader := bufio.NewReader(stdOut)
|
||||
var currentCommit *Commit
|
||||
var currentDiff *Diff
|
||||
|
||||
commitChan := make(chan Commit)
|
||||
|
||||
// TODO: Something about nil handling maybe? I don't remember
|
||||
go func() {
|
||||
for {
|
||||
line, _, err := outReader.ReadLine()
|
||||
if err != nil {
|
||||
fmt.Println(err)
|
||||
break
|
||||
}
|
||||
logrus.Debugf("Line: %s", line)
|
||||
switch {
|
||||
case isCommitLine(line):
|
||||
logrus.Debugf("commitline: %s", line)
|
||||
if currentCommit != nil {
|
||||
commitChan <- *currentCommit
|
||||
}
|
||||
currentCommit = &Commit{
|
||||
Message: strings.Builder{},
|
||||
}
|
||||
currentCommit.Hash = string(line[7:])
|
||||
case isAuthorLine(line):
|
||||
currentCommit.Author = string(line[8:])
|
||||
case isDateLine(line):
|
||||
date, err := time.Parse(DateFormat, strings.TrimSpace(string(line[6:])))
|
||||
if err != nil {
|
||||
errs(err)
|
||||
}
|
||||
currentCommit.Date = date
|
||||
case isDiffLine(line):
|
||||
if currentDiff != nil {
|
||||
currentCommit.Diffs = append(currentCommit.Diffs, *currentDiff)
|
||||
}
|
||||
currentDiff = &Diff{}
|
||||
case isModeLine(line):
|
||||
case isIndexLine(line):
|
||||
case isPlusFileLine(line):
|
||||
currentDiff.PathB = string(line[4:])
|
||||
case isMinusFileLine(line):
|
||||
currentDiff.PathA = string(line[4:])
|
||||
case isPlusDiffLine(line):
|
||||
currentDiff.Content.Write(line[1:])
|
||||
currentDiff.Content.Write([]byte("\n"))
|
||||
case isMinusDiffLine(line):
|
||||
case isMessageLine(line):
|
||||
currentCommit.Message.Write(line[4:])
|
||||
case isBinaryLine(line):
|
||||
currentDiff.IsBinary = true
|
||||
currentDiff.PathB = pathFromBinaryLine(line, source)
|
||||
}
|
||||
|
||||
}
|
||||
if currentDiff != nil && currentDiff.Content.Len() > 0 {
|
||||
currentCommit.Diffs = append(currentCommit.Diffs, *currentDiff)
|
||||
}
|
||||
if currentCommit != nil {
|
||||
commitChan <- *currentCommit
|
||||
}
|
||||
cmd.Wait()
|
||||
close(commitChan)
|
||||
}()
|
||||
return commitChan
|
||||
}
|
||||
|
||||
func isDateLine(line []byte) bool {
|
||||
if len(line) > 7 && bytes.Equal(line[:5], []byte("Date:")) {
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func isAuthorLine(line []byte) bool {
|
||||
if len(line) > 8 && bytes.Equal(line[:7], []byte("Author:")) {
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func isCommitLine(line []byte) bool {
|
||||
if len(line) > 7 && bytes.Equal(line[:6], []byte("commit")) {
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func isDiffLine(line []byte) bool {
|
||||
if len(line) > 5 && bytes.Equal(line[:4], []byte("diff")) {
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func isIndexLine(line []byte) bool {
|
||||
if len(line) > 6 && bytes.Equal(line[:5], []byte("index")) {
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func isModeLine(line []byte) bool {
|
||||
if len(line) > 8 && bytes.Equal(line[:8], []byte("new mode")) {
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func isMinusFileLine(line []byte) bool {
|
||||
if len(line) > 3 && bytes.Equal(line[:3], []byte("---")) {
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func isPlusFileLine(line []byte) bool {
|
||||
if len(line) > 3 && bytes.Equal(line[:3], []byte("+++")) {
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func isPlusDiffLine(line []byte) bool {
|
||||
if len(line) > 1 && bytes.Equal(line[:1], []byte("+")) {
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func isMinusDiffLine(line []byte) bool {
|
||||
if len(line) > 1 && bytes.Equal(line[:1], []byte("-")) {
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func isMessageLine(line []byte) bool {
|
||||
if len(line) > 4 && bytes.Equal(line[:4], []byte(" ")) {
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func isBinaryLine(line []byte) bool {
|
||||
if len(line) > 7 && bytes.Equal(line[:6], []byte("Binary")) {
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func pathFromBinaryLine(line []byte, source string) string {
|
||||
sbytes := bytes.Split(line, []byte(" and "))
|
||||
if len(sbytes) != 2 {
|
||||
logrus.Errorf("Expected binary line to be in 'Binary files a/filaA and b/fileB differ' format. Got: %s", line)
|
||||
return ""
|
||||
}
|
||||
bRaw := sbytes[1]
|
||||
return string(bRaw[2 : len(bRaw)-7]) // drop the "b/" and " differ"
|
||||
}
|
||||
|
||||
func errs(err error) {
|
||||
log.Println(err)
|
||||
}
|
||||
@@ -0,0 +1,30 @@
|
||||
package gitparse
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
glgo "github.com/zricethezav/gitleaks/v8/detect/git"
|
||||
)
|
||||
|
||||
func TestMain(t *testing.T) {
|
||||
source := "/home/hrich/go/src/github.com/hashicorp/terraform"
|
||||
start := time.Now()
|
||||
commitChan := BillParser(source)
|
||||
for _ = range commitChan {
|
||||
}
|
||||
fmt.Printf("BillParser took %f seconds.", time.Now().Sub(start).Seconds())
|
||||
start = time.Now()
|
||||
logOpts := glgo.LogOpts{
|
||||
DisableSafeDir: true,
|
||||
}
|
||||
errChan := make(chan error)
|
||||
fileChan, err := glgo.GitLog(source, logOpts, errChan)
|
||||
if err != nil {
|
||||
//errs(err)
|
||||
}
|
||||
for _ = range fileChan {
|
||||
}
|
||||
fmt.Printf("GitLeaksParser took %f seconds.", time.Now().Sub(start).Seconds())
|
||||
}
|
||||
@@ -72,8 +72,8 @@ func NormalizeOrgRepoURL(provider, repoURL string) (string, error) {
|
||||
return "", errors.Errorf("%s repo appears to be missing the repo name. Org: %q Repo url: %q", provider, org, repoURL)
|
||||
}
|
||||
|
||||
case len(parts) > 3:
|
||||
return "", errors.Errorf("%s repo appears to be too long or contains a trailing slash. Repo url: %q", provider, repoURL)
|
||||
case len(parts) > 3 && strings.HasSuffix(parsed.Path, "/"):
|
||||
return "", errors.Errorf("%s repo contains a trailing slash. Repo url: %q", provider, repoURL)
|
||||
}
|
||||
|
||||
// If we're here it's probably a provider repo without ".git" at the end, so add it and return
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user