Compare commits
31 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 Mike Vanbuskirk
|
9184507b10 | ||
|
Mike Vanbuskirk
|
8ad5b119e2 | ||
|
Mike Vanbuskirk
|
dffb89c29b | ||
|
Mike Vanbuskirk
|
1b42894abb | ||
|
Corben Leo
|
4cb67a571d | ||
|
Mike Vanbuskirk
|
7e876e070c | ||
|
Mike Vanbuskirk
|
a611b1dbe4 | ||
|
Mike Vanbuskirk
|
477dc39fe3 | ||
|
Mike Vanbuskirk
|
69c7ce9dac | ||
|
Cody Rose
|
7ac7fa8728 | ||
|
Richard Gomez
|
4b821e9732 | ||
|
Mike Vanbuskirk
|
777be4f713 | ||
|
Miccah
|
758344711a | ||
|
Corben Leo
|
8058006a92 | ||
|
ahrav
|
3d7207ddd5 | ||
|
Mike Vanbuskirk
|
45a006c903 | ||
|
Mike Vanbuskirk
|
ce32bac7dd | ||
|
Richard Gomez
|
b57b1c1aa7 | ||
|
Damanpreet Singh
|
a354cbd796 | ||
|
Miccah
|
23ae970bb0 | ||
|
Dustin Decker
|
93cf523760 | ||
|
Mike Vanbuskirk
|
6e9f6c58f9 | ||
|
Mike Vanbuskirk
|
00b696dfc7 | ||
|
Mike Vanbuskirk
|
a96e1f3180 | ||
|
Mike Vanbuskirk
|
39b0499d35 | ||
|
Mike Vanbuskirk
|
b528bf97e0 | ||
|
Mike Vanbuskirk
|
7126e35823 | ||
|
Mike Vanbuskirk
|
80d9d9565a | ||
|
Mike Vanbuskirk
|
ae06576592 | ||
|
Mike Vanbuskirk
|
1642f45c79 | ||
|
Mike Vanbuskirk
|
8b2778b052 |
@@ -32,7 +32,7 @@ require (
|
||||
github.com/felixge/fgprof v0.9.3
|
||||
github.com/getsentry/sentry-go v0.24.1
|
||||
github.com/go-errors/errors v1.4.2
|
||||
github.com/go-git/go-git/v5 v5.8.1
|
||||
github.com/go-git/go-git/v5 v5.8.0
|
||||
github.com/go-ldap/ldap/v3 v3.4.6
|
||||
github.com/go-logr/logr v1.2.4
|
||||
github.com/go-logr/zapr v1.2.4
|
||||
@@ -51,13 +51,14 @@ require (
|
||||
github.com/joho/godotenv v1.5.1
|
||||
github.com/jpillora/overseer v1.1.6
|
||||
github.com/kylelemons/godebug v1.1.0
|
||||
github.com/launchdarkly/go-server-sdk/v6 v6.1.0
|
||||
github.com/launchdarkly/go-server-sdk/v6 v6.0.0-alpha.pub.12
|
||||
github.com/lib/pq v1.10.9
|
||||
github.com/lrstanley/bubblezone v0.0.0-20221222153816-e95291e2243e
|
||||
github.com/marusama/semaphore/v2 v2.5.0
|
||||
github.com/mattn/go-isatty v0.0.19
|
||||
github.com/mattn/go-sqlite3 v1.14.17
|
||||
github.com/mholt/archiver/v4 v4.0.0-alpha.8
|
||||
github.com/mitchellh/go-ps v1.0.0
|
||||
github.com/muesli/reflow v0.3.0
|
||||
github.com/patrickmn/go-cache v2.1.0+incompatible
|
||||
github.com/paulbellamy/ratecounter v0.2.0
|
||||
@@ -65,7 +66,7 @@ require (
|
||||
github.com/prometheus/client_golang v1.16.0
|
||||
github.com/rabbitmq/amqp091-go v1.8.1
|
||||
github.com/sergi/go-diff v1.3.1
|
||||
github.com/snowflakedb/gosnowflake v1.6.23
|
||||
github.com/snowflakedb/gosnowflake v1.6.20
|
||||
github.com/stretchr/testify v1.8.4
|
||||
github.com/tailscale/depaware v0.0.0-20210622194025-720c4b409502
|
||||
github.com/xanzy/go-gitlab v0.92.3
|
||||
@@ -81,6 +82,7 @@ require (
|
||||
google.golang.org/protobuf v1.31.0
|
||||
gopkg.in/alecthomas/kingpin.v2 v2.2.6
|
||||
gopkg.in/h2non/gock.v1 v1.1.2
|
||||
pgregory.net/rapid v1.1.0
|
||||
sigs.k8s.io/yaml v1.3.0
|
||||
)
|
||||
|
||||
@@ -89,7 +91,6 @@ require (
|
||||
cloud.google.com/go/compute v1.20.1 // indirect
|
||||
cloud.google.com/go/compute/metadata v0.2.3 // indirect
|
||||
cloud.google.com/go/iam v1.1.0 // indirect
|
||||
dario.cat/mergo v1.0.0 // indirect
|
||||
github.com/99designs/go-keychain v0.0.0-20191008050251-8e49817e8af4 // indirect
|
||||
github.com/99designs/keyring v1.2.2 // indirect
|
||||
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.4.0 // indirect
|
||||
@@ -103,7 +104,6 @@ require (
|
||||
github.com/Azure/go-autorest/logger v0.2.1 // indirect
|
||||
github.com/Azure/go-autorest/tracing v0.6.0 // indirect
|
||||
github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 // indirect
|
||||
github.com/JohnCGriffin/overflow v0.0.0-20211019200055-46fa312c352c // indirect
|
||||
github.com/Microsoft/go-winio v0.6.1 // indirect
|
||||
github.com/ProtonMail/go-crypto v0.0.0-20230717121422-5aa5874ade95 // indirect
|
||||
github.com/acomagu/bufpipe v1.0.4 // indirect
|
||||
@@ -111,7 +111,7 @@ require (
|
||||
github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 // indirect
|
||||
github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 // indirect
|
||||
github.com/andybalholm/brotli v1.0.5 // indirect
|
||||
github.com/apache/arrow/go/v12 v12.0.1 // indirect
|
||||
github.com/apache/arrow/go/v10 v10.0.1 // indirect
|
||||
github.com/apache/thrift v0.16.0 // indirect
|
||||
github.com/atotto/clipboard v0.1.4 // indirect
|
||||
github.com/aws/aws-sdk-go-v2 v1.17.7 // indirect
|
||||
@@ -179,6 +179,7 @@ require (
|
||||
github.com/hashicorp/errwrap v1.0.0 // indirect
|
||||
github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
|
||||
github.com/hashicorp/go-multierror v1.1.1 // indirect
|
||||
github.com/imdario/mergo v0.3.15 // indirect
|
||||
github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect
|
||||
github.com/jmespath/go-jmespath v0.4.0 // indirect
|
||||
github.com/josharian/intern v1.0.0 // indirect
|
||||
@@ -239,7 +240,7 @@ require (
|
||||
github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d // indirect
|
||||
github.com/yuin/goldmark v1.5.2 // indirect
|
||||
github.com/yuin/goldmark-emoji v1.0.1 // indirect
|
||||
github.com/yusufpapurcu/wmi v1.2.2 // indirect
|
||||
github.com/yusufpapurcu/wmi v1.2.3 // indirect
|
||||
github.com/zeebo/xxh3 v1.0.2 // indirect
|
||||
go.opencensus.io v0.24.0 // indirect
|
||||
go.uber.org/atomic v1.7.0 // indirect
|
||||
|
||||
@@ -26,8 +26,6 @@ cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiy
|
||||
cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos=
|
||||
cloud.google.com/go/storage v1.33.0 h1:PVrDOkIC8qQVa1P3SXGpQvfuJhN2LHOoyZvWs8D2X5M=
|
||||
cloud.google.com/go/storage v1.33.0/go.mod h1:Hhh/dogNRGca7IWv1RC2YqEn0c0G77ctA/OxflYkiD8=
|
||||
dario.cat/mergo v1.0.0 h1:AGCNq9Evsj31mOgNPcLyXc+4PNABt905YmuqPYYpBWk=
|
||||
dario.cat/mergo v1.0.0/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk=
|
||||
dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
|
||||
github.com/99designs/go-keychain v0.0.0-20191008050251-8e49817e8af4 h1:/vQbFIOMbk2FiG/kXiLl8BRyzTWDw7gX/Hz7Dd5eDMs=
|
||||
github.com/99designs/go-keychain v0.0.0-20191008050251-8e49817e8af4/go.mod h1:hN7oaIRCjzsZ2dE+yG5k+rsdt3qcwykqK6HVGcKwsw4=
|
||||
@@ -96,8 +94,8 @@ github.com/andybalholm/brotli v1.0.5/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHG
|
||||
github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be h1:9AeTilPcZAjCFIImctFaOjnTIavg87rW78vTPkQqLI8=
|
||||
github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be/go.mod h1:ySMOLuWl6zY27l47sB3qLNK6tF2fkHG55UZxx8oIVo4=
|
||||
github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY=
|
||||
github.com/apache/arrow/go/v12 v12.0.1 h1:JsR2+hzYYjgSUkBSaahpqCetqZMr76djX80fF/DiJbg=
|
||||
github.com/apache/arrow/go/v12 v12.0.1/go.mod h1:weuTY7JvTG/HDPtMQxEUp7pU73vkLWMLpY67QwZ/WWw=
|
||||
github.com/apache/arrow/go/v10 v10.0.1 h1:n9dERvixoC/1JjDmBcs9FPaEryoANa2sCgVFo6ez9cI=
|
||||
github.com/apache/arrow/go/v10 v10.0.1/go.mod h1:YvhnlEePVnBS4+0z3fhPfUy7W1Ikj0Ih0vcRo/gZ1M0=
|
||||
github.com/apache/thrift v0.16.0 h1:qEy6UW60iVOlUy+b9ZR0d5WzUWYGOo4HfopoyBaNmoY=
|
||||
github.com/apache/thrift v0.16.0/go.mod h1:PHK3hniurgQaNMZYaCLEqXKsYK8upmhPbmdP2FXSqgU=
|
||||
github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio=
|
||||
@@ -277,8 +275,8 @@ github.com/go-git/go-billy/v5 v5.4.1 h1:Uwp5tDRkPr+l/TnbHOQzp+tmJfLceOlbVucgpTz8
|
||||
github.com/go-git/go-billy/v5 v5.4.1/go.mod h1:vjbugF6Fz7JIflbVpl1hJsGjSHNltrSw45YK/ukIvQg=
|
||||
github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20230305113008-0c11038e723f h1:Pz0DHeFij3XFhoBRGUDPzSJ+w2UcK5/0JvF8DRI58r8=
|
||||
github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20230305113008-0c11038e723f/go.mod h1:8LHG1a3SRW71ettAD/jW13h8c6AqjVSeL11RAdgaqpo=
|
||||
github.com/go-git/go-git/v5 v5.8.1 h1:Zo79E4p7TRk0xoRgMq0RShiTHGKcKI4+DI6BfJc/Q+A=
|
||||
github.com/go-git/go-git/v5 v5.8.1/go.mod h1:FHFuoD6yGz5OSKEBK+aWN9Oah0q54Jxl0abmj6GnqAo=
|
||||
github.com/go-git/go-git/v5 v5.8.0 h1:Rc543s6Tyq+YcyPwZRvU4jzZGM8rB/wWu94TnTIYALQ=
|
||||
github.com/go-git/go-git/v5 v5.8.0/go.mod h1:coJHKEOk5kUClpsNlXrUvPrDxY3w3gjHvhcZd8Fodw8=
|
||||
github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
|
||||
github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
|
||||
github.com/go-ldap/ldap/v3 v3.4.6 h1:ert95MdbiG7aWo/oPYp9btL3KJlMPKnP58r09rI8T+A=
|
||||
@@ -421,6 +419,8 @@ github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ
|
||||
github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
|
||||
github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
|
||||
github.com/ianlancetaylor/demangle v0.0.0-20210905161508-09a460cdf81d/go.mod h1:aYm2/VgdVmcIU8iMfdMvDMsRAQjcfZSKFby6HOFvi/w=
|
||||
github.com/imdario/mergo v0.3.15 h1:M8XP7IuFNsqUx6VPK2P9OSmsYsI/YFaGil0uD21V3dM=
|
||||
github.com/imdario/mergo v0.3.15/go.mod h1:WBLT9ZmE3lPoWsEzCh9LPo3TiwVN+ZKEjmz+hD27ysY=
|
||||
github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOlocH6Fxy8MmwDt+yVQYULKfN0RoTN8A=
|
||||
github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i6rXxKeerYnT8Nvf0QmHCRC1n8sfWVwXF2Frvo=
|
||||
github.com/jlaffaye/ftp v0.2.0 h1:lXNvW7cBu7R/68bknOX3MrRIIqZ61zELs1P2RAiA3lg=
|
||||
@@ -477,8 +477,8 @@ github.com/launchdarkly/go-semver v1.0.2 h1:sYVRnuKyvxlmQCnCUyDkAhtmzSFRoX6rG2Xa
|
||||
github.com/launchdarkly/go-semver v1.0.2/go.mod h1:xFmMwXba5Mb+3h72Z+VeSs9ahCvKo2QFUTHRNHVqR28=
|
||||
github.com/launchdarkly/go-server-sdk-evaluation/v2 v2.0.2 h1:PAM0GvE0nIUBeOkjdiymIEKI+8FFLJ+fEsWTupW1yGU=
|
||||
github.com/launchdarkly/go-server-sdk-evaluation/v2 v2.0.2/go.mod h1:Mztipcz+7ZMatXVun3k/IfPa8IOgUnAqiZawtFh2MRg=
|
||||
github.com/launchdarkly/go-server-sdk/v6 v6.1.0 h1:pp/VBIon5JNbtSw7ih7I5MXN9mXHfo6T5xikKVy52dI=
|
||||
github.com/launchdarkly/go-server-sdk/v6 v6.1.0/go.mod h1:3TfS6vNsNksPT4LGeGuGKtT3zTjTbbOsCUK3nrj1neA=
|
||||
github.com/launchdarkly/go-server-sdk/v6 v6.0.0-alpha.pub.12 h1:ES6o2EkrMjUDvJEKj6MaczFocPe9HetTNLgKdC89YDI=
|
||||
github.com/launchdarkly/go-server-sdk/v6 v6.0.0-alpha.pub.12/go.mod h1:N14BvkuLLq7zwoJhEaP6nqUEVmcmLU2EvtfMzYHV7ZU=
|
||||
github.com/launchdarkly/go-test-helpers/v2 v2.2.0 h1:L3kGILP/6ewikhzhdNkHy1b5y4zs50LueWenVF0sBbs=
|
||||
github.com/launchdarkly/go-test-helpers/v2 v2.2.0/go.mod h1:L7+th5govYp5oKU9iN7To5PgznBuIjBPn+ejqKR0avw=
|
||||
github.com/launchdarkly/go-test-helpers/v3 v3.0.2 h1:rh0085g1rVJM5qIukdaQ8z1XTWZztbJ49vRZuveqiuU=
|
||||
@@ -521,6 +521,8 @@ github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3 h1:+n/aFZefKZp7spd8D
|
||||
github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3/go.mod h1:RagcQ7I8IeTMnF8JTXieKnO4Z6JCsikNEzj0DwauVzE=
|
||||
github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y=
|
||||
github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
|
||||
github.com/mitchellh/go-ps v1.0.0 h1:i6ampVEEF4wQFF+bkYfwYgY+F/uYJDktmvLPf7qIgjc=
|
||||
github.com/mitchellh/go-ps v1.0.0/go.mod h1:J4lOc8z8yJs6vUwklHw2XEIiT4z4C40KtWVN3nvg8Pg=
|
||||
github.com/modocache/gover v0.0.0-20171022184752-b58185e213c5/go.mod h1:caMODM3PzxT8aQXRPkAt8xlV/e7d7w8GM5g0fa5F0D8=
|
||||
github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc=
|
||||
github.com/montanaflynn/stats v0.7.0 h1:r3y12KyNxj/Sb/iOE46ws+3mS1+MZca1wlHQFPsY/JU=
|
||||
@@ -614,8 +616,8 @@ github.com/smartystreets/assertions v1.0.1 h1:voD4ITNjPL5jjBfgR/r8fPIIBrliWrWHei
|
||||
github.com/smartystreets/assertions v1.0.1/go.mod h1:kHHU4qYBaI3q23Pp3VPrmWhuIUrLW/7eUrw0BU5VaoM=
|
||||
github.com/smartystreets/gunit v1.1.3 h1:32x+htJCu3aMswhPw3teoJ+PnWPONqdNgaGs6Qt8ZaU=
|
||||
github.com/smartystreets/gunit v1.1.3/go.mod h1:EH5qMBab2UclzXUcpR8b93eHsIlp9u+pDQIRp5DZNzQ=
|
||||
github.com/snowflakedb/gosnowflake v1.6.23 h1:uO+zMTXJcSHzOm6ks5To8ergNjt5Dy6cr5QtStpRFT8=
|
||||
github.com/snowflakedb/gosnowflake v1.6.23/go.mod h1:KfO4F7bk+aXPUIvBqYxvPhxLlu2/w4TtSC8Rw/yr5Mg=
|
||||
github.com/snowflakedb/gosnowflake v1.6.20 h1:WkWTTOnc2yQ/6LpCDZQQe0c9jquxsitvYYNTIgvy0lw=
|
||||
github.com/snowflakedb/gosnowflake v1.6.20/go.mod h1:FM1+PWUdwB9udFDsXdfD58NONC0m+MlOSmQRvimobSM=
|
||||
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||
github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
|
||||
github.com/stretchr/objx v0.5.0 h1:1zr/of2m5FGMsad5YfcqgdqdWrIhu+EBEJRhR1U7z/c=
|
||||
@@ -666,8 +668,9 @@ github.com/yuin/goldmark v1.5.2 h1:ALmeCk/px5FSm1MAcFBAsVKZjDuMVj8Tm7FFIlMJnqU=
|
||||
github.com/yuin/goldmark v1.5.2/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
|
||||
github.com/yuin/goldmark-emoji v1.0.1 h1:ctuWEyzGBwiucEqxzwe0SOYDXPAucOrE9NQC18Wa1os=
|
||||
github.com/yuin/goldmark-emoji v1.0.1/go.mod h1:2w1E6FEWLcDQkoTE+7HU6QF1F6SLlNGjRIBbIZQFqkQ=
|
||||
github.com/yusufpapurcu/wmi v1.2.2 h1:KBNDSne4vP5mbSWnJbO+51IMOXJB67QiYCSBrubbPRg=
|
||||
github.com/yusufpapurcu/wmi v1.2.2/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0=
|
||||
github.com/yusufpapurcu/wmi v1.2.3 h1:E1ctvB7uKFMOJw3fdOW32DwGE9I7t++CRUEMKvFoFiw=
|
||||
github.com/yusufpapurcu/wmi v1.2.3/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0=
|
||||
github.com/zeebo/assert v1.3.0 h1:g7C04CbJuIDKNPFHmsk4hwZDO5O+kntRxzaUoNXj+IQ=
|
||||
github.com/zeebo/assert v1.3.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0=
|
||||
github.com/zeebo/xxh3 v1.0.2 h1:xZmwmqxHZA8AI603jOQ0tMqmBr9lPeFwGg6d+xy9DC0=
|
||||
@@ -1013,6 +1016,8 @@ honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWh
|
||||
honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
|
||||
honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
|
||||
honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg=
|
||||
pgregory.net/rapid v1.1.0 h1:CMa0sjHSru3puNx+J0MIAuiiEV4N0qj8/cMWGBBCsjw=
|
||||
pgregory.net/rapid v1.1.0/go.mod h1:PY5XlDGj0+V1FCq0o192FdRhpKHGTRIWBgqjDBTrq04=
|
||||
rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=
|
||||
rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0=
|
||||
rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA=
|
||||
|
||||
@@ -5,6 +5,7 @@ import (
|
||||
"net/http"
|
||||
_ "net/http/pprof"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"strconv"
|
||||
"strings"
|
||||
@@ -176,6 +177,35 @@ func init() {
|
||||
}
|
||||
}
|
||||
|
||||
// Defines the interface for removing orphaned artifacts from aborted scans
|
||||
type CleanTemp interface {
|
||||
//Removes orphaned directories from sources like Git
|
||||
CleanTempDir(ctx context.Context, dirName string, pid int) error
|
||||
//Removes orphaned files/artifacts from sources like Artifactory
|
||||
CleanTempFiles(ctx context.Context, fileName string, pid int) error
|
||||
}
|
||||
|
||||
func CleanTempDir(ctx context.Context, dirName string, pid int) error {
|
||||
tempDir := os.TempDir()
|
||||
files, err := os.ReadDir(tempDir)
|
||||
if err != nil {
|
||||
return fmt.Errorf("Error reading temp dir: %w", err)
|
||||
}
|
||||
|
||||
pidStr := strconv.Itoa(pid)
|
||||
|
||||
for _, file := range files {
|
||||
if file.IsDir() && strings.Contains(file.Name(), dirName) && !strings.Contains(file.Name(), pidStr) {
|
||||
dirPath := filepath.Join(tempDir, file.Name())
|
||||
if err := os.RemoveAll(dirPath); err != nil {
|
||||
return fmt.Errorf("Error deleting temp directory: %s", dirPath)
|
||||
}
|
||||
ctx.Logger().V(1).Info("Deleted directory", "directory", dirPath)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func main() {
|
||||
// setup logger
|
||||
logFormat := log.WithConsoleSink
|
||||
@@ -213,6 +243,17 @@ func main() {
|
||||
if err != nil {
|
||||
logFatal(err, "error occurred with trufflehog updater 🐷")
|
||||
}
|
||||
|
||||
ctx := context.Background()
|
||||
|
||||
var execName = "trufflehog"
|
||||
|
||||
pid, _ := git.GetScannerPIDs(ctx, execName)
|
||||
|
||||
err = CleanTempDir(ctx, execName, pid)
|
||||
if err != nil {
|
||||
ctx.Logger().Error(err, "Error cleaning up orphaned directories ")
|
||||
}
|
||||
}
|
||||
|
||||
func run(state overseer.State) {
|
||||
|
||||
@@ -0,0 +1,120 @@
|
||||
package common
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"github.com/gobwas/glob"
|
||||
)
|
||||
|
||||
// Filter is a generic filter for excluding and including globs (limited
|
||||
// regular expressions). Exclusion takes precedence if both include and exclude
|
||||
// lists are provided.
|
||||
type Filter struct {
|
||||
exclude []glob.Glob
|
||||
include []glob.Glob
|
||||
}
|
||||
|
||||
type globFilterOpt func(*Filter) error
|
||||
|
||||
// WithExcludeGlobs adds exclude globs to the filter.
|
||||
func WithExcludeGlobs(excludes ...string) globFilterOpt {
|
||||
return func(f *Filter) error {
|
||||
for _, exclude := range excludes {
|
||||
g, err := glob.Compile(exclude)
|
||||
if err != nil {
|
||||
return fmt.Errorf("invalid exclude glob %q: %w", exclude, err)
|
||||
}
|
||||
f.exclude = append(f.exclude, g)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
// WithIncludeGlobs adds include globs to the filter.
|
||||
func WithIncludeGlobs(includes ...string) globFilterOpt {
|
||||
return func(f *Filter) error {
|
||||
for _, include := range includes {
|
||||
g, err := glob.Compile(include)
|
||||
if err != nil {
|
||||
return fmt.Errorf("invalid include glob %q: %w", include, err)
|
||||
}
|
||||
f.include = append(f.include, g)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
// NewGlobFilter creates a new Filter with the provided options.
|
||||
func NewGlobFilter(opts ...globFilterOpt) (*Filter, error) {
|
||||
filter := &Filter{}
|
||||
for _, opt := range opts {
|
||||
if err := opt(filter); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
return filter, nil
|
||||
}
|
||||
|
||||
// ShouldInclude returns whether the object is in the include list or not in
|
||||
// the exclude list (exclude taking precedence).
|
||||
func (f *Filter) ShouldInclude(object string) bool {
|
||||
if f == nil {
|
||||
return true
|
||||
}
|
||||
exclude, include := len(f.exclude), len(f.include)
|
||||
if exclude == 0 && include == 0 {
|
||||
return true
|
||||
} else if exclude > 0 && include == 0 {
|
||||
return f.shouldIncludeFromExclude(object)
|
||||
} else if exclude == 0 && include > 0 {
|
||||
return f.shouldIncludeFromInclude(object)
|
||||
} else {
|
||||
if ok, err := f.shouldIncludeFromBoth(object); err == nil {
|
||||
return ok
|
||||
}
|
||||
// Ambiguous case.
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
// shouldIncludeFromExclude checks for explicitly excluded paths. This should
|
||||
// only be called when the include list is empty.
|
||||
func (f *Filter) shouldIncludeFromExclude(object string) bool {
|
||||
for _, glob := range f.exclude {
|
||||
if glob.Match(object) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// shouldIncludeFromInclude checks for explicitly included paths. This should
|
||||
// only be called when the exclude list is empty.
|
||||
func (f *Filter) shouldIncludeFromInclude(object string) bool {
|
||||
for _, glob := range f.include {
|
||||
if glob.Match(object) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// shouldIncludeFromBoth checks for either excluded or included paths. Exclusion
|
||||
// takes precedence. If neither list contains the object, true is returned.
|
||||
func (f *Filter) shouldIncludeFromBoth(object string) (bool, error) {
|
||||
// Exclude takes precedence. If we find the object in the exclude list,
|
||||
// we should not match.
|
||||
for _, glob := range f.exclude {
|
||||
if glob.Match(object) {
|
||||
return false, nil
|
||||
}
|
||||
}
|
||||
// If we find the object in the include list, we should match.
|
||||
for _, glob := range f.include {
|
||||
if glob.Match(object) {
|
||||
return true, nil
|
||||
}
|
||||
}
|
||||
// If we find it in neither, return an error to let the caller decide.
|
||||
return false, fmt.Errorf("ambiguous match")
|
||||
}
|
||||
@@ -0,0 +1,138 @@
|
||||
package common
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"pgregory.net/rapid"
|
||||
)
|
||||
|
||||
type globTest struct {
|
||||
input string
|
||||
shouldInclude bool
|
||||
}
|
||||
|
||||
func testGlobs(t *testing.T, filter *Filter, tests ...globTest) {
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.input, func(t *testing.T) {
|
||||
// Invert because mentally it's easier to say whether an
|
||||
// input should be included.
|
||||
assert.Equal(t, tt.shouldInclude, filter.ShouldInclude(tt.input))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestGlobFilterExclude(t *testing.T) {
|
||||
filter, err := NewGlobFilter(WithExcludeGlobs("foo", "bar*"))
|
||||
assert.NoError(t, err)
|
||||
|
||||
testGlobs(t, filter,
|
||||
globTest{"foo", false},
|
||||
globTest{"bar", false},
|
||||
globTest{"bara", false},
|
||||
globTest{"barb", false},
|
||||
globTest{"barbosa", false},
|
||||
globTest{"foobar", true},
|
||||
globTest{"food", true},
|
||||
globTest{"anything else", true},
|
||||
)
|
||||
}
|
||||
|
||||
func TestGlobFilterInclude(t *testing.T) {
|
||||
filter, err := NewGlobFilter(WithIncludeGlobs("foo", "bar*"))
|
||||
assert.NoError(t, err)
|
||||
|
||||
testGlobs(t, filter,
|
||||
globTest{"foo", true},
|
||||
globTest{"bar", true},
|
||||
globTest{"bara", true},
|
||||
globTest{"barb", true},
|
||||
globTest{"barbosa", true},
|
||||
globTest{"foobar", false},
|
||||
globTest{"food", false},
|
||||
globTest{"anything else", false},
|
||||
)
|
||||
}
|
||||
|
||||
func TestGlobFilterEmpty(t *testing.T) {
|
||||
filter, err := NewGlobFilter()
|
||||
assert.NoError(t, err)
|
||||
|
||||
testGlobs(t, filter,
|
||||
globTest{"foo", true},
|
||||
globTest{"bar", true},
|
||||
globTest{"bara", true},
|
||||
globTest{"barb", true},
|
||||
globTest{"barbosa", true},
|
||||
globTest{"foobar", true},
|
||||
globTest{"food", true},
|
||||
globTest{"anything else", true},
|
||||
)
|
||||
}
|
||||
|
||||
func TestGlobFilterExcludeInclude(t *testing.T) {
|
||||
filter, err := NewGlobFilter(WithExcludeGlobs("/foo/bar/**"), WithIncludeGlobs("/foo/**"))
|
||||
assert.NoError(t, err)
|
||||
|
||||
testGlobs(t, filter,
|
||||
globTest{"/foo/a", true},
|
||||
globTest{"/foo/b", true},
|
||||
globTest{"/foo/c/d/e", true},
|
||||
globTest{"/foo/bar/a", false},
|
||||
globTest{"/foo/bar/b", false},
|
||||
globTest{"/foo/bar/c/d/e", false},
|
||||
globTest{"/any/other/path", false},
|
||||
)
|
||||
}
|
||||
|
||||
func TestGlobFilterExcludePrecedence(t *testing.T) {
|
||||
filter, err := NewGlobFilter(WithExcludeGlobs("foo"), WithIncludeGlobs("foo*"))
|
||||
assert.NoError(t, err)
|
||||
|
||||
testGlobs(t, filter,
|
||||
globTest{"foo", false},
|
||||
globTest{"foobar", true},
|
||||
)
|
||||
}
|
||||
|
||||
func TestGlobErrorContainsGlob(t *testing.T) {
|
||||
invalidGlob := "[this is invalid because it doesn't close the capture group"
|
||||
_, err := NewGlobFilter(WithExcludeGlobs(invalidGlob))
|
||||
assert.Error(t, err)
|
||||
assert.Contains(t, err.Error(), invalidGlob)
|
||||
}
|
||||
|
||||
// The filters in this test should be mutually exclusive because one includes
|
||||
// and the other excludes the same glob.
|
||||
func TestGlobInverse(t *testing.T) {
|
||||
for _, glob := range []string{
|
||||
"a",
|
||||
"a*",
|
||||
"a**",
|
||||
"*a",
|
||||
"**a",
|
||||
"*",
|
||||
} {
|
||||
include, err := NewGlobFilter(WithIncludeGlobs(glob))
|
||||
assert.NoError(t, err)
|
||||
exclude, err := NewGlobFilter(WithExcludeGlobs(glob))
|
||||
assert.NoError(t, err)
|
||||
rapid.Check(t, func(t *rapid.T) {
|
||||
input := rapid.String().Draw(t, "input")
|
||||
a, b := include.ShouldInclude(input), exclude.ShouldInclude(input)
|
||||
if a == b {
|
||||
t.Fatalf("Filter(Include(%q)) == Filter(Exclude(%q)) == %v for input %q", glob, glob, a, input)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestGlobDefaultFilters(t *testing.T) {
|
||||
for _, filter := range []*Filter{nil, {}} {
|
||||
rapid.Check(t, func(t *rapid.T) {
|
||||
if !filter.ShouldInclude(rapid.String().Draw(t, "input")) {
|
||||
t.Fatalf("filter %#v did not include input", filter)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -24,7 +24,7 @@ const verifyURL = "https://api.pagerduty.com/users"
|
||||
var (
|
||||
defaultClient = common.SaneHttpClient()
|
||||
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
|
||||
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"pagerduty", "pager_duty", "pd_", "pd-"}) + `\b([a-zA-Z0-9_+-]{20})\b`)
|
||||
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"pagerduty", "pager_duty", "pd_", "pd-"}) + `\b(u\+[a-zA-Z0-9_+-]{18})\b`)
|
||||
)
|
||||
|
||||
// Keywords are used for efficiently pre-filtering chunks.
|
||||
|
||||
@@ -2,6 +2,7 @@ package plaidkey
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"strings"
|
||||
@@ -53,32 +54,36 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
DetectorType: detectorspb.DetectorType_PlaidKey,
|
||||
Raw: []byte(resMatch),
|
||||
}
|
||||
|
||||
environments := []string{"development", "production"}
|
||||
if verify {
|
||||
payload := strings.NewReader(`{"client_id":"` + idresMatch + `","secret":"` + resMatch + `","user":{"client_user_id":"60e3ee4019a2660010f8bc54","phone_number_verified_time":"0001-01-01T00:00:00Z","email_address_verified_time":"0001-01-01T00:00:00Z"},"client_name":"Plaid Test App","products":["auth","transactions"],"country_codes":["US"],"webhook":"https://webhook-uri.com","account_filters":{"depository":{"account_subtypes":["checking","savings"]}},"language":"en","link_customization_name":"default"}`)
|
||||
req, err := http.NewRequestWithContext(ctx, "POST", "https://development.plaid.com/link/token/create", payload)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
req.Header.Add("Content-Type", "application/json")
|
||||
res, err := client.Do(req)
|
||||
if err == nil {
|
||||
defer res.Body.Close()
|
||||
if res.StatusCode >= 200 && res.StatusCode < 300 {
|
||||
s1.Verified = true
|
||||
} else {
|
||||
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
|
||||
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
|
||||
continue
|
||||
for _, env := range environments {
|
||||
payload := strings.NewReader(`{"client_id":"` + idresMatch + `","secret":"` + resMatch + `","user":{"client_user_id":"60e3ee4019a2660010f8bc54","phone_number_verified_time":"0001-01-01T00:00:00Z","email_address_verified_time":"0001-01-01T00:00:00Z"},"client_name":"Plaid Test App","products":["auth","transactions"],"country_codes":["US"],"webhook":"https://webhook-uri.com","account_filters":{"depository":{"account_subtypes":["checking","savings"]}},"language":"en","link_customization_name":"default"}`)
|
||||
req, err := http.NewRequestWithContext(ctx, "POST", "https://"+env+".plaid.com/link/token/create", payload)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
req.Header.Add("Content-Type", "application/json")
|
||||
res, err := client.Do(req)
|
||||
if err == nil {
|
||||
defer res.Body.Close()
|
||||
if res.StatusCode >= 200 && res.StatusCode < 300 {
|
||||
s1.Verified = true
|
||||
s1.ExtraData = map[string]string{"environment": fmt.Sprintf("https://%s.plaid.com", env)}
|
||||
} else {
|
||||
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
|
||||
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
|
||||
continue
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
results = append(results, s1)
|
||||
// if the environment is dev, we don't need to check production
|
||||
if s1.Verified {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
results = append(results, s1)
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
return results, nil
|
||||
|
||||
@@ -26,6 +26,8 @@ func TestPlaidKey_FromChunk(t *testing.T) {
|
||||
secret := testSecrets.MustGetField("PLAIDKEY_SECRET")
|
||||
inactiveSecret := testSecrets.MustGetField("PLAIDKEY_SECRET_INACTIVE")
|
||||
id := testSecrets.MustGetField("PLAIDKEY_CLIENTID")
|
||||
// env := testSecrets.MustGetField("PLAIDKEY_ENVIRONMENT") // development or production
|
||||
env := "development"
|
||||
|
||||
type args struct {
|
||||
ctx context.Context
|
||||
@@ -51,6 +53,9 @@ func TestPlaidKey_FromChunk(t *testing.T) {
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_PlaidKey,
|
||||
Verified: true,
|
||||
ExtraData: map[string]string{
|
||||
"environment": fmt.Sprintf("https://%s.plaid.com", env),
|
||||
},
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
|
||||
@@ -20,13 +20,13 @@ var (
|
||||
client = common.SaneHttpClient()
|
||||
|
||||
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
|
||||
keyPat = regexp.MustCompile(`\b(phc_[a-zA-Z0-9_]{43})\b`)
|
||||
keyPat = regexp.MustCompile(`\b(phx_[a-zA-Z0-9_]{43})\b`)
|
||||
)
|
||||
|
||||
// Keywords are used for efficiently pre-filtering chunks.
|
||||
// Use identifiers in the secret preferably, or the provider name.
|
||||
func (s Scanner) Keywords() []string {
|
||||
return []string{"phc_"}
|
||||
return []string{"phx_"}
|
||||
}
|
||||
|
||||
// FromData will find and optionally verify AppPosthog secrets in a given set of bytes.
|
||||
@@ -47,20 +47,33 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
}
|
||||
|
||||
if verify {
|
||||
payload := strings.NewReader(` {
|
||||
"api_key": "` + resMatch + `",
|
||||
"distinct_id": "1234"
|
||||
}`)
|
||||
req, err := http.NewRequestWithContext(ctx, "POST", "https://app.posthog.com/decide/", payload)
|
||||
if err != nil {
|
||||
req, err := http.NewRequestWithContext(ctx, "GET", "https://app.posthog.com/api/event/?personal_api_key="+resMatch, nil)
|
||||
reqEU, errEU := http.NewRequestWithContext(ctx, "GET", "https://eu.posthog.com/api/event/?personal_api_key="+resMatch, nil)
|
||||
|
||||
if err != nil || errEU != nil {
|
||||
continue
|
||||
}
|
||||
req.Header.Add("Content-Type", "application/json")
|
||||
reqEU.Header.Add("Content-Type", "application/json")
|
||||
|
||||
res, err := client.Do(req)
|
||||
if err == nil {
|
||||
defer res.Body.Close()
|
||||
if res.StatusCode >= 200 && res.StatusCode < 300 {
|
||||
s1.Verified = true
|
||||
} else if res.StatusCode == 401 {
|
||||
//Try EU Endpoint only if other one fails.
|
||||
res, err := client.Do(reqEU)
|
||||
if err == nil {
|
||||
defer res.Body.Close()
|
||||
if res.StatusCode >= 200 && res.StatusCode < 300 {
|
||||
s1.Verified = true
|
||||
} else {
|
||||
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
|
||||
continue
|
||||
}
|
||||
}
|
||||
}
|
||||
} else {
|
||||
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
|
||||
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
|
||||
|
||||
@@ -0,0 +1,89 @@
|
||||
package privacy
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
|
||||
)
|
||||
|
||||
type Scanner struct {
|
||||
client *http.Client
|
||||
}
|
||||
|
||||
// Ensure the Scanner satisfies the interface at compile time.
|
||||
var _ detectors.Detector = (*Scanner)(nil)
|
||||
|
||||
var (
|
||||
defaultClient = common.SaneHttpClient()
|
||||
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
|
||||
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"privacy"}) + `\b([a-z0-9]{8}-[a-z0-9]{4}-[a-z0-9]{4}-[a-z0-9]{4}-[a-z0-9]{12})\b`)
|
||||
)
|
||||
|
||||
// Keywords are used for efficiently pre-filtering chunks.
|
||||
// Use identifiers in the secret preferably, or the provider name.
|
||||
func (s Scanner) Keywords() []string {
|
||||
return []string{"privacy"}
|
||||
}
|
||||
|
||||
// FromData will find and optionally verify Privacy secrets in a given set of bytes.
|
||||
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
|
||||
dataStr := string(data)
|
||||
|
||||
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
|
||||
|
||||
for _, match := range matches {
|
||||
if len(match) != 2 {
|
||||
continue
|
||||
}
|
||||
resMatch := strings.TrimSpace(match[1])
|
||||
|
||||
s1 := detectors.Result{
|
||||
DetectorType: detectorspb.DetectorType_Privacy,
|
||||
Raw: []byte(resMatch),
|
||||
}
|
||||
|
||||
if verify {
|
||||
client := s.client
|
||||
if client == nil {
|
||||
client = defaultClient
|
||||
}
|
||||
req, err := http.NewRequestWithContext(ctx, "GET", "https://api.privacy.com/v1/card?page=1&page_size=50", nil)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
req.Header.Set("Authorization", "api-key "+resMatch)
|
||||
res, err := client.Do(req)
|
||||
if err == nil {
|
||||
defer res.Body.Close()
|
||||
if res.StatusCode >= 200 && res.StatusCode < 300 {
|
||||
s1.Verified = true
|
||||
} else if res.StatusCode == 401 {
|
||||
// The secret is determinately not verified (nothing to do)
|
||||
} else {
|
||||
s1.VerificationError = fmt.Errorf("unexpected HTTP response status %d", res.StatusCode)
|
||||
}
|
||||
} else {
|
||||
s1.VerificationError = err
|
||||
}
|
||||
}
|
||||
|
||||
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
|
||||
if !s1.Verified && detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
|
||||
continue
|
||||
}
|
||||
|
||||
results = append(results, s1)
|
||||
}
|
||||
|
||||
return results, nil
|
||||
}
|
||||
|
||||
func (s Scanner) Type() detectorspb.DetectorType {
|
||||
return detectorspb.DetectorType_Privacy
|
||||
}
|
||||
@@ -0,0 +1,161 @@
|
||||
//go:build detectors
|
||||
// +build detectors
|
||||
|
||||
package privacy
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"github.com/google/go-cmp/cmp"
|
||||
"github.com/google/go-cmp/cmp/cmpopts"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
|
||||
)
|
||||
|
||||
func TestPrivacy_FromChunk(t *testing.T) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
|
||||
defer cancel()
|
||||
testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")
|
||||
if err != nil {
|
||||
t.Fatalf("could not get test secrets from GCP: %s", err)
|
||||
}
|
||||
secret := testSecrets.MustGetField("PRIVACY")
|
||||
inactiveSecret := testSecrets.MustGetField("PRIVACY_INACTIVE")
|
||||
|
||||
type args struct {
|
||||
ctx context.Context
|
||||
data []byte
|
||||
verify bool
|
||||
}
|
||||
tests := []struct {
|
||||
name string
|
||||
s Scanner
|
||||
args args
|
||||
want []detectors.Result
|
||||
wantErr bool
|
||||
wantVerificationErr bool
|
||||
}{
|
||||
{
|
||||
name: "found, verified",
|
||||
s: Scanner{},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a privacy secret %s within", secret)),
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_Privacy,
|
||||
Verified: true,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
wantVerificationErr: false,
|
||||
},
|
||||
{
|
||||
name: "found, unverified",
|
||||
s: Scanner{},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a privacy secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_Privacy,
|
||||
Verified: false,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
wantVerificationErr: false,
|
||||
},
|
||||
{
|
||||
name: "not found",
|
||||
s: Scanner{},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte("You cannot find the secret within"),
|
||||
verify: true,
|
||||
},
|
||||
want: nil,
|
||||
wantErr: false,
|
||||
wantVerificationErr: false,
|
||||
},
|
||||
{
|
||||
name: "found, would be verified if not for timeout",
|
||||
s: Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a privacy secret %s within", secret)),
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_Privacy,
|
||||
Verified: false,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
wantVerificationErr: true,
|
||||
},
|
||||
{
|
||||
name: "found, verified but unexpected api surface",
|
||||
s: Scanner{client: common.ConstantResponseHttpClient(404, "")},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a privacy secret %s within", secret)),
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_Privacy,
|
||||
Verified: false,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
wantVerificationErr: true,
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
|
||||
if (err != nil) != tt.wantErr {
|
||||
t.Errorf("Privacy.FromData() error = %v, wantErr %v", err, tt.wantErr)
|
||||
return
|
||||
}
|
||||
for i := range got {
|
||||
if len(got[i].Raw) == 0 {
|
||||
t.Fatalf("no raw secret present: \n %+v", got[i])
|
||||
}
|
||||
if (got[i].VerificationError != nil) != tt.wantVerificationErr {
|
||||
t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError)
|
||||
}
|
||||
}
|
||||
ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "VerificationError")
|
||||
if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {
|
||||
t.Errorf("Privacy.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkFromData(benchmark *testing.B) {
|
||||
ctx := context.Background()
|
||||
s := Scanner{}
|
||||
for name, data := range detectors.MustGetBenchmarkData() {
|
||||
benchmark.Run(name, func(b *testing.B) {
|
||||
b.ResetTimer()
|
||||
for n := 0; n < b.N; n++ {
|
||||
_, err := s.FromData(ctx, false, data)
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -21,7 +21,7 @@ var (
|
||||
client = common.SaneHttpClient()
|
||||
|
||||
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives
|
||||
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"twist"}) + `\b([0-9a-f:]{40,47})\b`)
|
||||
accessToken = regexp.MustCompile(detectors.PrefixRegex([]string{"twist"}) + `\b(?:oauth2:)?([0-9a-f]{40})\b`)
|
||||
)
|
||||
|
||||
// Keywords are used for efficiently pre-filtering chunks.
|
||||
@@ -34,7 +34,7 @@ func (s Scanner) Keywords() []string {
|
||||
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
|
||||
dataStr := string(data)
|
||||
|
||||
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
|
||||
matches := accessToken.FindAllStringSubmatch(dataStr, -1)
|
||||
|
||||
for _, match := range matches {
|
||||
if len(match) != 2 {
|
||||
|
||||
@@ -0,0 +1,99 @@
|
||||
package voiceflow
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
|
||||
)
|
||||
|
||||
type Scanner struct {
|
||||
client *http.Client
|
||||
}
|
||||
|
||||
// Ensure the Scanner satisfies the interface at compile time.
|
||||
var _ detectors.Detector = (*Scanner)(nil)
|
||||
|
||||
var (
|
||||
defaultClient = common.SaneHttpClient()
|
||||
// Reference: https://developer.voiceflow.com/reference/project#dialog-manager-api-keys
|
||||
//
|
||||
//TODO: This includes Workspace and Legacy Workspace API keys; I haven't validated whether these actually work.
|
||||
// https://github.com/voiceflow/general-runtime/blob/master/tests/runtime/lib/DataAPI/utils.unit.ts
|
||||
keyPat = regexp.MustCompile(`\b(VF\.(?:(?:DM|WS)\.)?[a-fA-F0-9]{24}\.[a-zA-Z0-9]{16})\b`)
|
||||
)
|
||||
|
||||
// Keywords are used for efficiently pre-filtering chunks.
|
||||
// Use identifiers in the secret preferably, or the provider name.
|
||||
func (s Scanner) Keywords() []string {
|
||||
return []string{"vf", "dm"}
|
||||
}
|
||||
|
||||
// FromData will find and optionally verify Voiceflow secrets in a given set of bytes.
|
||||
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
|
||||
dataStr := string(data)
|
||||
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
|
||||
|
||||
for _, match := range matches {
|
||||
if len(match) != 2 {
|
||||
continue
|
||||
}
|
||||
resMatch := strings.TrimSpace(match[1])
|
||||
|
||||
s1 := detectors.Result{
|
||||
DetectorType: detectorspb.DetectorType_Voiceflow,
|
||||
Raw: []byte(resMatch),
|
||||
}
|
||||
|
||||
if verify {
|
||||
client := s.client
|
||||
if client == nil {
|
||||
client = defaultClient
|
||||
}
|
||||
// Fetch the state for a random user.
|
||||
payload := []byte(`{"question": "why is the sky blue?"}`)
|
||||
req, err := http.NewRequestWithContext(ctx, "POST", "https://general-runtime.voiceflow.com/knowledge-base/query", bytes.NewBuffer(payload))
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
req.Header.Set("Accept", "application/json")
|
||||
req.Header.Set("Authorization", resMatch)
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
|
||||
res, err := client.Do(req)
|
||||
if err == nil {
|
||||
if res.StatusCode == http.StatusOK {
|
||||
s1.Verified = true
|
||||
} else if res.StatusCode == http.StatusUnauthorized {
|
||||
// The secret is determinately not verified (nothing to do)
|
||||
} else {
|
||||
var buf bytes.Buffer
|
||||
var bodyString string
|
||||
_, err = io.Copy(&buf, res.Body)
|
||||
if err == nil {
|
||||
bodyString = buf.String()
|
||||
}
|
||||
s1.VerificationError = fmt.Errorf("unexpected HTTP response [status=%d, body=%s]", res.StatusCode, bodyString)
|
||||
}
|
||||
_ = res.Body.Close()
|
||||
} else {
|
||||
s1.VerificationError = err
|
||||
}
|
||||
}
|
||||
|
||||
results = append(results, s1)
|
||||
}
|
||||
|
||||
return results, nil
|
||||
}
|
||||
|
||||
func (s Scanner) Type() detectorspb.DetectorType {
|
||||
return detectorspb.DetectorType_Voiceflow
|
||||
}
|
||||
@@ -0,0 +1,302 @@
|
||||
//go:build detectors
|
||||
// +build detectors
|
||||
|
||||
package voiceflow
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"github.com/google/go-cmp/cmp"
|
||||
"github.com/google/go-cmp/cmp/cmpopts"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
|
||||
)
|
||||
|
||||
func TestVoiceflow_Pattern(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
data string
|
||||
shouldMatch bool
|
||||
match string
|
||||
}{
|
||||
// True positives
|
||||
// https://github.com/funDAOmental/endlessquest/blob/5c008f7c6a7e58c45a88b72fef4b965c258d665c/Voiceflow/agent-api/index.js#L6
|
||||
{
|
||||
name: `valid_result1`,
|
||||
data: `// z0MG IT'S NOT A SECRET (but we'll delete it)
|
||||
const API_KEY = "VF.DM.6469b4e5909a470007b96250.k4ip0SMy84jWlCsF"; // it should look like this: VF.DM.XXXXXXX.XXXXXX... keep this a secret!`,
|
||||
shouldMatch: true,
|
||||
match: `VF.DM.6469b4e5909a470007b96250.k4ip0SMy84jWlCsF`,
|
||||
},
|
||||
// https://github.com/sherifButt/ll-site/blob/b98b268214324da42a84e996e4c03c242e122680/src/components/Chatbot.jsx#L14
|
||||
{
|
||||
name: `valid_result2`,
|
||||
data: ` const runtime = useRuntime({
|
||||
verify: { authorization: 'VF.DM.652da078cde70b0008e1c5df.zsIo23VTxNXKfb9f' },
|
||||
session: { userID: 'user_123' },
|
||||
});`,
|
||||
shouldMatch: true,
|
||||
match: `VF.DM.652da078cde70b0008e1c5df.zsIo23VTxNXKfb9f`,
|
||||
},
|
||||
// https://github.com/the-vv/Voiceflow-chatbot/blob/324db17693dd46387ea7a020e92c4e79b94306c6/src/app/chat/chat.component.ts#L27
|
||||
{
|
||||
name: `valid_result3`,
|
||||
data: ` this.http.delete('https://general-runtime.voiceflow.com/state/user/TEST_USER', {
|
||||
headers: {
|
||||
Authorization: "VF.DM.652ecc210267ec00078fc726.ZFPdEwvU0d1jiIMq"
|
||||
}
|
||||
}).subscribe(res => {
|
||||
this.loading = false;
|
||||
this.doPrompt('', { action: { type: 'launch' } });
|
||||
})`,
|
||||
shouldMatch: true,
|
||||
match: `VF.DM.652ecc210267ec00078fc726.ZFPdEwvU0d1jiIMq`,
|
||||
},
|
||||
// https://github.com/legionX7/Graduation-Project-API/blob/451431771d3fba1d8c634b8855274b414d7aed6d/mainAPI.py#L547
|
||||
{
|
||||
name: `valid_result4`,
|
||||
data: `
|
||||
API_KEY = 'VF.DM.646388eb1419c80007bbbaa4.XHOqETFO3cvTxlGl'
|
||||
VERSION_ID = '646bc'`,
|
||||
shouldMatch: true,
|
||||
match: `VF.DM.646388eb1419c80007bbbaa4.XHOqETFO3cvTxlGl`,
|
||||
},
|
||||
// https://github.com/voiceflow/general-runtime/blob/master/tests/runtime/lib/DataAPI/utils.unit.ts
|
||||
{
|
||||
name: `valid_result5`,
|
||||
data: ` it('extracts ID from a Dialog Manager API key', () => {
|
||||
// eslint-disable-next-line no-secrets/no-secrets
|
||||
const key = 'VF.DM.628d5d92faf688001bda7907.dmC8KKO1oX8JO5ai';
|
||||
const result = utils.extractAPIKeyID(key);
|
||||
|
||||
expect(result).to.equal('628d5d92faf688001bda7907');
|
||||
});`,
|
||||
shouldMatch: true,
|
||||
match: `VF.DM.628d5d92faf688001bda7907.dmC8KKO1oX8JO5ai`,
|
||||
},
|
||||
{
|
||||
name: `valid_result6_legacy`,
|
||||
data: ` it('extracts ID from a Workspace API key', () => {
|
||||
// eslint-disable-next-line no-secrets/no-secrets
|
||||
const key = 'VF.WS.62bcb0cca5184300066f5ac7.egnKyyzZksiS5iGa';
|
||||
const result = utils.extractAPIKeyID(key);
|
||||
|
||||
expect(result).to.equal('62bcb0cca5184300066f5ac7');
|
||||
});
|
||||
`,
|
||||
shouldMatch: true,
|
||||
match: `VF.WS.62bcb0cca5184300066f5ac7.egnKyyzZksiS5iGa`,
|
||||
},
|
||||
{
|
||||
name: `valid_result7_legacy`,
|
||||
data: ` it('extracts ID from a Legacy Workspace API key', () => {
|
||||
// eslint-disable-next-line no-secrets/no-secrets
|
||||
const key = 'VF.62bcb0cca5184300066f5ac7.dmC8KKO1oX8JO5az';
|
||||
const result = utils.extractAPIKeyID(key);
|
||||
|
||||
expect(result).to.equal('62bcb0cca5184300066f5ac7');
|
||||
});`,
|
||||
shouldMatch: true,
|
||||
match: `VF.62bcb0cca5184300066f5ac7.dmC8KKO1oX8JO5az`,
|
||||
},
|
||||
|
||||
// False positives
|
||||
// https://github.com/ImperialCollegeLondon/voiceflow-integration-whatsapp/blob/0f3d6a5638b9acb4989d5bf8e77081cc78e9b976/README.md?plain=1#L155
|
||||
{
|
||||
name: `invalid_result1`,
|
||||
data: "Now, paste it in your .env file for the **VF_PROJECT_API** variable<br>\n```VF_PROJECT_API='VF.DM.62xxxxxxxxxxxxxxxxxxxxxxx'```",
|
||||
shouldMatch: false,
|
||||
},
|
||||
// https://github.com/voiceflow/api-examples/blob/c3d8ba9ee8eced7ec8d241973b1eb0284aaec212/rust/src/main.rs#L5
|
||||
{
|
||||
name: `invalid_result2`,
|
||||
data: `const API_KEY: &str = "YOUR_API_KEY_HERE"; // it should look like this: VF.DM.XXXXXXX.XXXXXX... keep this a secret!`,
|
||||
shouldMatch: false,
|
||||
},
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
s := Scanner{}
|
||||
|
||||
results, err := s.FromData(context.Background(), true, []byte(test.data))
|
||||
if err != nil {
|
||||
t.Errorf("CoinbaseWaaS.FromData() error = %v", err)
|
||||
return
|
||||
}
|
||||
|
||||
if test.shouldMatch {
|
||||
if len(results) == 0 {
|
||||
t.Errorf("%s: did not receive a match for '%v' when one was expected", test.name, test.data)
|
||||
return
|
||||
}
|
||||
expected := test.data
|
||||
if test.match != "" {
|
||||
expected = test.match
|
||||
}
|
||||
result := results[0]
|
||||
if result.VerificationError != nil {
|
||||
fmt.Printf("VerificationError: %v\n", result.VerificationError)
|
||||
}
|
||||
resultData := string(result.Raw)
|
||||
if resultData != expected {
|
||||
t.Errorf("%s: did not receive expected match.\n\texpected: '%s'\n\t actual: '%s'", test.name, expected, resultData)
|
||||
return
|
||||
}
|
||||
} else {
|
||||
if len(results) > 0 {
|
||||
t.Errorf("%s: received a match for '%v' when one wasn't wanted", test.name, test.data)
|
||||
return
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestVoiceflow_FromChunk(t *testing.T) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
|
||||
defer cancel()
|
||||
testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")
|
||||
if err != nil {
|
||||
t.Fatalf("could not get test secrets from GCP: %s", err)
|
||||
}
|
||||
secret := testSecrets.MustGetField("VOICEFLOW")
|
||||
inactiveSecret := testSecrets.MustGetField("VOICEFLOW_INACTIVE")
|
||||
|
||||
type args struct {
|
||||
ctx context.Context
|
||||
data []byte
|
||||
verify bool
|
||||
}
|
||||
tests := []struct {
|
||||
name string
|
||||
s Scanner
|
||||
args args
|
||||
want []detectors.Result
|
||||
wantErr bool
|
||||
wantVerificationErr bool
|
||||
}{
|
||||
{
|
||||
name: "found, verified",
|
||||
s: Scanner{},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a voiceflow secret %s within", secret)),
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_Voiceflow,
|
||||
Verified: true,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
wantVerificationErr: false,
|
||||
},
|
||||
{
|
||||
name: "found, unverified",
|
||||
s: Scanner{},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a voiceflow secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_Voiceflow,
|
||||
Verified: false,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
wantVerificationErr: false,
|
||||
},
|
||||
{
|
||||
name: "not found",
|
||||
s: Scanner{},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte("You cannot find the secret within"),
|
||||
verify: true,
|
||||
},
|
||||
want: nil,
|
||||
wantErr: false,
|
||||
wantVerificationErr: false,
|
||||
},
|
||||
{
|
||||
name: "found, would be verified if not for timeout",
|
||||
s: Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a voiceflow secret %s within", secret)),
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_Voiceflow,
|
||||
Verified: false,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
wantVerificationErr: true,
|
||||
},
|
||||
{
|
||||
name: "found, verified but unexpected api surface",
|
||||
s: Scanner{client: common.ConstantResponseHttpClient(404, "")},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a voiceflow secret %s within", secret)),
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_Voiceflow,
|
||||
Verified: false,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
wantVerificationErr: true,
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
|
||||
if (err != nil) != tt.wantErr {
|
||||
t.Errorf("Voiceflow.FromData() error = %v, wantErr %v", err, tt.wantErr)
|
||||
return
|
||||
}
|
||||
for i := range got {
|
||||
if len(got[i].Raw) == 0 {
|
||||
t.Fatalf("no raw secret present: \n %+v", got[i])
|
||||
}
|
||||
if (got[i].VerificationError != nil) != tt.wantVerificationErr {
|
||||
t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError)
|
||||
}
|
||||
}
|
||||
ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "VerificationError")
|
||||
if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {
|
||||
t.Errorf("Voiceflow.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkFromData(benchmark *testing.B) {
|
||||
ctx := context.Background()
|
||||
s := Scanner{}
|
||||
for name, data := range detectors.MustGetBenchmarkData() {
|
||||
benchmark.Run(name, func(b *testing.B) {
|
||||
b.ResetTimer()
|
||||
for n := 0; n < b.N; n++ {
|
||||
_, err := s.FromData(ctx, false, data)
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -11,6 +11,7 @@ import (
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/sourcegraphcody"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/tailscale"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/trufflehogenterprise"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/voiceflow"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/abbysale"
|
||||
@@ -1554,6 +1555,7 @@ func DefaultDetectors() []detectors.Detector {
|
||||
&ramp.Scanner{},
|
||||
&anthropic.Scanner{},
|
||||
&sourcegraphcody.Scanner{},
|
||||
voiceflow.Scanner{},
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -31,6 +31,10 @@ func (p *discardPrinter) Print(context.Context, *detectors.ResultWithMetadata) e
|
||||
func TestGitEngine(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
repoUrl := "https://github.com/dustin-decker/secretsandstuff.git"
|
||||
// Modifies the executable name in the testing context
|
||||
liveExecutableName := git.ExecutableName
|
||||
git.ExecutableName = "engine.test"
|
||||
defer func() {git.ExecutableName = liveExecutableName}()
|
||||
path, _, err := git.PrepareRepo(ctx, repoUrl)
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
|
||||
@@ -1008,6 +1008,8 @@ const (
|
||||
DetectorType_Ramp DetectorType = 934
|
||||
DetectorType_Klaviyo DetectorType = 935
|
||||
DetectorType_SourcegraphCody DetectorType = 936
|
||||
DetectorType_Voiceflow DetectorType = 937
|
||||
DetectorType_Privacy DetectorType = 938
|
||||
)
|
||||
|
||||
// Enum value maps for DetectorType.
|
||||
@@ -1946,6 +1948,8 @@ var (
|
||||
934: "Ramp",
|
||||
935: "Klaviyo",
|
||||
936: "SourcegraphCody",
|
||||
937: "Voiceflow",
|
||||
938: "Privacy",
|
||||
}
|
||||
DetectorType_value = map[string]int32{
|
||||
"Alibaba": 0,
|
||||
@@ -2881,6 +2885,8 @@ var (
|
||||
"Ramp": 934,
|
||||
"Klaviyo": 935,
|
||||
"SourcegraphCody": 936,
|
||||
"Voiceflow": 937,
|
||||
"Privacy": 938,
|
||||
}
|
||||
)
|
||||
|
||||
@@ -3259,7 +3265,7 @@ var file_detectors_proto_rawDesc = []byte{
|
||||
0x44, 0x65, 0x63, 0x6f, 0x64, 0x65, 0x72, 0x54, 0x79, 0x70, 0x65, 0x12, 0x0b, 0x0a, 0x07, 0x55,
|
||||
0x4e, 0x4b, 0x4e, 0x4f, 0x57, 0x4e, 0x10, 0x00, 0x12, 0x09, 0x0a, 0x05, 0x50, 0x4c, 0x41, 0x49,
|
||||
0x4e, 0x10, 0x01, 0x12, 0x0a, 0x0a, 0x06, 0x42, 0x41, 0x53, 0x45, 0x36, 0x34, 0x10, 0x02, 0x12,
|
||||
0x09, 0x0a, 0x05, 0x55, 0x54, 0x46, 0x31, 0x36, 0x10, 0x03, 0x2a, 0xa6, 0x75, 0x0a, 0x0c, 0x44,
|
||||
0x09, 0x0a, 0x05, 0x55, 0x54, 0x46, 0x31, 0x36, 0x10, 0x03, 0x2a, 0xc4, 0x75, 0x0a, 0x0c, 0x44,
|
||||
0x65, 0x74, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x54, 0x79, 0x70, 0x65, 0x12, 0x0b, 0x0a, 0x07, 0x41,
|
||||
0x6c, 0x69, 0x62, 0x61, 0x62, 0x61, 0x10, 0x00, 0x12, 0x08, 0x0a, 0x04, 0x41, 0x4d, 0x51, 0x50,
|
||||
0x10, 0x01, 0x12, 0x07, 0x0a, 0x03, 0x41, 0x57, 0x53, 0x10, 0x02, 0x12, 0x09, 0x0a, 0x05, 0x41,
|
||||
@@ -4198,11 +4204,13 @@ var file_detectors_proto_rawDesc = []byte{
|
||||
0x63, 0x10, 0xa5, 0x07, 0x12, 0x09, 0x0a, 0x04, 0x52, 0x61, 0x6d, 0x70, 0x10, 0xa6, 0x07, 0x12,
|
||||
0x0c, 0x0a, 0x07, 0x4b, 0x6c, 0x61, 0x76, 0x69, 0x79, 0x6f, 0x10, 0xa7, 0x07, 0x12, 0x14, 0x0a,
|
||||
0x0f, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x67, 0x72, 0x61, 0x70, 0x68, 0x43, 0x6f, 0x64, 0x79,
|
||||
0x10, 0xa8, 0x07, 0x42, 0x3d, 0x5a, 0x3b, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f,
|
||||
0x6d, 0x2f, 0x74, 0x72, 0x75, 0x66, 0x66, 0x6c, 0x65, 0x73, 0x65, 0x63, 0x75, 0x72, 0x69, 0x74,
|
||||
0x79, 0x2f, 0x74, 0x72, 0x75, 0x66, 0x66, 0x6c, 0x65, 0x68, 0x6f, 0x67, 0x2f, 0x76, 0x33, 0x2f,
|
||||
0x70, 0x6b, 0x67, 0x2f, 0x70, 0x62, 0x2f, 0x64, 0x65, 0x74, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x73,
|
||||
0x70, 0x62, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
|
||||
0x10, 0xa8, 0x07, 0x12, 0x0e, 0x0a, 0x09, 0x56, 0x6f, 0x69, 0x63, 0x65, 0x66, 0x6c, 0x6f, 0x77,
|
||||
0x10, 0xa9, 0x07, 0x12, 0x0c, 0x0a, 0x07, 0x50, 0x72, 0x69, 0x76, 0x61, 0x63, 0x79, 0x10, 0xaa,
|
||||
0x07, 0x42, 0x3d, 0x5a, 0x3b, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f,
|
||||
0x74, 0x72, 0x75, 0x66, 0x66, 0x6c, 0x65, 0x73, 0x65, 0x63, 0x75, 0x72, 0x69, 0x74, 0x79, 0x2f,
|
||||
0x74, 0x72, 0x75, 0x66, 0x66, 0x6c, 0x65, 0x68, 0x6f, 0x67, 0x2f, 0x76, 0x33, 0x2f, 0x70, 0x6b,
|
||||
0x67, 0x2f, 0x70, 0x62, 0x2f, 0x64, 0x65, 0x74, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x73, 0x70, 0x62,
|
||||
0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
|
||||
}
|
||||
|
||||
var (
|
||||
|
||||
+41
-1
@@ -9,6 +9,7 @@ import (
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
@@ -19,6 +20,7 @@ import (
|
||||
"github.com/go-git/go-git/v5/plumbing"
|
||||
"github.com/go-git/go-git/v5/plumbing/object"
|
||||
"github.com/google/go-github/v42/github"
|
||||
"github.com/mitchellh/go-ps"
|
||||
"golang.org/x/oauth2"
|
||||
"golang.org/x/sync/semaphore"
|
||||
"google.golang.org/protobuf/proto"
|
||||
@@ -319,6 +321,36 @@ type cloneParams struct {
|
||||
clonePath string
|
||||
}
|
||||
|
||||
// Defines the interface for returning a single PID value given an executable name
|
||||
type PID interface {
|
||||
getPIDs(ctx context.Context, executable string) (int, error)
|
||||
}
|
||||
|
||||
var ExecutableName = os.Args[0]
|
||||
|
||||
func GetScannerPIDs(ctx context.Context, executable string) (int, error) {
|
||||
var pids []int
|
||||
|
||||
procs, err := ps.Processes()
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("error getting jobs PIDs: %w", err)
|
||||
}
|
||||
|
||||
for _, proc := range procs {
|
||||
if strings.Contains(proc.Executable(), executable) {
|
||||
pids = append(pids, proc.Pid())
|
||||
}
|
||||
}
|
||||
|
||||
sort.Ints(pids)
|
||||
if len(pids) == 0 {
|
||||
return 0, fmt.Errorf("PID for %q not found", executable)
|
||||
} else {
|
||||
return pids[0], nil
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
// CloneRepo orchestrates the cloning of a given Git repository, returning its local path
|
||||
// and a git.Repository object for further operations. The function sets up error handling
|
||||
// infrastructure, ensuring that any encountered errors trigger a cleanup of resources.
|
||||
@@ -329,7 +361,15 @@ func CloneRepo(ctx context.Context, userInfo *url.Userinfo, gitURL string, args
|
||||
if err = GitCmdCheck(); err != nil {
|
||||
return "", nil, err
|
||||
}
|
||||
clonePath, err := os.MkdirTemp(os.TempDir(), "trufflehog")
|
||||
|
||||
pid, err := GetScannerPIDs(ctx, ExecutableName)
|
||||
if err != nil {
|
||||
return "", nil, err
|
||||
}
|
||||
|
||||
tmpdir := fmt.Sprintf("%s-%d-", ExecutableName, pid)
|
||||
|
||||
clonePath, err := os.MkdirTemp(os.TempDir(), tmpdir)
|
||||
if err != nil {
|
||||
return "", nil, err
|
||||
}
|
||||
|
||||
@@ -825,38 +825,51 @@ func (s *Source) scan(ctx context.Context, installationClient *github.Client, ch
|
||||
// Unauthenticated access to most github endpoints has a rate limit of 60 requests per hour.
|
||||
// This will likely only be exhausted if many users/orgs are scanned without auth
|
||||
func (s *Source) handleRateLimit(errIn error, res *github.Response) bool {
|
||||
limit, ok := errIn.(*github.RateLimitError)
|
||||
if !ok {
|
||||
var (
|
||||
knownWait = true
|
||||
remaining = 0
|
||||
retryAfter time.Duration
|
||||
)
|
||||
|
||||
// GitHub has both primary (RateLimit) and secondary (AbuseRateLimit) errors.
|
||||
var rateLimit *github.RateLimitError
|
||||
var abuseLimit *github.AbuseRateLimitError
|
||||
if errors.As(errIn, &rateLimit) {
|
||||
// Do nothing
|
||||
} else if errors.As(errIn, &abuseLimit) {
|
||||
retryAfter = abuseLimit.GetRetryAfter()
|
||||
} else {
|
||||
return false
|
||||
}
|
||||
|
||||
githubNumRateLimitEncountered.WithLabelValues(s.name).Inc()
|
||||
|
||||
if res != nil {
|
||||
knownWait := true
|
||||
remaining, err := strconv.Atoi(res.Header.Get("x-ratelimit-remaining"))
|
||||
// Parse retry information from response headers, unless a Retry-After value was already provided.
|
||||
// https://docs.github.com/en/rest/overview/resources-in-the-rest-api#exceeding-the-rate-limit
|
||||
if retryAfter <= 0 && res != nil {
|
||||
var err error
|
||||
remaining, err = strconv.Atoi(res.Header.Get("x-ratelimit-remaining"))
|
||||
if err != nil {
|
||||
knownWait = false
|
||||
}
|
||||
|
||||
resetTime, err := strconv.Atoi(res.Header.Get("x-ratelimit-reset"))
|
||||
if err != nil || resetTime == 0 {
|
||||
knownWait = false
|
||||
}
|
||||
|
||||
if knownWait && remaining == 0 {
|
||||
waitTime := int64(resetTime) - time.Now().Unix()
|
||||
if waitTime > 0 {
|
||||
duration := time.Duration(waitTime+1) * time.Second
|
||||
s.log.V(2).Info("rate limited", "resumeTime", time.Now().Add(duration).String())
|
||||
time.Sleep(duration)
|
||||
githubSecondsSpentRateLimited.WithLabelValues(s.name).Add(duration.Seconds())
|
||||
return true
|
||||
}
|
||||
} else if resetTime > 0 {
|
||||
retryAfter = time.Duration(int64(resetTime)-time.Now().Unix()) * time.Second
|
||||
}
|
||||
}
|
||||
|
||||
s.log.V(2).Info("handling rate limit (5 minutes retry)", "retry-after", limit.Message)
|
||||
time.Sleep(time.Minute * 5)
|
||||
resumeTime := time.Now().Add(retryAfter).String()
|
||||
if knownWait && remaining == 0 && retryAfter > 0 {
|
||||
s.log.V(2).Info("rate limited", "retry_after", retryAfter.String(), "resume_time", resumeTime)
|
||||
} else {
|
||||
// TODO: Use exponential backoff instead of static retry time.
|
||||
retryAfter = time.Minute * 5
|
||||
s.log.V(2).Error(errIn, "unexpected rate limit error", "retry_after", retryAfter.String(), "resume_time", resumeTime)
|
||||
}
|
||||
time.Sleep(retryAfter)
|
||||
githubSecondsSpentRateLimited.WithLabelValues(s.name).Add(retryAfter.Seconds())
|
||||
return true
|
||||
}
|
||||
|
||||
@@ -1131,9 +1144,10 @@ var (
|
||||
)
|
||||
|
||||
type repoInfo struct {
|
||||
owner string
|
||||
repo string
|
||||
repoPath string
|
||||
owner string
|
||||
repo string
|
||||
repoPath string
|
||||
visibility source_metadatapb.Visibility
|
||||
}
|
||||
|
||||
func (s *Source) processRepoComments(ctx context.Context, repoPath string, trimmedURL []string, repoURL *url.URL, chunksChan chan *sources.Chunk) error {
|
||||
@@ -1144,7 +1158,16 @@ func (s *Source) processRepoComments(ctx context.Context, repoPath string, trimm
|
||||
owner := trimmedURL[1]
|
||||
repo := trimmedURL[2]
|
||||
|
||||
repoInfo := repoInfo{owner: owner, repo: repo, repoPath: repoPath}
|
||||
if !(s.includeIssueComments || s.includePRComments) {
|
||||
return nil
|
||||
}
|
||||
|
||||
repoInfo := repoInfo{
|
||||
owner: owner,
|
||||
repo: repo,
|
||||
repoPath: repoPath,
|
||||
visibility: s.visibilityOf(ctx, repoPath),
|
||||
}
|
||||
|
||||
if s.includeIssueComments {
|
||||
if err := s.processIssueComments(ctx, repoInfo, chunksChan); err != nil {
|
||||
@@ -1191,7 +1214,7 @@ func (s *Source) processIssues(ctx context.Context, info repoInfo, chunksChan ch
|
||||
return err
|
||||
}
|
||||
|
||||
if err = s.chunkIssues(ctx, info.repo, info.repoPath, issues, chunksChan); err != nil {
|
||||
if err = s.chunkIssues(ctx, info, issues, chunksChan); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -1226,7 +1249,7 @@ func (s *Source) processIssueComments(ctx context.Context, info repoInfo, chunks
|
||||
return err
|
||||
}
|
||||
|
||||
if err = s.chunkIssueComments(ctx, info.repo, info.repoPath, issueComments, chunksChan); err != nil {
|
||||
if err = s.chunkIssueComments(ctx, info, issueComments, chunksChan); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -1262,7 +1285,7 @@ func (s *Source) processPRs(ctx context.Context, info repoInfo, chunksChan chan
|
||||
return err
|
||||
}
|
||||
|
||||
if err = s.chunkPullRequests(ctx, info.repo, prs, chunksChan); err != nil {
|
||||
if err = s.chunkPullRequests(ctx, info, prs, chunksChan); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -1297,7 +1320,7 @@ func (s *Source) processPRComments(ctx context.Context, info repoInfo, chunksCha
|
||||
return err
|
||||
}
|
||||
|
||||
if err = s.chunkPullRequestComments(ctx, info.repo, prComments, chunksChan); err != nil {
|
||||
if err = s.chunkPullRequestComments(ctx, info, prComments, chunksChan); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -1310,7 +1333,7 @@ func (s *Source) processPRComments(ctx context.Context, info repoInfo, chunksCha
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Source) chunkIssues(ctx context.Context, repo, repoPath string, issues []*github.Issue, chunksChan chan *sources.Chunk) error {
|
||||
func (s *Source) chunkIssues(ctx context.Context, repoInfo repoInfo, issues []*github.Issue, chunksChan chan *sources.Chunk) error {
|
||||
for _, issue := range issues {
|
||||
|
||||
// Skip pull requests since covered by processPRs.
|
||||
@@ -1330,9 +1353,9 @@ func (s *Source) chunkIssues(ctx context.Context, repo, repoPath string, issues
|
||||
Link: sanitizer.UTF8(issue.GetHTMLURL()),
|
||||
Username: sanitizer.UTF8(issue.GetUser().GetLogin()),
|
||||
Email: sanitizer.UTF8(issue.GetUser().GetEmail()),
|
||||
Repository: sanitizer.UTF8(repo),
|
||||
Repository: sanitizer.UTF8(repoInfo.repo),
|
||||
Timestamp: sanitizer.UTF8(issue.GetCreatedAt().String()),
|
||||
Visibility: s.visibilityOf(ctx, repoPath),
|
||||
Visibility: repoInfo.visibility,
|
||||
},
|
||||
},
|
||||
},
|
||||
@@ -1349,7 +1372,7 @@ func (s *Source) chunkIssues(ctx context.Context, repo, repoPath string, issues
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Source) chunkIssueComments(ctx context.Context, repo, repoPath string, comments []*github.IssueComment, chunksChan chan *sources.Chunk) error {
|
||||
func (s *Source) chunkIssueComments(ctx context.Context, repoInfo repoInfo, comments []*github.IssueComment, chunksChan chan *sources.Chunk) error {
|
||||
for _, comment := range comments {
|
||||
// Create chunk and send it to the channel.
|
||||
chunk := &sources.Chunk{
|
||||
@@ -1363,9 +1386,9 @@ func (s *Source) chunkIssueComments(ctx context.Context, repo, repoPath string,
|
||||
Link: sanitizer.UTF8(comment.GetHTMLURL()),
|
||||
Username: sanitizer.UTF8(comment.GetUser().GetLogin()),
|
||||
Email: sanitizer.UTF8(comment.GetUser().GetEmail()),
|
||||
Repository: sanitizer.UTF8(repo),
|
||||
Repository: sanitizer.UTF8(repoInfo.repo),
|
||||
Timestamp: sanitizer.UTF8(comment.GetCreatedAt().String()),
|
||||
Visibility: s.visibilityOf(ctx, repoPath),
|
||||
Visibility: repoInfo.visibility,
|
||||
},
|
||||
},
|
||||
},
|
||||
@@ -1382,7 +1405,7 @@ func (s *Source) chunkIssueComments(ctx context.Context, repo, repoPath string,
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Source) chunkPullRequestComments(ctx context.Context, repo string, comments []*github.PullRequestComment, chunksChan chan *sources.Chunk) error {
|
||||
func (s *Source) chunkPullRequestComments(ctx context.Context, repoInfo repoInfo, comments []*github.PullRequestComment, chunksChan chan *sources.Chunk) error {
|
||||
for _, comment := range comments {
|
||||
// Create chunk and send it to the channel.
|
||||
chunk := &sources.Chunk{
|
||||
@@ -1396,8 +1419,9 @@ func (s *Source) chunkPullRequestComments(ctx context.Context, repo string, comm
|
||||
Link: sanitizer.UTF8(comment.GetHTMLURL()),
|
||||
Username: sanitizer.UTF8(comment.GetUser().GetLogin()),
|
||||
Email: sanitizer.UTF8(comment.GetUser().GetEmail()),
|
||||
Repository: sanitizer.UTF8(repo),
|
||||
Repository: sanitizer.UTF8(repoInfo.repo),
|
||||
Timestamp: sanitizer.UTF8(comment.GetCreatedAt().String()),
|
||||
Visibility: repoInfo.visibility,
|
||||
},
|
||||
},
|
||||
},
|
||||
@@ -1414,7 +1438,7 @@ func (s *Source) chunkPullRequestComments(ctx context.Context, repo string, comm
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Source) chunkPullRequests(ctx context.Context, repo string, prs []*github.PullRequest, chunksChan chan *sources.Chunk) error {
|
||||
func (s *Source) chunkPullRequests(ctx context.Context, repoInfo repoInfo, prs []*github.PullRequest, chunksChan chan *sources.Chunk) error {
|
||||
for _, pr := range prs {
|
||||
// Create chunk and send it to the channel.
|
||||
chunk := &sources.Chunk{
|
||||
@@ -1428,8 +1452,9 @@ func (s *Source) chunkPullRequests(ctx context.Context, repo string, prs []*gith
|
||||
Link: sanitizer.UTF8(pr.GetHTMLURL()),
|
||||
Username: sanitizer.UTF8(pr.GetUser().GetLogin()),
|
||||
Email: sanitizer.UTF8(pr.GetUser().GetEmail()),
|
||||
Repository: sanitizer.UTF8(repo),
|
||||
Repository: sanitizer.UTF8(repoInfo.repo),
|
||||
Timestamp: sanitizer.UTF8(pr.GetCreatedAt().String()),
|
||||
Visibility: repoInfo.visibility,
|
||||
},
|
||||
},
|
||||
},
|
||||
@@ -1462,8 +1487,7 @@ func (s *Source) chunkGistComments(ctx context.Context, gistUrl string, comments
|
||||
Email: sanitizer.UTF8(comment.GetUser().GetEmail()),
|
||||
Repository: sanitizer.UTF8(gistUrl),
|
||||
Timestamp: sanitizer.UTF8(comment.GetCreatedAt().String()),
|
||||
// Fetching this information requires making an additional API call.
|
||||
// We may want to include this in the future.
|
||||
// TODO: Fetching this requires making an additional API call. We may want to include this in the future.
|
||||
// Visibility: s.visibilityOf(ctx, repoPath),
|
||||
},
|
||||
},
|
||||
|
||||
@@ -89,14 +89,14 @@ func (f Fatal) Unwrap() error { return f.error }
|
||||
// ChunkError is a custom error type for errors encountered during chunking of
|
||||
// a specific unit.
|
||||
type ChunkError struct {
|
||||
unit SourceUnit
|
||||
err error
|
||||
Unit SourceUnit
|
||||
Err error
|
||||
}
|
||||
|
||||
func (f ChunkError) Error() string {
|
||||
return fmt.Sprintf("error chunking unit %q: %s", f.unit.SourceUnitID(), f.err.Error())
|
||||
return fmt.Sprintf("error chunking unit %q: %s", f.Unit.SourceUnitID(), f.Err.Error())
|
||||
}
|
||||
func (f ChunkError) Unwrap() error { return f.err }
|
||||
func (f ChunkError) Unwrap() error { return f.Err }
|
||||
|
||||
// JobProgress aggregates information about a run of a Source.
|
||||
type JobProgress struct {
|
||||
@@ -351,3 +351,17 @@ func (m JobProgressMetrics) ElapsedTime() time.Duration {
|
||||
}
|
||||
return m.EndTime.Sub(m.StartTime)
|
||||
}
|
||||
|
||||
// ErrorsFor returns all the errors for the given SourceUnit. If there are no
|
||||
// errors, including the case that the unit has not been encountered, nil will
|
||||
// be returned.
|
||||
func (m JobProgressMetrics) ErrorsFor(unit SourceUnit) []error {
|
||||
var errs []error
|
||||
for _, err := range m.Errors {
|
||||
var chunkErr ChunkError
|
||||
if errors.As(err, &chunkErr) && chunkErr.Unit == unit {
|
||||
errs = append(errs, err)
|
||||
}
|
||||
}
|
||||
return errs
|
||||
}
|
||||
|
||||
@@ -126,3 +126,26 @@ func TestJobProgressElapsedTime(t *testing.T) {
|
||||
metrics.EndTime = metrics.StartTime.Add(1 * time.Hour)
|
||||
assert.Equal(t, metrics.ElapsedTime(), 1*time.Hour)
|
||||
}
|
||||
|
||||
func TestJobProgressErrorsFor(t *testing.T) {
|
||||
metrics := JobProgressMetrics{
|
||||
Errors: []error{
|
||||
Fatal{ChunkError{
|
||||
Unit: CommonSourceUnit{ID: "foo"},
|
||||
Err: fmt.Errorf("foo error"),
|
||||
}},
|
||||
ChunkError{
|
||||
Unit: CommonSourceUnit{ID: "foo"},
|
||||
Err: fmt.Errorf("foo again error"),
|
||||
},
|
||||
ChunkError{
|
||||
Unit: CommonSourceUnit{ID: "bar"},
|
||||
Err: fmt.Errorf("bar error"),
|
||||
},
|
||||
fmt.Errorf("hi there"),
|
||||
},
|
||||
}
|
||||
assert.Equal(t, 2, len(metrics.ErrorsFor(CommonSourceUnit{ID: "foo"})))
|
||||
assert.Equal(t, 1, len(metrics.ErrorsFor(CommonSourceUnit{ID: "bar"})))
|
||||
assert.Equal(t, 0, len(metrics.ErrorsFor(CommonSourceUnit{ID: "baz"})))
|
||||
}
|
||||
|
||||
@@ -945,6 +945,8 @@ enum DetectorType {
|
||||
Ramp = 934;
|
||||
Klaviyo = 935;
|
||||
SourcegraphCody = 936;
|
||||
Voiceflow = 937;
|
||||
Privacy = 938;
|
||||
}
|
||||
|
||||
message Result {
|
||||
|
||||
Reference in New Issue
Block a user