Compare commits
3 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 Zubair Khan
|
0cffc3b499 | ||
|
Zubair Khan
|
71cb6dd850 | ||
|
Zubair Khan
|
f800a99dcc |
@@ -6,7 +6,7 @@ replace github.com/jpillora/overseer => github.com/trufflesecurity/overseer v1.1
|
||||
|
||||
require (
|
||||
cloud.google.com/go/secretmanager v1.11.0
|
||||
cloud.google.com/go/storage v1.30.1
|
||||
cloud.google.com/go/storage v1.29.0
|
||||
github.com/Azure/go-autorest/autorest/azure/auth v0.5.11
|
||||
github.com/TheZeroSlave/zapsentry v1.15.0
|
||||
github.com/aws/aws-sdk-go v1.44.83
|
||||
@@ -22,7 +22,7 @@ require (
|
||||
github.com/getsentry/sentry-go v0.22.0
|
||||
github.com/go-errors/errors v1.4.2
|
||||
github.com/go-git/go-git/v5 v5.7.0
|
||||
github.com/go-ldap/ldap/v3 v3.4.5
|
||||
github.com/go-ldap/ldap/v3 v3.4.4
|
||||
github.com/go-logr/logr v1.2.4
|
||||
github.com/go-logr/zapr v1.2.4
|
||||
github.com/go-redis/redis v6.15.9+incompatible
|
||||
@@ -51,12 +51,12 @@ require (
|
||||
github.com/stretchr/testify v1.8.4
|
||||
github.com/tailscale/depaware v0.0.0-20210622194025-720c4b409502
|
||||
github.com/xanzy/go-gitlab v0.85.0
|
||||
go.mongodb.org/mongo-driver v1.12.0
|
||||
go.mongodb.org/mongo-driver v1.11.6
|
||||
go.uber.org/zap v1.24.0
|
||||
golang.org/x/crypto v0.10.0
|
||||
golang.org/x/exp v0.0.0-20221018205818-5c77f4b2bbd7
|
||||
golang.org/x/oauth2 v0.9.0
|
||||
golang.org/x/sync v0.3.0
|
||||
golang.org/x/sync v0.2.0
|
||||
golang.org/x/text v0.10.0
|
||||
google.golang.org/api v0.128.0
|
||||
google.golang.org/protobuf v1.30.0
|
||||
@@ -77,7 +77,7 @@ require (
|
||||
github.com/Azure/go-autorest/autorest/date v0.3.0 // indirect
|
||||
github.com/Azure/go-autorest/logger v0.2.1 // indirect
|
||||
github.com/Azure/go-autorest/tracing v0.6.0 // indirect
|
||||
github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 // indirect
|
||||
github.com/Azure/go-ntlmssp v0.0.0-20220621081337-cb9428e4ac1e // indirect
|
||||
github.com/Microsoft/go-winio v0.6.1 // indirect
|
||||
github.com/ProtonMail/go-crypto v0.0.0-20230518184743-7afd39499903 // indirect
|
||||
github.com/acomagu/bufpipe v1.0.4 // indirect
|
||||
@@ -153,8 +153,8 @@ require (
|
||||
github.com/vbatts/tar-split v0.11.3 // indirect
|
||||
github.com/xanzy/ssh-agent v0.3.3 // indirect
|
||||
github.com/xdg-go/pbkdf2 v1.0.0 // indirect
|
||||
github.com/xdg-go/scram v1.1.2 // indirect
|
||||
github.com/xdg-go/stringprep v1.0.4 // indirect
|
||||
github.com/xdg-go/scram v1.1.1 // indirect
|
||||
github.com/xdg-go/stringprep v1.0.3 // indirect
|
||||
github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d // indirect
|
||||
github.com/yusufpapurcu/wmi v1.2.2 // indirect
|
||||
go.opencensus.io v0.24.0 // indirect
|
||||
|
||||
@@ -25,8 +25,8 @@ cloud.google.com/go/secretmanager v1.11.0 h1:tNzxvs3fP57iTrhaS1XdEPwg4LSPKiDNgp+
|
||||
cloud.google.com/go/secretmanager v1.11.0/go.mod h1:qeQq0/jyJqrGeULu0GkRsVSPKTvf98AEqJnuEIQiJwA=
|
||||
cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw=
|
||||
cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos=
|
||||
cloud.google.com/go/storage v1.30.1 h1:uOdMxAs8HExqBlnLtnQyP0YkvbiDpdGShGKtx6U/oNM=
|
||||
cloud.google.com/go/storage v1.30.1/go.mod h1:NfxhC0UJE1aXSx7CIIbCf7y9HKT7BiccwkR7+P7gN8E=
|
||||
cloud.google.com/go/storage v1.29.0 h1:6weCgzRvMg7lzuUurI4697AqIRPU1SvzHhynwpW31jI=
|
||||
cloud.google.com/go/storage v1.29.0/go.mod h1:4puEjyTKnku6gfKoTfNOU/W+a9JyuVNxjpS5GBrB8h4=
|
||||
dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/azcore v0.19.0/go.mod h1:h6H6c8enJmmocHUbLiiGY6sx7f9i+X3m1CHdd5c6Rdw=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/azidentity v0.11.0/go.mod h1:HcM1YX14R7CJcghJGOYCgdezslRSVzqwLf/q+4Y2r/0=
|
||||
@@ -49,8 +49,8 @@ github.com/Azure/go-autorest/logger v0.2.1 h1:IG7i4p/mDa2Ce4TRyAO8IHnVhAVF3RFU+Z
|
||||
github.com/Azure/go-autorest/logger v0.2.1/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZmbF5NWuPV8+WeEW8=
|
||||
github.com/Azure/go-autorest/tracing v0.6.0 h1:TYi4+3m5t6K48TGI9AUdb+IzbnSxvnvUMfuitfgcfuo=
|
||||
github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU=
|
||||
github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 h1:mFRzDkZVAjdal+s7s0MwaRv9igoPqLRdzOLzw/8Xvq8=
|
||||
github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358/go.mod h1:chxPXzSsl7ZWRAuOIE23GDNzjWuZquvFlgA8xmpunjU=
|
||||
github.com/Azure/go-ntlmssp v0.0.0-20220621081337-cb9428e4ac1e h1:NeAW1fUYUEWhft7pkxDf6WoUvEZJ/uOKsvtpjLnn8MU=
|
||||
github.com/Azure/go-ntlmssp v0.0.0-20220621081337-cb9428e4ac1e/go.mod h1:chxPXzSsl7ZWRAuOIE23GDNzjWuZquvFlgA8xmpunjU=
|
||||
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
|
||||
github.com/BurntSushi/toml v1.2.1/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ=
|
||||
github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
|
||||
@@ -68,8 +68,6 @@ github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 h1:JYp7IbQjafo
|
||||
github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
|
||||
github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 h1:s6gZFSlWYmbqAuRjVTiNNhvNRfY2Wxp9nhfyel4rklc=
|
||||
github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137/go.mod h1:OMCwj8VM1Kc9e19TLln2VL61YJF0x1XFtfdL4JdbSyE=
|
||||
github.com/alexbrainman/sspi v0.0.0-20210105120005-909beea2cc74 h1:Kk6a4nehpJ3UuJRqlA3JxYxBZEqCeOmATOvrbT4p9RA=
|
||||
github.com/alexbrainman/sspi v0.0.0-20210105120005-909beea2cc74/go.mod h1:cEWa1LVoE5KvSD9ONXsZrj0z6KqySlCCNKHlLzbqAt4=
|
||||
github.com/andybalholm/brotli v1.0.5 h1:8uQZIdzKmjc/iuPu7O2ioW48L81FgatrcpfFmiq/cCs=
|
||||
github.com/andybalholm/brotli v1.0.5/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig=
|
||||
github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be h1:9AeTilPcZAjCFIImctFaOjnTIavg87rW78vTPkQqLI8=
|
||||
@@ -175,8 +173,8 @@ github.com/go-git/go-git/v5 v5.7.0 h1:t9AudWVLmqzlo+4bqdf7GY+46SUuRsx59SboFxkq2a
|
||||
github.com/go-git/go-git/v5 v5.7.0/go.mod h1:coJHKEOk5kUClpsNlXrUvPrDxY3w3gjHvhcZd8Fodw8=
|
||||
github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
|
||||
github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
|
||||
github.com/go-ldap/ldap/v3 v3.4.5 h1:ekEKmaDrpvR2yf5Nc/DClsGG9lAmdDixe44mLzlW5r8=
|
||||
github.com/go-ldap/ldap/v3 v3.4.5/go.mod h1:bMGIq3AGbytbaMwf8wdv5Phdxz0FWHTIYMSzyrYgnQs=
|
||||
github.com/go-ldap/ldap/v3 v3.4.4 h1:qPjipEpt+qDa6SI/h1fzuGWoRUY+qqQ9sOZq67/PYUs=
|
||||
github.com/go-ldap/ldap/v3 v3.4.4/go.mod h1:fe1MsuN5eJJ1FeLT/LEBVdWfNWKh459R7aXgXtJC+aI=
|
||||
github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ=
|
||||
github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
|
||||
github.com/go-logr/zapr v1.2.4 h1:QHVo+6stLbfJmYGkQ7uGHUCu5hnAFAj6mDe6Ea0SeOo=
|
||||
@@ -429,6 +427,7 @@ github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5
|
||||
github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
||||
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
||||
github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
||||
github.com/stretchr/testify v1.7.2/go.mod h1:R6va5+xMeoiuVRoj+gSkQ7d3FALtqAAGI1FQKckRals=
|
||||
github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
|
||||
github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
|
||||
github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
|
||||
@@ -437,6 +436,8 @@ github.com/tailscale/depaware v0.0.0-20210622194025-720c4b409502 h1:34icjjmqJ2HP
|
||||
github.com/tailscale/depaware v0.0.0-20210622194025-720c4b409502/go.mod h1:p9lPsd+cx33L3H9nNoecRRxPssFKUwwI50I3pZ0yT+8=
|
||||
github.com/therootcompany/xz v1.0.1 h1:CmOtsn1CbtmyYiusbfmhmkpAAETj0wBIH6kCYaX+xzw=
|
||||
github.com/therootcompany/xz v1.0.1/go.mod h1:3K3UH1yCKgBneZYhuQUvJ9HPD19UEXEI0BWbMn8qNMY=
|
||||
github.com/tidwall/pretty v1.0.0 h1:HsD+QiTn7sK6flMKIvNmpqz1qrpP3Ps6jOKIKMooyg4=
|
||||
github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk=
|
||||
github.com/trufflesecurity/overseer v1.1.7-custom5 h1:xu+Fg6fkSRifUPzUCl7N8HmobJ6WGOkIApGnM7mJS6w=
|
||||
github.com/trufflesecurity/overseer v1.1.7-custom5/go.mod h1:nT9w37AiO1Nop2VhVhNfzAFaPjthvxgpDV3XKsxYkcI=
|
||||
github.com/ulikunitz/xz v0.5.6/go.mod h1:2bypXElzHzzJZwzH67Y6wb67pO62Rzfn7BSiF4ABRW8=
|
||||
@@ -451,10 +452,10 @@ github.com/xanzy/ssh-agent v0.3.3 h1:+/15pJfg/RsTxqYcX6fHqOXZwwMP+2VyYWJeWM2qQFM
|
||||
github.com/xanzy/ssh-agent v0.3.3/go.mod h1:6dzNDKs0J9rVPHPhaGCukekBHKqfl+L3KghI1Bc68Uw=
|
||||
github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c=
|
||||
github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI=
|
||||
github.com/xdg-go/scram v1.1.2 h1:FHX5I5B4i4hKRVRBCFRxq1iQRej7WO3hhBuJf+UUySY=
|
||||
github.com/xdg-go/scram v1.1.2/go.mod h1:RT/sEzTbU5y00aCK8UOx6R7YryM0iF1N2MOmC3kKLN4=
|
||||
github.com/xdg-go/stringprep v1.0.4 h1:XLI/Ng3O1Atzq0oBs3TWm+5ZVgkq2aqdlvP9JtoZ6c8=
|
||||
github.com/xdg-go/stringprep v1.0.4/go.mod h1:mPGuuIYwz7CmR2bT9j4GbQqutWS1zV24gijq1dTyGkM=
|
||||
github.com/xdg-go/scram v1.1.1 h1:VOMT+81stJgXW3CpHyqHN3AXDYIMsx56mEFrB37Mb/E=
|
||||
github.com/xdg-go/scram v1.1.1/go.mod h1:RaEWvsqvNKKvBPvcKeFjrG2cJqOkHTiyTpzz23ni57g=
|
||||
github.com/xdg-go/stringprep v1.0.3 h1:kdwGpVNwPFtjs98xCGkHjQtGKh86rDcRZN17QEMCOIs=
|
||||
github.com/xdg-go/stringprep v1.0.3/go.mod h1:W3f5j4i+9rC0kuIEJL0ky1VpHXQU3ocBgklLGvcBnW8=
|
||||
github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d h1:splanxYIlg+5LfHAM6xpdFEAYOk8iySO56hMFq6uLyA=
|
||||
github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA=
|
||||
github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
|
||||
@@ -462,8 +463,8 @@ github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1
|
||||
github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
|
||||
github.com/yusufpapurcu/wmi v1.2.2 h1:KBNDSne4vP5mbSWnJbO+51IMOXJB67QiYCSBrubbPRg=
|
||||
github.com/yusufpapurcu/wmi v1.2.2/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0=
|
||||
go.mongodb.org/mongo-driver v1.12.0 h1:aPx33jmn/rQuJXPQLZQ8NtfPQG8CaqgLThFtqRb0PiE=
|
||||
go.mongodb.org/mongo-driver v1.12.0/go.mod h1:AZkxhPnFJUoH7kZlFkVKucV20K387miPfm7oimrSmK0=
|
||||
go.mongodb.org/mongo-driver v1.11.6 h1:XM7G6PjiGAO5betLF13BIa5TlLUUE3uJ/2Ox3Lz1K+o=
|
||||
go.mongodb.org/mongo-driver v1.11.6/go.mod h1:G9TgswdsWjX4tmDA5zfs2+6AEPpYJwqblyjsfuh8oXY=
|
||||
go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
|
||||
go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
|
||||
go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
|
||||
@@ -577,8 +578,8 @@ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJ
|
||||
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.3.0 h1:ftCYgMx6zT/asHUrPw8BLLscYtGznsLAnjq5RH9P66E=
|
||||
golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y=
|
||||
golang.org/x/sync v0.2.0 h1:PUR+T4wwASmuSTYdKjYHI5TD22Wy5ogLU5qZCOLxBrI=
|
||||
golang.org/x/sync v0.2.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
|
||||
@@ -6,7 +6,7 @@ import (
|
||||
"unicode"
|
||||
)
|
||||
|
||||
var DefaultFalsePositives = []FalsePositive{"example", "xxxxxx", "aaaaaa", "abcde", "00000", "sample", "www"}
|
||||
var DefaultFalsePositives = []FalsePositive{"example", "xxxxxx", "aaaaaa", "abcde", "00000", "sample"}
|
||||
|
||||
type FalsePositive string
|
||||
|
||||
@@ -31,9 +31,9 @@ var FalsePositiveWordlists = Wordlists{
|
||||
programmingBookWords: bytesToCleanWordList(programmingBookWords),
|
||||
}
|
||||
|
||||
// IsKnownFalsePositives will not return a valid secret finding if any of the disqualifying conditions are met
|
||||
// Currently that includes: No number, english word in key, or matches common example pattens.
|
||||
// Only the secret key material should be passed into this function
|
||||
//IsKnownFalsePositives will not return a valid secret finding if any of the disqualifying conditions are met
|
||||
//Currently that includes: No number, english word in key, or matches common example pattens.
|
||||
//Only the secret key material should be passed into this function
|
||||
func IsKnownFalsePositive(match string, falsePositives []FalsePositive, wordCheck bool) bool {
|
||||
|
||||
for _, fp := range falsePositives {
|
||||
|
||||
@@ -38,18 +38,19 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
keyMatches := keyPat.FindAllString(dataStr, -1)
|
||||
|
||||
for _, key := range keyMatches {
|
||||
secMatches := secretPat.FindAllString(dataStr, -1)
|
||||
|
||||
for _, secret := range secMatches {
|
||||
if verify {
|
||||
secMatches := secretPat.FindAllString(dataStr, -1)
|
||||
|
||||
s1 := detectors.Result{
|
||||
DetectorType: detectorspb.DetectorType_RazorPay,
|
||||
Raw: []byte(key),
|
||||
RawV2: []byte(key + secret),
|
||||
Redacted: key,
|
||||
}
|
||||
for _, secret := range secMatches {
|
||||
|
||||
s1 := detectors.Result{
|
||||
DetectorType: detectorspb.DetectorType_RazorPay,
|
||||
Raw: []byte(key),
|
||||
RawV2: []byte(key + secret),
|
||||
Redacted: key,
|
||||
}
|
||||
|
||||
if verify {
|
||||
req, err := http.NewRequest("GET", "https://api.razorpay.com/v1/items?count=1", nil)
|
||||
if err != nil {
|
||||
continue
|
||||
@@ -65,21 +66,25 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
if res.StatusCode >= 200 && res.StatusCode < 300 {
|
||||
if json.Valid(bodyBytes) {
|
||||
s1.Verified = true
|
||||
} else {
|
||||
s1.Verified = false
|
||||
}
|
||||
} else {
|
||||
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
|
||||
if detectors.IsKnownFalsePositive(key, detectors.DefaultFalsePositives, true) {
|
||||
continue
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if !s1.Verified && detectors.IsKnownFalsePositive(key, detectors.DefaultFalsePositives, true) {
|
||||
continue
|
||||
results = append(results, s1)
|
||||
}
|
||||
|
||||
results = append(results, s1)
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
return detectors.CleanResults(results), nil
|
||||
results = detectors.CleanResults(results)
|
||||
return
|
||||
}
|
||||
|
||||
func (s Scanner) Type() detectorspb.DetectorType {
|
||||
|
||||
@@ -62,7 +62,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
continue
|
||||
}
|
||||
body := string(bodyBytes)
|
||||
if (res.StatusCode >= 200 && res.StatusCode < 300) || (res.StatusCode == 400 && (strings.Contains(body, "no_text") || strings.Contains(body, "missing_text"))) {
|
||||
if (res.StatusCode >= 200 && res.StatusCode < 300) || (res.StatusCode == 400 && strings.Contains(body, "no_text")) {
|
||||
s1.Verified = true
|
||||
}
|
||||
}
|
||||
|
||||
@@ -21,7 +21,6 @@ var (
|
||||
identifierPat = regexp.MustCompile(`(?i)sid.{0,20}AC[0-9a-f]{32}`) // Should we have this? Seems restrictive.
|
||||
sidPat = regexp.MustCompile(`\bAC[0-9a-f]{32}\b`)
|
||||
keyPat = regexp.MustCompile(`\b[0-9a-f]{32}\b`)
|
||||
client = common.SaneHttpClient()
|
||||
)
|
||||
|
||||
// Keywords are used for efficiently pre-filtering chunks.
|
||||
@@ -44,15 +43,15 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
sidMatches := sidPat.FindAllString(dataStr, -1)
|
||||
|
||||
for _, sid := range sidMatches {
|
||||
for _, key := range keyMatches {
|
||||
s1 := detectors.Result{
|
||||
DetectorType: detectorspb.DetectorType_Twilio,
|
||||
Raw: []byte(sid),
|
||||
RawV2: []byte(sid + key),
|
||||
Redacted: sid,
|
||||
}
|
||||
s := detectors.Result{
|
||||
DetectorType: detectorspb.DetectorType_Twilio,
|
||||
Raw: []byte(sid),
|
||||
Redacted: sid,
|
||||
}
|
||||
|
||||
if verify {
|
||||
if verify {
|
||||
client := common.SaneHttpClient()
|
||||
for _, key := range keyMatches {
|
||||
|
||||
form := url.Values{}
|
||||
form.Add("FriendlyName", "MyServiceName")
|
||||
@@ -72,22 +71,22 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
res.Body.Close() // The request body is unused.
|
||||
|
||||
if res.StatusCode >= 200 && res.StatusCode < 300 {
|
||||
s1.Verified = true
|
||||
s.Verified = true
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if !s1.Verified && detectors.IsKnownFalsePositive(string(s1.Raw), detectors.DefaultFalsePositives, true) {
|
||||
continue
|
||||
}
|
||||
if !s.Verified && detectors.IsKnownFalsePositive(string(s.Raw), detectors.DefaultFalsePositives, true) {
|
||||
continue
|
||||
}
|
||||
|
||||
if len(keyMatches) > 0 {
|
||||
results = append(results, s1)
|
||||
}
|
||||
if len(keyMatches) > 0 {
|
||||
results = append(results, s)
|
||||
}
|
||||
}
|
||||
|
||||
return detectors.CleanResults(results), nil
|
||||
return
|
||||
}
|
||||
|
||||
func (s Scanner) Type() detectorspb.DetectorType {
|
||||
|
||||
@@ -98,7 +98,6 @@ func TestTwilio_FromChunk(t *testing.T) {
|
||||
t.Fatal("no raw secret present")
|
||||
}
|
||||
got[i].Raw = nil
|
||||
got[i].RawV2 = nil
|
||||
}
|
||||
if diff := pretty.Compare(got, tt.want); diff != "" {
|
||||
t.Errorf("Twilio.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
|
||||
|
||||
@@ -31,12 +31,10 @@ type Source struct {
|
||||
jobPool *errgroup.Group
|
||||
sources.Progress
|
||||
client *http.Client
|
||||
sources.CommonSourceUnitUnmarshaller
|
||||
}
|
||||
|
||||
// Ensure the Source satisfies the interfaces at compile time.
|
||||
// Ensure the Source satisfies the interface at compile time.
|
||||
var _ sources.Source = (*Source)(nil)
|
||||
var _ sources.SourceUnitUnmarshaller = (*Source)(nil)
|
||||
|
||||
// Type returns the type of source.
|
||||
// It is used for matching source types in configuration and job input.
|
||||
|
||||
@@ -30,14 +30,12 @@ type Source struct {
|
||||
verify bool
|
||||
conn sourcespb.Docker
|
||||
sources.Progress
|
||||
sources.CommonSourceUnitUnmarshaller
|
||||
}
|
||||
|
||||
var FilesizeLimitBytes int64 = 50 * 1024 * 1024 // 50MB
|
||||
|
||||
// Ensure the Source satisfies the interfaces at compile time.
|
||||
// Ensure the Source satisfies the interface at compile time.
|
||||
var _ sources.Source = (*Source)(nil)
|
||||
var _ sources.SourceUnitUnmarshaller = (*Source)(nil)
|
||||
|
||||
// Type returns the type of source.
|
||||
// It is used for matching source types in configuration and job input.
|
||||
|
||||
@@ -39,12 +39,10 @@ type Source struct {
|
||||
log logr.Logger
|
||||
filter *common.Filter
|
||||
sources.Progress
|
||||
sources.CommonSourceUnitUnmarshaller
|
||||
}
|
||||
|
||||
// Ensure the Source satisfies the interfaces at compile time
|
||||
// Ensure the Source satisfies the interface at compile time
|
||||
var _ sources.Source = (*Source)(nil)
|
||||
var _ sources.SourceUnitUnmarshaller = (*Source)(nil)
|
||||
|
||||
// Type returns the type of source.
|
||||
// It is used for matching source types in configuration and job input.
|
||||
|
||||
@@ -30,9 +30,8 @@ import (
|
||||
|
||||
const defaultCachePersistIncrement = 2500
|
||||
|
||||
// Ensure the Source satisfies the interfaces at compile time.
|
||||
// Ensure the Source satisfies the interface at compile time.
|
||||
var _ sources.Source = (*Source)(nil)
|
||||
var _ sources.SourceUnitUnmarshaller = (*Source)(nil)
|
||||
|
||||
// Type returns the type of source.
|
||||
// It is used for matching source types in configuration and job input.
|
||||
@@ -70,7 +69,6 @@ type Source struct {
|
||||
|
||||
mu sync.Mutex
|
||||
sources.Progress // progress is not thread safe
|
||||
sources.CommonSourceUnitUnmarshaller
|
||||
}
|
||||
|
||||
// persistableCache is a wrapper around cache.Cache that allows
|
||||
|
||||
@@ -67,9 +67,8 @@ func NewGit(sourceType sourcespb.SourceType, jobID, sourceID int64, sourceName s
|
||||
}
|
||||
}
|
||||
|
||||
// Ensure the Source satisfies the interfaces at compile time.
|
||||
// Ensure the Source satisfies the interface at compile time.
|
||||
var _ sources.Source = (*Source)(nil)
|
||||
var _ sources.SourceUnitUnmarshaller = (*Source)(nil)
|
||||
|
||||
// Type returns the type of source.
|
||||
// It is used for matching source types in configuration and job input.
|
||||
@@ -876,7 +875,3 @@ func handleBinary(ctx context.Context, repo *git.Repository, chunksChan chan *so
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Source) UnmarshalSourceUnit(data []byte) (sources.SourceUnit, error) {
|
||||
return UnmarshalUnit(data)
|
||||
}
|
||||
|
||||
@@ -1,40 +0,0 @@
|
||||
package git
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
|
||||
)
|
||||
|
||||
const (
|
||||
UnitRepo string = "repo"
|
||||
UnitDir string = "dir"
|
||||
)
|
||||
|
||||
// Ensure SourceUnit implements the interface at compile time.
|
||||
var _ sources.SourceUnit = SourceUnit{}
|
||||
|
||||
// A git source unit can be two kinds of units: either a local directory path
|
||||
// or a remote repository.
|
||||
type SourceUnit struct {
|
||||
Kind string `json:"kind"`
|
||||
ID string `json:"id"`
|
||||
}
|
||||
|
||||
// Implement sources.SourceUnit interface.
|
||||
func (u SourceUnit) SourceUnitID() string {
|
||||
return u.ID
|
||||
}
|
||||
|
||||
// Helper function to unmarshal raw bytes into our SourceUnit struct.
|
||||
func UnmarshalUnit(data []byte) (sources.SourceUnit, error) {
|
||||
var unit SourceUnit
|
||||
if err := json.Unmarshal(data, &unit); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if unit.ID == "" || (unit.Kind != UnitRepo && unit.Kind != UnitDir) {
|
||||
return nil, fmt.Errorf("not a git.SourceUnit")
|
||||
}
|
||||
return unit, nil
|
||||
}
|
||||
@@ -69,17 +69,14 @@ type Source struct {
|
||||
mu sync.Mutex
|
||||
publicMap map[string]source_metadatapb.Visibility
|
||||
sources.Progress
|
||||
sources.CommonSourceUnitUnmarshaller
|
||||
}
|
||||
|
||||
func (s *Source) WithScanOptions(scanOptions *git.ScanOptions) {
|
||||
s.scanOptions = scanOptions
|
||||
}
|
||||
|
||||
// Ensure the Source satisfies the interfaces at compile time
|
||||
// Ensure the Source satisfies the interface at compile time
|
||||
var _ sources.Source = (*Source)(nil)
|
||||
var _ sources.SourceUnitUnmarshaller = (*Source)(nil)
|
||||
|
||||
var endsWithGithub = regexp.MustCompile(`github\.com/?$`)
|
||||
|
||||
// Type returns the type of source.
|
||||
@@ -666,11 +663,17 @@ func (s *Source) scan(ctx context.Context, installationClient *github.Client, ch
|
||||
defer func(start time.Time) {
|
||||
logger.V(2).Info(fmt.Sprintf("scanned %d/%d repos", scanned, len(s.repos)), "repo_size", repoSize, "duration_seconds", time.Since(start).Seconds())
|
||||
}(now)
|
||||
|
||||
fmt.Printf("Scanning repoURL %s, path %+v, repo %+v \n ", repoURL, path, repo)
|
||||
if err = s.git.ScanRepo(ctx, repo, path, s.scanOptions, chunksChan); err != nil {
|
||||
scanErrs.Add(fmt.Errorf("error scanning repo %s: %w", repoURL, err))
|
||||
return nil
|
||||
}
|
||||
atomic.AddUint64(&scanned, 1)
|
||||
fmt.Println("SCANNNNN COMMENTS")
|
||||
if err = s.scanComments(ctx, repoURL, chunksChan); err != nil {
|
||||
fmt.Println("test")
|
||||
}
|
||||
|
||||
return nil
|
||||
})
|
||||
@@ -919,3 +922,82 @@ func (s *Source) setProgressCompleteWithRepo(index int, offset int, repoURL stri
|
||||
|
||||
s.SetProgressComplete(index+offset, len(s.repos)+offset, fmt.Sprintf("Repo: %s", repoURL), encodedResumeInfo)
|
||||
}
|
||||
|
||||
func (s *Source) scanComments(ctx context.Context, repoPath string, chunksChan chan *sources.Chunk) error {
|
||||
s.log.Info("scanning comments")
|
||||
fmt.Print("repoPath: ", repoPath, "\n")
|
||||
|
||||
trimmedURL := removeURLAndSplit(repoPath)
|
||||
owner := trimmedURL[1]
|
||||
repo := trimmedURL[2]
|
||||
|
||||
//for _, part := range trimmedURL {
|
||||
// fmt.Println(part)
|
||||
//}
|
||||
|
||||
sortType := "created"
|
||||
directionType := "desc"
|
||||
|
||||
issueOpts := &github.IssueListCommentsOptions{
|
||||
Sort: &sortType,
|
||||
Direction: &directionType,
|
||||
ListOptions: github.ListOptions{
|
||||
PerPage: defaultPagination,
|
||||
Page: 1,
|
||||
},
|
||||
}
|
||||
issueComments, resp, err := s.apiClient.Issues.ListComments(ctx, owner, repo, 0, issueOpts)
|
||||
|
||||
prOpts := &github.PullRequestListCommentsOptions{
|
||||
Sort: sortType,
|
||||
Direction: directionType,
|
||||
ListOptions: github.ListOptions{
|
||||
PerPage: defaultPagination,
|
||||
Page: 1,
|
||||
},
|
||||
}
|
||||
|
||||
prComments, resp, err := s.apiClient.PullRequests.ListComments(ctx, owner, repo, 0, prOpts)
|
||||
|
||||
fmt.Println("issue comments: ", issueComments)
|
||||
fmt.Println("pr comments: ", prComments)
|
||||
fmt.Println("resp: ", resp)
|
||||
fmt.Println("err: ", err)
|
||||
|
||||
//for _, comment := range comments {
|
||||
// // Create chunk and send it to the channel.
|
||||
// chunk := &sources.Chunk{
|
||||
// SourceName: s.name,
|
||||
// SourceID: s.SourceID(),
|
||||
// SourceType: s.Type(),
|
||||
// SourceMetadata: &source_metadatapb.MetaData{
|
||||
// Data: &source_metadatapb.MetaData_Github{
|
||||
// Github: &source_metadatapb.Github{
|
||||
// Link: sanitizer.UTF8(comment.GetHTMLURL()),
|
||||
// Username: sanitizer.UTF8(comment.GetUser().GetLogin()),
|
||||
// Email: sanitizer.UTF8(comment.GetUser().GetEmail()),
|
||||
// Repository: sanitizer.UTF8(repo),
|
||||
// Timestamp: sanitizer.UTF8(comment.GetCreatedAt().String()),
|
||||
// },
|
||||
// },
|
||||
// },
|
||||
// Data: []byte(sanitizer.UTF8(comment.GetBody())),
|
||||
// Verify: s.verify,
|
||||
// }
|
||||
//
|
||||
// select {
|
||||
// case <-ctx.Done():
|
||||
// return ctx.Err()
|
||||
// case chunksChan <- chunk:
|
||||
// }
|
||||
//}
|
||||
return nil
|
||||
}
|
||||
|
||||
func removeURLAndSplit(url string) []string {
|
||||
trimmedURL := strings.TrimPrefix(url, "https://")
|
||||
trimmedURL = strings.TrimSuffix(trimmedURL, ".git")
|
||||
splitURL := strings.Split(trimmedURL, "/")
|
||||
|
||||
return splitURL
|
||||
}
|
||||
|
||||
@@ -47,12 +47,10 @@ type Source struct {
|
||||
resumeInfoMutex sync.Mutex
|
||||
sources.Progress
|
||||
jobPool *errgroup.Group
|
||||
sources.CommonSourceUnitUnmarshaller
|
||||
}
|
||||
|
||||
// Ensure the Source satisfies the interfaces at compile time.
|
||||
// Ensure the Source satisfies the interface at compile time.
|
||||
var _ sources.Source = (*Source)(nil)
|
||||
var _ sources.SourceUnitUnmarshaller = (*Source)(nil)
|
||||
|
||||
// Type returns the type of source.
|
||||
// It is used for matching source types in configuration and job input.
|
||||
|
||||
@@ -46,12 +46,10 @@ type Source struct {
|
||||
conn *sourcespb.S3
|
||||
jobPool *errgroup.Group
|
||||
maxObjectSize int64
|
||||
sources.CommonSourceUnitUnmarshaller
|
||||
}
|
||||
|
||||
// Ensure the Source satisfies the interfaces at compile time
|
||||
// Ensure the Source satisfies the interface at compile time
|
||||
var _ sources.Source = (*Source)(nil)
|
||||
var _ sources.SourceUnitUnmarshaller = (*Source)(nil)
|
||||
|
||||
// Type returns the type of source
|
||||
func (s *Source) Type() sourcespb.SourceType {
|
||||
|
||||
@@ -1,37 +1,12 @@
|
||||
package sources
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
)
|
||||
|
||||
// Ensure CommonSourceUnit implements SourceUnit at compile time.
|
||||
var _ SourceUnit = CommonSourceUnit{}
|
||||
|
||||
// CommonSourceUnit is a common implementation of SourceUnit that Sources can
|
||||
// use instead of implementing their own types.
|
||||
type CommonSourceUnit struct {
|
||||
ID string `json:"source_unit_id"`
|
||||
}
|
||||
|
||||
// SourceUnitID implements the SourceUnit interface.
|
||||
// Implement the SourceUnit interface.
|
||||
func (c CommonSourceUnit) SourceUnitID() string {
|
||||
return c.ID
|
||||
}
|
||||
|
||||
// CommonSourceUnitUnmarshaller is an implementation of SourceUnitUnmarshaller
|
||||
// for the CommonSourceUnit. A source can embed this struct to gain the
|
||||
// functionality of converting []byte to a CommonSourceUnit.
|
||||
type CommonSourceUnitUnmarshaller struct{}
|
||||
|
||||
// UnmarshalSourceUnit implements the SourceUnitUnmarshaller interface.
|
||||
func (c CommonSourceUnitUnmarshaller) UnmarshalSourceUnit(data []byte) (SourceUnit, error) {
|
||||
var unit CommonSourceUnit
|
||||
if err := json.Unmarshal(data, &unit); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if unit.ID == "" {
|
||||
return nil, fmt.Errorf("not a CommonSourceUnit")
|
||||
}
|
||||
return unit, nil
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user