Compare commits

..

2 Commits

Author SHA1 Message Date
Zubair Khan a09c0f265f save ibm progress 2023-07-17 16:59:38 -04:00
Zubair Khan f0db55c548 init 2023-07-12 15:11:20 -04:00
70 changed files with 956 additions and 5505 deletions
+3 -2
View File
@@ -8,8 +8,9 @@ updates:
- package-ecosystem: "gomod" # See documentation for possible values
directory: "/" # Location of package manifests
schedule:
interval: "monthly"
interval: "weekly"
- package-ecosystem: "github-actions"
directory: "/"
schedule:
interval: "monthly"
# Check for updates to GitHub Actions every weekday
interval: "weekly"
+1 -1
View File
@@ -13,7 +13,7 @@ jobs:
- name: Install Go
uses: actions/setup-go@v4
with:
go-version: '1.20'
go-version: '1.18'
- name: Checkout code
uses: actions/checkout@v3
+1 -1
View File
@@ -11,7 +11,7 @@ jobs:
- name: Install Go
uses: actions/setup-go@v4
with:
go-version: '1.20'
go-version: '1.18'
- name: Checkout code
uses: actions/checkout@v3
- name: Smoke
+17 -20
View File
@@ -4,20 +4,16 @@ go 1.20
replace github.com/jpillora/overseer => github.com/trufflesecurity/overseer v1.1.7-custom5
// go wants to pull in the latest but after v10.2.5 has a breaking API change
replace github.com/couchbase/gocbcore/v10 => github.com/couchbase/gocbcore/v10 v10.2.4
require (
cloud.google.com/go/secretmanager v1.11.1
cloud.google.com/go/storage v1.31.0
github.com/Azure/go-autorest/autorest/azure/auth v0.5.11
github.com/BobuSumisu/aho-corasick v1.0.3
github.com/TheZeroSlave/zapsentry v1.17.0
github.com/aws/aws-sdk-go v1.44.83
github.com/bill-rich/disk-buffer-reader v0.1.7
github.com/bill-rich/go-syslog v0.0.0-20220413021637-49edb52a574c
github.com/bitfinexcom/bitfinex-api-go v0.0.0-20210608095005-9e0b26f200fb
github.com/bradleyfalzon/ghinstallation/v2 v2.6.0
github.com/bradleyfalzon/ghinstallation/v2 v2.4.0
github.com/couchbase/gocb/v2 v2.6.3
github.com/crewjam/rfc5424 v0.1.0
github.com/denisenkom/go-mssqldb v0.12.3
@@ -26,7 +22,7 @@ require (
github.com/felixge/fgprof v0.9.3
github.com/getsentry/sentry-go v0.22.0
github.com/go-errors/errors v1.4.2
github.com/go-git/go-git/v5 v5.8.0
github.com/go-git/go-git/v5 v5.7.0
github.com/go-ldap/ldap/v3 v3.4.5
github.com/go-logr/logr v1.2.4
github.com/go-logr/zapr v1.2.4
@@ -35,9 +31,9 @@ require (
github.com/gobwas/glob v0.2.3
github.com/golang-jwt/jwt v3.2.2+incompatible
github.com/google/go-cmp v0.5.9
github.com/google/go-containerregistry v0.15.2
github.com/google/go-containerregistry v0.14.0
github.com/google/go-github/v42 v42.0.0
github.com/googleapis/gax-go/v2 v2.12.0
github.com/googleapis/gax-go/v2 v2.11.0
github.com/hashicorp/go-retryablehttp v0.7.4
github.com/jlaffaye/ftp v0.2.0
github.com/joho/godotenv v1.5.1
@@ -48,21 +44,22 @@ require (
github.com/mholt/archiver/v4 v4.0.0-alpha.8
github.com/patrickmn/go-cache v2.1.0+incompatible
github.com/paulbellamy/ratecounter v0.2.0
github.com/petar-dambovaliev/aho-corasick v0.0.0-20211021192214-5ab2d9280aa9
github.com/pkg/errors v0.9.1
github.com/prometheus/client_golang v1.16.0
github.com/rabbitmq/amqp091-go v1.8.1
github.com/sergi/go-diff v1.3.1
github.com/stretchr/testify v1.8.4
github.com/tailscale/depaware v0.0.0-20210622194025-720c4b409502
github.com/xanzy/go-gitlab v0.88.0
github.com/xanzy/go-gitlab v0.86.0
go.mongodb.org/mongo-driver v1.12.0
go.uber.org/zap v1.24.0
golang.org/x/crypto v0.11.0
golang.org/x/exp v0.0.0-20221018205818-5c77f4b2bbd7
golang.org/x/oauth2 v0.10.0
golang.org/x/oauth2 v0.9.0
golang.org/x/sync v0.3.0
golang.org/x/text v0.11.0
google.golang.org/api v0.131.0
google.golang.org/api v0.130.0
google.golang.org/protobuf v1.31.0
gopkg.in/alecthomas/kingpin.v2 v2.2.6
gopkg.in/h2non/gock.v1 v1.1.2
@@ -71,7 +68,7 @@ require (
require (
cloud.google.com/go v0.110.2 // indirect
cloud.google.com/go/compute v1.20.1 // indirect
cloud.google.com/go/compute v1.19.3 // indirect
cloud.google.com/go/compute/metadata v0.2.3 // indirect
cloud.google.com/go/iam v1.1.0 // indirect
github.com/Azure/go-autorest v14.2.0+incompatible // indirect
@@ -83,7 +80,7 @@ require (
github.com/Azure/go-autorest/tracing v0.6.0 // indirect
github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 // indirect
github.com/Microsoft/go-winio v0.6.1 // indirect
github.com/ProtonMail/go-crypto v0.0.0-20230717121422-5aa5874ade95 // indirect
github.com/ProtonMail/go-crypto v0.0.0-20230518184743-7afd39499903 // indirect
github.com/acomagu/bufpipe v1.0.4 // indirect
github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 // indirect
github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 // indirect
@@ -97,7 +94,7 @@ require (
github.com/cloudflare/circl v1.3.3 // indirect
github.com/connesc/cipherio v0.2.1 // indirect
github.com/containerd/stargz-snapshotter/estargz v0.14.3 // indirect
github.com/couchbase/gocbcore/v10 v10.2.4 // indirect
github.com/couchbase/gocbcore/v10 v10.2.3 // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/dimchansky/utfbom v1.1.1 // indirect
github.com/docker/cli v23.0.5+incompatible // indirect
@@ -116,7 +113,7 @@ require (
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
github.com/golang/protobuf v1.5.3 // indirect
github.com/golang/snappy v0.0.4 // indirect
github.com/google/go-github/v53 v53.2.0 // indirect
github.com/google/go-github/v52 v52.0.0 // indirect
github.com/google/go-querystring v1.1.0 // indirect
github.com/google/pprof v0.0.0-20211214055906-6f57359322fd // indirect
github.com/google/s2a-go v0.1.4 // indirect
@@ -152,7 +149,7 @@ require (
github.com/prometheus/procfs v0.10.1 // indirect
github.com/rogpeppe/go-internal v1.10.0 // indirect
github.com/sirupsen/logrus v1.9.0 // indirect
github.com/skeema/knownhosts v1.2.0 // indirect
github.com/skeema/knownhosts v1.1.1 // indirect
github.com/therootcompany/xz v1.0.1 // indirect
github.com/ulikunitz/xz v0.5.10 // indirect
github.com/vbatts/tar-split v0.11.3 // indirect
@@ -173,10 +170,10 @@ require (
golang.org/x/tools v0.10.0 // indirect
golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect
google.golang.org/appengine v1.6.7 // indirect
google.golang.org/genproto v0.0.0-20230629202037-9506855d4529 // indirect
google.golang.org/genproto/googleapis/api v0.0.0-20230629202037-9506855d4529 // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20230706204954-ccb25ca9f130 // indirect
google.golang.org/grpc v1.56.2 // indirect
google.golang.org/genproto v0.0.0-20230530153820-e85fd2cbaebc // indirect
google.golang.org/genproto/googleapis/api v0.0.0-20230530153820-e85fd2cbaebc // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20230629202037-9506855d4529 // indirect
google.golang.org/grpc v1.56.1 // indirect
gopkg.in/warnings.v0 v0.1.2 // indirect
gopkg.in/yaml.v2 v2.4.0 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
+38 -44
View File
@@ -11,8 +11,8 @@ cloud.google.com/go v0.110.2 h1:sdFPBr6xG9/wkBbfhmUz/JmZC7X6LavQgcrVINrKiVA=
cloud.google.com/go v0.110.2/go.mod h1:k04UEeEtb6ZBRTv3dZz4CeJC3jKGxyhl0sAiVVquxiw=
cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o=
cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE=
cloud.google.com/go/compute v1.20.1 h1:6aKEtlUiwEpJzM001l0yFkpXmUVXaN8W+fbkb2AZNbg=
cloud.google.com/go/compute v1.20.1/go.mod h1:4tCnrn48xsqlwSAiLf1HXMQk8CONslYbdiEZc9FEIbM=
cloud.google.com/go/compute v1.19.3 h1:DcTwsFgGev/wV5+q8o2fzgcHOaac+DKGC91ZlvpsQds=
cloud.google.com/go/compute v1.19.3/go.mod h1:qxvISKp/gYnXkSAD1ppcSOveRAmzxicEv/JlizULFrI=
cloud.google.com/go/compute/metadata v0.2.0/go.mod h1:zFmK7XCadkQkj6TtorcaGlCW1hT1fIilQDwofLpJ20k=
cloud.google.com/go/compute/metadata v0.2.3 h1:mg4jlk7mCAj6xXp9UJ4fjI9VUI5rubuGBW5aJ7UnBMY=
cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA=
@@ -51,8 +51,6 @@ github.com/Azure/go-autorest/tracing v0.6.0 h1:TYi4+3m5t6K48TGI9AUdb+IzbnSxvnvUM
github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU=
github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 h1:mFRzDkZVAjdal+s7s0MwaRv9igoPqLRdzOLzw/8Xvq8=
github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358/go.mod h1:chxPXzSsl7ZWRAuOIE23GDNzjWuZquvFlgA8xmpunjU=
github.com/BobuSumisu/aho-corasick v1.0.3 h1:uuf+JHwU9CHP2Vx+wAy6jcksJThhJS9ehR8a+4nPE9g=
github.com/BobuSumisu/aho-corasick v1.0.3/go.mod h1:hm4jLcvZKI2vRF2WDU1N4p/jpWtpOzp3nLmi9AzX/XE=
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
github.com/BurntSushi/toml v1.2.1/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ=
github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
@@ -60,8 +58,8 @@ github.com/Microsoft/go-winio v0.5.2/go.mod h1:WpS1mjBmmwHBEWmogvA2mj8546UReBk4v
github.com/Microsoft/go-winio v0.6.1 h1:9/kr64B9VUZrLm5YYwbGtUJnMgqWVOdUAXu6Migciow=
github.com/Microsoft/go-winio v0.6.1/go.mod h1:LRdKpFKfdobln8UmuiYcKPot9D2v6svN5+sAH+4kjUM=
github.com/ProtonMail/go-crypto v0.0.0-20230217124315-7d5c6f04bbb8/go.mod h1:I0gYDMZ6Z5GRU7l58bNFSkPTFN6Yl12dsUlAZ8xy98g=
github.com/ProtonMail/go-crypto v0.0.0-20230717121422-5aa5874ade95 h1:KLq8BE0KwCL+mmXnjLWEAOYO+2l2AE4YMmqG1ZpZHBs=
github.com/ProtonMail/go-crypto v0.0.0-20230717121422-5aa5874ade95/go.mod h1:EjAoLdwvbIOoOQr3ihjnSoLZRtE8azugULFRteWMNc0=
github.com/ProtonMail/go-crypto v0.0.0-20230518184743-7afd39499903 h1:ZK3C5DtzV2nVAQTx5S5jQvMeDqWtD1By5mOoyY/xJek=
github.com/ProtonMail/go-crypto v0.0.0-20230518184743-7afd39499903/go.mod h1:8TI4H3IbrackdNgv+92dI+rhpCaLqM0IfpgCgenFvRE=
github.com/TheZeroSlave/zapsentry v1.17.0 h1:RIQCG89U7vWWZVmmCxeUz/g32WEcAYXUrXHoMlbSDmc=
github.com/TheZeroSlave/zapsentry v1.17.0/go.mod h1:D1YMfSuu6xnkhwFXxrronesmsiyDhIqo+86I3Ok+r64=
github.com/acomagu/bufpipe v1.0.4 h1:e3H4WUzM3npvo5uv95QuJM3cQspFNtFBzvJ2oNjKIDQ=
@@ -95,10 +93,9 @@ github.com/bodgit/sevenzip v1.3.0 h1:1ljgELgtHqvgIp8W8kgeEGHIWP4ch3xGI8uOBZgLVKY
github.com/bodgit/sevenzip v1.3.0/go.mod h1:omwNcgZTEooWM8gA/IJ2Nk/+ZQ94+GsytRzOJJ8FBlM=
github.com/bodgit/windows v1.0.0 h1:rLQ/XjsleZvx4fR1tB/UxQrK+SJ2OFHzfPjLWWOhDIA=
github.com/bodgit/windows v1.0.0/go.mod h1:a6JLwrB4KrTR5hBpp8FI9/9W9jJfeQ2h4XDXU74ZCdM=
github.com/bradleyfalzon/ghinstallation/v2 v2.6.0 h1:IRY7Xy588KylkoycsUhFpW7cdGpy5Y5BPsz4IfuJtGk=
github.com/bradleyfalzon/ghinstallation/v2 v2.6.0/go.mod h1:oQ3etOwN3TRH4EwgW5/7MxSVMGlMlzG/O8TU7eYdoSk=
github.com/bradleyfalzon/ghinstallation/v2 v2.4.0 h1:zYSzkoIwekCQAr6GT6KxISLt4YRS6kd4/ixfzMN+7yc=
github.com/bradleyfalzon/ghinstallation/v2 v2.4.0/go.mod h1:4MwZLSgBJJgg4i3nJwZJ95AMooSqN8fJDmegLVn9Q2U=
github.com/bwesterb/go-ristretto v1.2.0/go.mod h1:fUIoIZaG73pV5biE2Blr2xEzDoMj7NFEuV9ekS419A0=
github.com/bwesterb/go-ristretto v1.2.3/go.mod h1:fUIoIZaG73pV5biE2Blr2xEzDoMj7NFEuV9ekS419A0=
github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44=
@@ -122,8 +119,8 @@ github.com/containerd/stargz-snapshotter/estargz v0.14.3 h1:OqlDCK3ZVUO6C3B/5FSk
github.com/containerd/stargz-snapshotter/estargz v0.14.3/go.mod h1:KY//uOCIkSuNAHhJogcZtrNHdKrA99/FCCRjE3HD36o=
github.com/couchbase/gocb/v2 v2.6.3 h1:5RsMo+RRfK0mVxHLAfpBz3/tHlgXZb1WBNItLk9Ab+c=
github.com/couchbase/gocb/v2 v2.6.3/go.mod h1:yF5F6BHTZ/ZowhEuZbySbXrlI4rHd1TIhm5azOaMbJU=
github.com/couchbase/gocbcore/v10 v10.2.4 h1:TDTQ1mSBUw9eajuV71ZDJcLRbmfGcsvZV9SMeUDAVGc=
github.com/couchbase/gocbcore/v10 v10.2.4/go.mod h1:lYQIIk+tzoMcwtwU5GzPbDdqEkwkH3isI2rkSpfL0oM=
github.com/couchbase/gocbcore/v10 v10.2.3 h1:PEkRSNSkKjUBXx82Ucr094+anoiCG5GleOOQZOHo6D4=
github.com/couchbase/gocbcore/v10 v10.2.3/go.mod h1:lYQIIk+tzoMcwtwU5GzPbDdqEkwkH3isI2rkSpfL0oM=
github.com/couchbaselabs/gocaves/client v0.0.0-20230307083111-cc3960c624b1/go.mod h1:AVekAZwIY2stsJOMWLAS/0uA/+qdp7pjO8EHnl61QkY=
github.com/couchbaselabs/gocaves/client v0.0.0-20230404095311-05e3ba4f0259 h1:2TXy68EGEzIMHOx9UvczR5ApVecwCfQZ0LjkmwMI6g4=
github.com/couchbaselabs/gocaves/client v0.0.0-20230404095311-05e3ba4f0259/go.mod h1:AVekAZwIY2stsJOMWLAS/0uA/+qdp7pjO8EHnl61QkY=
@@ -181,8 +178,8 @@ github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376/go.mod h1:an3vInlBmS
github.com/go-git/go-billy/v5 v5.4.1 h1:Uwp5tDRkPr+l/TnbHOQzp+tmJfLceOlbVucgpTz8ix4=
github.com/go-git/go-billy/v5 v5.4.1/go.mod h1:vjbugF6Fz7JIflbVpl1hJsGjSHNltrSw45YK/ukIvQg=
github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20230305113008-0c11038e723f h1:Pz0DHeFij3XFhoBRGUDPzSJ+w2UcK5/0JvF8DRI58r8=
github.com/go-git/go-git/v5 v5.8.0 h1:Rc543s6Tyq+YcyPwZRvU4jzZGM8rB/wWu94TnTIYALQ=
github.com/go-git/go-git/v5 v5.8.0/go.mod h1:coJHKEOk5kUClpsNlXrUvPrDxY3w3gjHvhcZd8Fodw8=
github.com/go-git/go-git/v5 v5.7.0 h1:t9AudWVLmqzlo+4bqdf7GY+46SUuRsx59SboFxkq2aE=
github.com/go-git/go-git/v5 v5.7.0/go.mod h1:coJHKEOk5kUClpsNlXrUvPrDxY3w3gjHvhcZd8Fodw8=
github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
github.com/go-ldap/ldap/v3 v3.4.5 h1:ekEKmaDrpvR2yf5Nc/DClsGG9lAmdDixe44mLzlW5r8=
@@ -258,12 +255,12 @@ github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/
github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
github.com/google/go-containerregistry v0.15.2 h1:MMkSh+tjSdnmJZO7ljvEqV1DjfekB6VUEAZgy3a+TQE=
github.com/google/go-containerregistry v0.15.2/go.mod h1:wWK+LnOv4jXMM23IT/F1wdYftGWGr47Is8CG+pmHK1Q=
github.com/google/go-containerregistry v0.14.0 h1:z58vMqHxuwvAsVwvKEkmVBz2TlgBgH5k6koEXBtlYkw=
github.com/google/go-containerregistry v0.14.0/go.mod h1:aiJ2fp/SXvkWgmYHioXnbMdlgB8eXiiYOY55gfN91Wk=
github.com/google/go-github/v42 v42.0.0 h1:YNT0FwjPrEysRkLIiKuEfSvBPCGKphW5aS5PxwaoLec=
github.com/google/go-github/v42 v42.0.0/go.mod h1:jgg/jvyI0YlDOM1/ps6XYh04HNQ3vKf0CVko62/EhRg=
github.com/google/go-github/v53 v53.2.0 h1:wvz3FyF53v4BK+AsnvCmeNhf8AkTaeh2SoYu/XUvTtI=
github.com/google/go-github/v53 v53.2.0/go.mod h1:XhFRObz+m/l+UCm9b7KSIC3lT3NWSXGt7mOsAWEloao=
github.com/google/go-github/v52 v52.0.0 h1:uyGWOY+jMQ8GVGSX8dkSwCzlehU3WfdxQ7GweO/JP7M=
github.com/google/go-github/v52 v52.0.0/go.mod h1:WJV6VEEUPuMo5pXqqa2ZCZEdbQqua4zAk2MZTIo+m+4=
github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8=
github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU=
github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPgecwXBIDzw5no=
@@ -284,8 +281,8 @@ github.com/googleapis/enterprise-certificate-proxy v0.2.5 h1:UR4rDjcgpgEnqpIEvki
github.com/googleapis/enterprise-certificate-proxy v0.2.5/go.mod h1:RxW0N9901Cko1VOCW3SXCpWP+mlIEkk2tP7jnHy9a3w=
github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
github.com/googleapis/gax-go/v2 v2.12.0 h1:A+gCJKdRfqXkr+BIRGtZLibNXf0m1f9E4HG56etFpas=
github.com/googleapis/gax-go/v2 v2.12.0/go.mod h1:y+aIqrI5eb1YGMVJfuV3185Ts/D7qKpsEkdD5+I6QGU=
github.com/googleapis/gax-go/v2 v2.11.0 h1:9V9PWXEsWnPpQhu/PeQIkS4eGzMlTLGgt80cUUI8Ki4=
github.com/googleapis/gax-go/v2 v2.11.0/go.mod h1:DxmR61SGKkGLa2xigwuZIQpkCI2S5iydzRfb3peWZJI=
github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw=
github.com/h2non/parth v0.0.0-20190131123155-b4df798d6542 h1:2VTzZjLZBgl62/EtslCrtky5vbi9dd7HrQPQIx6wqiw=
@@ -385,6 +382,8 @@ github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaR
github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ=
github.com/paulbellamy/ratecounter v0.2.0 h1:2L/RhJq+HA8gBQImDXtLPrDXK5qAj6ozWVK/zFXVJGs=
github.com/paulbellamy/ratecounter v0.2.0/go.mod h1:Hfx1hDpSGoqxkVVpBi/IlYD7kChlfo5C6hzIHwPqfFE=
github.com/petar-dambovaliev/aho-corasick v0.0.0-20211021192214-5ab2d9280aa9 h1:lL+y4Xv20pVlCGyLzNHRC0I0rIHhIL1lTvHizoS/dU8=
github.com/petar-dambovaliev/aho-corasick v0.0.0-20211021192214-5ab2d9280aa9/go.mod h1:EHPiTAKtiFmrMldLUNswFwfZ2eJIYBHktdaUTZxYWRw=
github.com/pierrec/lz4/v4 v4.1.15 h1:MO0/ucJhngq7299dKLwIMtgTfbkoSPF6AoMYDd8Q4q0=
github.com/pierrec/lz4/v4 v4.1.15/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
github.com/pingcap/errors v0.11.4 h1:lFuQV/oaUMGcD2tqt+01ROSmJs75VG1ToEOkZIZ4nE4=
@@ -421,8 +420,8 @@ github.com/sergi/go-diff v1.3.1/go.mod h1:aMJSSKb2lpPvRNec0+w3fl7LP9IOFzdc9Pa4NF
github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0=
github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
github.com/skeema/knownhosts v1.2.0 h1:h9r9cf0+u7wSE+M183ZtMGgOJKiL96brpaz5ekfJCpM=
github.com/skeema/knownhosts v1.2.0/go.mod h1:g4fPeYpque7P0xefxtGzV81ihjC8sX2IqpAoNkjxbMo=
github.com/skeema/knownhosts v1.1.1 h1:MTk78x9FPgDFVFkDLTrsnnfCJl7g1C/nnKvePgrIngE=
github.com/skeema/knownhosts v1.1.1/go.mod h1:g4fPeYpque7P0xefxtGzV81ihjC8sX2IqpAoNkjxbMo=
github.com/smartystreets/assertions v1.0.1 h1:voD4ITNjPL5jjBfgR/r8fPIIBrliWrWHeiJApdr3r4w=
github.com/smartystreets/assertions v1.0.1/go.mod h1:kHHU4qYBaI3q23Pp3VPrmWhuIUrLW/7eUrw0BU5VaoM=
github.com/smartystreets/gunit v1.1.3 h1:32x+htJCu3aMswhPw3teoJ+PnWPONqdNgaGs6Qt8ZaU=
@@ -455,8 +454,8 @@ github.com/ulikunitz/xz v0.5.10/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0o
github.com/urfave/cli v1.22.12/go.mod h1:sSBEIC79qR6OvcmsD4U3KABeOTxDqQtdDnaFuUN30b8=
github.com/vbatts/tar-split v0.11.3 h1:hLFqsOLQ1SsppQNTMpkpPXClLDfC2A3Zgy9OUU+RVck=
github.com/vbatts/tar-split v0.11.3/go.mod h1:9QlHN18E+fEH7RdG+QAJJcuya3rqT7eXSTY7wGrAokY=
github.com/xanzy/go-gitlab v0.88.0 h1:9GHBrxyCUNZZNuAsbJ1NbEH6XAYsKyTn6NfE0wYO5SY=
github.com/xanzy/go-gitlab v0.88.0/go.mod h1:5ryv+MnpZStBH8I/77HuQBsMbBGANtVpLWC15qOjWAw=
github.com/xanzy/go-gitlab v0.86.0 h1:jR8V9cK9jXRQDb46KOB20NCF3ksY09luaG0IfXE6p7w=
github.com/xanzy/go-gitlab v0.86.0/go.mod h1:5ryv+MnpZStBH8I/77HuQBsMbBGANtVpLWC15qOjWAw=
github.com/xanzy/ssh-agent v0.3.3 h1:+/15pJfg/RsTxqYcX6fHqOXZwwMP+2VyYWJeWM2qQFM=
github.com/xanzy/ssh-agent v0.3.3/go.mod h1:6dzNDKs0J9rVPHPhaGCukekBHKqfl+L3KghI1Bc68Uw=
github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c=
@@ -502,7 +501,6 @@ golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5y
golang.org/x/crypto v0.0.0-20211215153901-e495a2d5b3d3/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
golang.org/x/crypto v0.0.0-20220314234659-1baeb1ce4c0b/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
golang.org/x/crypto v0.3.1-0.20221117191849-2c476679df9a/go.mod h1:hebNnKkNXi2UzZN1eVRvBB7co0a+JxK6XbPiWVs/3J4=
golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU=
golang.org/x/crypto v0.11.0 h1:6Ewdq3tDic1mg5xRO4milcWCfMVQhI4NkqWWvqejpuA=
golang.org/x/crypto v0.11.0/go.mod h1:xgJhtzW8F9jGdVFWZESrid1U1bjeNy4zgy5cRr/CIio=
@@ -565,10 +563,9 @@ golang.org/x/net v0.0.0-20210610132358-84b48f89b13b/go.mod h1:9nx3DQGgdP8bBQD5qx
golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY=
golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc=
golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg=
golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns=
golang.org/x/net v0.12.0 h1:cfawfvKITfUsFCeJIHJrbSxpeu/E81khclypR0GVT50=
golang.org/x/net v0.12.0/go.mod h1:zEVYFnQC7m/vmpQFELhcD1EWkZlX69l4oqgmer6hfKA=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
@@ -576,9 +573,9 @@ golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4Iltr
golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
golang.org/x/oauth2 v0.8.0/go.mod h1:yr7u4HXZRm1R1kBWqr/xKNqewf0plRYoB7sla+BCIXE=
golang.org/x/oauth2 v0.10.0 h1:zHCpF2Khkwy4mMB4bv0U37YtJdTGW8jI0glAApi0Kh8=
golang.org/x/oauth2 v0.10.0/go.mod h1:kTpgurOux7LqtuxjuyZa4Gj2gdezIt/jQtGnNFfypQI=
golang.org/x/oauth2 v0.7.0/go.mod h1:hPLQkd9LyjfXTiRohC/41GhcFqxisoUQ99sCUOHO9x4=
golang.org/x/oauth2 v0.9.0 h1:BPpt2kU7oMRq3kCHAA1tbSEshXRw1LpG2ztgDwrzuAs=
golang.org/x/oauth2 v0.9.0/go.mod h1:qYgFZaFiu6Wg24azG8bdV52QJXJGbZzIIsRCdVKzbLw=
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@@ -627,19 +624,17 @@ golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBc
golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220906165534-d0df966e6959/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.10.0 h1:SqMFp9UcQJZa+pmYuAKjd9xq1f0j5rLcDIk0mj4qAsA=
golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc=
golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U=
golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo=
golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY=
golang.org/x/term v0.10.0 h1:3R7pNqamzBraeqj/Tj8qt1aQ2HpmlC+Cx/qL/7hn4/c=
golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
@@ -649,7 +644,6 @@ golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ=
golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
@@ -704,8 +698,8 @@ google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsb
google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI=
google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI=
google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
google.golang.org/api v0.131.0 h1:AcgWS2edQ4chVEt/SxgDKubVu/9/idCJy00tBGuGB4M=
google.golang.org/api v0.131.0/go.mod h1:7vtkbKv2REjJbxmHSkBTBQ5LUGvPdAqjjvt84XAfhpA=
google.golang.org/api v0.130.0 h1:A50ujooa1h9iizvfzA4rrJr2B7uRmWexwbekQ2+5FPQ=
google.golang.org/api v0.130.0/go.mod h1:J/LCJMYSDFvAVREGCbrESb53n4++NMBDetSHGL5I5RY=
google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
@@ -728,12 +722,12 @@ google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvx
google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=
google.golang.org/genproto v0.0.0-20230629202037-9506855d4529 h1:9JucMWR7sPvCxUFd6UsOUNmA5kCcWOfORaT3tpAsKQs=
google.golang.org/genproto v0.0.0-20230629202037-9506855d4529/go.mod h1:xZnkP7mREFX5MORlOPEzLMr+90PPZQ2QWzrVTWfAq64=
google.golang.org/genproto/googleapis/api v0.0.0-20230629202037-9506855d4529 h1:s5YSX+ZH5b5vS9rnpGymvIyMpLRJizowqDlOuyjXnTk=
google.golang.org/genproto/googleapis/api v0.0.0-20230629202037-9506855d4529/go.mod h1:vHYtlOoi6TsQ3Uk2yxR7NI5z8uoV+3pZtR4jmHIkRig=
google.golang.org/genproto/googleapis/rpc v0.0.0-20230706204954-ccb25ca9f130 h1:2FZP5XuJY9zQyGM5N0rtovnoXjiMUEIUMvw0m9wlpLc=
google.golang.org/genproto/googleapis/rpc v0.0.0-20230706204954-ccb25ca9f130/go.mod h1:8mL13HKkDa+IuJ8yruA3ci0q+0vsUz4m//+ottjwS5o=
google.golang.org/genproto v0.0.0-20230530153820-e85fd2cbaebc h1:8DyZCyvI8mE1IdLy/60bS+52xfymkE72wv1asokgtao=
google.golang.org/genproto v0.0.0-20230530153820-e85fd2cbaebc/go.mod h1:xZnkP7mREFX5MORlOPEzLMr+90PPZQ2QWzrVTWfAq64=
google.golang.org/genproto/googleapis/api v0.0.0-20230530153820-e85fd2cbaebc h1:kVKPf/IiYSBWEWtkIn6wZXwWGCnLKcC8oWfZvXjsGnM=
google.golang.org/genproto/googleapis/api v0.0.0-20230530153820-e85fd2cbaebc/go.mod h1:vHYtlOoi6TsQ3Uk2yxR7NI5z8uoV+3pZtR4jmHIkRig=
google.golang.org/genproto/googleapis/rpc v0.0.0-20230629202037-9506855d4529 h1:DEH99RbiLZhMxrpEJCZ0A+wdTe0EOgou/poSLx9vWf4=
google.golang.org/genproto/googleapis/rpc v0.0.0-20230629202037-9506855d4529/go.mod h1:66JfowdXAEgad5O9NnYcsNPLCPZJD++2L9X0PCMODrA=
google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
@@ -746,8 +740,8 @@ google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTp
google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc=
google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU=
google.golang.org/grpc v1.45.0/go.mod h1:lN7owxKUQEqMfSyQikvvk5tf/6zMPsrK+ONuO11+0rQ=
google.golang.org/grpc v1.56.2 h1:fVRFRnXvU+x6C4IlHZewvJOVHoOv1TUuQyoRsYnB4bI=
google.golang.org/grpc v1.56.2/go.mod h1:I9bI3vqKfayGqPUAwGdOSu7kt6oIJLixfffKrpXqQ9s=
google.golang.org/grpc v1.56.1 h1:z0dNfjIl0VpaZ9iSVjA6daGatAYwPGstTjt5vkRMFkQ=
google.golang.org/grpc v1.56.1/go.mod h1:I9bI3vqKfayGqPUAwGdOSu7kt6oIJLixfffKrpXqQ9s=
google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
+3 -19
View File
@@ -62,31 +62,15 @@ If you think that something should be included outside of these guidelines, plea
1. Add the test secret to GCP Secrets. See [managing test secrets](#managing-test-secrets)
2. Update the pattern regex and keywords. Try iterating with [regex101.com](http://regex101.com/).
3. Update the verifier code to use a non-destructive API call that can determine whether the secret is valid or not.
* Make sure you understand [verification indeterminacy](#verification-indeterminacy).
4. Update the tests with these test cases at minimum:
1. Found and verified (using a credential loaded from GCP Secrets)
2. Found and unverified (determinately, i.e. the secret is invalid)
3. Found and unverified (indeterminately due to timeout)
4. Found and unverified (indeterminately due to an unexpected API response)
5. Not found
6. Any false positive cases that you come across
2. Found and unverified
3. Not found
4. Any false positive cases that you come across
5. Create a merge request for review. CI tests must be passing.
## Addendum
### Verification indeterminacy
There are two types of reasons that secret verification can fail:
* The candidate secret is not actually a valid secret.
* Something went wrong in the process unrelated to the candidate secret, such as a transient network error or an unexpected API response.
In Trufflehog parlance, the first type of verification response is called _determinate_ and the second type is called _indeterminate_. Verification code should distinguish between the two by returning an error object in the result struct **only** for indeterminate failures. In general, a verifier should return an error (indicating an indeterminate failure) in all cases that haven't been explicitly identified as determinate failure states.
For example, consider a hypothetical authentication endpoint that returns `200 OK` for valid credentials and `403 Forbidden` for invalid credentials. The verifier for this endpoint could make an HTTP request and use the response status code to decide what to return:
* A `200` response would indicate that verification succeeded. (Or maybe any `2xx` response.)
* A `403` response would indicate that verification failed **determinately** and no error object should be returned.
* Any other response would indicate that verification failed **indeterminately** and an error object should be returned.
### Managing Test Secrets
Do not embed test credentials in the test code. Instead, use GCP Secrets Manager.
+3 -19
View File
@@ -62,31 +62,15 @@ If you think that something should be included outside of these guidelines, plea
1. Create a [test secrets file, and export the variable](#using-a-test-secret-file)
2. Update the pattern regex and keywords. Try iterating with [regex101.com](http://regex101.com/).
3. Update the verifier code to use a non-destructive API call that can determine whether the secret is valid or not.
* Make sure you understand [verification indeterminacy](#verification-indeterminacy).
4. Update the tests with these test cases at minimum:
1. Found and verified (using a credential loaded from GCP Secrets)
2. Found and unverified (determinately, i.e. the secret is invalid)
3. Found and unverified (indeterminately due to timeout)
4. Found and unverified (indeterminately due to an unexpected API response)
5. Not found
6. Any false positive cases that you come across
2. Found and unverified
3. Not found
4. Any false positive cases that you come across
5. Create a pull request for review.
## Addendum
### Verification indeterminacy
There are two types of reasons that secret verification can fail:
* The candidate secret is not actually a valid secret.
* Something went wrong in the process unrelated to the candidate secret, such as a transient network error or an unexpected API response.
In Trufflehog parlance, the first type of verification response is called _determinate_ and the second type is called _indeterminate_. Verification code should distinguish between the two by returning an error object in the result struct **only** for indeterminate failures. In general, a verifier should return an error (indicating an indeterminate failure) in all cases that haven't been explicitly identified as determinate failure states.
For example, consider a hypothetical authentication endpoint that returns `200 OK` for valid credentials and `403 Forbidden` for invalid credentials. The verifier for this endpoint could make an HTTP request and use the response status code to decide what to return:
* A `200` response would indicate that verification succeeded. (Or maybe any `2xx` response.)
* A `403` response would indicate that verification failed **determinately** and no error object should be returned.
* Any other response would indicate that verification failed **indeterminately** and an error object should be returned.
### Using a test secret file
1. Create a file called `.env` with this env file format:
+4 -4
View File
@@ -34,14 +34,14 @@ func main() {
case "detector":
mustWriteTemplates([]templateJob{
{
TemplatePath: "pkg/detectors/alchemy/alchemy.go",
TemplatePath: "pkg/detectors/heroku/heroku.go",
WritePath: filepath.Join(folderPath(), nameLower+".go"),
ReplaceString: []string{"alchemy"},
ReplaceString: []string{"heroku"},
},
{
TemplatePath: "pkg/detectors/alchemy/alchemy_test.go",
TemplatePath: "pkg/detectors/heroku/heroku_test.go",
WritePath: filepath.Join(folderPath(), nameLower+"_test.go"),
ReplaceString: []string{"alchemy"},
ReplaceString: []string{"heroku"},
},
})
// case "source":
+1 -1
View File
@@ -207,7 +207,7 @@ func run(state overseer.State) {
if *profile {
go func() {
router := http.NewServeMux()
router.Handle("/debug/pprof/", http.DefaultServeMux)
router.Handle("/debug/pprof", http.DefaultServeMux)
router.Handle("/debug/fgprof", fgprof.Handler())
logger.Info("starting pprof and fgprof server on :18066 /debug/pprof and /debug/fgprof")
if err := http.ListenAndServe(":18066", router); err != nil {
+2 -26
View File
@@ -3,7 +3,6 @@ package common
import (
"crypto/tls"
"crypto/x509"
"io"
"net"
"net/http"
"strings"
@@ -76,14 +75,6 @@ func PinnedCertPool() *x509.CertPool {
return trustedCerts
}
type FakeTransport struct {
CreateResponse func(req *http.Request) (*http.Response, error)
}
func (t FakeTransport) RoundTrip(req *http.Request) (*http.Response, error) {
return t.CreateResponse(req)
}
type CustomTransport struct {
T http.RoundTripper
}
@@ -100,21 +91,6 @@ func NewCustomTransport(T http.RoundTripper) *CustomTransport {
return &CustomTransport{T}
}
func ConstantStatusHttpClient(statusCode int) *http.Client {
return &http.Client{
Timeout: DefaultResponseTimeout,
Transport: FakeTransport{
CreateResponse: func(req *http.Request) (*http.Response, error) {
return &http.Response{
Request: req,
Body: io.NopCloser(strings.NewReader("")),
StatusCode: statusCode,
}, nil
},
},
}
}
func PinnedRetryableHttpClient() *http.Client {
httpClient := retryablehttp.NewClient()
httpClient.Logger = nil
@@ -176,9 +152,9 @@ func SaneHttpClient() *http.Client {
}
// SaneHttpClientTimeOut adds a custom timeout for some scanners
func SaneHttpClientTimeOut(timeout time.Duration) *http.Client {
func SaneHttpClientTimeOut(timeOutSeconds int64) *http.Client {
httpClient := &http.Client{}
httpClient.Timeout = timeout
httpClient.Timeout = time.Second * time.Duration(timeOutSeconds)
httpClient.Transport = NewCustomTransport(nil)
return httpClient
}
+16 -10
View File
@@ -4,7 +4,6 @@ import (
"context"
"fmt"
"runtime/debug"
"sync"
"time"
"github.com/go-logr/logr"
@@ -19,19 +18,29 @@ var (
type Context interface {
context.Context
Logger() logr.Logger
Parent() context.Context
SetParent(ctx context.Context) Context
}
// CancelFunc is a type alias to context.CancelFunc to allow use as if they are
// the same types.
type CancelFunc = context.CancelFunc
// Parent returns the parent context.
func (l logCtx) Parent() context.Context {
return l.Context
}
// SetParent sets the parent context on the context.
func (l logCtx) SetParent(ctx context.Context) Context {
l.Context = ctx
return l
}
type CancelFunc context.CancelFunc
// logCtx implements Context.
type logCtx struct {
// Embed context.Context to get all methods for free.
context.Context
log logr.Logger
err *error
errLock *sync.Mutex
log logr.Logger
err *error
}
// Logger returns a structured logger.
@@ -147,11 +156,8 @@ func captureCancelCallstack(ctx logCtx, f context.CancelFunc) (Context, context.
if ctx.err == nil {
var err error
ctx.err = &err
ctx.errLock = &sync.Mutex{}
}
return ctx, func() {
ctx.errLock.Lock()
defer ctx.errLock.Unlock()
// We must check Err() before calling f() since f() sets the error.
// If there's already an error, do nothing special.
if ctx.Err() != nil {
-6
View File
@@ -212,9 +212,3 @@ func TestErrCallstackTimeoutCancel(t *testing.T) {
cancel()
assert.Equal(t, err, ctx.Err())
}
func TestRace(t *testing.T) {
_, cancel := WithCancel(Background())
go cancel()
cancel()
}
+3 -15
View File
@@ -2,7 +2,6 @@ package alchemy
import (
"context"
"fmt"
"net/http"
"regexp"
"strings"
@@ -12,15 +11,14 @@ import (
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
)
type Scanner struct {
client *http.Client
}
type Scanner struct{}
// Ensure the Scanner satisfies the interface at compile time.
var _ detectors.Detector = (*Scanner)(nil)
var (
defaultClient = common.SaneHttpClient()
client = common.SaneHttpClient()
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"alchemy"}) + `\b([0-9a-zA-Z]{23}_[0-9a-zA-Z]{8})\b`)
)
@@ -49,10 +47,6 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
}
if verify {
client := s.client
if client == nil {
client = defaultClient
}
req, err := http.NewRequestWithContext(ctx, "GET", "https://eth-mainnet.g.alchemy.com/v2/"+resMatch+"/getNFTs/?owner=vitalik.eth", nil)
if err != nil {
continue
@@ -67,13 +61,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
continue
}
if res.StatusCode != 401 {
s1.VerificationError = fmt.Errorf("request to %v returned unexpected status %d", res.Request.URL, res.StatusCode)
}
}
} else {
s1.VerificationError = err
}
}
+11 -52
View File
@@ -32,12 +32,11 @@ func TestAlchemy_FromChunk(t *testing.T) {
verify bool
}
tests := []struct {
name string
s Scanner
args args
want []detectors.Result
wantErr bool
wantVerificationErr bool
name string
s Scanner
args args
want []detectors.Result
wantErr bool
}{
{
name: "found, verified",
@@ -53,8 +52,7 @@ func TestAlchemy_FromChunk(t *testing.T) {
Verified: true,
},
},
wantErr: false,
wantVerificationErr: false,
wantErr: false,
},
{
name: "found, unverified",
@@ -70,8 +68,7 @@ func TestAlchemy_FromChunk(t *testing.T) {
Verified: false,
},
},
wantErr: false,
wantVerificationErr: false,
wantErr: false,
},
{
name: "not found",
@@ -81,48 +78,14 @@ func TestAlchemy_FromChunk(t *testing.T) {
data: []byte("You cannot find the secret within"),
verify: true,
},
want: nil,
wantErr: false,
wantVerificationErr: false,
},
{
name: "found, would be verified if not for timeout",
s: Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a alchemy secret %s within", secret)),
verify: true,
},
want: []detectors.Result{
{
DetectorType: detectorspb.DetectorType_Alchemy,
Verified: false,
},
},
wantErr: false,
wantVerificationErr: true,
},
{
name: "found, verified but unexpected api surface",
s: Scanner{client: common.ConstantStatusHttpClient(404)},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a alchemy secret %s within", secret)),
verify: true,
},
want: []detectors.Result{
{
DetectorType: detectorspb.DetectorType_Alchemy,
Verified: false,
},
},
wantErr: false,
wantVerificationErr: true,
want: nil,
wantErr: false,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
s := Scanner{}
got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
if (err != nil) != tt.wantErr {
t.Errorf("Alchemy.FromData() error = %v, wantErr %v", err, tt.wantErr)
return
@@ -132,10 +95,6 @@ func TestAlchemy_FromChunk(t *testing.T) {
t.Fatalf("no raw secret present: \n %+v", got[i])
}
got[i].Raw = nil
if (got[i].VerificationError != nil) != tt.wantVerificationErr {
t.Fatalf("verification error = %v, wantVerificationError %v", got[i].VerificationError, tt.wantVerificationErr)
}
got[i].VerificationError = nil
}
if diff := pretty.Compare(got, tt.want); diff != "" {
t.Errorf("Alchemy.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
-2
View File
@@ -184,8 +184,6 @@ func (s scanner) FromData(ctx context.Context, verify bool, data []byte) (result
"user_id": identityInfo.GetCallerIdentityResponse.GetCallerIdentityResult.UserID,
"arn": identityInfo.GetCallerIdentityResponse.GetCallerIdentityResult.Arn,
}
} else {
s1.VerificationError = err
}
res.Body.Close()
} else {
-95
View File
@@ -1,95 +0,0 @@
package dockerhub
import (
"context"
"fmt"
"io"
"net/http"
"regexp"
"strings"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
)
type Scanner struct{}
// Ensure the Scanner satisfies the interface at compile time.
var _ detectors.Detector = (*Scanner)(nil)
var (
client = common.SaneHttpClient()
// Can use email or username for login.
usernamePat = regexp.MustCompile(`(?im)(?:user|usr|-u)\S{0,40}?[:=\s]{1,3}[ '"=]?([a-zA-Z0-9]{4,40})\b`)
emailPat = regexp.MustCompile(common.EmailPattern)
// Can use password or personal access token (PAT) for login, but this scanner will only check for PATs.
accessTokenPat = regexp.MustCompile(`\bdckr_pat_([a-zA-Z0-9_-]){27}\b`)
)
// Keywords are used for efficiently pre-filtering chunks.
// Use identifiers in the secret preferably, or the provider name.
func (s Scanner) Keywords() []string {
return []string{"dckr_pat_"}
}
// FromData will find and optionally verify Dockerhub secrets in a given set of bytes.
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
dataStr := string(data)
emailMatches := emailPat.FindAllString(dataStr, -1)
dataStr = emailPat.ReplaceAllString(dataStr, "")
usernameMatches := usernamePat.FindAllStringSubmatch(dataStr, -1)
accessTokenMatches := accessTokenPat.FindAllString(dataStr, -1)
userMatches := emailMatches
for _, usernameMatch := range usernameMatches {
if len(usernameMatch) > 1 {
userMatches = append(userMatches, usernameMatch[1])
}
}
for _, resUserMatch := range userMatches {
for _, resAccessTokenMatch := range accessTokenMatches {
s1 := detectors.Result{
DetectorType: detectorspb.DetectorType_Dockerhub,
Raw: []byte(fmt.Sprintf("%s: %s", resUserMatch, resAccessTokenMatch)),
}
if verify {
payload := strings.NewReader(fmt.Sprintf(`{"username": "%s", "password": "%s"}`, resUserMatch, resAccessTokenMatch))
req, err := http.NewRequestWithContext(ctx, "GET", "https://hub.docker.com/v2/users/login", payload)
if err != nil {
continue
}
req.Header.Add("Content-Type", "application/json")
res, err := client.Do(req)
if err == nil {
defer res.Body.Close()
body, err := io.ReadAll(res.Body)
if err != nil {
continue
}
// Valid credentials can still return a 401 status code if 2FA is enabled
if (res.StatusCode >= 200 && res.StatusCode < 300) || (res.StatusCode == 401 && strings.Contains(string(body), "login_2fa_token")) {
s1.Verified = true
}
}
}
results = append(results, s1)
}
}
return results, nil
}
func (s Scanner) Type() detectorspb.DetectorType {
return detectorspb.DetectorType_Dockerhub
}
-138
View File
@@ -1,138 +0,0 @@
//go:build detectors
// +build detectors
package dockerhub
import (
"context"
"fmt"
"testing"
"time"
"github.com/kylelemons/godebug/pretty"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
)
func TestDockerhub_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors4")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
username := testSecrets.MustGetField("DOCKERHUB_USERNAME")
email := testSecrets.MustGetField("DOCKERHUB_EMAIL")
pat := testSecrets.MustGetField("DOCKERHUB_PAT")
inactivePat := testSecrets.MustGetField("DOCKERHUB_INACTIVE_PAT")
type args struct {
ctx context.Context
data []byte
verify bool
}
tests := []struct {
name string
s Scanner
args args
want []detectors.Result
wantErr bool
}{
{
name: "found, verified",
s: Scanner{},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("docker login -u %s -p %s", username, pat)),
verify: true,
},
want: []detectors.Result{
{
DetectorType: detectorspb.DetectorType_Dockerhub,
Verified: true,
},
},
wantErr: false,
},
{
name: "found, verified (email)",
s: Scanner{},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("docker login -u %s -p %s", email, pat)),
verify: true,
},
want: []detectors.Result{
{
DetectorType: detectorspb.DetectorType_Dockerhub,
Verified: true,
},
},
wantErr: false,
},
{
name: "found, unverified",
s: Scanner{},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("docker login -u %s -p %s", username, inactivePat)), // the secret would satisfy the regex but not pass validation
verify: true,
},
want: []detectors.Result{
{
DetectorType: detectorspb.DetectorType_Dockerhub,
Verified: false,
},
},
wantErr: false,
},
{
name: "not found",
s: Scanner{},
args: args{
ctx: context.Background(),
data: []byte("You cannot find the secret within"),
verify: true,
},
want: nil,
wantErr: false,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
s := Scanner{}
got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
if (err != nil) != tt.wantErr {
t.Errorf("Dockerhub.FromData() error = %v, wantErr %v", err, tt.wantErr)
return
}
for i := range got {
if len(got[i].Raw) == 0 {
t.Fatalf("no raw secret present: \n %+v", got[i])
}
got[i].Raw = nil
}
if diff := pretty.Compare(got, tt.want); diff != "" {
t.Errorf("Dockerhub.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
}
})
}
}
func BenchmarkFromData(benchmark *testing.B) {
ctx := context.Background()
s := Scanner{}
for name, data := range detectors.MustGetBenchmarkData() {
benchmark.Run(name, func(b *testing.B) {
for n := 0; n < b.N; n++ {
_, err := s.FromData(ctx, false, data)
if err != nil {
b.Fatal(err)
}
}
})
}
}
+1 -2
View File
@@ -7,7 +7,6 @@ import (
"net/http"
"regexp"
"strings"
"time"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
@@ -20,7 +19,7 @@ type Scanner struct{}
var _ detectors.Detector = (*Scanner)(nil)
var (
client = common.SaneHttpClientTimeOut(5 * time.Second)
client = common.SaneHttpClientTimeOut(5)
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"getemail"}) + `\b([a-zA-Z0-9-]{20})\b`)
@@ -1,8 +1,8 @@
package envoyapikey
package ibmcloud
import (
"context"
"io"
"fmt"
"net/http"
"regexp"
"strings"
@@ -21,16 +21,16 @@ var (
client = common.SaneHttpClient()
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"envoy"}) + `\b([a-zA-Z0-9]{220})\b`)
keyPat = regexp.MustCompile(`\b([A-Za-z0-9_-]{44})\b`)
)
// Keywords are used for efficiently pre-filtering chunks.
// Use identifiers in the secret preferably, or the provider name.
func (s Scanner) Keywords() []string {
return []string{"envoy"}
return []string{"ibm"}
}
// FromData will find and optionally verify Envoy secrets in a given set of bytes.
// FromData will find and optionally verify IBM Cloud secrets in a given set of bytes.
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
dataStr := string(data)
@@ -43,27 +43,22 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
resMatch := strings.TrimSpace(match[1])
s1 := detectors.Result{
DetectorType: detectorspb.DetectorType_EnvoyApiKey,
DetectorType: detectorspb.DetectorType_Ibmcloud,
Raw: []byte(resMatch),
}
if verify {
req, err := http.NewRequestWithContext(ctx, "GET", "https://api.envoy.com/v1/locations", nil)
req, err := http.NewRequestWithContext(ctx, "GET", "https://api.ibmcloud.com/apps", nil)
if err != nil {
continue
}
req.Header.Add("Accept", "application/vnd.envoy+json; version=3")
req.Header.Add("X-Api-Key", resMatch)
req.Header.Add("Accept", "application/vnd.ibmcloud+json; version=3")
req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", resMatch))
res, err := client.Do(req)
if err == nil {
defer res.Body.Close()
body, _ := io.ReadAll(res.Body)
// Invalid API keys can also return status code 200, so check for presence of 'status 401' in response body.
if res.StatusCode >= 200 && res.StatusCode < 300 || res.StatusCode == 403 {
if !strings.Contains(string(body), `"status":401`) {
s1.Verified = true
}
if res.StatusCode >= 200 && res.StatusCode < 300 {
s1.Verified = true
} else {
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
@@ -80,5 +75,5 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
}
func (s Scanner) Type() detectorspb.DetectorType {
return detectorspb.DetectorType_EnvoyApiKey
return detectorspb.DetectorType_Ibmcloud
}
@@ -1,7 +1,7 @@
//go:build detectors
// +build detectors
package envoyapikey
package ibmcloud
import (
"context"
@@ -10,22 +10,21 @@ import (
"time"
"github.com/kylelemons/godebug/pretty"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
)
func TestEnvoyapikey_FromChunk(t *testing.T) {
func TestIbmcloud_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors4")
testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
secret := testSecrets.MustGetField("ENVOYAPIKEY")
inactiveSecret := testSecrets.MustGetField("ENVOYAPIKEY_INACTIVE")
secret := testSecrets.MustGetField("IBM_CLOUD_KEY")
inactiveSecret := testSecrets.MustGetField("IBM_CLOUD_KEY_INACTIVE")
type args struct {
ctx context.Context
@@ -44,12 +43,12 @@ func TestEnvoyapikey_FromChunk(t *testing.T) {
s: Scanner{},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a envoyapikey secret %s within", secret)),
data: []byte(fmt.Sprintf("You can find a ibmcloud secret %s within", secret)),
verify: true,
},
want: []detectors.Result{
{
DetectorType: detectorspb.DetectorType_EnvoyApiKey,
DetectorType: detectorspb.DetectorType_IbmClou,
Verified: true,
},
},
@@ -60,12 +59,12 @@ func TestEnvoyapikey_FromChunk(t *testing.T) {
s: Scanner{},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a envoyapikey secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
data: []byte(fmt.Sprintf("You can find a ibmcloud secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
verify: true,
},
want: []detectors.Result{
{
DetectorType: detectorspb.DetectorType_EnvoyApiKey,
DetectorType: detectorspb.DetectorType_Ibmcloud,
Verified: false,
},
},
@@ -88,7 +87,7 @@ func TestEnvoyapikey_FromChunk(t *testing.T) {
s := Scanner{}
got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
if (err != nil) != tt.wantErr {
t.Errorf("Envoyapikey.FromData() error = %v, wantErr %v", err, tt.wantErr)
t.Errorf("Ibmcloud.FromData() error = %v, wantErr %v", err, tt.wantErr)
return
}
for i := range got {
@@ -98,7 +97,7 @@ func TestEnvoyapikey_FromChunk(t *testing.T) {
got[i].Raw = nil
}
if diff := pretty.Compare(got, tt.want); diff != "" {
t.Errorf("Envoyapikey.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
t.Errorf("Ibmcloud.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
}
})
}
+11 -28
View File
@@ -86,14 +86,7 @@ matchLoop:
}
ctx, cancel := context.WithTimeout(ctx, 5*time.Second)
defer cancel()
pingRes := j.ping(ctx)
s.Verified = pingRes.err == nil
// If there's a ping error that is marked as "determinate" we throw it away. We do this because this was the
// behavior before tri-state verification was introduced and preserving it allows us to gradually migrate
// detectors to use tri-state verification.
if pingRes.err != nil && !pingRes.determinate {
s.VerificationError = pingRes.err
}
s.Verified = j.ping(ctx)
// TODO: specialized redaction
}
@@ -108,15 +101,15 @@ matchLoop:
}
func tryRedactAnonymousJDBC(conn string) string {
if s, ok := tryRedactBasicAuth(conn); ok {
return s
}
if s, ok := tryRedactURLParams(conn); ok {
return s
}
if s, ok := tryRedactODBC(conn); ok {
return s
}
if s, ok := tryRedactBasicAuth(conn); ok {
return s
}
if s, ok := tryRedactRegex(conn); ok {
return s
}
@@ -165,8 +158,8 @@ func tryRedactODBC(conn string) (string, bool) {
var found bool
var newParams []string
for _, param := range strings.Split(conn, ";") {
key, val, isKvp := strings.Cut(param, "=")
if isKvp && strings.Contains(strings.ToLower(key), "pass") {
key, val, _ := strings.Cut(param, "=")
if strings.Contains(strings.ToLower(key), "pass") {
newParams = append(newParams, key+"="+strings.Repeat("*", len(val)))
found = true
continue
@@ -205,13 +198,8 @@ var supportedSubprotocols = map[string]func(string) (jdbc, error){
"sqlserver": parseSqlServer,
}
type pingResult struct {
err error
determinate bool
}
type jdbc interface {
ping(context.Context) pingResult
ping(context.Context) bool
}
func newJDBC(conn string) (jdbc, error) {
@@ -232,16 +220,11 @@ func newJDBC(conn string) (jdbc, error) {
return parser(subname)
}
func ping(ctx context.Context, driverName string, isDeterminate func(error) bool, candidateConns ...string) pingResult {
var indeterminateErrors []error
for _, c := range candidateConns {
err := pingErr(ctx, driverName, c)
if err == nil || isDeterminate(err) {
return pingResult{err, true}
}
indeterminateErrors = append(indeterminateErrors, err)
func ping(ctx context.Context, driverName, conn string) bool {
if err := pingErr(ctx, driverName, conn); err != nil {
return false
}
return pingResult{errors.Join(indeterminateErrors...), false}
return true
}
func pingErr(ctx context.Context, driverName, conn string) error {
@@ -105,22 +105,6 @@ func TestJdbcVerified(t *testing.T) {
},
wantErr: false,
},
{
name: "sql server verified",
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("jdbc:sqlserver://odbc:server=localhost;database=%s;password=%s", sqlServerDatabase, sqlServerPass)),
verify: true,
},
want: []detectors.Result{
{
DetectorType: detectorspb.DetectorType_JDBC,
Verified: true,
Redacted: "jdbc:sqlserver://odbc:server=localhost;database=master;password=**************",
},
},
wantErr: false,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
-46
View File
@@ -5,7 +5,6 @@ package jdbc
import (
"context"
"github.com/stretchr/testify/assert"
"os"
"testing"
@@ -208,51 +207,6 @@ func TestJdbc_FromDataWithIgnorePattern(t *testing.T) {
}
}
func TestJdbc_Redact(t *testing.T) {
tests := []struct {
name string
conn string
want string
}{
{
name: "basic auth'",
conn: "//user:secret@tcp(127.0.0.1:3306)/",
want: "//user:******@tcp(127.0.0.1:3306)/",
},
{
name: "basic auth including raw string 'pass'",
conn: "//wrongUser:wrongPass@tcp(127.0.0.1:3306)/",
want: "//wrongUser:*********@tcp(127.0.0.1:3306)/",
},
{
name: "basic auth including raw string 'pass' with unfortunate db name",
conn: "//wrongUser:wrongPass@tcp(127.0.0.1:3306)/passwords",
want: "//wrongUser:*********@tcp(127.0.0.1:3306)/passwords",
},
{
name: "url param-style",
conn: "jdbc:postgresql://localhost:5432/foo?sslmode=disable&password=p@ssw04d",
want: "jdbc:postgresql://localhost:5432/foo?sslmode=disable&password=********",
},
{
name: "odbc-style without server",
conn: "//odbc:server=localhost;user id=sa;database=master;password=/p?s=sw&rd",
want: "//odbc:server=localhost;user id=sa;database=master;password=**********",
},
{
name: "odbc-style with server",
conn: "jdbc:sqlserver://a.b.c.net;database=database-name;spring.datasource.password=super-secret-password",
want: "jdbc:sqlserver://a.b.c.net;database=database-name;spring.datasource.password=*********************",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got := tryRedactAnonymousJDBC(tt.conn)
assert.Equal(t, tt.want, got)
})
}
}
func BenchmarkFromData(benchmark *testing.B) {
ctx := context.Background()
s := Scanner{}
+19 -28
View File
@@ -3,8 +3,9 @@ package jdbc
import (
"context"
"errors"
"github.com/go-sql-driver/mysql"
"strings"
_ "github.com/go-sql-driver/mysql"
)
type mysqlJDBC struct {
@@ -15,40 +16,30 @@ type mysqlJDBC struct {
params string
}
func (s *mysqlJDBC) ping(ctx context.Context) pingResult {
return ping(ctx, "mysql", isMySQLErrorDeterminate,
s.conn,
buildMySQLConnectionString(s.host, s.database, s.userPass, s.params),
buildMySQLConnectionString(s.host, "", s.userPass, s.params))
func (s *mysqlJDBC) ping(ctx context.Context) bool {
if ping(ctx, "mysql", s.conn) {
return true
}
// try building connection string (should be same as s.conn though)
if ping(ctx, "mysql", s.build()) {
return true
}
// try removing database
s.database = ""
return ping(ctx, "mysql", s.build())
}
func buildMySQLConnectionString(host, database, userPass, params string) string {
conn := host + "/" + database
if userPass != "" {
conn = userPass + "@" + conn
func (s *mysqlJDBC) build() string {
conn := s.host + "/" + s.database
if s.userPass != "" {
conn = s.userPass + "@" + conn
}
if params != "" {
conn = conn + "?" + params
if s.params != "" {
conn = conn + "?" + s.params
}
return conn
}
func isMySQLErrorDeterminate(err error) bool {
// MySQL error numbers from https://dev.mysql.com/doc/mysql-errors/8.0/en/server-error-reference.html
if mySQLErr, isMySQLErr := err.(*mysql.MySQLError); isMySQLErr {
switch mySQLErr.Number {
case 1044:
// User access denied to a particular database
return false // "Indeterminate" so that other connection variations will be tried
case 1045:
// User access denied
return true
}
}
return false
}
func parseMySQL(subname string) (jdbc, error) {
// expected form: [subprotocol:]//[user:password@]HOST[/DB][?key=val[&key=val]]
hostAndDB, params, _ := strings.Cut(subname, "?")
+19 -28
View File
@@ -21,54 +21,45 @@ const (
)
func TestMySQL(t *testing.T) {
type result struct {
parseErr bool
pingOk bool
pingDeterminate bool
}
tests := []struct {
input string
want result
input string
wantErr bool
wantPing bool
}{
{
input: "",
want: result{parseErr: true},
input: "",
wantErr: true,
},
{
input: "//" + mysqlUser + ":" + mysqlPass + "@tcp(127.0.0.1:3306)/" + mysqlDatabase,
want: result{pingOk: true, pingDeterminate: true},
input: "//" + mysqlUser + ":" + mysqlPass + "@tcp(127.0.0.1:3306)/" + mysqlDatabase,
wantPing: true,
},
{
input: "//wrongUser:wrongPass@tcp(127.0.0.1:3306)/" + mysqlDatabase,
want: result{pingOk: false, pingDeterminate: true},
input: "//wrongUser:wrongPass@tcp(127.0.0.1:3306)/" + mysqlDatabase,
wantPing: false,
},
{
input: "//" + mysqlUser + ":wrongPass@tcp(127.0.0.1:3306)/" + mysqlDatabase,
want: result{pingOk: false, pingDeterminate: true},
input: "//" + mysqlUser + ":wrongPass@tcp(127.0.0.1:3306)/" + mysqlDatabase,
wantPing: false,
},
{
input: "//" + mysqlUser + ":" + mysqlPass + "@tcp(127.0.0.1:3306)/",
want: result{pingOk: true, pingDeterminate: true},
input: "//" + mysqlUser + ":" + mysqlPass + "@tcp(127.0.0.1:3306)/",
wantPing: true,
},
{
input: "//" + mysqlUser + ":" + mysqlPass + "@tcp(127.0.0.1:3306)/wrongDB",
want: result{pingOk: true, pingDeterminate: true},
input: "//" + mysqlUser + ":" + mysqlPass + "@tcp(127.0.0.1:3306)/wrongDB",
wantPing: true,
},
}
for _, tt := range tests {
t.Run(tt.input, func(t *testing.T) {
j, err := parseMySQL(tt.input)
if err != nil {
got := result{parseErr: true}
assert.Equal(t, tt.want, got)
if tt.wantErr {
assert.Error(t, err)
return
}
pr := j.ping(context.Background())
got := result{pingOk: pr.err == nil, pingDeterminate: pr.determinate}
assert.Equal(t, tt.want, got)
assert.NoError(t, err)
assert.Equal(t, tt.wantPing, j.ping(context.Background()))
})
}
}
+32 -51
View File
@@ -4,8 +4,9 @@ import (
"context"
"errors"
"fmt"
"github.com/lib/pq"
"strings"
_ "github.com/lib/pq"
)
type postgresJDBC struct {
@@ -13,35 +14,37 @@ type postgresJDBC struct {
params map[string]string
}
func (s *postgresJDBC) ping(ctx context.Context) pingResult {
// It is crucial that we try to build a connection string ourselves before using the one we found. This is because
// if the found connection string doesn't include a username, the driver will attempt to connect using the current
// user's name, which will fail in a way that looks like a determinate failure, thus terminating the waterfall. In
// contrast, when we build a connection string ourselves, if there's no username, we try 'postgres' instead, which
// actually has a chance of working.
return ping(ctx, "postgres", isPostgresErrorDeterminate,
buildPostgresConnectionString(s.params, true),
buildPostgresConnectionString(s.params, false),
s.conn,
"postgres://"+s.conn)
}
func isPostgresErrorDeterminate(err error) bool {
// Postgres codes from https://www.postgresql.org/docs/current/errcodes-appendix.html
if pqErr, isPostgresError := err.(*pq.Error); isPostgresError {
switch pqErr.Code {
case "28P01":
// Invalid username/password
return true
case "3D000":
// Unknown database
return false // "Indeterminate" so that other connection variations will be tried
case "3F000":
// Unknown schema
return false // "Indeterminate" so that other connection variations will be tried
}
func (s *postgresJDBC) ping(ctx context.Context) bool {
// try the provided connection string directly
if ping(ctx, "postgres", s.conn) {
return true
}
// try as a URL
if ping(ctx, "postgres", "postgres://"+s.conn) {
return true
}
// build a connection string
data := map[string]string{
// default user
"user": "postgres",
}
for key, val := range s.params {
if key == "host" {
if h, p, found := strings.Cut(val, ":"); found {
data["host"] = h
data["port"] = p
continue
}
}
data[key] = val
}
if ping(ctx, "postgres", joinKeyValues(data, " ")) {
return true
}
if s.params["dbname"] != "" {
delete(s.params, "dbname")
return s.ping(ctx)
}
return false
}
@@ -76,25 +79,3 @@ func parsePostgres(subname string) (jdbc, error) {
return &postgresJDBC{subname[2:], params}, nil
}
func buildPostgresConnectionString(params map[string]string, includeDbName bool) string {
data := map[string]string{
// default user
"user": "postgres",
}
for key, val := range params {
if key == "host" {
if h, p, found := strings.Cut(val, ":"); found {
data["host"] = h
data["port"] = p
continue
}
}
if key == "dbname" && !includeDbName {
continue
}
data[key] = val
}
return joinKeyValues(data, " ")
}
+21 -34
View File
@@ -20,62 +20,49 @@ const (
)
func TestPostgres(t *testing.T) {
type result struct {
parseErr bool
pingOk bool
pingDeterminate bool
}
tests := []struct {
input string
want result
input string
wantErr bool
wantPing bool
}{
{
input: "//localhost:5432/foo?sslmode=disable&password=" + postgresPass,
want: result{pingOk: true, pingDeterminate: true},
input: "//localhost:5432/foo?sslmode=disable&password=" + postgresPass,
wantPing: true,
},
{
input: "//localhost:5432/foo?sslmode=disable&user=" + postgresUser + "&password=" + postgresPass,
want: result{pingOk: true, pingDeterminate: true},
input: "//localhost:5432/foo?sslmode=disable&user=" + postgresUser + "&password=" + postgresPass,
wantPing: true,
},
{
input: "//localhost/foo?sslmode=disable&port=5432&password=" + postgresPass,
want: result{pingOk: true, pingDeterminate: true},
input: "//localhost/foo?sslmode=disable&port=5432&password=" + postgresPass,
wantPing: true,
},
{
input: "//localhost:5432/foo?password=" + postgresPass,
want: result{pingOk: false, pingDeterminate: false},
input: "//localhost:5432/foo?password=" + postgresPass,
wantPing: false,
},
{
input: "//localhost:5432/foo?sslmode=disable&password=foo",
want: result{pingOk: false, pingDeterminate: true},
input: "//localhost:5432/foo?sslmode=disable&password=foo",
wantPing: false,
},
{
input: "//localhost:5432/foo?sslmode=disable&user=foo&password=" + postgresPass,
want: result{pingOk: false, pingDeterminate: true},
input: "//localhost:5432/foo?sslmode=disable&user=foo&password=" + postgresPass,
wantPing: false,
},
{
input: "//badhost:5432/foo?sslmode=disable&user=foo&password=" + postgresPass,
want: result{pingOk: false, pingDeterminate: false},
},
{
input: "invalid",
want: result{parseErr: true},
input: "invalid",
wantErr: true,
},
}
for _, tt := range tests {
t.Run(tt.input, func(t *testing.T) {
j, err := parsePostgres(tt.input)
if err != nil {
got := result{parseErr: true}
assert.Equal(t, tt.want, got)
if tt.wantErr {
assert.Error(t, err)
return
}
pr := j.ping(context.Background())
got := result{pingOk: pr.err == nil, pingDeterminate: pr.determinate}
assert.Equal(t, tt.want, got)
assert.NoError(t, err)
assert.Equal(t, tt.wantPing, j.ping(context.Background()))
})
}
}
+3 -9
View File
@@ -14,18 +14,12 @@ type sqliteJDBC struct {
testing bool
}
var cannotVerifySqliteError error = errors.New("sqlite credentials cannot be verified")
func (s *sqliteJDBC) ping(ctx context.Context) pingResult {
func (s *sqliteJDBC) ping(ctx context.Context) bool {
if !s.testing {
// sqlite is not a networked database, so we cannot verify
return pingResult{cannotVerifySqliteError, true}
return false
}
return ping(ctx, "sqlite3", isSqliteErrorDeterminate, s.filename)
}
func isSqliteErrorDeterminate(err error) bool {
return true
return ping(ctx, "sqlite3", s.filename)
}
func parseSqlite(subname string) (jdbc, error) {
+1 -1
View File
@@ -39,7 +39,7 @@ func TestParseSqlite(t *testing.T) {
assert.Error(t, err)
} else {
assert.NoError(t, err)
assert.True(t, j.ping(context.Background()).err == nil)
assert.True(t, j.ping(context.Background()))
}
})
}
+10 -19
View File
@@ -3,8 +3,9 @@ package jdbc
import (
"context"
"errors"
mssql "github.com/denisenkom/go-mssqldb"
"strings"
_ "github.com/denisenkom/go-mssqldb"
)
type sqlServerJDBC struct {
@@ -12,25 +13,15 @@ type sqlServerJDBC struct {
params map[string]string
}
func (s *sqlServerJDBC) ping(ctx context.Context) pingResult {
return ping(ctx, "mssql", isSqlServerErrorDeterminate,
joinKeyValues(s.params, ";"),
s.conn,
"sqlserver://"+s.conn)
}
func isSqlServerErrorDeterminate(err error) bool {
// Error numbers from https://learn.microsoft.com/en-us/sql/relational-databases/errors-events/database-engine-events-and-errors?view=sql-server-ver16
if mssqlError, isMssqlError := err.(mssql.Error); isMssqlError {
switch mssqlError.Number {
case 18456:
// Login failed
// This is a determinate failure iff we tried to use a real user
return mssqlError.Message != "login error: Login failed for user ''."
}
func (s *sqlServerJDBC) ping(ctx context.Context) bool {
if ping(ctx, "mssql", s.conn) {
return true
}
return false
if ping(ctx, "mssql", joinKeyValues(s.params, ";")) {
return true
}
// try URL format
return ping(ctx, "mssql", "sqlserver://"+s.conn)
}
func parseSqlServer(subname string) (jdbc, error) {
@@ -21,50 +21,33 @@ const (
)
func TestSqlServer(t *testing.T) {
type result struct {
parseErr bool
pingOk bool
pingDeterminate bool
}
tests := []struct {
input string
want result
input string
wantErr bool
wantPing bool
}{
{
input: "",
want: result{parseErr: true},
input: "",
wantErr: true,
},
{
input: "//server=localhost;user id=sa;database=master;password=" + sqlServerPass,
want: result{pingOk: true, pingDeterminate: true},
input: "//odbc:server=localhost;user id=sa;database=master;password=" + sqlServerPass,
wantPing: true,
},
{
input: "//server=badhost;user id=sa;database=master;password=" + sqlServerPass,
want: result{pingOk: false, pingDeterminate: false},
},
{
input: "//localhost;database=master;spring.datasource.password=" + sqlServerPass,
want: result{pingOk: true, pingDeterminate: true},
},
{
input: "//localhost;database=master;spring.datasource.password=badpassword",
want: result{pingOk: false, pingDeterminate: true},
input: "//localhost;database= master;spring.datasource.password=" + sqlServerPass,
wantPing: true,
},
}
for _, tt := range tests {
t.Run(tt.input, func(t *testing.T) {
j, err := parseSqlServer(tt.input)
if err != nil {
got := result{parseErr: true}
assert.Equal(t, tt.want, got)
if tt.wantErr {
assert.Error(t, err)
return
}
pr := j.ping(context.Background())
got := result{pingOk: pr.err == nil, pingDeterminate: pr.determinate}
assert.Equal(t, tt.want, got)
assert.NoError(t, err)
assert.Equal(t, tt.wantPing, j.ping(context.Background()))
})
}
}
+2 -2
View File
@@ -20,13 +20,13 @@ var (
client = common.SaneHttpClient()
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"launchdarkly", "launch_darkly"}) + `\b([a-z0-9-]{40})\b`)
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"launchdarkly"}) + `\b([a-z0-9-]{40})\b`)
)
// Keywords are used for efficiently pre-filtering chunks.
// Use identifiers in the secret preferably, or the provider name.
func (s Scanner) Keywords() []string {
return []string{"launchdarkly", "launch_darkly"}
return []string{"launchdarkly"}
}
// FromData will find and optionally verify LaunchDarkly secrets in a given set of bytes.
@@ -6,7 +6,6 @@ import (
"net/http"
"regexp"
"strings"
"time"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
@@ -19,7 +18,7 @@ type Scanner struct{}
var _ detectors.Detector = (*Scanner)(nil)
var (
client = common.SaneHttpClientTimeOut(5 * time.Second)
client = common.SaneHttpClientTimeOut(5)
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
keyPat = regexp.MustCompile(`(https:\/\/[a-zA-Z-0-9]+\.webhook\.office\.com\/webhookb2\/[a-zA-Z-0-9]{8}-[a-zA-Z-0-9]{4}-[a-zA-Z-0-9]{4}-[a-zA-Z-0-9]{4}-[a-zA-Z-0-9]{12}\@[a-zA-Z-0-9]{8}-[a-zA-Z-0-9]{4}-[a-zA-Z-0-9]{4}-[a-zA-Z-0-9]{4}-[a-zA-Z-0-9]{12}\/IncomingWebhook\/[a-zA-Z-0-9]{32}\/[a-zA-Z-0-9]{8}-[a-zA-Z-0-9]{4}-[a-zA-Z-0-9]{4}-[a-zA-Z-0-9]{4}-[a-zA-Z-0-9]{12})`)
+1 -1
View File
@@ -21,7 +21,7 @@ var _ detectors.Detector = (*Scanner)(nil)
var (
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
keyPat = regexp.MustCompile(`\b(mongodb(\+srv)?://[\S]{3,50}:([\S]{3,88})@[-.%\w\/:]+)\b`)
keyPat = regexp.MustCompile(`\b(mongodb(\+srv)?://[\S]{3,50}:([\S]{3,50})@[-.%\w\/:]+)\b`)
// TODO: Add support for sharded cluster, replica set and Atlas Deployment.
)
+1
View File
@@ -36,6 +36,7 @@ func (s Scanner) Keywords() []string {
// FromData will find and optionally verify NpmToken secrets in a given set of bytes.
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
dataStr := string(data)
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
for _, match := range matches {
if len(match) != 2 {
+2 -2
View File
@@ -19,8 +19,8 @@ type Scanner struct{}
var _ detectors.Detector = (*Scanner)(nil)
var (
domainPat = regexp.MustCompile(`\b[a-z0-9-]{1,40}\.okta(?:preview|-emea){0,1}\.com\b`)
tokenPat = regexp.MustCompile(`\b00[a-zA-Z0-9_-]{40}\b`)
domainPat = regexp.MustCompile(`[a-z0-9-]{1,40}\.okta(?:preview|-emea){0,1}\.com`)
tokenPat = regexp.MustCompile(`00[a-zA-Z0-9_-]{40}`)
// TODO: Oauth client secrets
)
+1 -2
View File
@@ -7,7 +7,6 @@ import (
"net/http"
"regexp"
"strings"
"time"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
@@ -20,7 +19,7 @@ type Scanner struct{}
var _ detectors.Detector = (*Scanner)(nil)
var (
client = common.SaneHttpClientTimeOut(10 * time.Second)
client = common.SaneHttpClientTimeOut(10)
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"scrapersite"}) + `\b([a-zA-Z0-9]{45})\b`)
@@ -7,7 +7,6 @@ import (
"net/http"
"regexp"
"strings"
"time"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
@@ -20,7 +19,7 @@ type Scanner struct{}
var _ detectors.Detector = (*Scanner)(nil)
var (
client = common.SaneHttpClientTimeOut(10 * time.Second)
client = common.SaneHttpClientTimeOut(10)
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"screenshotlayer"}) + `\b([a-zA-Z0-9_]{32})\b`)
@@ -1,108 +0,0 @@
package trufflehogenterprise
import (
"context"
"fmt"
"net/http"
"regexp"
"strings"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
)
type Scanner struct{}
// Ensure the Scanner satisfies the interface at compile time.
var _ detectors.Detector = (*Scanner)(nil)
var (
client = common.SaneHttpClient()
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
keyPat = regexp.MustCompile(`\bthog-key-[0-9a-f]{16}\b`)
secretPat = regexp.MustCompile(`\bthog-secret-[0-9a-f]{32}\b`)
hostnamePat = regexp.MustCompile(`\b[a-z]+-[a-z]+-[a-z]+\.[a-z][0-9]\.[a-z]+\.trufflehog\.org\b`)
)
// Keywords are used for efficiently pre-filtering chunks.
// Use identifiers in the secret preferably, or the provider name.
func (s Scanner) Keywords() []string {
return []string{"thog"}
}
// FromData will find and optionally verify Trufflehog Enterprise secrets in a given set of bytes.
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
dataStr := string(data)
keyMatches := keyPat.FindAllStringSubmatch(dataStr, -1)
secretMatches := secretPat.FindAllStringSubmatch(dataStr, -1)
hostnameMatches := hostnamePat.FindAllStringSubmatch(dataStr, -1)
for _, keyMatch := range keyMatches {
if len(keyMatch) != 1 {
continue
}
resKeyMatch := strings.TrimSpace(keyMatch[0])
for _, secretMatch := range secretMatches {
if len(secretMatch) != 1 {
continue
}
resSecretMatch := strings.TrimSpace(secretMatch[0])
for _, hostnameMatch := range hostnameMatches {
if len(hostnameMatch) != 1 {
continue
}
resHostnameMatch := strings.TrimSpace(hostnameMatch[0])
s1 := detectors.Result{
DetectorType: detectorspb.DetectorType_TrufflehogEnterprise,
Raw: []byte(resKeyMatch),
}
if verify {
endpoint := fmt.Sprintf("https://%s/api/v1/sources", resHostnameMatch)
req, err := http.NewRequestWithContext(ctx, "GET", endpoint, nil)
if err != nil {
continue
}
req.Header.Add("Accept", "application/vnd.trufflehogenterprise+json; version=3")
req.Header.Add("X-Thog-Secret", resSecretMatch)
req.Header.Add("X-Thog-Key", resKeyMatch)
res, err := client.Do(req)
if err == nil {
verifiedBodyResponse, err := common.ResponseContainsSubstring(res.Body, "data")
if err != nil {
return nil, err
}
defer res.Body.Close()
if res.StatusCode >= 200 && res.StatusCode < 300 && verifiedBodyResponse {
s1.Verified = true
} else {
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
if detectors.IsKnownFalsePositive(resSecretMatch, detectors.DefaultFalsePositives, true) {
continue
}
}
}
}
results = append(results, s1)
}
}
}
return results, nil
}
func (s Scanner) Type() detectorspb.DetectorType {
return detectorspb.DetectorType_TrufflehogEnterprise
}
@@ -1,121 +0,0 @@
//go:build detectors
// +build detectors
package trufflehogenterprise
import (
"context"
"fmt"
"testing"
"time"
"github.com/kylelemons/godebug/pretty"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
)
func TestTrufflehogenterprise_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
host := testSecrets.MustGetField("THOG_HOSTNAME")
key := testSecrets.MustGetField("THOG_WEB_KEY")
secret := testSecrets.MustGetField("THOG_WEB_SECRET")
inactiveSecret := testSecrets.MustGetField("THOG_WEB_INACTIVE_SECRET")
type args struct {
ctx context.Context
data []byte
verify bool
}
tests := []struct {
name string
s Scanner
args args
want []detectors.Result
wantErr bool
}{
{
name: "found, verified",
s: Scanner{},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a thog secret %s for %s with key %s within", secret, host, key)),
verify: true,
},
want: []detectors.Result{
{
DetectorType: detectorspb.DetectorType_TrufflehogEnterprise,
Verified: true,
},
},
wantErr: false,
},
{
name: "found, unverified",
s: Scanner{},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a trufflehogenterprise secret %s for %s with key %s within but not valid", inactiveSecret, host, key)), // the secret would satisfy the regex but not pass validation
verify: true,
},
want: []detectors.Result{
{
DetectorType: detectorspb.DetectorType_TrufflehogEnterprise,
Verified: false,
},
},
wantErr: false,
},
{
name: "not found",
s: Scanner{},
args: args{
ctx: context.Background(),
data: []byte("You cannot find the secret within"),
verify: true,
},
want: nil,
wantErr: false,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
s := Scanner{}
got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
if (err != nil) != tt.wantErr {
t.Errorf("Trufflehogenterprise.FromData() error = %v, wantErr %v", err, tt.wantErr)
return
}
for i := range got {
if len(got[i].Raw) == 0 {
t.Fatalf("no raw secret present: \n %+v", got[i])
}
got[i].Raw = nil
}
if diff := pretty.Compare(got, tt.want); diff != "" {
t.Errorf("Trufflehogenterprise.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
}
})
}
}
func BenchmarkFromData(benchmark *testing.B) {
ctx := context.Background()
s := Scanner{}
for name, data := range detectors.MustGetBenchmarkData() {
benchmark.Run(name, func(b *testing.B) {
for n := 0; n < b.N; n++ {
_, err := s.FromData(ctx, false, data)
if err != nil {
b.Fatal(err)
}
}
})
}
}
+9 -1
View File
@@ -3,7 +3,9 @@ package twilio
import (
"context"
"net/http"
"net/url"
"regexp"
"strings"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
@@ -51,8 +53,14 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
}
if verify {
form := url.Values{}
form.Add("FriendlyName", "MyServiceName")
req, err := http.NewRequestWithContext(
ctx, "GET", "https://verify.twilio.com/v2/Services", nil)
ctx, "POST", "https://verify.twilio.com/v2/Services",
strings.NewReader(form.Encode()),
)
if err != nil {
continue
}
+1 -2
View File
@@ -53,8 +53,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
password := match[1]
// Skip findings where the password only has "*" characters, this is a redacted password
// Also include the url encoded "*" characters: "%2A"
if strings.Trim(password, "*") == "" || strings.Trim(password, "%2A") == "" {
if strings.Trim(password, "*") == "" {
continue
}
+2 -13
View File
@@ -31,14 +31,14 @@ func TestURI_FromChunk(t *testing.T) {
s: Scanner{},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a uri secret %s within", "https://user:pass@httpwatch.com/httpgallery/authentication/authenticatedimage/default.aspx")),
data: []byte(fmt.Sprintf("You can find a uri secret %s within", "https://user:pass@www.httpwatch.com/httpgallery/authentication/authenticatedimage/default.aspx")),
verify: true,
},
want: []detectors.Result{
{
DetectorType: detectorspb.DetectorType_URI,
Verified: false,
Redacted: "https://user:********@httpwatch.com",
Redacted: "https://user:********@www.httpwatch.com",
},
},
wantErr: false,
@@ -87,17 +87,6 @@ func TestURI_FromChunk(t *testing.T) {
},
wantErr: false,
},
{
name: "nothing found, password was redacted two different ways",
s: Scanner{},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("Both %s and %s have been redacted within", "https://httpwatch::********@www.httpwatch.com/httpgallery/authentication/authenticatedimage/default.aspx?foo=bar", "https://httpwatch::%2A%2A%2A%2A%2A%2A@www.httpwatch.com/httpgallery/authentication/authenticatedimage/default.aspx?foo=bar")),
verify: true,
},
want: []detectors.Result{},
wantErr: false,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
+1 -2
View File
@@ -6,7 +6,6 @@ import (
"net/http"
"regexp"
"strings"
"time"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
@@ -19,7 +18,7 @@ type Scanner struct{}
var _ detectors.Detector = (*Scanner)(nil)
var (
client = common.SaneHttpClientTimeOut(5 * time.Second)
client = common.SaneHttpClientTimeOut(5)
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"zenserp"}) + `\b([0-9a-z-]{36})\b`)
-7
View File
@@ -1,8 +1,6 @@
package engine
import (
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/envoyapikey"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/abbysale"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/abuseipdb"
@@ -161,7 +159,6 @@ import (
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/convertkit"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/convier"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/copper"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/couchbase"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/countrylayer"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/courier"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/coveralls"
@@ -201,7 +198,6 @@ import (
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/disqus"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/ditto"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/dnscheck"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/dockerhub"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/docparser"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/documo"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/docusign"
@@ -1527,9 +1523,6 @@ func DefaultDetectors() []detectors.Detector {
prefect.Scanner{},
buildkitev2.Scanner{},
opsgenie.Scanner{},
dockerhub.Scanner{},
couchbase.Scanner{},
envoyapikey.Scanner{},
}
}
+34 -23
View File
@@ -10,7 +10,7 @@ import (
"sync/atomic"
"time"
ahocorasick "github.com/BobuSumisu/aho-corasick"
ahocorasick "github.com/petar-dambovaliev/aho-corasick"
"golang.org/x/sync/errgroup"
"google.golang.org/protobuf/proto"
@@ -43,7 +43,7 @@ type Engine struct {
// prefilter is a ahocorasick struct used for doing efficient string
// matching given a set of words (keywords from the rules in the config)
prefilter ahocorasick.Trie
prefilter ahocorasick.AhoCorasick
}
type EngineOption func(*Engine)
@@ -113,7 +113,6 @@ func Start(ctx context.Context, options ...EngineOption) *Engine {
chunks: make(chan *sources.Chunk),
results: make(chan detectors.ResultWithMetadata),
detectorAvgTime: sync.Map{},
sourcesWg: &errgroup.Group{},
}
for _, option := range options {
@@ -128,8 +127,10 @@ func Start(ctx context.Context, options ...EngineOption) *Engine {
}
ctx.Logger().V(2).Info("engine started", "workers", e.concurrency)
// Limit number of concurrent goroutines dedicated to chunking a source.
e.sourcesWg.SetLimit(e.concurrency)
sourcesWg, egCtx := errgroup.WithContext(ctx)
sourcesWg.SetLimit(e.concurrency)
e.sourcesWg = sourcesWg
ctx.SetParent(egCtx)
if len(e.decoders) == 0 {
e.decoders = decoders.DefaultDecoders()
@@ -145,16 +146,18 @@ func Start(ctx context.Context, options ...EngineOption) *Engine {
// on keywords
keywords := []string{}
for _, d := range e.detectors[false] {
for _, kw := range d.Keywords() {
keywords = append(keywords, strings.ToLower(kw))
}
keywords = append(keywords, d.Keywords()...)
}
for _, d := range e.detectors[true] {
for _, kw := range d.Keywords() {
keywords = append(keywords, strings.ToLower(kw))
}
keywords = append(keywords, d.Keywords()...)
}
e.prefilter = *ahocorasick.NewTrieBuilder().AddStrings(keywords).Build()
builder := ahocorasick.NewAhoCorasickBuilder(ahocorasick.Opts{
AsciiCaseInsensitive: true,
MatchOnlyWholeWords: false,
MatchKind: ahocorasick.LeftMostLongestMatch,
DFA: true,
})
e.prefilter = builder.Build(keywords)
ctx.Logger().Info("loaded decoders", "count", len(e.decoders))
ctx.Logger().Info("loaded detectors",
@@ -182,7 +185,7 @@ func Start(ctx context.Context, options ...EngineOption) *Engine {
for i := 0; i < e.concurrency; i++ {
e.workersWg.Add(1)
go func() {
defer common.Recover(ctx)
defer common.RecoverWithExit(ctx)
defer e.workersWg.Done()
e.detectorWorker(ctx)
}()
@@ -295,8 +298,8 @@ func (e *Engine) detectorWorker(ctx context.Context) {
}
// build a map of all keywords that were matched in the chunk
for _, m := range e.prefilter.MatchString(strings.ToLower(string(decoded.Data))) {
matchedKeywords[strings.ToLower(m.MatchString())] = struct{}{}
for _, m := range e.prefilter.FindAll(string(decoded.Data)) {
matchedKeywords[strings.ToLower(string(decoded.Data[m.Start():m.End()]))] = struct{}{}
}
for verify, detectorsSet := range e.detectors {
@@ -375,23 +378,31 @@ func (e *Engine) detectorWorker(ctx context.Context) {
}
}
// SupportsLineNumbers determines if a line number can be found for a source type.
func SupportsLineNumbers(sourceType sourcespb.SourceType) bool {
switch sourceType {
case sourcespb.SourceType_SOURCE_TYPE_GIT,
// lineNumberSupportedSources is a list of sources that support line numbers.
// It is stored this way because slice consts are not supported.
func lineNumberSupportedSources() []sourcespb.SourceType {
return []sourcespb.SourceType{
sourcespb.SourceType_SOURCE_TYPE_GIT,
sourcespb.SourceType_SOURCE_TYPE_GITHUB,
sourcespb.SourceType_SOURCE_TYPE_GITLAB,
sourcespb.SourceType_SOURCE_TYPE_BITBUCKET,
sourcespb.SourceType_SOURCE_TYPE_GERRIT,
sourcespb.SourceType_SOURCE_TYPE_GITHUB_UNAUTHENTICATED_ORG,
sourcespb.SourceType_SOURCE_TYPE_PUBLIC_GIT,
sourcespb.SourceType_SOURCE_TYPE_FILESYSTEM:
return true
default:
return false
sourcespb.SourceType_SOURCE_TYPE_FILESYSTEM,
}
}
// SupportsLineNumbers determines if a line number can be found for a source type.
func SupportsLineNumbers(sourceType sourcespb.SourceType) bool {
for _, i := range lineNumberSupportedSources() {
if i == sourceType {
return true
}
}
return false
}
// FragmentLineOffset sets the line number for a provided source chunk with a given detector result.
func FragmentLineOffset(chunk *sources.Chunk, result *detectors.Result) (int64, bool) {
lines := bytes.Split(chunk.Data, []byte("\n"))
-28
View File
@@ -5,7 +5,6 @@ import (
"github.com/trufflesecurity/trufflehog/v3/pkg/decoders"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
)
@@ -73,30 +72,3 @@ func TestDefaultDecoders(t *testing.T) {
t.Errorf("DefaultDecoders() = %v, expected UTF8 decoder to be first", ds)
}
}
func TestSupportsLineNumbers(t *testing.T) {
testCases := []struct {
name string
input sourcespb.SourceType
expected bool
}{
{"Git source type", sourcespb.SourceType_SOURCE_TYPE_GIT, true},
{"Github source type", sourcespb.SourceType_SOURCE_TYPE_GITHUB, true},
{"Gitlab source type", sourcespb.SourceType_SOURCE_TYPE_GITLAB, true},
}
for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
if result := SupportsLineNumbers(tc.input); result != tc.expected {
t.Errorf("Expected %v for input %v, got %v", tc.expected, tc.input, result)
}
})
}
}
func BenchmarkSupportsLineNumbersLoop(b *testing.B) {
sourceType := sourcespb.SourceType_SOURCE_TYPE_GITHUB
for i := 0; i < b.N; i++ {
_ = SupportsLineNumbers(sourceType)
}
}
+1 -16
View File
@@ -28,22 +28,6 @@ func (e *Engine) ScanFileSystem(ctx context.Context, c sources.FilesystemConfig)
}
fileSystemSource := filesystem.Source{}
fileSystemSource.WithFilter(c.Filter)
// TODO: This is how it would be used; everything after these comments would be removed.
// handle, err := e.sourceManager.Enroll(ctx, "trufflehog - filesystem", fileSystemSource.Type(),
// func(ctx context.Context, jobID, sourceID int64) (sources.Source, error) {
// fileSystemSource := filesystem.Source{}
// fileSystemSource.WithFilter(c.Filter)
// if err := fileSystemSource.Init(ctx, "trufflehog - filesystem", jobID, sourceID, true, &conn, runtime.NumCPU()); err != nil {
// return nil, err
// }
// return &fileSystemSource, nil
// })
// if err != nil {
// return err
// }
// return e.sourceManager.ScheduleRun(ctx, handle)
ctx = context.WithValues(ctx,
"source_type", fileSystemSource.Type().String(),
@@ -53,6 +37,7 @@ func (e *Engine) ScanFileSystem(ctx context.Context, c sources.FilesystemConfig)
if err != nil {
return errors.WrapPrefix(err, "could not init filesystem source", 0)
}
fileSystemSource.WithFilter(c.Filter)
e.sourcesWg.Go(func() error {
defer common.RecoverWithExit(ctx)
err := fileSystemSource.Chunks(ctx, e.ChunksChan())
+7 -17
View File
@@ -7,7 +7,6 @@ import (
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/context"
"github.com/trufflesecurity/trufflehog/v3/pkg/decoders"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/source_metadatapb"
"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
"github.com/trufflesecurity/trufflehog/v3/pkg/sources/git"
@@ -16,7 +15,6 @@ import (
type expResult struct {
B string
LineNumber int64
Verified bool
}
func TestGitEngine(t *testing.T) {
@@ -40,16 +38,16 @@ func TestGitEngine(t *testing.T) {
for tName, tTest := range map[string]testProfile{
"all_secrets": {
expected: map[string]expResult{
"70001020fab32b1fcf2f1f0e5c66424eae649826": {"AKIAXYZDQCEN4B6JSJQI", 2, true},
"84e9c75e388ae3e866e121087ea2dd45a71068f2": {"AKIAILE3JG6KMS3HZGCA", 4, true},
"8afb0ecd4998b1179e428db5ebbcdc8221214432": {"369963c1434c377428ca8531fbc46c0c43d037a0", 3, false},
"27fbead3bf883cdb7de9d7825ed401f28f9398f1": {"ffc7e0f9400fb6300167009e42d2f842cd7956e2", 7, false},
"70001020fab32b1fcf2f1f0e5c66424eae649826": {"AKIAXYZDQCEN4B6JSJQI", 2},
"84e9c75e388ae3e866e121087ea2dd45a71068f2": {"AKIAILE3JG6KMS3HZGCA", 4},
"8afb0ecd4998b1179e428db5ebbcdc8221214432": {"369963c1434c377428ca8531fbc46c0c43d037a0", 3},
"27fbead3bf883cdb7de9d7825ed401f28f9398f1": {"ffc7e0f9400fb6300167009e42d2f842cd7956e2", 7},
},
filter: common.FilterEmpty(),
},
"base_commit": {
expected: map[string]expResult{
"70001020fab32b1fcf2f1f0e5c66424eae649826": {"AKIAXYZDQCEN4B6JSJQI", 2, true},
"70001020fab32b1fcf2f1f0e5c66424eae649826": {"AKIAXYZDQCEN4B6JSJQI", 2},
},
filter: common.FilterEmpty(),
base: "2f251b8c1e72135a375b659951097ec7749d4af9",
@@ -59,12 +57,8 @@ func TestGitEngine(t *testing.T) {
e := Start(ctx,
WithConcurrency(1),
WithDecoders(decoders.DefaultDecoders()...),
WithDetectors(true, DefaultDetectors()...),
WithDetectors(false, DefaultDetectors()...),
)
// Make the channels buffered so Finish returns.
e.chunks = make(chan *sources.Chunk, 10)
e.results = make(chan detectors.ResultWithMetadata, 10)
cfg := sources.GitConfig{
RepoPath: path,
HeadRef: tTest.branch,
@@ -79,8 +73,7 @@ func TestGitEngine(t *testing.T) {
logFatalFunc := func(_ error, _ string, _ ...any) {
t.Fatalf("error logging function should not have been called")
}
// Wait for all the chunks to be processed.
e.Finish(ctx, logFatalFunc)
go e.Finish(ctx, logFatalFunc)
resultCount := 0
for result := range e.ResultsChan() {
switch meta := result.SourceMetadata.GetData().(type) {
@@ -91,9 +84,6 @@ func TestGitEngine(t *testing.T) {
if tTest.expected[meta.Git.Commit].LineNumber != result.SourceMetadata.GetGit().Line {
t.Errorf("%s: unexpected line number. Got: %d, Expected: %d", tName, result.SourceMetadata.GetGit().Line, tTest.expected[meta.Git.Commit].LineNumber)
}
if tTest.expected[meta.Git.Commit].Verified != result.Verified {
t.Errorf("%s: unexpected verification. Got: %v, Expected: %v", tName, result.Verified, tTest.expected[meta.Git.Commit].Verified)
}
}
resultCount++
+114 -369
View File
@@ -51,49 +51,6 @@ type Parser struct {
dateFormat string
}
type ParseState int
const (
Initial ParseState = iota
CommitLine
MergeLine
AuthorLine
DateLine
MessageStartLine
MessageLine
MessageEndLine
DiffLine
ModeLine
IndexLine
FromFileLine
ToFileLine
BinaryFileLine
HunkLineNumberLine
HunkContentLine
ParseFailure
)
func (state ParseState) String() string {
return [...]string{
"Initial",
"CommitLine",
"MergeLine",
"AuthorLine",
"DateLine",
"MessageStartLine",
"MessageLine",
"MessageEndLine",
"DiffLine",
"ModeLine",
"IndexLine",
"FromFileLine",
"ToFileLine",
"BinaryFileLine",
"HunkLineNumberLine",
"HunkContentLine",
}[state]
}
// WithMaxDiffSize sets maxDiffSize option. Diffs larger than maxDiffSize will
// be truncated.
func WithMaxDiffSize(maxDiffSize int) Option {
@@ -180,7 +137,7 @@ func (c *Parser) RepoPath(ctx context.Context, source string, head string, abbre
cmd.Env = append(cmd.Env, fmt.Sprintf("GIT_DIR=%s", filepath.Join(absPath, ".git")))
}
return c.executeCommand(ctx, cmd, false)
return c.executeCommand(ctx, cmd)
}
// Staged parses the output of the `git diff` command for the `source` path.
@@ -195,11 +152,11 @@ func (c *Parser) Staged(ctx context.Context, source string) (chan Commit, error)
cmd.Env = append(cmd.Env, fmt.Sprintf("GIT_DIR=%s", filepath.Join(absPath, ".git")))
}
return c.executeCommand(ctx, cmd, true)
return c.executeCommand(ctx, cmd)
}
// executeCommand runs an exec.Cmd, reads stdout and stderr, and waits for the Cmd to complete.
func (c *Parser) executeCommand(ctx context.Context, cmd *exec.Cmd, isStaged bool) (chan Commit, error) {
func (c *Parser) executeCommand(ctx context.Context, cmd *exec.Cmd) (chan Commit, error) {
commitChan := make(chan Commit, 64)
stdOut, err := cmd.StdoutPipe()
@@ -224,7 +181,7 @@ func (c *Parser) executeCommand(ctx context.Context, cmd *exec.Cmd, isStaged boo
}()
go func() {
c.FromReader(ctx, stdOut, commitChan, isStaged)
c.FromReader(ctx, stdOut, commitChan)
if err := cmd.Wait(); err != nil {
ctx.Logger().V(2).Info("Error waiting for git command to complete.", "error", err)
}
@@ -233,15 +190,14 @@ func (c *Parser) executeCommand(ctx context.Context, cmd *exec.Cmd, isStaged boo
return commitChan, nil
}
func (c *Parser) FromReader(ctx context.Context, stdOut io.Reader, commitChan chan Commit, isStaged bool) {
func (c *Parser) FromReader(ctx context.Context, stdOut io.Reader, commitChan chan Commit) {
outReader := bufio.NewReader(stdOut)
var (
currentCommit *Commit
currentDiff *Diff
totalLogSize int
currentCommit *Commit
currentDiff *Diff
recentlyPassedAuthor bool
totalLogSize int
)
var latestState = Initial
defer common.RecoverWithExit(ctx)
defer close(commitChan)
@@ -249,16 +205,12 @@ func (c *Parser) FromReader(ctx context.Context, stdOut io.Reader, commitChan ch
if common.IsDone(ctx) {
break
}
line, err := outReader.ReadBytes([]byte("\n")[0])
if err != nil && len(line) == 0 {
break
}
switch {
case isCommitLine(isStaged, latestState, line):
latestState = CommitLine
case isCommitLine(line):
// If there is a currentDiff, add it to currentCommit.
if currentDiff != nil && currentDiff.Content.Len() > 0 {
currentCommit.Diffs = append(currentCommit.Diffs, *currentDiff)
@@ -278,33 +230,16 @@ func (c *Parser) FromReader(ctx context.Context, stdOut io.Reader, commitChan ch
if len(line) >= 47 {
currentCommit.Hash = string(line[7:47])
}
case isMergeLine(isStaged, latestState, line):
latestState = MergeLine
case isAuthorLine(isStaged, latestState, line):
latestState = AuthorLine
case isAuthorLine(line):
currentCommit.Author = strings.TrimRight(string(line[8:]), "\n")
case isDateLine(isStaged, latestState, line):
latestState = DateLine
recentlyPassedAuthor = true
case isDateLine(line):
date, err := time.Parse(c.dateFormat, strings.TrimSpace(string(line[6:])))
if err != nil {
ctx.Logger().V(2).Info("Could not parse date from git stream.", "error", err)
}
currentCommit.Date = date
case isMessageStartLine(isStaged, latestState, line):
latestState = MessageStartLine
// NoOp
case isMessageLine(isStaged, latestState, line):
latestState = MessageLine
currentCommit.Message.Write(line[4:])
case isMessageEndLine(isStaged, latestState, line):
latestState = MessageEndLine
// NoOp
case isDiffLine(isStaged, latestState, line):
latestState = DiffLine
case isDiffLine(line):
// This should never be nil, but check in case the stdin stream is messed up.
if currentCommit == nil {
currentCommit = &Commit{}
@@ -332,29 +267,27 @@ func (c *Parser) FromReader(ctx context.Context, stdOut io.Reader, commitChan ch
}
}
currentDiff = &Diff{}
case isModeLine(isStaged, latestState, line):
latestState = ModeLine
case isModeLine(line):
// NoOp
case isIndexLine(isStaged, latestState, line):
latestState = IndexLine
case isIndexLine(line):
recentlyPassedAuthor = false
// NoOp
case isBinaryLine(isStaged, latestState, line):
latestState = BinaryFileLine
case isPlusFileLine(line):
currentDiff.PathB = strings.TrimRight(strings.TrimRight(string(line[6:]), "\n"), "\t") // Trim the newline and tab characters. https://github.com/trufflesecurity/trufflehog/issues/1060
case isMinusFileLine(line):
// NoOp
case isPlusDiffLine(line):
currentDiff.Content.Write(line[1:])
case isMinusDiffLine(line):
// NoOp. We only care about additions.
case (isMessageLine(line) && recentlyPassedAuthor):
currentCommit.Message.Write(line[4:])
case isContextDiffLine(line):
currentDiff.Content.Write([]byte("\n"))
case isBinaryLine(line):
currentDiff.IsBinary = true
currentDiff.PathB = pathFromBinaryLine(line)
case isFromFileLine(isStaged, latestState, line):
latestState = FromFileLine
// NoOp
case isToFileLine(isStaged, latestState, line):
latestState = ToFileLine
// TODO: Is this fix still required?
currentDiff.PathB = strings.TrimRight(strings.TrimRight(string(line[6:]), "\n"), "\t") // Trim the newline and tab characters. https://github.com/trufflesecurity/trufflehog/issues/1060
case isHunkLineNumberLine(isStaged, latestState, line):
latestState = HunkLineNumberLine
// TODO: Is it still necessary to check whether the currentDiff is nil?
case isLineNumberDiffLine(line):
if currentDiff != nil && currentDiff.Content.Len() > 0 {
currentCommit.Diffs = append(currentCommit.Diffs, *currentDiff)
}
@@ -372,54 +305,7 @@ func (c *Parser) FromReader(ctx context.Context, stdOut io.Reader, commitChan ch
currentDiff.LineStart = lineStart
}
}
case isHunkContextLine(isStaged, latestState, line):
if latestState != HunkContentLine {
latestState = HunkContentLine
}
// TODO: Why do we care about this? It creates empty lines in the diff. If there are no plusLines, it's just newlines.
currentDiff.Content.Write([]byte("\n"))
case isHunkPlusLine(isStaged, latestState, line):
if latestState != HunkContentLine {
latestState = HunkContentLine
}
currentDiff.Content.Write(line[1:])
case isHunkMinusLine(isStaged, latestState, line):
if latestState != HunkContentLine {
latestState = HunkContentLine
}
// NoOp. We only care about additions.
case isHunkNewlineWarningLine(isStaged, latestState, line):
if latestState != HunkContentLine {
latestState = HunkContentLine
}
// NoOp
case isHunkEmptyLine(isStaged, latestState, line):
if latestState != HunkContentLine {
latestState = HunkContentLine
}
// NoOp
case isCommitSeparatorLine(isStaged, latestState, line):
// NoOp
default:
// Skip ahead until we find the next diff or commit.
if latestState == ParseFailure {
continue
}
// Here be dragons...
// Build an informative error message.
var err error
if currentCommit != nil && currentCommit.Hash != "" {
err = fmt.Errorf(`failed to parse line "%s" after state "%s" (commit=%s)`, line, latestState, currentCommit.Hash)
} else {
err = fmt.Errorf(`failed to parse line "%s" after state "%s"`, line, latestState)
}
ctx.Logger().V(2).Error(err, "Recovering at the latest commit or diff...\n")
latestState = ParseFailure
}
if currentDiff.Content.Len() > c.maxDiffSize {
ctx.Logger().V(2).Info(fmt.Sprintf(
"Diff for %s exceeded MaxDiffSize(%d)", currentDiff.PathB, c.maxDiffSize,
@@ -432,157 +318,130 @@ func (c *Parser) FromReader(ctx context.Context, stdOut io.Reader, commitChan ch
ctx.Logger().V(2).Info("finished parsing git log.", "total_log_size", totalLogSize)
}
func isMergeLine(isStaged bool, latestState ParseState, line []byte) bool {
if isStaged || latestState != CommitLine {
return false
func cleanupParse(currentCommit *Commit, currentDiff *Diff, commitChan chan Commit, totalLogSize *int) {
if currentDiff != nil && currentDiff.Content.Len() > 0 {
currentCommit.Diffs = append(currentCommit.Diffs, *currentDiff)
}
if len(line) > 6 && bytes.Equal(line[:6], []byte("Merge:")) {
return true
if currentCommit != nil {
commitChan <- *currentCommit
if totalLogSize != nil {
*totalLogSize += currentCommit.Size
}
}
return false
}
// commit 7a95bbf0199e280a0e42dbb1d1a3f56cdd0f6e05
func isCommitLine(isStaged bool, latestState ParseState, line []byte) bool {
if isStaged || !(latestState == Initial ||
latestState == MessageStartLine ||
latestState == MessageEndLine ||
latestState == ModeLine ||
latestState == IndexLine ||
latestState == BinaryFileLine ||
latestState == ToFileLine ||
latestState == HunkContentLine ||
latestState == ParseFailure) {
return false
}
if len(line) > 7 && bytes.Equal(line[:7], []byte("commit ")) {
return true
}
return false
}
// Author: Bill Rich <bill.rich@trufflesec.com>
func isAuthorLine(isStaged bool, latestState ParseState, line []byte) bool {
if isStaged || !(latestState == CommitLine || latestState == MergeLine) {
return false
}
if len(line) > 8 && bytes.Equal(line[:7], []byte("Author:")) {
return true
}
return false
}
// Date: Tue Aug 10 15:20:40 2021 +0100
func isDateLine(isStaged bool, latestState ParseState, line []byte) bool {
if isStaged || latestState != AuthorLine {
return false
}
func isDateLine(line []byte) bool {
if len(line) > 7 && bytes.Equal(line[:5], []byte("Date:")) {
return true
}
return false
}
// Line directly after Date with only a newline.
func isMessageStartLine(isStaged bool, latestState ParseState, line []byte) bool {
if isStaged || latestState != DateLine {
return false
}
// TODO: Improve the implementation of this and isMessageEndLine
if len(strings.TrimRight(string(line[:]), "\r\n")) == 0 {
// Author: Bill Rich <bill.rich@trufflesec.com>
func isAuthorLine(line []byte) bool {
if len(line) > 8 && bytes.Equal(line[:7], []byte("Author:")) {
return true
}
return false
}
// Line that starts with 4 spaces
func isMessageLine(isStaged bool, latestState ParseState, line []byte) bool {
if isStaged || !(latestState == MessageStartLine || latestState == MessageLine) {
return false
}
if len(line) > 4 && bytes.Equal(line[:4], []byte(" ")) {
return true
}
return false
}
// Line directly after MessageLine with only a newline.
func isMessageEndLine(isStaged bool, latestState ParseState, line []byte) bool {
if isStaged || latestState != MessageLine {
return false
}
if len(strings.TrimRight(string(line[:]), "\r\n")) == 0 {
// commit 7a95bbf0199e280a0e42dbb1d1a3f56cdd0f6e05
func isCommitLine(line []byte) bool {
if len(line) > 7 && bytes.Equal(line[:6], []byte("commit")) {
return true
}
return false
}
// diff --git a/internal/addrs/move_endpoint_module.go b/internal/addrs/move_endpoint_module.go
func isDiffLine(isStaged bool, latestState ParseState, line []byte) bool {
if !(latestState == MessageStartLine || // Empty commit messages can go from MessageStart->Diff
latestState == MessageEndLine ||
latestState == BinaryFileLine ||
latestState == IndexLine ||
latestState == HunkContentLine ||
latestState == ParseFailure) {
if latestState == Initial && !isStaged {
return false
}
}
if len(line) > 11 && bytes.Equal(line[:11], []byte("diff --git ")) {
return true
}
return false
}
// old mode 100644
// new mode 100755
// new file mode 100644
// similarity index 100%
// rename from old.txt
// rename to new.txt
// deleted file mode 100644
func isModeLine(isStaged bool, latestState ParseState, line []byte) bool {
if !(latestState == DiffLine || latestState == ModeLine) {
return false
}
// This could probably be better written.
if (len(line) > 17 && bytes.Equal(line[:17], []byte("deleted file mode"))) ||
(len(line) > 16 && bytes.Equal(line[:16], []byte("similarity index"))) ||
(len(line) > 13 && bytes.Equal(line[:13], []byte("new file mode"))) ||
(len(line) > 11 && bytes.Equal(line[:11], []byte("rename from"))) ||
(len(line) > 9 && bytes.Equal(line[:9], []byte("rename to"))) ||
(len(line) > 8 && bytes.Equal(line[:8], []byte("old mode"))) ||
(len(line) > 8 && bytes.Equal(line[:8], []byte("new mode"))) {
func isDiffLine(line []byte) bool {
if len(line) > 5 && bytes.Equal(line[:4], []byte("diff")) {
return true
}
return false
}
// index 1ed6fbee1..aea1e643a 100644
// index 00000000..e69de29b
func isIndexLine(isStaged bool, latestState ParseState, line []byte) bool {
if !(latestState == DiffLine || latestState == ModeLine) {
return false
func isIndexLine(line []byte) bool {
if len(line) > 6 && bytes.Equal(line[:5], []byte("index")) {
return true
}
if len(line) > 6 && bytes.Equal(line[:6], []byte("index ")) {
return false
}
// new file mode 100644
func isModeLine(line []byte) bool {
if len(line) > 13 && bytes.Equal(line[:13], []byte("new file mode")) {
return true
}
return false
}
// --- a/internal/addrs/move_endpoint_module.go
func isMinusFileLine(line []byte) bool {
if len(line) >= 6 && bytes.Equal(line[:3], []byte("---")) {
return true
}
return false
}
// +++ b/internal/addrs/move_endpoint_module.go
func isPlusFileLine(line []byte) bool {
if len(line) >= 6 && bytes.Equal(line[:4], []byte("+++ ")) {
return true
}
return false
}
// +fmt.Println("ok")
func isPlusDiffLine(line []byte) bool {
if len(line) >= 1 && bytes.Equal(line[:1], []byte("+")) {
return true
}
return false
}
// -fmt.Println("ok")
func isMinusDiffLine(line []byte) bool {
if len(line) >= 1 && bytes.Equal(line[:1], []byte("-")) {
return true
}
return false
}
// fmt.Println("ok")
func isContextDiffLine(line []byte) bool {
if len(line) >= 1 && bytes.Equal(line[:1], []byte(" ")) {
return true
}
return false
}
// Line that starts with 4 spaces
func isMessageLine(line []byte) bool {
if len(line) > 4 && bytes.Equal(line[:4], []byte(" ")) {
return true
}
return false
}
// Binary files /dev/null and b/plugin.sig differ
func isBinaryLine(isStaged bool, latestState ParseState, line []byte) bool {
if latestState != IndexLine {
return false
}
func isBinaryLine(line []byte) bool {
if len(line) > 7 && bytes.Equal(line[:6], []byte("Binary")) {
return true
}
return false
}
// @@ -298 +298 @@ func maxRetryErrorHandler(resp *http.Response, err error, numTries int)
func isLineNumberDiffLine(line []byte) bool {
if len(line) >= 8 && bytes.Equal(line[:2], []byte("@@")) {
return true
}
return false
}
// Get the b/ file path. Ignoring the edge case of files having `and /b` in the name for simplicity.
func pathFromBinaryLine(line []byte) string {
logger := context.Background().Logger()
@@ -594,117 +453,3 @@ func pathFromBinaryLine(line []byte) string {
bRaw := sbytes[1]
return string(bRaw[:len(bRaw)-7]) // drop the "b/" and " differ"
}
// --- a/internal/addrs/move_endpoint_module.go
func isFromFileLine(isStaged bool, latestState ParseState, line []byte) bool {
if latestState != IndexLine {
return false
}
if len(line) >= 6 && bytes.Equal(line[:4], []byte("--- ")) {
return true
}
return false
}
// +++ b/internal/addrs/move_endpoint_module.go
func isToFileLine(isStaged bool, latestState ParseState, line []byte) bool {
if latestState != FromFileLine {
return false
}
if len(line) >= 6 && bytes.Equal(line[:4], []byte("+++ ")) {
return true
}
return false
}
// @@ -298 +298 @@ func maxRetryErrorHandler(resp *http.Response, err error, numTries int)
func isHunkLineNumberLine(isStaged bool, latestState ParseState, line []byte) bool {
if !(latestState == ToFileLine || latestState == HunkContentLine) {
return false
}
if len(line) >= 8 && bytes.Equal(line[:2], []byte("@@")) {
return true
}
return false
}
// fmt.Println("ok")
// (There's a space before `fmt` that gets removed by the formatter.)
func isHunkContextLine(isStaged bool, latestState ParseState, line []byte) bool {
if !(latestState == HunkLineNumberLine || latestState == HunkContentLine) {
return false
}
if len(line) >= 1 && bytes.Equal(line[:1], []byte(" ")) {
return true
}
return false
}
// +fmt.Println("ok")
func isHunkPlusLine(isStaged bool, latestState ParseState, line []byte) bool {
if !(latestState == HunkLineNumberLine || latestState == HunkContentLine) {
return false
}
if len(line) >= 1 && bytes.Equal(line[:1], []byte("+")) {
return true
}
return false
}
// -fmt.Println("ok")
func isHunkMinusLine(isStaged bool, latestState ParseState, line []byte) bool {
if !(latestState == HunkLineNumberLine || latestState == HunkContentLine) {
return false
}
if len(line) >= 1 && bytes.Equal(line[:1], []byte("-")) {
return true
}
return false
}
// \ No newline at end of file
func isHunkNewlineWarningLine(isStaged bool, latestState ParseState, line []byte) bool {
if latestState != HunkContentLine {
return false
}
if len(line) >= 27 && bytes.Equal(line[:27], []byte("\\ No newline at end of file")) {
return true
}
return false
}
// Newline after hunk, or an empty line, e.g.
// +}
//
// commit 00920984e3435057f09cee5468850f7546dfa637 (tag: v3.42.0)
func isHunkEmptyLine(isStaged bool, latestState ParseState, line []byte) bool {
if !(latestState == HunkLineNumberLine || latestState == HunkContentLine) {
return false
}
// TODO: Can this also be `\n\r`?
if len(line) == 1 && bytes.Equal(line[:1], []byte("\n")) {
return true
}
return false
}
func isCommitSeparatorLine(isStaged bool, latestState ParseState, line []byte) bool {
if (latestState == ModeLine || latestState == IndexLine || latestState == BinaryFileLine || latestState == ToFileLine) &&
len(line) == 1 && bytes.Equal(line[:1], []byte("\n")) {
return true
}
return false
}
func cleanupParse(currentCommit *Commit, currentDiff *Diff, commitChan chan Commit, totalLogSize *int) {
// Ignore empty or binary diffs (this condition may be redundant).
if currentDiff != nil && currentDiff.Content.Len() > 0 {
currentCommit.Diffs = append(currentCommit.Diffs, *currentDiff)
}
if currentCommit != nil {
commitChan <- *currentCommit
if totalLogSize != nil {
*totalLogSize += currentCommit.Size
}
}
}
File diff suppressed because it is too large Load Diff
-413
View File
@@ -1,413 +0,0 @@
diff --git a/server/src/main/java/org/opensearch/common/lucene/search/MultiPhrasePrefixQuery.java b/server/src/main/java/org/opensearch/common/lucene/search/MultiPhrasePrefixQuery.java
index 98ebb34fc040..cc0468efb243 100644
--- a/server/src/main/java/org/opensearch/common/lucene/search/MultiPhrasePrefixQuery.java
+++ b/server/src/main/java/org/opensearch/common/lucene/search/MultiPhrasePrefixQuery.java
@@ -39,6 +39,7 @@
import org.apache.lucene.index.TermsEnum;
import org.apache.lucene.search.BooleanClause;
import org.apache.lucene.search.BooleanQuery;
+import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.MatchNoDocsQuery;
import org.apache.lucene.search.MultiPhraseQuery;
import org.apache.lucene.search.Query;
@@ -159,8 +160,8 @@ public int[] getPositions() {
}
@Override
- public Query rewrite(IndexReader reader) throws IOException {
- Query rewritten = super.rewrite(reader);
+ public Query rewrite(IndexSearcher searcher) throws IOException {
+ Query rewritten = super.rewrite(searcher);
if (rewritten != this) {
return rewritten;
}
@@ -177,7 +178,7 @@ public Query rewrite(IndexReader reader) throws IOException {
int position = positions.get(sizeMinus1);
Set<Term> terms = new HashSet<>();
for (Term term : suffixTerms) {
- getPrefixTerms(terms, term, reader);
+ getPrefixTerms(terms, term, searcher.getIndexReader());
if (terms.size() > maxExpansions) {
break;
}
diff --git a/server/src/main/java/org/opensearch/common/lucene/search/function/FunctionScoreQuery.java b/server/src/main/java/org/opensearch/common/lucene/search/function/FunctionScoreQuery.java
index ebd155b8a567..db92d4f1cba9 100644
--- a/server/src/main/java/org/opensearch/common/lucene/search/function/FunctionScoreQuery.java
+++ b/server/src/main/java/org/opensearch/common/lucene/search/function/FunctionScoreQuery.java
@@ -128,7 +128,7 @@ protected int doHashCode() {
@Override
protected ScoreFunction rewrite(IndexReader reader) throws IOException {
- Query newFilter = filter.rewrite(reader);
+ Query newFilter = filter.rewrite(new IndexSearcher(reader));
if (newFilter == filter) {
return this;
}
@@ -322,16 +322,16 @@ public void visit(QueryVisitor visitor) {
}
@Override
- public Query rewrite(IndexReader reader) throws IOException {
- Query rewritten = super.rewrite(reader);
+ public Query rewrite(IndexSearcher searcher) throws IOException {
+ Query rewritten = super.rewrite(searcher);
if (rewritten != this) {
return rewritten;
}
- Query newQ = subQ2023-07-28T08:38:41-07:00 info-0 forager/scanner loaded decoders {"count": 3}
uery.rewrite(reader);
+ Query newQ = subQuery.rewrite(searcher);
ScoreFunction[] newFunctions = new ScoreFunction[functions.length];
boolean needsRewrite = (newQ != subQuery);
for (int i = 0; i < functions.length; i++) {
- newFunctions[i] = functions[i].rewrite(reader);
+ newFunctions[i] = functions[i].rewrite(searcher.getIndexReader());
needsRewrite |= (newFunctions[i] != functions[i]);
}
if (needsRewrite) {
diff --git a/server/src/main/java/org/opensearch/common/lucene/search/function/ScriptScoreQuery.java b/server/src/main/java/org/opensearch/common/lucene/search/function/ScriptScoreQuery.java
index 8bf5fc0f89d3..07084a6f690e 100644
--- a/server/src/main/java/org/opensearch/common/lucene/search/function/ScriptScoreQuery.java
+++ b/server/src/main/java/org/opensearch/common/lucene/search/function/ScriptScoreQuery.java
@@ -32,7 +32,6 @@
package org.opensearch.common.lucene.search.function;
-import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.search.BooleanClause;
import org.apache.lucene.search.DocIdSetIterator;
@@ -105,12 +104,12 @@ public ScriptScoreQuery(
}
@Override
- public Query rewrite(IndexReader reader) throws IOException {
- Query newQ = subQuery.rewrite(reader);
+ public Query rewrite(IndexSearcher searcher) throws IOException {
+ Query newQ = subQuery.rewrite(searcher);
if (newQ != subQuery) {
return new ScriptScoreQuery(newQ, queryName, script, scriptBuilder, minScore, indexName, shardId, indexVersion);
}
- return super.rewrite(reader);
+ return super.rewrite(searcher);
}
@Override
diff --git a/server/src/main/java/org/opensearch/index/document/SortedUnsignedLongDocValuesRangeQuery.java b/server/src/main/java/org/opensearch/index/document/SortedUnsignedLongDocValuesRangeQuery.java
index 04dd669d6088..234c67cc637f 100644
--- a/server/src/main/java/org/opensearch/index/document/SortedUnsignedLongDocValuesRangeQuery.java
+++ b/server/src/main/java/org/opensearch/index/document/SortedUnsignedLongDocValuesRangeQuery.java
@@ -10,7 +10,6 @@
import org.apache.lucene.index.DocValues;
import org.apache.lucene.index.FieldInfo;
-import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.LeafReader;
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.NumericDocValues;
@@ -87,12 +86,12 @@ public String toString(String field) {
}
@Override
- public Query rewrite(IndexReader reader) throws IOException {
+ public Query rewrite(IndexSearcher searcher) throws IOException {
if (Long.compareUnsigned(lowerValue, Numbers.MIN_UNSIGNED_LONG_VALUE_AS_LONG) == 0
&& Long.compareUnsigned(upperValue, Numbers.MAX_UNSIGNED_LONG_VALUE_AS_LONG) == 0) {
return new FieldExistsQuery(field);
}
- return super.rewrite(reader);
+ return super.rewrite(searcher);
}
abstract SortedNumericDocValues getValues(LeafReader reader, String field) throws IOException;
diff --git a/server/src/main/java/org/opensearch/index/query/DateRangeIncludingNowQuery.java b/server/src/main/java/org/opensearch/index/query/DateRangeIncludingNowQuery.java
index 2a7dbf314103..001d77facb82 100644
--- a/server/src/main/java/org/opensearch/index/query/DateRangeIncludingNowQuery.java
+++ b/server/src/main/java/org/opensearch/index/query/DateRangeIncludingNowQuery.java
@@ -32,8 +32,8 @@
package org.opensearch.index.query;
-import org.apache.lucene.index.IndexReader;
import org.apache.lucene.search.BooleanClause;
+import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.QueryVisitor;
@@ -60,7 +60,7 @@ public Query getQuery() {
}
@Override
- public Query r2023-07-28T08:38:41-07:00 info-0 forager/scanner loaded detectors {"total": 748, "verification_enabled": 748, "verification_disabled": 0}
ewrite(IndexReader reader) throws IOException {
+ public Query rewrite(IndexSearcher searcher) throws IOException {
return in;
}
diff --git a/server/src/main/java/org/opensearch/index/search/OpenSearchToParentBlockJoinQuery.java b/server/src/main/java/org/opensearch/index/search/OpenSearchToParentBlockJoinQuery.java
index 770301629d96..cd28d8b6ef04 100644
--- a/server/src/main/java/org/opensearch/index/search/OpenSearchToParentBlockJoinQuery.java
+++ b/server/src/main/java/org/opensearch/index/search/OpenSearchToParentBlockJoinQuery.java
@@ -32,7 +32,6 @@
package org.opensearch.index.search;
-import org.apache.lucene.index.IndexReader;
import org.apache.lucene.search.BooleanClause;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.Query;
@@ -82,8 +81,8 @@ public ScoreMode getScoreMode() {
}
@Override
- public Query rewrite(IndexReader reader) throws IOException {
- Query innerRewrite = query.rewrite(reader);
+ public Query rewrite(IndexSearcher searcher) throws IOException {
+ Query innerRewrite = query.rewrite(searcher);
if (innerRewrite != query) {
// Right now ToParentBlockJoinQuery always rewrites to a ToParentBlockJoinQuery
// so the else block will never be used. It is useful in the case that
@@ -97,7 +96,7 @@ public Query rewrite(IndexReader reader) throws IOException {
return innerRewrite;
}
}
- return super.rewrite(reader);
+ return super.rewrite(searcher);
}
@Override
diff --git a/server/src/main/java/org/opensearch/lucene/queries/MinDocQuery.java b/server/src/main/java/org/opensearch/lucene/queries/MinDocQuery.java
index 2af0237e800c..5c904d8a7770 100644
--- a/server/src/main/java/org/opensearch/lucene/queries/MinDocQuery.java
+++ b/server/src/main/java/org/opensearch/lucene/queries/MinDocQuery.java
@@ -86,7 +86,8 @@ public boolean equals(Object obj) {
}
@Override
- public Query rewrite(IndexReader reader) throws IOException {
+ public Query rewrite(IndexSearcher searcher) throws IOException {
+ final IndexReader reader = searcher.getIndexReader();
if (Objects.equals(reader.getContext().id(), readerId) == false) {
return new MinDocQuery(minDoc, reader.getContext().id());
}
diff --git a/server/src/main/java/org/opensearch/search/internal/ContextIndexSearcher.java b/server/src/main/java/org/opensearch/search/internal/ContextIndexSearcher.java
index 5384b47cc69e..575133fcffad 100644
--- a/server/src/main/java/org/opensearch/search/internal/ContextIndexSearcher.java
+++ b/server/src/main/java/org/opensearch/search/internal/ContextIndexSearcher.java
@@ -339,14 +339,9 @@ private void searchLeaf(LeafReaderContext ctx, Weight weight, Collector collecto
// continue with the following leaf
} catch (QueryPhase.TimeExceededException e) {
searchContext.setSearchTimedOut(true);
- return;
}
}
}
-
- // Note: this is called if collection ran successfully, including the above special cases of
- // CollectionTerminatedException and TimeExceededException, but no other exception.
- leafCollector.finish();
}
private Weight wrapWeight(Weight weight) {
diff --git a/server/src/main/java/org/opensearch/search/suggest/completion/CompletionSuggester.java b/server/src/main/java/org/opensearch/search/suggest/completion/CompletionSuggester.java
index df18e874be9c..af7b1a798a77 100644
--- a/server/src/main/java/org/opensearch/search/suggest/completion/CompletionSuggester.java
+++ b/server/src/main/java/org/opensearch/search/suggest/completion/CompletionSuggester.java
@@ -35,7 +35,6 @@
import org.apache.lucene.search.BulkScorer;
import org.apache.lucene.search.CollectionTerminatedException;
import org.apache.lucene.search.IndexSearcher;
-import org.apache.lucene.search.LeafCollector;
import org.apache.lucene.search.Weight;
import org.apache.lucene.search.suggest.document.CompletionQuery;
import org.apache.lucene.search.suggest.document.TopSuggestDocs;
@@ -104,24 +103,17 @@ protected Suggest.Suggestion<? extends Suggest.Suggestion.Entry<? extends Sugges
}
private static void suggest(IndexSearcher searcher, CompletionQuery query, TopSuggestDocsCollector collector) throws IOException {
- query = (CompletionQuery) query.rewrite(searcher.getIndexReader());
+ query = (CompletionQuery) query.rewrite(searcher);
Weight weight = query.createWeight(searcher, collector.scoreMode(), 1f);
for (LeafReaderContext context : searcher.getIndexReader().leaves()) {
BulkScorer scorer = weight.bulkScorer(context);
if (scorer != null) {
- LeafCollector leafCollector = null;
try {
- leafCollector = collector.getLeafCollector(context);
- scorer.score(leafCollector, context.reader().getLiveDocs());
+ scorer.score(collector.getLeafCollector(context), context.reader().getLiveDocs());
} catch (CollectionTerminatedException e) {
// collection was terminated prematurely
// continue with the following leaf
}
- // Note: this is called if collection ran successfully, including the above special cases of
- // CollectionTerminatedException and TimeExceededException, but no other exception.
- if (leafCollector != null) {
- leafCollector.finish();
- }
}
}
}
diff --git a/server/src/test/java/org/opensearch/deps/lucene/VectorHighlighterTests.java b/server/src/test/java/org/opensearch/deps/lucene/VectorHighlighterTests.java
index 0f831ff28af7..880eaeca99bb 100644
--- a/server/src/test/java/org/opensearch/deps/lucene/VectorHighlighterTests.java
+++ b/server/src/test/java/org/opensearch/deps/lucene/VectorHighlighterTests.java
@@ -125,7 +125,7 @@ public void testVectorHighlighterPrefixQuery() throws Exception {
assertThat(fragment, nullValue());
prefixQuery = new PrefixQuery(new Term("content", "ba"), PrefixQuery.SCORING_BOOLEAN_REWRITE);
- Query rewriteQuery = prefixQuery.rewrite(reader);
+ Query rewriteQuery = prefixQuery.rewrite(searcher);
fragment = highlighter.getBestFragment(highlighter.getFieldQuery(rewriteQuery), reader, topDocs.scoreDocs[0].doc, "content", 30);
assertThat(fragment, notNullValue());
diff --git a/server/src/test/java/org/opensearch/index/mapper/DateFieldTypeTests.java b/server/src/test/java/org/opensearch/index/mapper/DateFieldTypeTests.java
index d360a2a767e8..510dd3456924 100644
--- a/server/src/test/java/org/opensearch/index/mapper/DateFieldTypeTests.java
+++ b/server/src/test/java/org/opensearch/index/mapper/DateFieldTypeTests.java
@@ -40,6 +40,7 @@
import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.index.MultiReader;
import org.apache.lucene.index.SortedNumericDocValues;
+import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.IndexSortSortedNumericDocValuesRangeQuery;
import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.search.IndexOrDocValuesQuery;
@@ -259,7 +260,10 @@ public void testRangeQuery() throws IOException {
LongPoint.newRangeQuery("field", instant1, instant2),
SortedNumericDocValuesField.newSlowRangeQuery("field", instant1, instant2)
);
- assertEquals(expected, ft.rangeQuery(date1, date2, true, true, null, null, null, context).rewrite(new MultiReader()));
+ assertEquals(
+ expected,
+ ft.rangeQuery(date1, date2, true, true, null, null, null, context).rewrite(new IndexSearcher(new MultiReader()))
+ );
instant1 = nowInMillis;
instant2 = instant1 + 100;
diff --git a/server/src/test/java/org/opensearch/index/query/SpanMultiTermQueryBuilderTests.java b/server/src/test/java/org/opensearch/index/query/SpanMultiTermQueryBuilderTests.java
index bc514208c803..be87f7ca1590 100644
--- a/server/src/test/java/org/opensearch/index/query/SpanMultiTermQueryBuilderTests.java
+++ b/server/src/test/java/org/opensearch/index/query/SpanMultiTermQueryBuilderTests.java
@@ -284,8 +284,9 @@ public void testTermExpansionExceptionOnSpanFailure() throws Exception {
BooleanQuery.setMaxClauseCount(1);
try {
QueryBuilder queryBuilder = new SpanMultiTermQueryBuilder(QueryBuilders.prefixQuery("body", "bar"));
- Query query = queryBuilder.toQuery(createShardContext(new IndexSearcher(reader)));
- RuntimeException exc = expectThrows(RuntimeException.class, () -> query.rewrite(reader));
+ IndexSearcher searcher = new IndexSearcher(reader);
+ Query query = queryBuilder.toQuery(createShardContext(searcher));
+ RuntimeException exc = expectThrows(RuntimeException.class, () -> query.rewrite(searcher));
assertThat(exc.getMessage(), containsString("maxClauseCount"));
} finally {
BooleanQuery.setMaxClauseCount(origBoolMaxClauseCount);
diff --git a/server/src/test/java/org/opensearch/index/search/OpenSearchToParentBlockJoinQueryTests.java b/server/src/test/java/org/opensearch/index/search/OpenSearchToParentBlockJoinQueryTests.java
index 39160b1ac49b..45172c946ab3 100644
--- a/server/src/test/java/org/opensearch/index/search/OpenSearchToParentBlockJoinQueryTests.java
+++ b/server/src/test/java/org/opensearch/index/search/OpenSearchToParentBlockJoinQueryTests.java
@@ -34,6 +34,7 @@
import org.apache.lucene.index.MultiReader;
import org.apache.lucene.index.Term;
+import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.PhraseQuery;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.TermQuery;
@@ -112,7 +113,7 @@ public void testRewrite() throws IOException {
ScoreMode.Avg,
"nested"
);
- Query rewritten = q.rewrite(new MultiReader());
+ Query rewritten = q.rewrite(new IndexSearcher(new MultiReader()));
assertEquals(expected, rewritten);
}
}
diff --git a/server/src/test/java/org/opensearch/lucene/queries/MinDocQueryTests.java b/server/src/test/java/org/opensearch/lucene/queries/MinDocQueryTests.java
index a75a1d5b2fdb..701ba5a756b4 100644
--- a/server/src/test/java/org/opensearch/lucene/queries/MinDocQueryTests.java
+++ b/server/src/test/java/org/opensearch/lucene/queries/MinDocQueryTests.java
@@ -61,10 +61,11 @@ public void testBasics() {
public void testRewrite() throws Exception {
IndexReader reader = new MultiReader();
+ IndexSearcher searcher = new IndexSearcher(reader);
MinDocQuery query = new MinDocQuery(42);
- Query rewritten = query.rewrite(reader);
+ Query rewritten = query.rewrite(searcher);
QueryUtils.checkUnequal(query, rewritten);
- Query rewritten2 = rewritten.rewrite(reader);
+ Query rewritten2 = rewritten.rewrite(searcher);
assertSame(rewritten, rewritten2);
}
diff --git a/server/src/test/java/org/opensearch/lucene/queries/SpanMatchNoDocsQueryTests.java b/server/src/test/java/org/opensearch/lucene/queries/SpanMatchNoDocsQueryTests.java
index 727377c9a5d5..81f4290b6588 100644
--- a/server/src/test/java/org/opensearch/lucene/queries/SpanMatchNoDocsQueryTests.java
+++ b/server/src/test/java/org/opensearch/lucene/queries/SpanMatchNoDocsQueryTests.java
@@ -56,19 +56,11 @@
public class SpanMatchNoDocsQueryTests extends OpenSearchTestCase {
public void testSimple() throws Exception {
- Directory dir = newDirectory();
- IndexWriter iw = new IndexWriter(dir, newIndexWriterConfig());
- IndexReader ir = DirectoryReader.open(iw);
-
SpanMatchNoDocsQuery query = new SpanMatchNoDocsQuery("field", "a good reason");
assertEquals(query.toString(), "SpanMatchNoDocsQuery(\"a good reason\")");
- Query rewrite = query.rewrite(ir);
+ Query rewrite = query.rewrite((IndexSearcher) null);
assertTrue(rewrite instanceof SpanMatchNoDocsQuery);
assertEquals(rewrite.toString(), "SpanMatchNoDocsQuery(\"a good reason\")");
-
- iw.close();
- ir.close();
- dir.close();
}
public void testQuery() throws Exception {
@@ -101,7 +93,7 @@ public void testQuery() throws Exception {
assertEquals(searcher.count(orQuery), 1);
hits = searcher.search(orQuery, 1000).scoreDocs;
assertEquals(1, hits.length);
- Query rewrite = orQuery.rewrite(ir);
+ Query rewrite = orQuery.rewrite(searcher);
assertEquals(rewrite, orQuery);
SpanNearQuery nearQuery = new SpanNearQuery(
@@ -112,7 +104,7 @@ public void testQuery() throws Exception {
assertEquals(searcher.count(nearQuery), 0);
hits = searcher.search(nearQuery, 1000).scoreDocs;
assertEquals(0, hits.length);
- rewrite = nearQuery.rewrite(ir);
+ rewrite = nearQuery.rewrite(searcher);
assertEquals(rewrite, nearQuery);
iw.close();
diff --git a/server/src/test/java/org/opensearch/search/internal/ContextIndexSearcherTests.java b/server/src/test/java/org/opensearch/search/internal/ContextIndexSearcherTests.java
index f3907355ac6e..823fc6b46390 100644
--- a/server/src/test/java/org/opensearch/search/internal/ContextIndexSearcherTests.java
+++ b/server/src/test/java/org/opensearch/search/internal/ContextIndexSearcherTests.java
@@ -40,7 +40,6 @@
import org.apache.lucene.document.StringField;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.FilterDirectoryReader;
-import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.index.LeafReader;
@@ -466,12 +465,12 @@ public String toString(String field) {
}
@Override
- public Query rewrite(IndexReader reader) throws IOException {
- Query queryRewritten = query.rewrite(reader);
+ public Query rewrite(IndexSearcher searcher) throws IOException {
+ Query queryRewritten = query.rewrite(searcher);
if (query != queryRewritten) {
return new CreateScorerOnceQuery(queryRewritten);
}
- return super.rewrite(reader);
+ return super.rewrite(searcher);
}
@Override
+6 -19
View File
@@ -992,9 +992,6 @@ const (
DetectorType_Prefect DetectorType = 918
DetectorType_Docusign DetectorType = 919
DetectorType_Couchbase DetectorType = 920
DetectorType_Dockerhub DetectorType = 921
DetectorType_TrufflehogEnterprise DetectorType = 922
DetectorType_EnvoyApiKey DetectorType = 923
)
// Enum value maps for DetectorType.
@@ -1917,9 +1914,6 @@ var (
918: "Prefect",
919: "Docusign",
920: "Couchbase",
921: "Dockerhub",
922: "TrufflehogEnterprise",
923: "EnvoyApiKey",
}
DetectorType_value = map[string]int32{
"Alibaba": 0,
@@ -2839,9 +2833,6 @@ var (
"Prefect": 918,
"Docusign": 919,
"Couchbase": 920,
"Dockerhub": 921,
"TrufflehogEnterprise": 922,
"EnvoyApiKey": 923,
}
)
@@ -3220,7 +3211,7 @@ var file_detectors_proto_rawDesc = []byte{
0x44, 0x65, 0x63, 0x6f, 0x64, 0x65, 0x72, 0x54, 0x79, 0x70, 0x65, 0x12, 0x0b, 0x0a, 0x07, 0x55,
0x4e, 0x4b, 0x4e, 0x4f, 0x57, 0x4e, 0x10, 0x00, 0x12, 0x09, 0x0a, 0x05, 0x50, 0x4c, 0x41, 0x49,
0x4e, 0x10, 0x01, 0x12, 0x0a, 0x0a, 0x06, 0x42, 0x41, 0x53, 0x45, 0x36, 0x34, 0x10, 0x02, 0x12,
0x09, 0x0a, 0x05, 0x55, 0x54, 0x46, 0x31, 0x36, 0x10, 0x03, 0x2a, 0xc6, 0x73, 0x0a, 0x0c, 0x44,
0x09, 0x0a, 0x05, 0x55, 0x54, 0x46, 0x31, 0x36, 0x10, 0x03, 0x2a, 0x89, 0x73, 0x0a, 0x0c, 0x44,
0x65, 0x74, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x54, 0x79, 0x70, 0x65, 0x12, 0x0b, 0x0a, 0x07, 0x41,
0x6c, 0x69, 0x62, 0x61, 0x62, 0x61, 0x10, 0x00, 0x12, 0x08, 0x0a, 0x04, 0x41, 0x4d, 0x51, 0x50,
0x10, 0x01, 0x12, 0x07, 0x0a, 0x03, 0x41, 0x57, 0x53, 0x10, 0x02, 0x12, 0x09, 0x0a, 0x05, 0x41,
@@ -4141,15 +4132,11 @@ var file_detectors_proto_rawDesc = []byte{
0x05, 0x41, 0x69, 0x76, 0x65, 0x6e, 0x10, 0x95, 0x07, 0x12, 0x0c, 0x0a, 0x07, 0x50, 0x72, 0x65,
0x66, 0x65, 0x63, 0x74, 0x10, 0x96, 0x07, 0x12, 0x0d, 0x0a, 0x08, 0x44, 0x6f, 0x63, 0x75, 0x73,
0x69, 0x67, 0x6e, 0x10, 0x97, 0x07, 0x12, 0x0e, 0x0a, 0x09, 0x43, 0x6f, 0x75, 0x63, 0x68, 0x62,
0x61, 0x73, 0x65, 0x10, 0x98, 0x07, 0x12, 0x0e, 0x0a, 0x09, 0x44, 0x6f, 0x63, 0x6b, 0x65, 0x72,
0x68, 0x75, 0x62, 0x10, 0x99, 0x07, 0x12, 0x19, 0x0a, 0x14, 0x54, 0x72, 0x75, 0x66, 0x66, 0x6c,
0x65, 0x68, 0x6f, 0x67, 0x45, 0x6e, 0x74, 0x65, 0x72, 0x70, 0x72, 0x69, 0x73, 0x65, 0x10, 0x9a,
0x07, 0x12, 0x10, 0x0a, 0x0b, 0x45, 0x6e, 0x76, 0x6f, 0x79, 0x41, 0x70, 0x69, 0x4b, 0x65, 0x79,
0x10, 0x9b, 0x07, 0x42, 0x3d, 0x5a, 0x3b, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f,
0x6d, 0x2f, 0x74, 0x72, 0x75, 0x66, 0x66, 0x6c, 0x65, 0x73, 0x65, 0x63, 0x75, 0x72, 0x69, 0x74,
0x79, 0x2f, 0x74, 0x72, 0x75, 0x66, 0x66, 0x6c, 0x65, 0x68, 0x6f, 0x67, 0x2f, 0x76, 0x33, 0x2f,
0x70, 0x6b, 0x67, 0x2f, 0x70, 0x62, 0x2f, 0x64, 0x65, 0x74, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x73,
0x70, 0x62, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
0x61, 0x73, 0x65, 0x10, 0x98, 0x07, 0x42, 0x3d, 0x5a, 0x3b, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62,
0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x74, 0x72, 0x75, 0x66, 0x66, 0x6c, 0x65, 0x73, 0x65, 0x63, 0x75,
0x72, 0x69, 0x74, 0x79, 0x2f, 0x74, 0x72, 0x75, 0x66, 0x66, 0x6c, 0x65, 0x68, 0x6f, 0x67, 0x2f,
0x76, 0x33, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x70, 0x62, 0x2f, 0x64, 0x65, 0x74, 0x65, 0x63, 0x74,
0x6f, 0x72, 0x73, 0x70, 0x62, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
}
var (
+136 -327
View File
@@ -2294,7 +2294,6 @@ type GoogleDrive struct {
Timestamp string `protobuf:"bytes,4,opt,name=timestamp,proto3" json:"timestamp,omitempty"`
Shared bool `protobuf:"varint,5,opt,name=shared,proto3" json:"shared,omitempty"`
LastModifiedBy string `protobuf:"bytes,6,opt,name=last_modified_by,json=lastModifiedBy,proto3" json:"last_modified_by,omitempty"`
Path string `protobuf:"bytes,7,opt,name=path,proto3" json:"path,omitempty"`
}
func (x *GoogleDrive) Reset() {
@@ -2371,140 +2370,6 @@ func (x *GoogleDrive) GetLastModifiedBy() string {
return ""
}
func (x *GoogleDrive) GetPath() string {
if x != nil {
return x.Path
}
return ""
}
type AzureRepos struct {
state protoimpl.MessageState
sizeCache protoimpl.SizeCache
unknownFields protoimpl.UnknownFields
Link string `protobuf:"bytes,1,opt,name=link,proto3" json:"link,omitempty"`
Username string `protobuf:"bytes,2,opt,name=username,proto3" json:"username,omitempty"`
Repository string `protobuf:"bytes,3,opt,name=repository,proto3" json:"repository,omitempty"`
Commit string `protobuf:"bytes,4,opt,name=commit,proto3" json:"commit,omitempty"`
Email string `protobuf:"bytes,5,opt,name=email,proto3" json:"email,omitempty"`
File string `protobuf:"bytes,6,opt,name=file,proto3" json:"file,omitempty"`
Timestamp string `protobuf:"bytes,7,opt,name=timestamp,proto3" json:"timestamp,omitempty"`
Line int64 `protobuf:"varint,8,opt,name=line,proto3" json:"line,omitempty"`
Visibility Visibility `protobuf:"varint,9,opt,name=visibility,proto3,enum=source_metadata.Visibility" json:"visibility,omitempty"`
Project string `protobuf:"bytes,10,opt,name=project,proto3" json:"project,omitempty"`
Organization string `protobuf:"bytes,11,opt,name=organization,proto3" json:"organization,omitempty"`
}
func (x *AzureRepos) Reset() {
*x = AzureRepos{}
if protoimpl.UnsafeEnabled {
mi := &file_source_metadata_proto_msgTypes[26]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
}
func (x *AzureRepos) String() string {
return protoimpl.X.MessageStringOf(x)
}
func (*AzureRepos) ProtoMessage() {}
func (x *AzureRepos) ProtoReflect() protoreflect.Message {
mi := &file_source_metadata_proto_msgTypes[26]
if protoimpl.UnsafeEnabled && x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
ms.StoreMessageInfo(mi)
}
return ms
}
return mi.MessageOf(x)
}
// Deprecated: Use AzureRepos.ProtoReflect.Descriptor instead.
func (*AzureRepos) Descriptor() ([]byte, []int) {
return file_source_metadata_proto_rawDescGZIP(), []int{26}
}
func (x *AzureRepos) GetLink() string {
if x != nil {
return x.Link
}
return ""
}
func (x *AzureRepos) GetUsername() string {
if x != nil {
return x.Username
}
return ""
}
func (x *AzureRepos) GetRepository() string {
if x != nil {
return x.Repository
}
return ""
}
func (x *AzureRepos) GetCommit() string {
if x != nil {
return x.Commit
}
return ""
}
func (x *AzureRepos) GetEmail() string {
if x != nil {
return x.Email
}
return ""
}
func (x *AzureRepos) GetFile() string {
if x != nil {
return x.File
}
return ""
}
func (x *AzureRepos) GetTimestamp() string {
if x != nil {
return x.Timestamp
}
return ""
}
func (x *AzureRepos) GetLine() int64 {
if x != nil {
return x.Line
}
return 0
}
func (x *AzureRepos) GetVisibility() Visibility {
if x != nil {
return x.Visibility
}
return Visibility_public
}
func (x *AzureRepos) GetProject() string {
if x != nil {
return x.Project
}
return ""
}
func (x *AzureRepos) GetOrganization() string {
if x != nil {
return x.Organization
}
return ""
}
type MetaData struct {
state protoimpl.MessageState
sizeCache protoimpl.SizeCache
@@ -2537,14 +2402,13 @@ type MetaData struct {
// *MetaData_PublicEventMonitoring
// *MetaData_Sharepoint
// *MetaData_GoogleDrive
// *MetaData_AzureRepos
Data isMetaData_Data `protobuf_oneof:"data"`
}
func (x *MetaData) Reset() {
*x = MetaData{}
if protoimpl.UnsafeEnabled {
mi := &file_source_metadata_proto_msgTypes[27]
mi := &file_source_metadata_proto_msgTypes[26]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
@@ -2557,7 +2421,7 @@ func (x *MetaData) String() string {
func (*MetaData) ProtoMessage() {}
func (x *MetaData) ProtoReflect() protoreflect.Message {
mi := &file_source_metadata_proto_msgTypes[27]
mi := &file_source_metadata_proto_msgTypes[26]
if protoimpl.UnsafeEnabled && x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
@@ -2570,7 +2434,7 @@ func (x *MetaData) ProtoReflect() protoreflect.Message {
// Deprecated: Use MetaData.ProtoReflect.Descriptor instead.
func (*MetaData) Descriptor() ([]byte, []int) {
return file_source_metadata_proto_rawDescGZIP(), []int{27}
return file_source_metadata_proto_rawDescGZIP(), []int{26}
}
func (m *MetaData) GetData() isMetaData_Data {
@@ -2762,13 +2626,6 @@ func (x *MetaData) GetGoogleDrive() *GoogleDrive {
return nil
}
func (x *MetaData) GetAzureRepos() *AzureRepos {
if x, ok := x.GetData().(*MetaData_AzureRepos); ok {
return x.AzureRepos
}
return nil
}
type isMetaData_Data interface {
isMetaData_Data()
}
@@ -2877,10 +2734,6 @@ type MetaData_GoogleDrive struct {
GoogleDrive *GoogleDrive `protobuf:"bytes,26,opt,name=googleDrive,proto3,oneof"`
}
type MetaData_AzureRepos struct {
AzureRepos *AzureRepos `protobuf:"bytes,27,opt,name=azureRepos,proto3,oneof"`
}
func (*MetaData_Azure) isMetaData_Data() {}
func (*MetaData_Bitbucket) isMetaData_Data() {}
@@ -2933,8 +2786,6 @@ func (*MetaData_Sharepoint) isMetaData_Data() {}
func (*MetaData_GoogleDrive) isMetaData_Data() {}
func (*MetaData_AzureRepos) isMetaData_Data() {}
var File_source_metadata_proto protoreflect.FileDescriptor
var file_source_metadata_proto_rawDesc = []byte{
@@ -3198,7 +3049,7 @@ var file_source_metadata_proto_rawDesc = []byte{
0x73, 0x18, 0x05, 0x20, 0x01, 0x28, 0x03, 0x52, 0x05, 0x76, 0x69, 0x65, 0x77, 0x73, 0x12, 0x14,
0x0a, 0x05, 0x64, 0x6f, 0x63, 0x69, 0x64, 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x64,
0x6f, 0x63, 0x69, 0x64, 0x12, 0x14, 0x0a, 0x05, 0x65, 0x6d, 0x61, 0x69, 0x6c, 0x18, 0x07, 0x20,
0x01, 0x28, 0x09, 0x52, 0x05, 0x65, 0x6d, 0x61, 0x69, 0x6c, 0x22, 0xbf, 0x01, 0x0a, 0x0b, 0x47,
0x01, 0x28, 0x09, 0x52, 0x05, 0x65, 0x6d, 0x61, 0x69, 0x6c, 0x22, 0xab, 0x01, 0x0a, 0x0b, 0x47,
0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x44, 0x72, 0x69, 0x76, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x66, 0x69,
0x6c, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x66, 0x69, 0x6c, 0x65, 0x12, 0x12,
0x0a, 0x04, 0x6c, 0x69, 0x6e, 0x6b, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6c, 0x69,
@@ -3209,130 +3060,104 @@ var file_source_metadata_proto_rawDesc = []byte{
0x18, 0x05, 0x20, 0x01, 0x28, 0x08, 0x52, 0x06, 0x73, 0x68, 0x61, 0x72, 0x65, 0x64, 0x12, 0x28,
0x0a, 0x10, 0x6c, 0x61, 0x73, 0x74, 0x5f, 0x6d, 0x6f, 0x64, 0x69, 0x66, 0x69, 0x65, 0x64, 0x5f,
0x62, 0x79, 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0e, 0x6c, 0x61, 0x73, 0x74, 0x4d, 0x6f,
0x64, 0x69, 0x66, 0x69, 0x65, 0x64, 0x42, 0x79, 0x12, 0x12, 0x0a, 0x04, 0x70, 0x61, 0x74, 0x68,
0x18, 0x07, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x70, 0x61, 0x74, 0x68, 0x22, 0xcb, 0x02, 0x0a,
0x0a, 0x41, 0x7a, 0x75, 0x72, 0x65, 0x52, 0x65, 0x70, 0x6f, 0x73, 0x12, 0x12, 0x0a, 0x04, 0x6c,
0x69, 0x6e, 0x6b, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6c, 0x69, 0x6e, 0x6b, 0x12,
0x1a, 0x0a, 0x08, 0x75, 0x73, 0x65, 0x72, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28,
0x09, 0x52, 0x08, 0x75, 0x73, 0x65, 0x72, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x1e, 0x0a, 0x0a, 0x72,
0x65, 0x70, 0x6f, 0x73, 0x69, 0x74, 0x6f, 0x72, 0x79, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52,
0x0a, 0x72, 0x65, 0x70, 0x6f, 0x73, 0x69, 0x74, 0x6f, 0x72, 0x79, 0x12, 0x16, 0x0a, 0x06, 0x63,
0x6f, 0x6d, 0x6d, 0x69, 0x74, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x63, 0x6f, 0x6d,
0x6d, 0x69, 0x74, 0x12, 0x14, 0x0a, 0x05, 0x65, 0x6d, 0x61, 0x69, 0x6c, 0x18, 0x05, 0x20, 0x01,
0x28, 0x09, 0x52, 0x05, 0x65, 0x6d, 0x61, 0x69, 0x6c, 0x12, 0x12, 0x0a, 0x04, 0x66, 0x69, 0x6c,
0x65, 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x66, 0x69, 0x6c, 0x65, 0x12, 0x1c, 0x0a,
0x09, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x18, 0x07, 0x20, 0x01, 0x28, 0x09,
0x52, 0x09, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x12, 0x12, 0x0a, 0x04, 0x6c,
0x69, 0x6e, 0x65, 0x18, 0x08, 0x20, 0x01, 0x28, 0x03, 0x52, 0x04, 0x6c, 0x69, 0x6e, 0x65, 0x12,
0x3b, 0x0a, 0x0a, 0x76, 0x69, 0x73, 0x69, 0x62, 0x69, 0x6c, 0x69, 0x74, 0x79, 0x18, 0x09, 0x20,
0x01, 0x28, 0x0e, 0x32, 0x1b, 0x2e, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f, 0x6d, 0x65, 0x74,
0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x56, 0x69, 0x73, 0x69, 0x62, 0x69, 0x6c, 0x69, 0x74, 0x79,
0x52, 0x0a, 0x76, 0x69, 0x73, 0x69, 0x62, 0x69, 0x6c, 0x69, 0x74, 0x79, 0x12, 0x18, 0x0a, 0x07,
0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x70,
0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x12, 0x22, 0x0a, 0x0c, 0x6f, 0x72, 0x67, 0x61, 0x6e, 0x69,
0x7a, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x0b, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0c, 0x6f, 0x72,
0x67, 0x61, 0x6e, 0x69, 0x7a, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x22, 0xbe, 0x0b, 0x0a, 0x08, 0x4d,
0x65, 0x74, 0x61, 0x44, 0x61, 0x74, 0x61, 0x12, 0x2e, 0x0a, 0x05, 0x61, 0x7a, 0x75, 0x72, 0x65,
0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x16, 0x2e, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f,
0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x41, 0x7a, 0x75, 0x72, 0x65, 0x48, 0x00,
0x52, 0x05, 0x61, 0x7a, 0x75, 0x72, 0x65, 0x12, 0x3a, 0x0a, 0x09, 0x62, 0x69, 0x74, 0x62, 0x75,
0x63, 0x6b, 0x65, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x73, 0x6f, 0x75,
0x72, 0x63, 0x65, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x42, 0x69, 0x74,
0x62, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x48, 0x00, 0x52, 0x09, 0x62, 0x69, 0x74, 0x62, 0x75, 0x63,
0x6b, 0x65, 0x74, 0x12, 0x37, 0x0a, 0x08, 0x63, 0x69, 0x72, 0x63, 0x6c, 0x65, 0x63, 0x69, 0x18,
0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x19, 0x2e, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f, 0x6d,
0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x43, 0x69, 0x72, 0x63, 0x6c, 0x65, 0x43, 0x49,
0x48, 0x00, 0x52, 0x08, 0x63, 0x69, 0x72, 0x63, 0x6c, 0x65, 0x63, 0x69, 0x12, 0x3d, 0x0a, 0x0a,
0x63, 0x6f, 0x6e, 0x66, 0x6c, 0x75, 0x65, 0x6e, 0x63, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b,
0x32, 0x1b, 0x2e, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61,
0x74, 0x61, 0x2e, 0x43, 0x6f, 0x6e, 0x66, 0x6c, 0x75, 0x65, 0x6e, 0x63, 0x65, 0x48, 0x00, 0x52,
0x0a, 0x63, 0x6f, 0x6e, 0x66, 0x6c, 0x75, 0x65, 0x6e, 0x63, 0x65, 0x12, 0x31, 0x0a, 0x06, 0x64,
0x6f, 0x63, 0x6b, 0x65, 0x72, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x17, 0x2e, 0x73, 0x6f,
0x75, 0x72, 0x63, 0x65, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x44, 0x6f,
0x63, 0x6b, 0x65, 0x72, 0x48, 0x00, 0x52, 0x06, 0x64, 0x6f, 0x63, 0x6b, 0x65, 0x72, 0x12, 0x28,
0x0a, 0x03, 0x65, 0x63, 0x72, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x73, 0x6f,
0x75, 0x72, 0x63, 0x65, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x45, 0x43,
0x52, 0x48, 0x00, 0x52, 0x03, 0x65, 0x63, 0x72, 0x12, 0x28, 0x0a, 0x03, 0x67, 0x63, 0x73, 0x18,
0x07, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f, 0x6d,
0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x47, 0x43, 0x53, 0x48, 0x00, 0x52, 0x03, 0x67,
0x63, 0x73, 0x12, 0x31, 0x0a, 0x06, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x18, 0x08, 0x20, 0x01,
0x28, 0x0b, 0x32, 0x17, 0x2e, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f, 0x6d, 0x65, 0x74, 0x61,
0x64, 0x61, 0x74, 0x61, 0x2e, 0x47, 0x69, 0x74, 0x68, 0x75, 0x62, 0x48, 0x00, 0x52, 0x06, 0x67,
0x69, 0x74, 0x68, 0x75, 0x62, 0x12, 0x31, 0x0a, 0x06, 0x67, 0x69, 0x74, 0x6c, 0x61, 0x62, 0x18,
0x09, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x17, 0x2e, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f, 0x6d,
0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x47, 0x69, 0x74, 0x6c, 0x61, 0x62, 0x48, 0x00,
0x52, 0x06, 0x67, 0x69, 0x74, 0x6c, 0x61, 0x62, 0x12, 0x2b, 0x0a, 0x04, 0x6a, 0x69, 0x72, 0x61,
0x18, 0x0a, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x15, 0x2e, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f,
0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x4a, 0x69, 0x72, 0x61, 0x48, 0x00, 0x52,
0x04, 0x6a, 0x69, 0x72, 0x61, 0x12, 0x28, 0x0a, 0x03, 0x6e, 0x70, 0x6d, 0x18, 0x0b, 0x20, 0x01,
0x28, 0x0b, 0x32, 0x14, 0x2e, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f, 0x6d, 0x65, 0x74, 0x61,
0x64, 0x61, 0x74, 0x61, 0x2e, 0x4e, 0x50, 0x4d, 0x48, 0x00, 0x52, 0x03, 0x6e, 0x70, 0x6d, 0x12,
0x2b, 0x0a, 0x04, 0x70, 0x79, 0x70, 0x69, 0x18, 0x0c, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x15, 0x2e,
0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e,
0x50, 0x79, 0x50, 0x69, 0x48, 0x00, 0x52, 0x04, 0x70, 0x79, 0x70, 0x69, 0x12, 0x25, 0x0a, 0x02,
0x73, 0x33, 0x18, 0x0d, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x13, 0x2e, 0x73, 0x6f, 0x75, 0x72, 0x63,
0x65, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x53, 0x33, 0x48, 0x00, 0x52,
0x02, 0x73, 0x33, 0x12, 0x2e, 0x0a, 0x05, 0x73, 0x6c, 0x61, 0x63, 0x6b, 0x18, 0x0e, 0x20, 0x01,
0x28, 0x0b, 0x32, 0x16, 0x2e, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f, 0x6d, 0x65, 0x74, 0x61,
0x64, 0x61, 0x74, 0x61, 0x2e, 0x53, 0x6c, 0x61, 0x63, 0x6b, 0x48, 0x00, 0x52, 0x05, 0x73, 0x6c,
0x61, 0x63, 0x6b, 0x12, 0x3d, 0x0a, 0x0a, 0x66, 0x69, 0x6c, 0x65, 0x73, 0x79, 0x73, 0x74, 0x65,
0x6d, 0x18, 0x0f, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1b, 0x2e, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65,
0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x46, 0x69, 0x6c, 0x65, 0x73, 0x79,
0x73, 0x74, 0x65, 0x6d, 0x48, 0x00, 0x52, 0x0a, 0x66, 0x69, 0x6c, 0x65, 0x73, 0x79, 0x73, 0x74,
0x65, 0x6d, 0x12, 0x28, 0x0a, 0x03, 0x67, 0x69, 0x74, 0x18, 0x10, 0x20, 0x01, 0x28, 0x0b, 0x32,
0x14, 0x2e, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74,
0x61, 0x2e, 0x47, 0x69, 0x74, 0x48, 0x00, 0x52, 0x03, 0x67, 0x69, 0x74, 0x12, 0x2b, 0x0a, 0x04,
0x74, 0x65, 0x73, 0x74, 0x18, 0x11, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x15, 0x2e, 0x73, 0x6f, 0x75,
0x72, 0x63, 0x65, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x54, 0x65, 0x73,
0x74, 0x48, 0x00, 0x52, 0x04, 0x74, 0x65, 0x73, 0x74, 0x12, 0x3a, 0x0a, 0x09, 0x62, 0x75, 0x69,
0x6c, 0x64, 0x6b, 0x69, 0x74, 0x65, 0x18, 0x12, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x73,
0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x42,
0x75, 0x69, 0x6c, 0x64, 0x6b, 0x69, 0x74, 0x65, 0x48, 0x00, 0x52, 0x09, 0x62, 0x75, 0x69, 0x6c,
0x64, 0x6b, 0x69, 0x74, 0x65, 0x12, 0x31, 0x0a, 0x06, 0x67, 0x65, 0x72, 0x72, 0x69, 0x74, 0x18,
0x13, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x17, 0x2e, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f, 0x6d,
0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x47, 0x65, 0x72, 0x72, 0x69, 0x74, 0x48, 0x00,
0x52, 0x06, 0x67, 0x65, 0x72, 0x72, 0x69, 0x74, 0x12, 0x34, 0x0a, 0x07, 0x6a, 0x65, 0x6e, 0x6b,
0x69, 0x6e, 0x73, 0x18, 0x14, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x18, 0x2e, 0x73, 0x6f, 0x75, 0x72,
0x63, 0x65, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x4a, 0x65, 0x6e, 0x6b,
0x69, 0x6e, 0x73, 0x48, 0x00, 0x52, 0x07, 0x6a, 0x65, 0x6e, 0x6b, 0x69, 0x6e, 0x73, 0x12, 0x2e,
0x0a, 0x05, 0x74, 0x65, 0x61, 0x6d, 0x73, 0x18, 0x15, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x16, 0x2e,
0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e,
0x54, 0x65, 0x61, 0x6d, 0x73, 0x48, 0x00, 0x52, 0x05, 0x74, 0x65, 0x61, 0x6d, 0x73, 0x12, 0x40,
0x0a, 0x0b, 0x61, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x6f, 0x72, 0x79, 0x18, 0x16, 0x20,
0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f, 0x6d, 0x65, 0x74,
0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x6f, 0x72,
0x79, 0x48, 0x00, 0x52, 0x0b, 0x61, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x6f, 0x72, 0x79,
0x12, 0x31, 0x0a, 0x06, 0x73, 0x79, 0x73, 0x6c, 0x6f, 0x67, 0x18, 0x17, 0x20, 0x01, 0x28, 0x0b,
0x64, 0x69, 0x66, 0x69, 0x65, 0x64, 0x42, 0x79, 0x22, 0xff, 0x0a, 0x0a, 0x08, 0x4d, 0x65, 0x74,
0x61, 0x44, 0x61, 0x74, 0x61, 0x12, 0x2e, 0x0a, 0x05, 0x61, 0x7a, 0x75, 0x72, 0x65, 0x18, 0x01,
0x20, 0x01, 0x28, 0x0b, 0x32, 0x16, 0x2e, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f, 0x6d, 0x65,
0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x41, 0x7a, 0x75, 0x72, 0x65, 0x48, 0x00, 0x52, 0x05,
0x61, 0x7a, 0x75, 0x72, 0x65, 0x12, 0x3a, 0x0a, 0x09, 0x62, 0x69, 0x74, 0x62, 0x75, 0x63, 0x6b,
0x65, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x73, 0x6f, 0x75, 0x72, 0x63,
0x65, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x42, 0x69, 0x74, 0x62, 0x75,
0x63, 0x6b, 0x65, 0x74, 0x48, 0x00, 0x52, 0x09, 0x62, 0x69, 0x74, 0x62, 0x75, 0x63, 0x6b, 0x65,
0x74, 0x12, 0x37, 0x0a, 0x08, 0x63, 0x69, 0x72, 0x63, 0x6c, 0x65, 0x63, 0x69, 0x18, 0x03, 0x20,
0x01, 0x28, 0x0b, 0x32, 0x19, 0x2e, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f, 0x6d, 0x65, 0x74,
0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x43, 0x69, 0x72, 0x63, 0x6c, 0x65, 0x43, 0x49, 0x48, 0x00,
0x52, 0x08, 0x63, 0x69, 0x72, 0x63, 0x6c, 0x65, 0x63, 0x69, 0x12, 0x3d, 0x0a, 0x0a, 0x63, 0x6f,
0x6e, 0x66, 0x6c, 0x75, 0x65, 0x6e, 0x63, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1b,
0x2e, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61,
0x2e, 0x43, 0x6f, 0x6e, 0x66, 0x6c, 0x75, 0x65, 0x6e, 0x63, 0x65, 0x48, 0x00, 0x52, 0x0a, 0x63,
0x6f, 0x6e, 0x66, 0x6c, 0x75, 0x65, 0x6e, 0x63, 0x65, 0x12, 0x31, 0x0a, 0x06, 0x64, 0x6f, 0x63,
0x6b, 0x65, 0x72, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x17, 0x2e, 0x73, 0x6f, 0x75, 0x72,
0x63, 0x65, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x44, 0x6f, 0x63, 0x6b,
0x65, 0x72, 0x48, 0x00, 0x52, 0x06, 0x64, 0x6f, 0x63, 0x6b, 0x65, 0x72, 0x12, 0x28, 0x0a, 0x03,
0x65, 0x63, 0x72, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x73, 0x6f, 0x75, 0x72,
0x63, 0x65, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x45, 0x43, 0x52, 0x48,
0x00, 0x52, 0x03, 0x65, 0x63, 0x72, 0x12, 0x28, 0x0a, 0x03, 0x67, 0x63, 0x73, 0x18, 0x07, 0x20,
0x01, 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f, 0x6d, 0x65, 0x74,
0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x47, 0x43, 0x53, 0x48, 0x00, 0x52, 0x03, 0x67, 0x63, 0x73,
0x12, 0x31, 0x0a, 0x06, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x18, 0x08, 0x20, 0x01, 0x28, 0x0b,
0x32, 0x17, 0x2e, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61,
0x74, 0x61, 0x2e, 0x53, 0x79, 0x73, 0x6c, 0x6f, 0x67, 0x48, 0x00, 0x52, 0x06, 0x73, 0x79, 0x73,
0x6c, 0x6f, 0x67, 0x12, 0x5e, 0x0a, 0x15, 0x70, 0x75, 0x62, 0x6c, 0x69, 0x63, 0x45, 0x76, 0x65,
0x6e, 0x74, 0x4d, 0x6f, 0x6e, 0x69, 0x74, 0x6f, 0x72, 0x69, 0x6e, 0x67, 0x18, 0x18, 0x20, 0x01,
0x28, 0x0b, 0x32, 0x26, 0x2e, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f, 0x6d, 0x65, 0x74, 0x61,
0x64, 0x61, 0x74, 0x61, 0x2e, 0x50, 0x75, 0x62, 0x6c, 0x69, 0x63, 0x45, 0x76, 0x65, 0x6e, 0x74,
0x4d, 0x6f, 0x6e, 0x69, 0x74, 0x6f, 0x72, 0x69, 0x6e, 0x67, 0x48, 0x00, 0x52, 0x15, 0x70, 0x75,
0x62, 0x6c, 0x69, 0x63, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x4d, 0x6f, 0x6e, 0x69, 0x74, 0x6f, 0x72,
0x69, 0x6e, 0x67, 0x12, 0x3d, 0x0a, 0x0a, 0x73, 0x68, 0x61, 0x72, 0x65, 0x70, 0x6f, 0x69, 0x6e,
0x74, 0x18, 0x19, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1b, 0x2e, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65,
0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x53, 0x68, 0x61, 0x72, 0x65, 0x50,
0x6f, 0x69, 0x6e, 0x74, 0x48, 0x00, 0x52, 0x0a, 0x73, 0x68, 0x61, 0x72, 0x65, 0x70, 0x6f, 0x69,
0x6e, 0x74, 0x12, 0x40, 0x0a, 0x0b, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x44, 0x72, 0x69, 0x76,
0x65, 0x18, 0x1a, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65,
0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x47, 0x6f, 0x6f, 0x67, 0x6c, 0x65,
0x44, 0x72, 0x69, 0x76, 0x65, 0x48, 0x00, 0x52, 0x0b, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x44,
0x72, 0x69, 0x76, 0x65, 0x12, 0x3d, 0x0a, 0x0a, 0x61, 0x7a, 0x75, 0x72, 0x65, 0x52, 0x65, 0x70,
0x6f, 0x73, 0x18, 0x1b, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1b, 0x2e, 0x73, 0x6f, 0x75, 0x72, 0x63,
0x65, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x41, 0x7a, 0x75, 0x72, 0x65,
0x52, 0x65, 0x70, 0x6f, 0x73, 0x48, 0x00, 0x52, 0x0a, 0x61, 0x7a, 0x75, 0x72, 0x65, 0x52, 0x65,
0x70, 0x6f, 0x73, 0x42, 0x06, 0x0a, 0x04, 0x64, 0x61, 0x74, 0x61, 0x2a, 0x3e, 0x0a, 0x0a, 0x56,
0x69, 0x73, 0x69, 0x62, 0x69, 0x6c, 0x69, 0x74, 0x79, 0x12, 0x0a, 0x0a, 0x06, 0x70, 0x75, 0x62,
0x6c, 0x69, 0x63, 0x10, 0x00, 0x12, 0x0b, 0x0a, 0x07, 0x70, 0x72, 0x69, 0x76, 0x61, 0x74, 0x65,
0x10, 0x01, 0x12, 0x0a, 0x0a, 0x06, 0x73, 0x68, 0x61, 0x72, 0x65, 0x64, 0x10, 0x02, 0x12, 0x0b,
0x0a, 0x07, 0x75, 0x6e, 0x6b, 0x6e, 0x6f, 0x77, 0x6e, 0x10, 0x03, 0x42, 0x43, 0x5a, 0x41, 0x67,
0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x74, 0x72, 0x75, 0x66, 0x66, 0x6c,
0x65, 0x73, 0x65, 0x63, 0x75, 0x72, 0x69, 0x74, 0x79, 0x2f, 0x74, 0x72, 0x75, 0x66, 0x66, 0x6c,
0x65, 0x68, 0x6f, 0x67, 0x2f, 0x76, 0x33, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x70, 0x62, 0x2f, 0x73,
0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x70, 0x62,
0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
0x74, 0x61, 0x2e, 0x47, 0x69, 0x74, 0x68, 0x75, 0x62, 0x48, 0x00, 0x52, 0x06, 0x67, 0x69, 0x74,
0x68, 0x75, 0x62, 0x12, 0x31, 0x0a, 0x06, 0x67, 0x69, 0x74, 0x6c, 0x61, 0x62, 0x18, 0x09, 0x20,
0x01, 0x28, 0x0b, 0x32, 0x17, 0x2e, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f, 0x6d, 0x65, 0x74,
0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x47, 0x69, 0x74, 0x6c, 0x61, 0x62, 0x48, 0x00, 0x52, 0x06,
0x67, 0x69, 0x74, 0x6c, 0x61, 0x62, 0x12, 0x2b, 0x0a, 0x04, 0x6a, 0x69, 0x72, 0x61, 0x18, 0x0a,
0x20, 0x01, 0x28, 0x0b, 0x32, 0x15, 0x2e, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f, 0x6d, 0x65,
0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x4a, 0x69, 0x72, 0x61, 0x48, 0x00, 0x52, 0x04, 0x6a,
0x69, 0x72, 0x61, 0x12, 0x28, 0x0a, 0x03, 0x6e, 0x70, 0x6d, 0x18, 0x0b, 0x20, 0x01, 0x28, 0x0b,
0x32, 0x14, 0x2e, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61,
0x74, 0x61, 0x2e, 0x4e, 0x50, 0x4d, 0x48, 0x00, 0x52, 0x03, 0x6e, 0x70, 0x6d, 0x12, 0x2b, 0x0a,
0x04, 0x70, 0x79, 0x70, 0x69, 0x18, 0x0c, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x15, 0x2e, 0x73, 0x6f,
0x75, 0x72, 0x63, 0x65, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x50, 0x79,
0x50, 0x69, 0x48, 0x00, 0x52, 0x04, 0x70, 0x79, 0x70, 0x69, 0x12, 0x25, 0x0a, 0x02, 0x73, 0x33,
0x18, 0x0d, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x13, 0x2e, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f,
0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x53, 0x33, 0x48, 0x00, 0x52, 0x02, 0x73,
0x33, 0x12, 0x2e, 0x0a, 0x05, 0x73, 0x6c, 0x61, 0x63, 0x6b, 0x18, 0x0e, 0x20, 0x01, 0x28, 0x0b,
0x32, 0x16, 0x2e, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61,
0x74, 0x61, 0x2e, 0x53, 0x6c, 0x61, 0x63, 0x6b, 0x48, 0x00, 0x52, 0x05, 0x73, 0x6c, 0x61, 0x63,
0x6b, 0x12, 0x3d, 0x0a, 0x0a, 0x66, 0x69, 0x6c, 0x65, 0x73, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x18,
0x0f, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1b, 0x2e, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f, 0x6d,
0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x46, 0x69, 0x6c, 0x65, 0x73, 0x79, 0x73, 0x74,
0x65, 0x6d, 0x48, 0x00, 0x52, 0x0a, 0x66, 0x69, 0x6c, 0x65, 0x73, 0x79, 0x73, 0x74, 0x65, 0x6d,
0x12, 0x28, 0x0a, 0x03, 0x67, 0x69, 0x74, 0x18, 0x10, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x14, 0x2e,
0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e,
0x47, 0x69, 0x74, 0x48, 0x00, 0x52, 0x03, 0x67, 0x69, 0x74, 0x12, 0x2b, 0x0a, 0x04, 0x74, 0x65,
0x73, 0x74, 0x18, 0x11, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x15, 0x2e, 0x73, 0x6f, 0x75, 0x72, 0x63,
0x65, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x54, 0x65, 0x73, 0x74, 0x48,
0x00, 0x52, 0x04, 0x74, 0x65, 0x73, 0x74, 0x12, 0x3a, 0x0a, 0x09, 0x62, 0x75, 0x69, 0x6c, 0x64,
0x6b, 0x69, 0x74, 0x65, 0x18, 0x12, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x73, 0x6f, 0x75,
0x72, 0x63, 0x65, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x42, 0x75, 0x69,
0x6c, 0x64, 0x6b, 0x69, 0x74, 0x65, 0x48, 0x00, 0x52, 0x09, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x6b,
0x69, 0x74, 0x65, 0x12, 0x31, 0x0a, 0x06, 0x67, 0x65, 0x72, 0x72, 0x69, 0x74, 0x18, 0x13, 0x20,
0x01, 0x28, 0x0b, 0x32, 0x17, 0x2e, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f, 0x6d, 0x65, 0x74,
0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x47, 0x65, 0x72, 0x72, 0x69, 0x74, 0x48, 0x00, 0x52, 0x06,
0x67, 0x65, 0x72, 0x72, 0x69, 0x74, 0x12, 0x34, 0x0a, 0x07, 0x6a, 0x65, 0x6e, 0x6b, 0x69, 0x6e,
0x73, 0x18, 0x14, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x18, 0x2e, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65,
0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x4a, 0x65, 0x6e, 0x6b, 0x69, 0x6e,
0x73, 0x48, 0x00, 0x52, 0x07, 0x6a, 0x65, 0x6e, 0x6b, 0x69, 0x6e, 0x73, 0x12, 0x2e, 0x0a, 0x05,
0x74, 0x65, 0x61, 0x6d, 0x73, 0x18, 0x15, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x16, 0x2e, 0x73, 0x6f,
0x75, 0x72, 0x63, 0x65, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x54, 0x65,
0x61, 0x6d, 0x73, 0x48, 0x00, 0x52, 0x05, 0x74, 0x65, 0x61, 0x6d, 0x73, 0x12, 0x40, 0x0a, 0x0b,
0x61, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x6f, 0x72, 0x79, 0x18, 0x16, 0x20, 0x01, 0x28,
0x0b, 0x32, 0x1c, 0x2e, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64,
0x61, 0x74, 0x61, 0x2e, 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x6f, 0x72, 0x79, 0x48,
0x00, 0x52, 0x0b, 0x61, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x6f, 0x72, 0x79, 0x12, 0x31,
0x0a, 0x06, 0x73, 0x79, 0x73, 0x6c, 0x6f, 0x67, 0x18, 0x17, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x17,
0x2e, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61,
0x2e, 0x53, 0x79, 0x73, 0x6c, 0x6f, 0x67, 0x48, 0x00, 0x52, 0x06, 0x73, 0x79, 0x73, 0x6c, 0x6f,
0x67, 0x12, 0x5e, 0x0a, 0x15, 0x70, 0x75, 0x62, 0x6c, 0x69, 0x63, 0x45, 0x76, 0x65, 0x6e, 0x74,
0x4d, 0x6f, 0x6e, 0x69, 0x74, 0x6f, 0x72, 0x69, 0x6e, 0x67, 0x18, 0x18, 0x20, 0x01, 0x28, 0x0b,
0x32, 0x26, 0x2e, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61,
0x74, 0x61, 0x2e, 0x50, 0x75, 0x62, 0x6c, 0x69, 0x63, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x4d, 0x6f,
0x6e, 0x69, 0x74, 0x6f, 0x72, 0x69, 0x6e, 0x67, 0x48, 0x00, 0x52, 0x15, 0x70, 0x75, 0x62, 0x6c,
0x69, 0x63, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x4d, 0x6f, 0x6e, 0x69, 0x74, 0x6f, 0x72, 0x69, 0x6e,
0x67, 0x12, 0x3d, 0x0a, 0x0a, 0x73, 0x68, 0x61, 0x72, 0x65, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x18,
0x19, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1b, 0x2e, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f, 0x6d,
0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x53, 0x68, 0x61, 0x72, 0x65, 0x50, 0x6f, 0x69,
0x6e, 0x74, 0x48, 0x00, 0x52, 0x0a, 0x73, 0x68, 0x61, 0x72, 0x65, 0x70, 0x6f, 0x69, 0x6e, 0x74,
0x12, 0x40, 0x0a, 0x0b, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x44, 0x72, 0x69, 0x76, 0x65, 0x18,
0x1a, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f, 0x6d,
0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x47, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x44, 0x72,
0x69, 0x76, 0x65, 0x48, 0x00, 0x52, 0x0b, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x44, 0x72, 0x69,
0x76, 0x65, 0x42, 0x06, 0x0a, 0x04, 0x64, 0x61, 0x74, 0x61, 0x2a, 0x3e, 0x0a, 0x0a, 0x56, 0x69,
0x73, 0x69, 0x62, 0x69, 0x6c, 0x69, 0x74, 0x79, 0x12, 0x0a, 0x0a, 0x06, 0x70, 0x75, 0x62, 0x6c,
0x69, 0x63, 0x10, 0x00, 0x12, 0x0b, 0x0a, 0x07, 0x70, 0x72, 0x69, 0x76, 0x61, 0x74, 0x65, 0x10,
0x01, 0x12, 0x0a, 0x0a, 0x06, 0x73, 0x68, 0x61, 0x72, 0x65, 0x64, 0x10, 0x02, 0x12, 0x0b, 0x0a,
0x07, 0x75, 0x6e, 0x6b, 0x6e, 0x6f, 0x77, 0x6e, 0x10, 0x03, 0x42, 0x43, 0x5a, 0x41, 0x67, 0x69,
0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x74, 0x72, 0x75, 0x66, 0x66, 0x6c, 0x65,
0x73, 0x65, 0x63, 0x75, 0x72, 0x69, 0x74, 0x79, 0x2f, 0x74, 0x72, 0x75, 0x66, 0x66, 0x6c, 0x65,
0x68, 0x6f, 0x67, 0x2f, 0x76, 0x33, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x70, 0x62, 0x2f, 0x73, 0x6f,
0x75, 0x72, 0x63, 0x65, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x70, 0x62, 0x62,
0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
}
var (
@@ -3348,7 +3173,7 @@ func file_source_metadata_proto_rawDescGZIP() []byte {
}
var file_source_metadata_proto_enumTypes = make([]protoimpl.EnumInfo, 1)
var file_source_metadata_proto_msgTypes = make([]protoimpl.MessageInfo, 28)
var file_source_metadata_proto_msgTypes = make([]protoimpl.MessageInfo, 27)
var file_source_metadata_proto_goTypes = []interface{}{
(Visibility)(0), // 0: source_metadata.Visibility
(*Azure)(nil), // 1: source_metadata.Azure
@@ -3377,46 +3202,43 @@ var file_source_metadata_proto_goTypes = []interface{}{
(*PublicEventMonitoring)(nil), // 24: source_metadata.PublicEventMonitoring
(*SharePoint)(nil), // 25: source_metadata.SharePoint
(*GoogleDrive)(nil), // 26: source_metadata.GoogleDrive
(*AzureRepos)(nil), // 27: source_metadata.AzureRepos
(*MetaData)(nil), // 28: source_metadata.MetaData
(*MetaData)(nil), // 27: source_metadata.MetaData
}
var file_source_metadata_proto_depIdxs = []int32{
0, // 0: source_metadata.Github.visibility:type_name -> source_metadata.Visibility
0, // 1: source_metadata.Slack.visibility:type_name -> source_metadata.Visibility
10, // 2: source_metadata.PublicEventMonitoring.github:type_name -> source_metadata.Github
0, // 3: source_metadata.AzureRepos.visibility:type_name -> source_metadata.Visibility
1, // 4: source_metadata.MetaData.azure:type_name -> source_metadata.Azure
2, // 5: source_metadata.MetaData.bitbucket:type_name -> source_metadata.Bitbucket
4, // 6: source_metadata.MetaData.circleci:type_name -> source_metadata.CircleCI
5, // 7: source_metadata.MetaData.confluence:type_name -> source_metadata.Confluence
6, // 8: source_metadata.MetaData.docker:type_name -> source_metadata.Docker
7, // 9: source_metadata.MetaData.ecr:type_name -> source_metadata.ECR
12, // 10: source_metadata.MetaData.gcs:type_name -> source_metadata.GCS
10, // 11: source_metadata.MetaData.github:type_name -> source_metadata.Github
11, // 12: source_metadata.MetaData.gitlab:type_name -> source_metadata.Gitlab
13, // 13: source_metadata.MetaData.jira:type_name -> source_metadata.Jira
14, // 14: source_metadata.MetaData.npm:type_name -> source_metadata.NPM
15, // 15: source_metadata.MetaData.pypi:type_name -> source_metadata.PyPi
16, // 16: source_metadata.MetaData.s3:type_name -> source_metadata.S3
17, // 17: source_metadata.MetaData.slack:type_name -> source_metadata.Slack
8, // 18: source_metadata.MetaData.filesystem:type_name -> source_metadata.Filesystem
9, // 19: source_metadata.MetaData.git:type_name -> source_metadata.Git
19, // 20: source_metadata.MetaData.test:type_name -> source_metadata.Test
3, // 21: source_metadata.MetaData.buildkite:type_name -> source_metadata.Buildkite
18, // 22: source_metadata.MetaData.gerrit:type_name -> source_metadata.Gerrit
20, // 23: source_metadata.MetaData.jenkins:type_name -> source_metadata.Jenkins
21, // 24: source_metadata.MetaData.teams:type_name -> source_metadata.Teams
22, // 25: source_metadata.MetaData.artifactory:type_name -> source_metadata.Artifactory
23, // 26: source_metadata.MetaData.syslog:type_name -> source_metadata.Syslog
24, // 27: source_metadata.MetaData.publicEventMonitoring:type_name -> source_metadata.PublicEventMonitoring
25, // 28: source_metadata.MetaData.sharepoint:type_name -> source_metadata.SharePoint
26, // 29: source_metadata.MetaData.googleDrive:type_name -> source_metadata.GoogleDrive
27, // 30: source_metadata.MetaData.azureRepos:type_name -> source_metadata.AzureRepos
31, // [31:31] is the sub-list for method output_type
31, // [31:31] is the sub-list for method input_type
31, // [31:31] is the sub-list for extension type_name
31, // [31:31] is the sub-list for extension extendee
0, // [0:31] is the sub-list for field type_name
1, // 3: source_metadata.MetaData.azure:type_name -> source_metadata.Azure
2, // 4: source_metadata.MetaData.bitbucket:type_name -> source_metadata.Bitbucket
4, // 5: source_metadata.MetaData.circleci:type_name -> source_metadata.CircleCI
5, // 6: source_metadata.MetaData.confluence:type_name -> source_metadata.Confluence
6, // 7: source_metadata.MetaData.docker:type_name -> source_metadata.Docker
7, // 8: source_metadata.MetaData.ecr:type_name -> source_metadata.ECR
12, // 9: source_metadata.MetaData.gcs:type_name -> source_metadata.GCS
10, // 10: source_metadata.MetaData.github:type_name -> source_metadata.Github
11, // 11: source_metadata.MetaData.gitlab:type_name -> source_metadata.Gitlab
13, // 12: source_metadata.MetaData.jira:type_name -> source_metadata.Jira
14, // 13: source_metadata.MetaData.npm:type_name -> source_metadata.NPM
15, // 14: source_metadata.MetaData.pypi:type_name -> source_metadata.PyPi
16, // 15: source_metadata.MetaData.s3:type_name -> source_metadata.S3
17, // 16: source_metadata.MetaData.slack:type_name -> source_metadata.Slack
8, // 17: source_metadata.MetaData.filesystem:type_name -> source_metadata.Filesystem
9, // 18: source_metadata.MetaData.git:type_name -> source_metadata.Git
19, // 19: source_metadata.MetaData.test:type_name -> source_metadata.Test
3, // 20: source_metadata.MetaData.buildkite:type_name -> source_metadata.Buildkite
18, // 21: source_metadata.MetaData.gerrit:type_name -> source_metadata.Gerrit
20, // 22: source_metadata.MetaData.jenkins:type_name -> source_metadata.Jenkins
21, // 23: source_metadata.MetaData.teams:type_name -> source_metadata.Teams
22, // 24: source_metadata.MetaData.artifactory:type_name -> source_metadata.Artifactory
23, // 25: source_metadata.MetaData.syslog:type_name -> source_metadata.Syslog
24, // 26: source_metadata.MetaData.publicEventMonitoring:type_name -> source_metadata.PublicEventMonitoring
25, // 27: source_metadata.MetaData.sharepoint:type_name -> source_metadata.SharePoint
26, // 28: source_metadata.MetaData.googleDrive:type_name -> source_metadata.GoogleDrive
29, // [29:29] is the sub-list for method output_type
29, // [29:29] is the sub-list for method input_type
29, // [29:29] is the sub-list for extension type_name
29, // [29:29] is the sub-list for extension extendee
0, // [0:29] is the sub-list for field type_name
}
func init() { file_source_metadata_proto_init() }
@@ -3738,18 +3560,6 @@ func file_source_metadata_proto_init() {
}
}
file_source_metadata_proto_msgTypes[26].Exporter = func(v interface{}, i int) interface{} {
switch v := v.(*AzureRepos); i {
case 0:
return &v.state
case 1:
return &v.sizeCache
case 2:
return &v.unknownFields
default:
return nil
}
}
file_source_metadata_proto_msgTypes[27].Exporter = func(v interface{}, i int) interface{} {
switch v := v.(*MetaData); i {
case 0:
return &v.state
@@ -3765,7 +3575,7 @@ func file_source_metadata_proto_init() {
file_source_metadata_proto_msgTypes[23].OneofWrappers = []interface{}{
(*PublicEventMonitoring_Github)(nil),
}
file_source_metadata_proto_msgTypes[27].OneofWrappers = []interface{}{
file_source_metadata_proto_msgTypes[26].OneofWrappers = []interface{}{
(*MetaData_Azure)(nil),
(*MetaData_Bitbucket)(nil),
(*MetaData_Circleci)(nil),
@@ -3792,7 +3602,6 @@ func file_source_metadata_proto_init() {
(*MetaData_PublicEventMonitoring)(nil),
(*MetaData_Sharepoint)(nil),
(*MetaData_GoogleDrive)(nil),
(*MetaData_AzureRepos)(nil),
}
type x struct{}
out := protoimpl.TypeBuilder{
@@ -3800,7 +3609,7 @@ func file_source_metadata_proto_init() {
GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
RawDescriptor: file_source_metadata_proto_rawDesc,
NumEnums: 1,
NumMessages: 28,
NumMessages: 27,
NumExtensions: 0,
NumServices: 0,
},
@@ -2865,8 +2865,6 @@ func (m *GoogleDrive) validate(all bool) error {
// no validation rules for LastModifiedBy
// no validation rules for Path
if len(errors) > 0 {
return GoogleDriveMultiError(errors)
}
@@ -2944,127 +2942,6 @@ var _ interface {
ErrorName() string
} = GoogleDriveValidationError{}
// Validate checks the field values on AzureRepos with the rules defined in the
// proto definition for this message. If any rules are violated, the first
// error encountered is returned, or nil if there are no violations.
func (m *AzureRepos) Validate() error {
return m.validate(false)
}
// ValidateAll checks the field values on AzureRepos with the rules defined in
// the proto definition for this message. If any rules are violated, the
// result is a list of violation errors wrapped in AzureReposMultiError, or
// nil if none found.
func (m *AzureRepos) ValidateAll() error {
return m.validate(true)
}
func (m *AzureRepos) validate(all bool) error {
if m == nil {
return nil
}
var errors []error
// no validation rules for Link
// no validation rules for Username
// no validation rules for Repository
// no validation rules for Commit
// no validation rules for Email
// no validation rules for File
// no validation rules for Timestamp
// no validation rules for Line
// no validation rules for Visibility
// no validation rules for Project
// no validation rules for Organization
if len(errors) > 0 {
return AzureReposMultiError(errors)
}
return nil
}
// AzureReposMultiError is an error wrapping multiple validation errors
// returned by AzureRepos.ValidateAll() if the designated constraints aren't met.
type AzureReposMultiError []error
// Error returns a concatenation of all the error messages it wraps.
func (m AzureReposMultiError) Error() string {
var msgs []string
for _, err := range m {
msgs = append(msgs, err.Error())
}
return strings.Join(msgs, "; ")
}
// AllErrors returns a list of validation violation errors.
func (m AzureReposMultiError) AllErrors() []error { return m }
// AzureReposValidationError is the validation error returned by
// AzureRepos.Validate if the designated constraints aren't met.
type AzureReposValidationError struct {
field string
reason string
cause error
key bool
}
// Field function returns field value.
func (e AzureReposValidationError) Field() string { return e.field }
// Reason function returns reason value.
func (e AzureReposValidationError) Reason() string { return e.reason }
// Cause function returns cause value.
func (e AzureReposValidationError) Cause() error { return e.cause }
// Key function returns key value.
func (e AzureReposValidationError) Key() bool { return e.key }
// ErrorName returns error name.
func (e AzureReposValidationError) ErrorName() string { return "AzureReposValidationError" }
// Error satisfies the builtin error interface
func (e AzureReposValidationError) Error() string {
cause := ""
if e.cause != nil {
cause = fmt.Sprintf(" | caused by: %v", e.cause)
}
key := ""
if e.key {
key = "key for "
}
return fmt.Sprintf(
"invalid %sAzureRepos.%s: %s%s",
key,
e.field,
e.reason,
cause)
}
var _ error = AzureReposValidationError{}
var _ interface {
Field() string
Reason() string
Key() bool
Cause() error
ErrorName() string
} = AzureReposValidationError{}
// Validate checks the field values on MetaData with the rules defined in the
// proto definition for this message. If any rules are violated, the first
// error encountered is returned, or nil if there are no violations.
@@ -3895,37 +3772,6 @@ func (m *MetaData) validate(all bool) error {
}
}
case *MetaData_AzureRepos:
if all {
switch v := interface{}(m.GetAzureRepos()).(type) {
case interface{ ValidateAll() error }:
if err := v.ValidateAll(); err != nil {
errors = append(errors, MetaDataValidationError{
field: "AzureRepos",
reason: "embedded message failed validation",
cause: err,
})
}
case interface{ Validate() error }:
if err := v.Validate(); err != nil {
errors = append(errors, MetaDataValidationError{
field: "AzureRepos",
reason: "embedded message failed validation",
cause: err,
})
}
}
} else if v, ok := interface{}(m.GetAzureRepos()).(interface{ Validate() error }); ok {
if err := v.Validate(); err != nil {
return MetaDataValidationError{
field: "AzureRepos",
reason: "embedded message failed validation",
cause: err,
}
}
}
}
if len(errors) > 0 {
+124 -326
View File
@@ -59,7 +59,6 @@ const (
SourceType_SOURCE_TYPE_GOOGLE_DRIVE SourceType = 28
SourceType_SOURCE_TYPE_SHAREPOINT SourceType = 29
SourceType_SOURCE_TYPE_GCS_UNAUTHED SourceType = 30
SourceType_SOURCE_TYPE_AZURE_REPOS SourceType = 31
)
// Enum value maps for SourceType.
@@ -96,7 +95,6 @@ var (
28: "SOURCE_TYPE_GOOGLE_DRIVE",
29: "SOURCE_TYPE_SHAREPOINT",
30: "SOURCE_TYPE_GCS_UNAUTHED",
31: "SOURCE_TYPE_AZURE_REPOS",
}
SourceType_value = map[string]int32{
"SOURCE_TYPE_AZURE_STORAGE": 0,
@@ -130,7 +128,6 @@ var (
"SOURCE_TYPE_GOOGLE_DRIVE": 28,
"SOURCE_TYPE_SHAREPOINT": 29,
"SOURCE_TYPE_GCS_UNAUTHED": 30,
"SOURCE_TYPE_AZURE_REPOS": 31,
}
)
@@ -2980,158 +2977,6 @@ type Sharepoint_Oauth struct {
func (*Sharepoint_Oauth) isSharepoint_Credential() {}
type AzureRepos struct {
state protoimpl.MessageState
sizeCache protoimpl.SizeCache
unknownFields protoimpl.UnknownFields
Endpoint string `protobuf:"bytes,1,opt,name=endpoint,proto3" json:"endpoint,omitempty"`
// Types that are assignable to Credential:
// *AzureRepos_Token
// *AzureRepos_Oauth
Credential isAzureRepos_Credential `protobuf_oneof:"credential"`
Repositories []string `protobuf:"bytes,4,rep,name=repositories,proto3" json:"repositories,omitempty"`
Organizations []string `protobuf:"bytes,5,rep,name=organizations,proto3" json:"organizations,omitempty"`
Projects []string `protobuf:"bytes,6,rep,name=projects,proto3" json:"projects,omitempty"`
IncludeForks bool `protobuf:"varint,7,opt,name=includeForks,proto3" json:"includeForks,omitempty"`
IgnoreRepos []string `protobuf:"bytes,8,rep,name=ignoreRepos,proto3" json:"ignoreRepos,omitempty"`
IncludeRepos []string `protobuf:"bytes,9,rep,name=includeRepos,proto3" json:"includeRepos,omitempty"`
IncludeProjects []string `protobuf:"bytes,10,rep,name=includeProjects,proto3" json:"includeProjects,omitempty"`
IgnoreProjects []string `protobuf:"bytes,11,rep,name=ignoreProjects,proto3" json:"ignoreProjects,omitempty"`
}
func (x *AzureRepos) Reset() {
*x = AzureRepos{}
if protoimpl.UnsafeEnabled {
mi := &file_sources_proto_msgTypes[28]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
}
func (x *AzureRepos) String() string {
return protoimpl.X.MessageStringOf(x)
}
func (*AzureRepos) ProtoMessage() {}
func (x *AzureRepos) ProtoReflect() protoreflect.Message {
mi := &file_sources_proto_msgTypes[28]
if protoimpl.UnsafeEnabled && x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
ms.StoreMessageInfo(mi)
}
return ms
}
return mi.MessageOf(x)
}
// Deprecated: Use AzureRepos.ProtoReflect.Descriptor instead.
func (*AzureRepos) Descriptor() ([]byte, []int) {
return file_sources_proto_rawDescGZIP(), []int{28}
}
func (x *AzureRepos) GetEndpoint() string {
if x != nil {
return x.Endpoint
}
return ""
}
func (m *AzureRepos) GetCredential() isAzureRepos_Credential {
if m != nil {
return m.Credential
}
return nil
}
func (x *AzureRepos) GetToken() string {
if x, ok := x.GetCredential().(*AzureRepos_Token); ok {
return x.Token
}
return ""
}
func (x *AzureRepos) GetOauth() *credentialspb.Oauth2 {
if x, ok := x.GetCredential().(*AzureRepos_Oauth); ok {
return x.Oauth
}
return nil
}
func (x *AzureRepos) GetRepositories() []string {
if x != nil {
return x.Repositories
}
return nil
}
func (x *AzureRepos) GetOrganizations() []string {
if x != nil {
return x.Organizations
}
return nil
}
func (x *AzureRepos) GetProjects() []string {
if x != nil {
return x.Projects
}
return nil
}
func (x *AzureRepos) GetIncludeForks() bool {
if x != nil {
return x.IncludeForks
}
return false
}
func (x *AzureRepos) GetIgnoreRepos() []string {
if x != nil {
return x.IgnoreRepos
}
return nil
}
func (x *AzureRepos) GetIncludeRepos() []string {
if x != nil {
return x.IncludeRepos
}
return nil
}
func (x *AzureRepos) GetIncludeProjects() []string {
if x != nil {
return x.IncludeProjects
}
return nil
}
func (x *AzureRepos) GetIgnoreProjects() []string {
if x != nil {
return x.IgnoreProjects
}
return nil
}
type isAzureRepos_Credential interface {
isAzureRepos_Credential()
}
type AzureRepos_Token struct {
Token string `protobuf:"bytes,2,opt,name=token,proto3,oneof"`
}
type AzureRepos_Oauth struct {
Oauth *credentialspb.Oauth2 `protobuf:"bytes,3,opt,name=oauth,proto3,oneof"`
}
func (*AzureRepos_Token) isAzureRepos_Credential() {}
func (*AzureRepos_Oauth) isAzureRepos_Credential() {}
var File_sources_proto protoreflect.FileDescriptor
var file_sources_proto_rawDesc = []byte{
@@ -3536,95 +3381,66 @@ var file_sources_proto_rawDesc = []byte{
0x6e, 0x74, 0x69, 0x61, 0x6c, 0x73, 0x2e, 0x4f, 0x61, 0x75, 0x74, 0x68, 0x32, 0x48, 0x00, 0x52,
0x05, 0x6f, 0x61, 0x75, 0x74, 0x68, 0x12, 0x19, 0x0a, 0x08, 0x73, 0x69, 0x74, 0x65, 0x5f, 0x75,
0x72, 0x6c, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x73, 0x69, 0x74, 0x65, 0x55, 0x72,
0x6c, 0x42, 0x0c, 0x0a, 0x0a, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x22,
0xa7, 0x03, 0x0a, 0x0a, 0x41, 0x7a, 0x75, 0x72, 0x65, 0x52, 0x65, 0x70, 0x6f, 0x73, 0x12, 0x24,
0x0a, 0x08, 0x65, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09,
0x42, 0x08, 0xfa, 0x42, 0x05, 0x72, 0x03, 0x90, 0x01, 0x01, 0x52, 0x08, 0x65, 0x6e, 0x64, 0x70,
0x6f, 0x69, 0x6e, 0x74, 0x12, 0x16, 0x0a, 0x05, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x18, 0x02, 0x20,
0x01, 0x28, 0x09, 0x48, 0x00, 0x52, 0x05, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x12, 0x2b, 0x0a, 0x05,
0x6f, 0x61, 0x75, 0x74, 0x68, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x13, 0x2e, 0x63, 0x72,
0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x73, 0x2e, 0x4f, 0x61, 0x75, 0x74, 0x68, 0x32,
0x48, 0x00, 0x52, 0x05, 0x6f, 0x61, 0x75, 0x74, 0x68, 0x12, 0x22, 0x0a, 0x0c, 0x72, 0x65, 0x70,
0x6f, 0x73, 0x69, 0x74, 0x6f, 0x72, 0x69, 0x65, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x09, 0x52,
0x0c, 0x72, 0x65, 0x70, 0x6f, 0x73, 0x69, 0x74, 0x6f, 0x72, 0x69, 0x65, 0x73, 0x12, 0x24, 0x0a,
0x0d, 0x6f, 0x72, 0x67, 0x61, 0x6e, 0x69, 0x7a, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x05,
0x20, 0x03, 0x28, 0x09, 0x52, 0x0d, 0x6f, 0x72, 0x67, 0x61, 0x6e, 0x69, 0x7a, 0x61, 0x74, 0x69,
0x6f, 0x6e, 0x73, 0x12, 0x1a, 0x0a, 0x08, 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x18,
0x06, 0x20, 0x03, 0x28, 0x09, 0x52, 0x08, 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x12,
0x22, 0x0a, 0x0c, 0x69, 0x6e, 0x63, 0x6c, 0x75, 0x64, 0x65, 0x46, 0x6f, 0x72, 0x6b, 0x73, 0x18,
0x07, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0c, 0x69, 0x6e, 0x63, 0x6c, 0x75, 0x64, 0x65, 0x46, 0x6f,
0x72, 0x6b, 0x73, 0x12, 0x20, 0x0a, 0x0b, 0x69, 0x67, 0x6e, 0x6f, 0x72, 0x65, 0x52, 0x65, 0x70,
0x6f, 0x73, 0x18, 0x08, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0b, 0x69, 0x67, 0x6e, 0x6f, 0x72, 0x65,
0x52, 0x65, 0x70, 0x6f, 0x73, 0x12, 0x22, 0x0a, 0x0c, 0x69, 0x6e, 0x63, 0x6c, 0x75, 0x64, 0x65,
0x52, 0x65, 0x70, 0x6f, 0x73, 0x18, 0x09, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0c, 0x69, 0x6e, 0x63,
0x6c, 0x75, 0x64, 0x65, 0x52, 0x65, 0x70, 0x6f, 0x73, 0x12, 0x28, 0x0a, 0x0f, 0x69, 0x6e, 0x63,
0x6c, 0x75, 0x64, 0x65, 0x50, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x18, 0x0a, 0x20, 0x03,
0x28, 0x09, 0x52, 0x0f, 0x69, 0x6e, 0x63, 0x6c, 0x75, 0x64, 0x65, 0x50, 0x72, 0x6f, 0x6a, 0x65,
0x63, 0x74, 0x73, 0x12, 0x26, 0x0a, 0x0e, 0x69, 0x67, 0x6e, 0x6f, 0x72, 0x65, 0x50, 0x72, 0x6f,
0x6a, 0x65, 0x63, 0x74, 0x73, 0x18, 0x0b, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0e, 0x69, 0x67, 0x6e,
0x6f, 0x72, 0x65, 0x50, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x42, 0x0c, 0x0a, 0x0a, 0x63,
0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x2a, 0x84, 0x07, 0x0a, 0x0a, 0x53, 0x6f,
0x75, 0x72, 0x63, 0x65, 0x54, 0x79, 0x70, 0x65, 0x12, 0x1d, 0x0a, 0x19, 0x53, 0x4f, 0x55, 0x52,
0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x41, 0x5a, 0x55, 0x52, 0x45, 0x5f, 0x53, 0x54,
0x4f, 0x52, 0x41, 0x47, 0x45, 0x10, 0x00, 0x12, 0x19, 0x0a, 0x15, 0x53, 0x4f, 0x55, 0x52, 0x43,
0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x42, 0x49, 0x54, 0x42, 0x55, 0x43, 0x4b, 0x45, 0x54,
0x10, 0x01, 0x12, 0x18, 0x0a, 0x14, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50,
0x45, 0x5f, 0x43, 0x49, 0x52, 0x43, 0x4c, 0x45, 0x43, 0x49, 0x10, 0x02, 0x12, 0x1a, 0x0a, 0x16,
0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x43, 0x4f, 0x4e, 0x46,
0x4c, 0x55, 0x45, 0x4e, 0x43, 0x45, 0x10, 0x03, 0x12, 0x16, 0x0a, 0x12, 0x53, 0x4f, 0x55, 0x52,
0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x44, 0x4f, 0x43, 0x4b, 0x45, 0x52, 0x10, 0x04,
0x12, 0x13, 0x0a, 0x0f, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f,
0x45, 0x43, 0x52, 0x10, 0x05, 0x12, 0x13, 0x0a, 0x0f, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f,
0x54, 0x59, 0x50, 0x45, 0x5f, 0x47, 0x43, 0x53, 0x10, 0x06, 0x12, 0x16, 0x0a, 0x12, 0x53, 0x4f,
0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x47, 0x49, 0x54, 0x48, 0x55, 0x42,
0x10, 0x07, 0x12, 0x1a, 0x0a, 0x16, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50,
0x45, 0x5f, 0x50, 0x55, 0x42, 0x4c, 0x49, 0x43, 0x5f, 0x47, 0x49, 0x54, 0x10, 0x08, 0x12, 0x16,
0x0a, 0x12, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x47, 0x49,
0x54, 0x4c, 0x41, 0x42, 0x10, 0x09, 0x12, 0x14, 0x0a, 0x10, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45,
0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x4a, 0x49, 0x52, 0x41, 0x10, 0x0a, 0x12, 0x24, 0x0a, 0x20,
0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x4e, 0x50, 0x4d, 0x5f,
0x55, 0x4e, 0x41, 0x55, 0x54, 0x48, 0x44, 0x5f, 0x50, 0x41, 0x43, 0x4b, 0x41, 0x47, 0x45, 0x53,
0x10, 0x0b, 0x12, 0x25, 0x0a, 0x21, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50,
0x45, 0x5f, 0x50, 0x59, 0x50, 0x49, 0x5f, 0x55, 0x4e, 0x41, 0x55, 0x54, 0x48, 0x44, 0x5f, 0x50,
0x41, 0x43, 0x4b, 0x41, 0x47, 0x45, 0x53, 0x10, 0x0c, 0x12, 0x12, 0x0a, 0x0e, 0x53, 0x4f, 0x55,
0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x53, 0x33, 0x10, 0x0d, 0x12, 0x15, 0x0a,
0x11, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x53, 0x4c, 0x41,
0x43, 0x4b, 0x10, 0x0e, 0x12, 0x1a, 0x0a, 0x16, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54,
0x59, 0x50, 0x45, 0x5f, 0x46, 0x49, 0x4c, 0x45, 0x53, 0x59, 0x53, 0x54, 0x45, 0x4d, 0x10, 0x0f,
0x12, 0x13, 0x0a, 0x0f, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f,
0x47, 0x49, 0x54, 0x10, 0x10, 0x12, 0x14, 0x0a, 0x10, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f,
0x54, 0x59, 0x50, 0x45, 0x5f, 0x54, 0x45, 0x53, 0x54, 0x10, 0x11, 0x12, 0x1b, 0x0a, 0x17, 0x53,
0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x53, 0x33, 0x5f, 0x55, 0x4e,
0x41, 0x55, 0x54, 0x48, 0x45, 0x44, 0x10, 0x12, 0x12, 0x2a, 0x0a, 0x26, 0x53, 0x4f, 0x55, 0x52,
0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x47, 0x49, 0x54, 0x48, 0x55, 0x42, 0x5f, 0x55,
0x4e, 0x41, 0x55, 0x54, 0x48, 0x45, 0x4e, 0x54, 0x49, 0x43, 0x41, 0x54, 0x45, 0x44, 0x5f, 0x4f,
0x52, 0x47, 0x10, 0x13, 0x12, 0x19, 0x0a, 0x15, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54,
0x59, 0x50, 0x45, 0x5f, 0x42, 0x55, 0x49, 0x4c, 0x44, 0x4b, 0x49, 0x54, 0x45, 0x10, 0x14, 0x12,
0x16, 0x0a, 0x12, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x47,
0x45, 0x52, 0x52, 0x49, 0x54, 0x10, 0x15, 0x12, 0x17, 0x0a, 0x13, 0x53, 0x4f, 0x55, 0x52, 0x43,
0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x4a, 0x45, 0x4e, 0x4b, 0x49, 0x4e, 0x53, 0x10, 0x16,
0x12, 0x15, 0x0a, 0x11, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f,
0x54, 0x45, 0x41, 0x4d, 0x53, 0x10, 0x17, 0x12, 0x21, 0x0a, 0x1d, 0x53, 0x4f, 0x55, 0x52, 0x43,
0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x4a, 0x46, 0x52, 0x4f, 0x47, 0x5f, 0x41, 0x52, 0x54,
0x49, 0x46, 0x41, 0x43, 0x54, 0x4f, 0x52, 0x59, 0x10, 0x18, 0x12, 0x16, 0x0a, 0x12, 0x53, 0x4f,
0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x53, 0x59, 0x53, 0x4c, 0x4f, 0x47,
0x10, 0x19, 0x12, 0x27, 0x0a, 0x23, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50,
0x45, 0x5f, 0x50, 0x55, 0x42, 0x4c, 0x49, 0x43, 0x5f, 0x45, 0x56, 0x45, 0x4e, 0x54, 0x5f, 0x4d,
0x4f, 0x4e, 0x49, 0x54, 0x4f, 0x52, 0x49, 0x4e, 0x47, 0x10, 0x1a, 0x12, 0x1e, 0x0a, 0x1a, 0x53,
0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x53, 0x4c, 0x41, 0x43, 0x4b,
0x5f, 0x52, 0x45, 0x41, 0x4c, 0x54, 0x49, 0x4d, 0x45, 0x10, 0x1b, 0x12, 0x1c, 0x0a, 0x18, 0x53,
0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x47, 0x4f, 0x4f, 0x47, 0x4c,
0x45, 0x5f, 0x44, 0x52, 0x49, 0x56, 0x45, 0x10, 0x1c, 0x12, 0x1a, 0x0a, 0x16, 0x53, 0x4f, 0x55,
0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x53, 0x48, 0x41, 0x52, 0x45, 0x50, 0x4f,
0x49, 0x4e, 0x54, 0x10, 0x1d, 0x12, 0x1c, 0x0a, 0x18, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f,
0x54, 0x59, 0x50, 0x45, 0x5f, 0x47, 0x43, 0x53, 0x5f, 0x55, 0x4e, 0x41, 0x55, 0x54, 0x48, 0x45,
0x44, 0x10, 0x1e, 0x12, 0x1b, 0x0a, 0x17, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59,
0x50, 0x45, 0x5f, 0x41, 0x5a, 0x55, 0x52, 0x45, 0x5f, 0x52, 0x45, 0x50, 0x4f, 0x53, 0x10, 0x1f,
0x42, 0x3b, 0x5a, 0x39, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x74,
0x72, 0x75, 0x66, 0x66, 0x6c, 0x65, 0x73, 0x65, 0x63, 0x75, 0x72, 0x69, 0x74, 0x79, 0x2f, 0x74,
0x72, 0x75, 0x66, 0x66, 0x6c, 0x65, 0x68, 0x6f, 0x67, 0x2f, 0x76, 0x33, 0x2f, 0x70, 0x6b, 0x67,
0x2f, 0x70, 0x62, 0x2f, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x73, 0x70, 0x62, 0x62, 0x06, 0x70,
0x72, 0x6f, 0x74, 0x6f, 0x33,
0x6c, 0x42, 0x0c, 0x0a, 0x0a, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x2a,
0xe7, 0x06, 0x0a, 0x0a, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x54, 0x79, 0x70, 0x65, 0x12, 0x1d,
0x0a, 0x19, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x41, 0x5a,
0x55, 0x52, 0x45, 0x5f, 0x53, 0x54, 0x4f, 0x52, 0x41, 0x47, 0x45, 0x10, 0x00, 0x12, 0x19, 0x0a,
0x15, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x42, 0x49, 0x54,
0x42, 0x55, 0x43, 0x4b, 0x45, 0x54, 0x10, 0x01, 0x12, 0x18, 0x0a, 0x14, 0x53, 0x4f, 0x55, 0x52,
0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x43, 0x49, 0x52, 0x43, 0x4c, 0x45, 0x43, 0x49,
0x10, 0x02, 0x12, 0x1a, 0x0a, 0x16, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50,
0x45, 0x5f, 0x43, 0x4f, 0x4e, 0x46, 0x4c, 0x55, 0x45, 0x4e, 0x43, 0x45, 0x10, 0x03, 0x12, 0x16,
0x0a, 0x12, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x44, 0x4f,
0x43, 0x4b, 0x45, 0x52, 0x10, 0x04, 0x12, 0x13, 0x0a, 0x0f, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45,
0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x45, 0x43, 0x52, 0x10, 0x05, 0x12, 0x13, 0x0a, 0x0f, 0x53,
0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x47, 0x43, 0x53, 0x10, 0x06,
0x12, 0x16, 0x0a, 0x12, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f,
0x47, 0x49, 0x54, 0x48, 0x55, 0x42, 0x10, 0x07, 0x12, 0x1a, 0x0a, 0x16, 0x53, 0x4f, 0x55, 0x52,
0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x50, 0x55, 0x42, 0x4c, 0x49, 0x43, 0x5f, 0x47,
0x49, 0x54, 0x10, 0x08, 0x12, 0x16, 0x0a, 0x12, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54,
0x59, 0x50, 0x45, 0x5f, 0x47, 0x49, 0x54, 0x4c, 0x41, 0x42, 0x10, 0x09, 0x12, 0x14, 0x0a, 0x10,
0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x4a, 0x49, 0x52, 0x41,
0x10, 0x0a, 0x12, 0x24, 0x0a, 0x20, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50,
0x45, 0x5f, 0x4e, 0x50, 0x4d, 0x5f, 0x55, 0x4e, 0x41, 0x55, 0x54, 0x48, 0x44, 0x5f, 0x50, 0x41,
0x43, 0x4b, 0x41, 0x47, 0x45, 0x53, 0x10, 0x0b, 0x12, 0x25, 0x0a, 0x21, 0x53, 0x4f, 0x55, 0x52,
0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x50, 0x59, 0x50, 0x49, 0x5f, 0x55, 0x4e, 0x41,
0x55, 0x54, 0x48, 0x44, 0x5f, 0x50, 0x41, 0x43, 0x4b, 0x41, 0x47, 0x45, 0x53, 0x10, 0x0c, 0x12,
0x12, 0x0a, 0x0e, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x53,
0x33, 0x10, 0x0d, 0x12, 0x15, 0x0a, 0x11, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59,
0x50, 0x45, 0x5f, 0x53, 0x4c, 0x41, 0x43, 0x4b, 0x10, 0x0e, 0x12, 0x1a, 0x0a, 0x16, 0x53, 0x4f,
0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x46, 0x49, 0x4c, 0x45, 0x53, 0x59,
0x53, 0x54, 0x45, 0x4d, 0x10, 0x0f, 0x12, 0x13, 0x0a, 0x0f, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45,
0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x47, 0x49, 0x54, 0x10, 0x10, 0x12, 0x14, 0x0a, 0x10, 0x53,
0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x54, 0x45, 0x53, 0x54, 0x10,
0x11, 0x12, 0x1b, 0x0a, 0x17, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45,
0x5f, 0x53, 0x33, 0x5f, 0x55, 0x4e, 0x41, 0x55, 0x54, 0x48, 0x45, 0x44, 0x10, 0x12, 0x12, 0x2a,
0x0a, 0x26, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x47, 0x49,
0x54, 0x48, 0x55, 0x42, 0x5f, 0x55, 0x4e, 0x41, 0x55, 0x54, 0x48, 0x45, 0x4e, 0x54, 0x49, 0x43,
0x41, 0x54, 0x45, 0x44, 0x5f, 0x4f, 0x52, 0x47, 0x10, 0x13, 0x12, 0x19, 0x0a, 0x15, 0x53, 0x4f,
0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x42, 0x55, 0x49, 0x4c, 0x44, 0x4b,
0x49, 0x54, 0x45, 0x10, 0x14, 0x12, 0x16, 0x0a, 0x12, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f,
0x54, 0x59, 0x50, 0x45, 0x5f, 0x47, 0x45, 0x52, 0x52, 0x49, 0x54, 0x10, 0x15, 0x12, 0x17, 0x0a,
0x13, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x4a, 0x45, 0x4e,
0x4b, 0x49, 0x4e, 0x53, 0x10, 0x16, 0x12, 0x15, 0x0a, 0x11, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45,
0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x54, 0x45, 0x41, 0x4d, 0x53, 0x10, 0x17, 0x12, 0x21, 0x0a,
0x1d, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x4a, 0x46, 0x52,
0x4f, 0x47, 0x5f, 0x41, 0x52, 0x54, 0x49, 0x46, 0x41, 0x43, 0x54, 0x4f, 0x52, 0x59, 0x10, 0x18,
0x12, 0x16, 0x0a, 0x12, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f,
0x53, 0x59, 0x53, 0x4c, 0x4f, 0x47, 0x10, 0x19, 0x12, 0x27, 0x0a, 0x23, 0x53, 0x4f, 0x55, 0x52,
0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x50, 0x55, 0x42, 0x4c, 0x49, 0x43, 0x5f, 0x45,
0x56, 0x45, 0x4e, 0x54, 0x5f, 0x4d, 0x4f, 0x4e, 0x49, 0x54, 0x4f, 0x52, 0x49, 0x4e, 0x47, 0x10,
0x1a, 0x12, 0x1e, 0x0a, 0x1a, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45,
0x5f, 0x53, 0x4c, 0x41, 0x43, 0x4b, 0x5f, 0x52, 0x45, 0x41, 0x4c, 0x54, 0x49, 0x4d, 0x45, 0x10,
0x1b, 0x12, 0x1c, 0x0a, 0x18, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45,
0x5f, 0x47, 0x4f, 0x4f, 0x47, 0x4c, 0x45, 0x5f, 0x44, 0x52, 0x49, 0x56, 0x45, 0x10, 0x1c, 0x12,
0x1a, 0x0a, 0x16, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x53,
0x48, 0x41, 0x52, 0x45, 0x50, 0x4f, 0x49, 0x4e, 0x54, 0x10, 0x1d, 0x12, 0x1c, 0x0a, 0x18, 0x53,
0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x47, 0x43, 0x53, 0x5f, 0x55,
0x4e, 0x41, 0x55, 0x54, 0x48, 0x45, 0x44, 0x10, 0x1e, 0x42, 0x3b, 0x5a, 0x39, 0x67, 0x69, 0x74,
0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x74, 0x72, 0x75, 0x66, 0x66, 0x6c, 0x65, 0x73,
0x65, 0x63, 0x75, 0x72, 0x69, 0x74, 0x79, 0x2f, 0x74, 0x72, 0x75, 0x66, 0x66, 0x6c, 0x65, 0x68,
0x6f, 0x67, 0x2f, 0x76, 0x33, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x70, 0x62, 0x2f, 0x73, 0x6f, 0x75,
0x72, 0x63, 0x65, 0x73, 0x70, 0x62, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
}
var (
@@ -3640,7 +3456,7 @@ func file_sources_proto_rawDescGZIP() []byte {
}
var file_sources_proto_enumTypes = make([]protoimpl.EnumInfo, 2)
var file_sources_proto_msgTypes = make([]protoimpl.MessageInfo, 29)
var file_sources_proto_msgTypes = make([]protoimpl.MessageInfo, 28)
var file_sources_proto_goTypes = []interface{}{
(SourceType)(0), // 0: sources.SourceType
(Confluence_GetAllSpacesScope)(0), // 1: sources.Confluence.GetAllSpacesScope
@@ -3672,73 +3488,71 @@ var file_sources_proto_goTypes = []interface{}{
(*PublicEventMonitoring)(nil), // 27: sources.PublicEventMonitoring
(*SlackRealtime)(nil), // 28: sources.SlackRealtime
(*Sharepoint)(nil), // 29: sources.Sharepoint
(*AzureRepos)(nil), // 30: sources.AzureRepos
(*durationpb.Duration)(nil), // 31: google.protobuf.Duration
(*anypb.Any)(nil), // 32: google.protobuf.Any
(*credentialspb.BasicAuth)(nil), // 33: credentials.BasicAuth
(*credentialspb.Unauthenticated)(nil), // 34: credentials.Unauthenticated
(*credentialspb.Oauth2)(nil), // 35: credentials.Oauth2
(*credentialspb.KeySecret)(nil), // 36: credentials.KeySecret
(*credentialspb.CloudEnvironment)(nil), // 37: credentials.CloudEnvironment
(*credentialspb.SSHAuth)(nil), // 38: credentials.SSHAuth
(*credentialspb.GitHubApp)(nil), // 39: credentials.GitHubApp
(*credentialspb.AWSSessionTokenSecret)(nil), // 40: credentials.AWSSessionTokenSecret
(*credentialspb.SlackTokens)(nil), // 41: credentials.SlackTokens
(*credentialspb.Header)(nil), // 42: credentials.Header
(*credentialspb.ClientCredentials)(nil), // 43: credentials.ClientCredentials
(*timestamppb.Timestamp)(nil), // 44: google.protobuf.Timestamp
(*durationpb.Duration)(nil), // 30: google.protobuf.Duration
(*anypb.Any)(nil), // 31: google.protobuf.Any
(*credentialspb.BasicAuth)(nil), // 32: credentials.BasicAuth
(*credentialspb.Unauthenticated)(nil), // 33: credentials.Unauthenticated
(*credentialspb.Oauth2)(nil), // 34: credentials.Oauth2
(*credentialspb.KeySecret)(nil), // 35: credentials.KeySecret
(*credentialspb.CloudEnvironment)(nil), // 36: credentials.CloudEnvironment
(*credentialspb.SSHAuth)(nil), // 37: credentials.SSHAuth
(*credentialspb.GitHubApp)(nil), // 38: credentials.GitHubApp
(*credentialspb.AWSSessionTokenSecret)(nil), // 39: credentials.AWSSessionTokenSecret
(*credentialspb.SlackTokens)(nil), // 40: credentials.SlackTokens
(*credentialspb.Header)(nil), // 41: credentials.Header
(*credentialspb.ClientCredentials)(nil), // 42: credentials.ClientCredentials
(*timestamppb.Timestamp)(nil), // 43: google.protobuf.Timestamp
}
var file_sources_proto_depIdxs = []int32{
31, // 0: sources.LocalSource.scan_interval:type_name -> google.protobuf.Duration
32, // 1: sources.LocalSource.connection:type_name -> google.protobuf.Any
33, // 2: sources.AzureStorage.basic_auth:type_name -> credentials.BasicAuth
34, // 3: sources.AzureStorage.unauthenticated:type_name -> credentials.Unauthenticated
35, // 4: sources.Bitbucket.oauth:type_name -> credentials.Oauth2
33, // 5: sources.Bitbucket.basic_auth:type_name -> credentials.BasicAuth
34, // 6: sources.Confluence.unauthenticated:type_name -> credentials.Unauthenticated
33, // 7: sources.Confluence.basic_auth:type_name -> credentials.BasicAuth
30, // 0: sources.LocalSource.scan_interval:type_name -> google.protobuf.Duration
31, // 1: sources.LocalSource.connection:type_name -> google.protobuf.Any
32, // 2: sources.AzureStorage.basic_auth:type_name -> credentials.BasicAuth
33, // 3: sources.AzureStorage.unauthenticated:type_name -> credentials.Unauthenticated
34, // 4: sources.Bitbucket.oauth:type_name -> credentials.Oauth2
32, // 5: sources.Bitbucket.basic_auth:type_name -> credentials.BasicAuth
33, // 6: sources.Confluence.unauthenticated:type_name -> credentials.Unauthenticated
32, // 7: sources.Confluence.basic_auth:type_name -> credentials.BasicAuth
1, // 8: sources.Confluence.spaces_scope:type_name -> sources.Confluence.GetAllSpacesScope
34, // 9: sources.Docker.unauthenticated:type_name -> credentials.Unauthenticated
33, // 10: sources.Docker.basic_auth:type_name -> credentials.BasicAuth
36, // 11: sources.ECR.access_key:type_name -> credentials.KeySecret
34, // 12: sources.GCS.unauthenticated:type_name -> credentials.Unauthenticated
37, // 13: sources.GCS.adc:type_name -> credentials.CloudEnvironment
35, // 14: sources.GCS.oauth:type_name -> credentials.Oauth2
33, // 15: sources.Git.basic_auth:type_name -> credentials.BasicAuth
34, // 16: sources.Git.unauthenticated:type_name -> credentials.Unauthenticated
38, // 17: sources.Git.ssh_auth:type_name -> credentials.SSHAuth
35, // 18: sources.GitLab.oauth:type_name -> credentials.Oauth2
33, // 19: sources.GitLab.basic_auth:type_name -> credentials.BasicAuth
39, // 20: sources.GitHub.github_app:type_name -> credentials.GitHubApp
34, // 21: sources.GitHub.unauthenticated:type_name -> credentials.Unauthenticated
33, // 22: sources.GitHub.basic_auth:type_name -> credentials.BasicAuth
33, // 23: sources.JIRA.basic_auth:type_name -> credentials.BasicAuth
34, // 24: sources.JIRA.unauthenticated:type_name -> credentials.Unauthenticated
35, // 25: sources.JIRA.oauth:type_name -> credentials.Oauth2
34, // 26: sources.NPMUnauthenticatedPackage.unauthenticated:type_name -> credentials.Unauthenticated
34, // 27: sources.PyPIUnauthenticatedPackage.unauthenticated:type_name -> credentials.Unauthenticated
36, // 28: sources.S3.access_key:type_name -> credentials.KeySecret
34, // 29: sources.S3.unauthenticated:type_name -> credentials.Unauthenticated
37, // 30: sources.S3.cloud_environment:type_name -> credentials.CloudEnvironment
40, // 31: sources.S3.session_token:type_name -> credentials.AWSSessionTokenSecret
41, // 32: sources.Slack.tokens:type_name -> credentials.SlackTokens
33, // 33: sources.Gerrit.basic_auth:type_name -> credentials.BasicAuth
34, // 34: sources.Gerrit.unauthenticated:type_name -> credentials.Unauthenticated
33, // 35: sources.Jenkins.basic_auth:type_name -> credentials.BasicAuth
42, // 36: sources.Jenkins.header:type_name -> credentials.Header
43, // 37: sources.Teams.authenticated:type_name -> credentials.ClientCredentials
35, // 38: sources.Teams.oauth:type_name -> credentials.Oauth2
33, // 39: sources.Artifactory.basic_auth:type_name -> credentials.BasicAuth
34, // 40: sources.PublicEventMonitoring.unauthenticated:type_name -> credentials.Unauthenticated
44, // 41: sources.PublicEventMonitoring.since:type_name -> google.protobuf.Timestamp
41, // 42: sources.SlackRealtime.tokens:type_name -> credentials.SlackTokens
35, // 43: sources.Sharepoint.oauth:type_name -> credentials.Oauth2
35, // 44: sources.AzureRepos.oauth:type_name -> credentials.Oauth2
45, // [45:45] is the sub-list for method output_type
45, // [45:45] is the sub-list for method input_type
45, // [45:45] is the sub-list for extension type_name
45, // [45:45] is the sub-list for extension extendee
0, // [0:45] is the sub-list for field type_name
33, // 9: sources.Docker.unauthenticated:type_name -> credentials.Unauthenticated
32, // 10: sources.Docker.basic_auth:type_name -> credentials.BasicAuth
35, // 11: sources.ECR.access_key:type_name -> credentials.KeySecret
33, // 12: sources.GCS.unauthenticated:type_name -> credentials.Unauthenticated
36, // 13: sources.GCS.adc:type_name -> credentials.CloudEnvironment
34, // 14: sources.GCS.oauth:type_name -> credentials.Oauth2
32, // 15: sources.Git.basic_auth:type_name -> credentials.BasicAuth
33, // 16: sources.Git.unauthenticated:type_name -> credentials.Unauthenticated
37, // 17: sources.Git.ssh_auth:type_name -> credentials.SSHAuth
34, // 18: sources.GitLab.oauth:type_name -> credentials.Oauth2
32, // 19: sources.GitLab.basic_auth:type_name -> credentials.BasicAuth
38, // 20: sources.GitHub.github_app:type_name -> credentials.GitHubApp
33, // 21: sources.GitHub.unauthenticated:type_name -> credentials.Unauthenticated
32, // 22: sources.GitHub.basic_auth:type_name -> credentials.BasicAuth
32, // 23: sources.JIRA.basic_auth:type_name -> credentials.BasicAuth
33, // 24: sources.JIRA.unauthenticated:type_name -> credentials.Unauthenticated
34, // 25: sources.JIRA.oauth:type_name -> credentials.Oauth2
33, // 26: sources.NPMUnauthenticatedPackage.unauthenticated:type_name -> credentials.Unauthenticated
33, // 27: sources.PyPIUnauthenticatedPackage.unauthenticated:type_name -> credentials.Unauthenticated
35, // 28: sources.S3.access_key:type_name -> credentials.KeySecret
33, // 29: sources.S3.unauthenticated:type_name -> credentials.Unauthenticated
36, // 30: sources.S3.cloud_environment:type_name -> credentials.CloudEnvironment
39, // 31: sources.S3.session_token:type_name -> credentials.AWSSessionTokenSecret
40, // 32: sources.Slack.tokens:type_name -> credentials.SlackTokens
32, // 33: sources.Gerrit.basic_auth:type_name -> credentials.BasicAuth
33, // 34: sources.Gerrit.unauthenticated:type_name -> credentials.Unauthenticated
32, // 35: sources.Jenkins.basic_auth:type_name -> credentials.BasicAuth
41, // 36: sources.Jenkins.header:type_name -> credentials.Header
42, // 37: sources.Teams.authenticated:type_name -> credentials.ClientCredentials
34, // 38: sources.Teams.oauth:type_name -> credentials.Oauth2
32, // 39: sources.Artifactory.basic_auth:type_name -> credentials.BasicAuth
33, // 40: sources.PublicEventMonitoring.unauthenticated:type_name -> credentials.Unauthenticated
43, // 41: sources.PublicEventMonitoring.since:type_name -> google.protobuf.Timestamp
40, // 42: sources.SlackRealtime.tokens:type_name -> credentials.SlackTokens
34, // 43: sources.Sharepoint.oauth:type_name -> credentials.Oauth2
44, // [44:44] is the sub-list for method output_type
44, // [44:44] is the sub-list for method input_type
44, // [44:44] is the sub-list for extension type_name
44, // [44:44] is the sub-list for extension extendee
0, // [0:44] is the sub-list for field type_name
}
func init() { file_sources_proto_init() }
@@ -4083,18 +3897,6 @@ func file_sources_proto_init() {
return nil
}
}
file_sources_proto_msgTypes[28].Exporter = func(v interface{}, i int) interface{} {
switch v := v.(*AzureRepos); i {
case 0:
return &v.state
case 1:
return &v.sizeCache
case 2:
return &v.unknownFields
default:
return nil
}
}
}
file_sources_proto_msgTypes[1].OneofWrappers = []interface{}{
(*AzureStorage_ConnectionString)(nil),
@@ -4202,17 +4004,13 @@ func file_sources_proto_init() {
file_sources_proto_msgTypes[27].OneofWrappers = []interface{}{
(*Sharepoint_Oauth)(nil),
}
file_sources_proto_msgTypes[28].OneofWrappers = []interface{}{
(*AzureRepos_Token)(nil),
(*AzureRepos_Oauth)(nil),
}
type x struct{}
out := protoimpl.TypeBuilder{
File: protoimpl.DescBuilder{
GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
RawDescriptor: file_sources_proto_rawDesc,
NumEnums: 2,
NumMessages: 29,
NumMessages: 28,
NumExtensions: 0,
NumServices: 0,
},
-151
View File
@@ -4459,154 +4459,3 @@ var _ interface {
Cause() error
ErrorName() string
} = SharepointValidationError{}
// Validate checks the field values on AzureRepos with the rules defined in the
// proto definition for this message. If any rules are violated, the first
// error encountered is returned, or nil if there are no violations.
func (m *AzureRepos) Validate() error {
return m.validate(false)
}
// ValidateAll checks the field values on AzureRepos with the rules defined in
// the proto definition for this message. If any rules are violated, the
// result is a list of violation errors wrapped in AzureReposMultiError, or
// nil if none found.
func (m *AzureRepos) ValidateAll() error {
return m.validate(true)
}
func (m *AzureRepos) validate(all bool) error {
if m == nil {
return nil
}
var errors []error
if _, err := url.Parse(m.GetEndpoint()); err != nil {
err = AzureReposValidationError{
field: "Endpoint",
reason: "value must be a valid URI",
cause: err,
}
if !all {
return err
}
errors = append(errors, err)
}
// no validation rules for IncludeForks
switch m.Credential.(type) {
case *AzureRepos_Token:
// no validation rules for Token
case *AzureRepos_Oauth:
if all {
switch v := interface{}(m.GetOauth()).(type) {
case interface{ ValidateAll() error }:
if err := v.ValidateAll(); err != nil {
errors = append(errors, AzureReposValidationError{
field: "Oauth",
reason: "embedded message failed validation",
cause: err,
})
}
case interface{ Validate() error }:
if err := v.Validate(); err != nil {
errors = append(errors, AzureReposValidationError{
field: "Oauth",
reason: "embedded message failed validation",
cause: err,
})
}
}
} else if v, ok := interface{}(m.GetOauth()).(interface{ Validate() error }); ok {
if err := v.Validate(); err != nil {
return AzureReposValidationError{
field: "Oauth",
reason: "embedded message failed validation",
cause: err,
}
}
}
}
if len(errors) > 0 {
return AzureReposMultiError(errors)
}
return nil
}
// AzureReposMultiError is an error wrapping multiple validation errors
// returned by AzureRepos.ValidateAll() if the designated constraints aren't met.
type AzureReposMultiError []error
// Error returns a concatenation of all the error messages it wraps.
func (m AzureReposMultiError) Error() string {
var msgs []string
for _, err := range m {
msgs = append(msgs, err.Error())
}
return strings.Join(msgs, "; ")
}
// AllErrors returns a list of validation violation errors.
func (m AzureReposMultiError) AllErrors() []error { return m }
// AzureReposValidationError is the validation error returned by
// AzureRepos.Validate if the designated constraints aren't met.
type AzureReposValidationError struct {
field string
reason string
cause error
key bool
}
// Field function returns field value.
func (e AzureReposValidationError) Field() string { return e.field }
// Reason function returns reason value.
func (e AzureReposValidationError) Reason() string { return e.reason }
// Cause function returns cause value.
func (e AzureReposValidationError) Cause() error { return e.cause }
// Key function returns key value.
func (e AzureReposValidationError) Key() bool { return e.key }
// ErrorName returns error name.
func (e AzureReposValidationError) ErrorName() string { return "AzureReposValidationError" }
// Error satisfies the builtin error interface
func (e AzureReposValidationError) Error() string {
cause := ""
if e.cause != nil {
cause = fmt.Sprintf(" | caused by: %v", e.cause)
}
key := ""
if e.key {
key = "key for "
}
return fmt.Sprintf(
"invalid %sAzureRepos.%s: %s%s",
key,
e.field,
e.reason,
cause)
}
var _ error = AzureReposValidationError{}
var _ interface {
Field() string
Reason() string
Key() bool
Cause() error
ErrorName() string
} = AzureReposValidationError{}
+9 -15
View File
@@ -12,8 +12,6 @@ const (
ChunkSize = 10 * 1024
// PeekSize is the size of the peek into the previous chunk.
PeekSize = 3 * 1024
// TotalChunkSize is the total size of a chunk with peek data.
TotalChunkSize = ChunkSize + PeekSize
)
// Chunker takes a chunk and splits it into chunks of ChunkSize.
@@ -21,31 +19,27 @@ func Chunker(originalChunk *Chunk) chan *Chunk {
chunkChan := make(chan *Chunk)
go func() {
defer close(chunkChan)
if len(originalChunk.Data) <= TotalChunkSize {
if len(originalChunk.Data) <= ChunkSize+PeekSize {
chunkChan <- originalChunk
return
}
r := bytes.NewReader(originalChunk.Data)
reader := bufio.NewReaderSize(bufio.NewReader(r), ChunkSize)
for {
chunkBytes := make([]byte, TotalChunkSize)
chunkBytes := make([]byte, ChunkSize)
chunk := *originalChunk
chunkBytes = chunkBytes[:ChunkSize]
n, err := reader.Read(chunkBytes)
if err != nil && !errors.Is(err, io.EOF) {
break
}
if n == 0 {
if errors.Is(err, io.EOF) {
break
}
continue
}
peekData, _ := reader.Peek(PeekSize)
copy(chunkBytes[n:], peekData)
chunk.Data = chunkBytes[:n+len(peekData)]
chunkChan <- &chunk
chunk.Data = append(chunkBytes[:n], peekData...)
if n > 0 {
chunkChan <- &chunk
}
if errors.Is(err, io.EOF) {
break
}
}
}()
return chunkChan
-12
View File
@@ -72,15 +72,3 @@ func TestChunker(t *testing.T) {
}
}
func BenchmarkChunker(b *testing.B) {
data := bytes.Repeat([]byte("a"), ChunkSize*100)
chunk := &Chunk{
Data: data,
}
b.ResetTimer()
for i := 0; i < b.N; i++ {
for range Chunker(chunk) {
}
}
}
+3 -45
View File
@@ -46,7 +46,6 @@ type Source struct {
var _ sources.Source = (*Source)(nil)
var _ sources.SourceUnitUnmarshaller = (*Source)(nil)
var _ sources.SourceUnitEnumerator = (*Source)(nil)
var _ sources.SourceUnitChunker = (*Source)(nil)
// Type returns the type of source.
// It is used for matching source types in configuration and job input.
@@ -221,53 +220,12 @@ func (s *Source) scanFile(ctx context.Context, path string, chunksChan chan *sou
// Enumerate implements SourceUnitEnumerator interface. This implementation simply
// passes the configured paths as the source unit, whether it be a single
// filepath or a directory.
func (s *Source) Enumerate(ctx context.Context, reporter sources.UnitReporter) error {
func (s *Source) Enumerate(ctx context.Context, units chan<- sources.EnumerationResult) error {
for _, path := range s.paths {
item := sources.CommonSourceUnit{ID: path}
if err := reporter.UnitOk(ctx, item); err != nil {
item := sources.CommonEnumerationOk(path)
if err := common.CancellableWrite(ctx, units, item); err != nil {
return err
}
}
return nil
}
// ChunkUnit implements SourceUnitChunker interface.
func (s *Source) ChunkUnit(ctx context.Context, unit sources.SourceUnit, reporter sources.ChunkReporter) error {
path := unit.SourceUnitID()
logger := ctx.Logger().WithValues("path", path)
cleanPath := filepath.Clean(path)
fileInfo, err := os.Stat(cleanPath)
if err != nil {
return reporter.ChunkErr(ctx, fmt.Errorf("unable to get file info: %w", err))
}
ch := make(chan *sources.Chunk)
var scanErr error
go func() {
defer close(ch)
if fileInfo.IsDir() {
// TODO: Finer grain error tracking of individual chunks.
scanErr = s.scanDir(ctx, cleanPath, ch)
} else {
// TODO: Finer grain error tracking of individual
// chunks (in the case of archives).
scanErr = s.scanFile(ctx, cleanPath, ch)
}
}()
for chunk := range ch {
if chunk == nil {
continue
}
if err := reporter.ChunkOk(ctx, *chunk); err != nil {
return err
}
}
if scanErr != nil && scanErr != io.EOF {
logger.Info("error scanning filesystem", "error", scanErr)
return reporter.ChunkErr(ctx, scanErr)
}
return nil
}
+2 -12
View File
@@ -51,14 +51,9 @@ type Git struct {
jobID int64
sourceMetadataFunc func(file, email, commit, timestamp, repository string, line int64) *source_metadatapb.MetaData
verify bool
metrics metrics
concurrency *semaphore.Weighted
}
type metrics struct {
commitsScanned int
}
func NewGit(sourceType sourcespb.SourceType, jobID, sourceID int64, sourceName string, verify bool, concurrency int,
sourceMetadataFunc func(file, email, commit, timestamp, repository string, line int64) *source_metadatapb.MetaData,
) *Git {
@@ -344,10 +339,6 @@ func CloneRepoUsingSSH(ctx context.Context, gitUrl string, args ...string) (stri
return CloneRepo(ctx, userInfo, gitUrl, args...)
}
func (s *Git) CommitsScanned() int {
return s.metrics.commitsScanned
}
func (s *Git) ScanCommits(ctx context.Context, repo *git.Repository, path string, scanOptions *ScanOptions, chunksChan chan *sources.Chunk) error {
if err := GitCmdCheck(); err != nil {
return err
@@ -380,7 +371,6 @@ func (s *Git) ScanCommits(ctx context.Context, repo *git.Repository, path string
break
}
depth++
s.metrics.commitsScanned++
logger.V(5).Info("scanning commit", "commit", commit.Hash)
for _, diff := range commit.Diffs {
if !scanOptions.Filter.Pass(diff.PathB) {
@@ -765,7 +755,7 @@ func PrepareRepoSinceCommit(ctx context.Context, uriString, commitHash string) (
}
path, _, err = CloneRepoUsingToken(ctx, password, remotePath, uri.User.Username(), "--shallow-since", timestamp)
if err != nil {
return path, true, fmt.Errorf("failed to clone authenticated Git repo (%s): %s", uri.Redacted(), err)
return path, true, fmt.Errorf("failed to clone authenticated Git repo (%s): %s", remotePath, err)
}
default:
ctx.Logger().V(1).Info("cloning repo without authentication", "uri", uri)
@@ -803,7 +793,7 @@ func PrepareRepo(ctx context.Context, uriString string) (string, bool, error) {
}
path, _, err = CloneRepoUsingToken(ctx, password, remotePath, uri.User.Username())
if err != nil {
return path, remote, fmt.Errorf("failed to clone authenticated Git repo (%s): %s", uri.Redacted(), err)
return path, remote, fmt.Errorf("failed to clone authenticated Git repo (%s): %s", remotePath, err)
}
default:
ctx.Logger().V(1).Info("cloning repo without authentication", "uri", uri)
+21 -46
View File
@@ -356,11 +356,11 @@ func (s *Source) enumerate(ctx context.Context, apiEndpoint string) (*github.Cli
switch cred := s.conn.GetCredential().(type) {
case *sourcespb.GitHub_BasicAuth:
if err = s.enumerateBasicAuth(ctx, apiEndpoint, cred.BasicAuth); err != nil {
if err = s.enumerateBasicAuth(ctx, cred.BasicAuth); err != nil {
return nil, err
}
case *sourcespb.GitHub_Unauthenticated:
s.enumerateUnauthenticated(ctx, apiEndpoint)
s.enumerateUnauthenticated(ctx)
case *sourcespb.GitHub_Token:
if err = s.enumerateWithToken(ctx, apiEndpoint, cred.Token); err != nil {
return nil, err
@@ -382,16 +382,11 @@ func (s *Source) enumerate(ctx context.Context, apiEndpoint string) (*github.Cli
return installationClient, nil
}
func (s *Source) enumerateBasicAuth(ctx context.Context, apiEndpoint string, basicAuth *credentialspb.BasicAuth) error {
s.httpClient.Transport = &github.BasicAuthTransport{
func (s *Source) enumerateBasicAuth(ctx context.Context, basicAuth *credentialspb.BasicAuth) error {
s.apiClient = github.NewClient(&http.Client{Transport: &github.BasicAuthTransport{
Username: basicAuth.Username,
Password: basicAuth.Password,
}
ghClient, err := createGitHubClient(s.httpClient, apiEndpoint)
if err != nil {
s.log.Error(err, "error creating GitHub client")
}
s.apiClient = ghClient
}})
for _, org := range s.orgsCache.Keys() {
if err := s.getReposByOrg(ctx, org); err != nil {
@@ -402,12 +397,8 @@ func (s *Source) enumerateBasicAuth(ctx context.Context, apiEndpoint string, bas
return nil
}
func (s *Source) enumerateUnauthenticated(ctx context.Context, apiEndpoint string) {
ghClient, err := createGitHubClient(s.httpClient, apiEndpoint)
if err != nil {
s.log.Error(err, "error creating GitHub client")
}
s.apiClient = ghClient
func (s *Source) enumerateUnauthenticated(ctx context.Context) {
s.apiClient = github.NewClient(s.httpClient)
if s.orgsCache.Count() > unauthGithubOrgRateLimt {
s.log.Info("You may experience rate limiting when using the unauthenticated GitHub api. Consider using an authenticated scan instead.")
}
@@ -441,14 +432,19 @@ func (s *Source) enumerateWithToken(ctx context.Context, apiEndpoint, token stri
Source: oauth2.ReuseTokenSource(nil, ts),
}
// If we're using public GitHub, make a regular client.
var err error
// If we're using public Github, make a regular client.
// Otherwise, make an enterprise client.
var isGHE bool = apiEndpoint != "https://api.github.com"
ghClient, err := createGitHubClient(s.httpClient, apiEndpoint)
if err != nil {
s.log.Error(err, "error creating GitHub client")
var isGHE bool
if apiEndpoint == "https://api.github.com" {
s.apiClient = github.NewClient(s.httpClient)
} else {
isGHE = true
s.apiClient, err = github.NewEnterpriseClient(apiEndpoint, apiEndpoint, s.httpClient)
if err != nil {
return errors.New(err)
}
}
s.apiClient = ghClient
// TODO: this should support scanning users too
@@ -564,9 +560,7 @@ func (s *Source) enumerateWithApp(ctx context.Context, apiEndpoint string, app *
return nil, errors.New(err)
}
itr.BaseURL = apiEndpoint
s.httpClient.Transport = itr
s.apiClient, err = github.NewEnterpriseClient(apiEndpoint, apiEndpoint, s.httpClient)
s.apiClient, err = github.NewEnterpriseClient(apiEndpoint, apiEndpoint, &http.Client{Transport: itr})
if err != nil {
return nil, errors.New(err)
}
@@ -581,11 +575,7 @@ func (s *Source) enumerateWithApp(ctx context.Context, apiEndpoint string, app *
return nil, errors.New(err)
}
appItr.BaseURL = apiEndpoint
// Does this need to be separate from |s.httpClient|?
instHttpClient := common.RetryableHttpClientTimeout(60)
instHttpClient.Transport = appItr
installationClient, err = github.NewEnterpriseClient(apiEndpoint, apiEndpoint, instHttpClient)
installationClient, err = github.NewEnterpriseClient(apiEndpoint, apiEndpoint, &http.Client{Transport: appItr})
if err != nil {
return nil, errors.New(err)
}
@@ -618,21 +608,6 @@ func (s *Source) enumerateWithApp(ctx context.Context, apiEndpoint string, app *
return installationClient, nil
}
func createGitHubClient(httpClient *http.Client, apiEndpoint string) (ghClient *github.Client, err error) {
// If we're using public GitHub, make a regular client.
// Otherwise, make an enterprise client.
if apiEndpoint == "https://api.github.com" {
ghClient = github.NewClient(httpClient)
} else {
ghClient, err = github.NewEnterpriseClient(apiEndpoint, apiEndpoint, httpClient)
if err != nil {
return nil, errors.New(err)
}
}
return ghClient, err
}
func (s *Source) scan(ctx context.Context, installationClient *github.Client, chunksChan chan *sources.Chunk) error {
var scanned uint64
@@ -957,7 +932,7 @@ func (s *Source) setProgressCompleteWithRepo(index int, offset int, repoURL stri
}
func (s *Source) scanComments(ctx context.Context, repoPath string, chunksChan chan *sources.Chunk) error {
s.log.Info("scanning comments", "repository", repoPath)
s.log.Info("scanning comments")
// Support ssh and https URLs
repoURL, err := git.GitURLParse(repoPath)
+2 -3
View File
@@ -339,8 +339,7 @@ func TestHandleRateLimit(t *testing.T) {
func TestEnumerateUnauthenticated(t *testing.T) {
defer gock.Off()
apiEndpoint := "https://api.github.com"
gock.New(apiEndpoint).
gock.New("https://api.github.com").
Get("/orgs/super-secret-org/repos").
Reply(200).
JSON([]map[string]string{{"clone_url": "https://github.com/super-secret-repo.git", "full_name": "super-secret-repo"}})
@@ -348,7 +347,7 @@ func TestEnumerateUnauthenticated(t *testing.T) {
s := initTestSource(nil)
s.orgsCache = memory.New()
s.orgsCache.Set("super-secret-org", "super-secret-org")
s.enumerateUnauthenticated(context.Background(), apiEndpoint)
s.enumerateUnauthenticated(context.Background())
assert.Equal(t, 1, s.filteredRepoCache.Count())
ok := s.filteredRepoCache.Exists("super-secret-repo")
assert.True(t, ok)
-37
View File
@@ -1,37 +0,0 @@
package sources
import (
"errors"
"sync"
"time"
)
// JobReport aggregates information about a run of a Source.
type JobReport struct {
SourceID int64
JobID int64
StartTime time.Time
EndTime time.Time
TotalChunks uint64
errors []error
errorsLock sync.Mutex
}
// AddError adds a non-nil error to the aggregate of errors encountered during
// scanning.
func (jr *JobReport) AddError(err error) {
if err == nil {
return
}
jr.errorsLock.Lock()
defer jr.errorsLock.Unlock()
jr.errors = append(jr.errors, err)
}
// Errors joins all aggregated errors into one. If there were no errors, nil is
// returned. errors.Is can be used to check for specific errors.
func (jr *JobReport) Errors() error {
jr.errorsLock.Lock()
defer jr.errorsLock.Unlock()
return errors.Join(jr.errors...)
}
-404
View File
@@ -1,404 +0,0 @@
package sources
import (
"errors"
"fmt"
"sync"
"sync/atomic"
"time"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/context"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
"golang.org/x/sync/errgroup"
)
// handle uniquely identifies a Source given to the manager to manage. If the
// SourceManager is connected to the API, it will be equivalent to the unique
// source ID, otherwise it behaves as a counter.
type handle int64
// SourceInitFunc is a function that takes a source and job ID and returns an
// initialized Source.
type SourceInitFunc func(ctx context.Context, sourceID int64, jobID int64) (Source, error)
type SourceManager struct {
api apiClient
// Map of handle to source initializer.
handles map[handle]SourceInitFunc
handlesLock sync.Mutex
// Map of handle to job reports.
// TODO: Manage culling and flushing to the API.
report map[handle]*JobReport
reportLock sync.Mutex
// Pool limiting the amount of concurrent sources running.
pool errgroup.Group
concurrentUnits int
// Run the sources using source unit enumeration / chunking if available.
useSourceUnits bool
// Downstream chunks channel to be scanned.
outputChunks chan *Chunk
// Set when Wait() returns.
done bool
doneErr error
}
// apiClient is an interface for optionally communicating with an external API.
type apiClient interface {
// RegisterSource lets the API know we have a source and it returns a unique source ID for it.
RegisterSource(ctx context.Context, name string, kind sourcespb.SourceType) (int64, error)
// GetJobID queries the API for an existing job or new job ID.
GetJobID(ctx context.Context, id int64) (int64, error)
}
// WithAPI adds an API client to the manager for tracking jobs and progress.
func WithAPI(api apiClient) func(*SourceManager) {
return func(mgr *SourceManager) { mgr.api = api }
}
// WithConcurrency limits the concurrent number of sources a manager can run.
func WithConcurrency(concurrency int) func(*SourceManager) {
return func(mgr *SourceManager) { mgr.pool.SetLimit(concurrency) }
}
// WithBufferedOutput sets the size of the buffer used for the Chunks() channel.
func WithBufferedOutput(size int) func(*SourceManager) {
return func(mgr *SourceManager) { mgr.outputChunks = make(chan *Chunk, size) }
}
// WithSourceUnits enables using source unit enumeration and chunking if the
// source supports it.
func WithSourceUnits() func(*SourceManager) {
return func(mgr *SourceManager) { mgr.useSourceUnits = true }
}
// WithConcurrentUnits limits the number of units to be scanned concurrently.
// The default is unlimited.
func WithConcurrentUnits(n int) func(*SourceManager) {
return func(mgr *SourceManager) { mgr.concurrentUnits = n }
}
// NewManager creates a new manager with the provided options.
func NewManager(opts ...func(*SourceManager)) *SourceManager {
mgr := SourceManager{
// Default to the headless API. Can be overwritten by the WithAPI option.
api: &headlessAPI{},
handles: make(map[handle]SourceInitFunc),
report: make(map[handle]*JobReport),
outputChunks: make(chan *Chunk),
}
for _, opt := range opts {
opt(&mgr)
}
return &mgr
}
// Enroll informs the SourceManager to track and manage a Source.
func (s *SourceManager) Enroll(ctx context.Context, name string, kind sourcespb.SourceType, f SourceInitFunc) (handle, error) {
if s.done {
return 0, fmt.Errorf("manager is done")
}
id, err := s.api.RegisterSource(ctx, name, kind)
if err != nil {
return 0, err
}
handleID := handle(id)
s.handlesLock.Lock()
defer s.handlesLock.Unlock()
if _, ok := s.handles[handleID]; ok {
// TODO: smartly handle this?
return 0, fmt.Errorf("handle ID '%d' already in use", handleID)
}
s.handles[handleID] = f
return handleID, nil
}
// Run blocks until a resource is available to run the source, then
// synchronously runs it.
func (s *SourceManager) Run(ctx context.Context, handle handle) error {
// Do preflight checks before waiting on the pool.
if err := s.preflightChecks(ctx, handle); err != nil {
return err
}
ch := make(chan error)
s.pool.Go(func() error {
defer common.Recover(ctx)
report, err := s.run(ctx, handle)
if report != nil {
s.reportLock.Lock()
s.report[handle] = report
s.reportLock.Unlock()
}
if err != nil {
ch <- err
return nil
}
ch <- report.Errors()
return nil
})
return <-ch
}
// ScheduleRun blocks until a resource is available to run the source, then
// asynchronously runs it. Error information is stored and returned by Wait().
func (s *SourceManager) ScheduleRun(ctx context.Context, handle handle) error {
// Do preflight checks before waiting on the pool.
if err := s.preflightChecks(ctx, handle); err != nil {
return err
}
s.pool.Go(func() error {
defer common.Recover(ctx)
// The error is already saved in the report, so we can ignore
// it here.
report, _ := s.run(ctx, handle)
if report != nil {
s.reportLock.Lock()
s.report[handle] = report
s.reportLock.Unlock()
}
return nil
})
// TODO: Maybe wait for a signal here that initialization was successful?
return nil
}
// Chunks returns the read only channel of all the chunks produced by all of
// the sources managed by this manager.
func (s *SourceManager) Chunks() <-chan *Chunk {
return s.outputChunks
}
// Wait blocks until all running sources are completed and returns an error if
// any of the sources had fatal errors. It also closes the channel returned by
// Chunks(). The manager should not be reused after calling this method.
func (s *SourceManager) Wait() error {
// Check if the manager has been Waited.
if s.done {
return s.doneErr
}
defer close(s.outputChunks)
defer func() { s.done = true }()
// We are only using the errgroup for limiting concurrency.
// TODO: Maybe switch to using a semaphore.Weighted.
_ = s.pool.Wait()
// Aggregate all errors from all job reports.
// TODO: This should probably only be the fatal errors. We'll also need
// to rewrite this for when the reports start getting culled.
s.reportLock.Lock()
defer s.reportLock.Unlock()
errs := make([]error, 0, len(s.report))
for _, report := range s.report {
errs = append(errs, report.Errors())
}
s.doneErr = errors.Join(errs...)
return s.doneErr
}
// Report retrieves a scan report for a given handle. If no report exists or
// the Source has not finished, nil will be returned.
func (s *SourceManager) Report(handle handle) *JobReport {
s.reportLock.Lock()
defer s.reportLock.Unlock()
return s.report[handle]
}
// preflightChecks is a helper method to check the Manager or the context isn't
// done and that the handle is valid.
func (s *SourceManager) preflightChecks(ctx context.Context, handle handle) error {
// Check if the manager has been Waited.
if s.done {
return fmt.Errorf("manager is done")
}
// Check the handle is valid.
if _, ok := s.getInitFunc(handle); !ok {
return fmt.Errorf("unrecognized handle")
}
return ctx.Err()
}
// run is a helper method to sychronously run the source. It does not check for
// acquired resources. Possible return values are:
//
// - *JobReport, nil
// Successfully ran the source, but the report could have errors.
//
// - *JobReport, error
// There was an error calling Init or Chunks. This sort of error indicates
// a fatal error and is also recorded in the report.
//
// - nil, error:
// There was an error from the API or the handle is invalid. The latter of
// which should never happen due to the preflightChecks.
func (s *SourceManager) run(ctx context.Context, handle handle) (*JobReport, error) {
jobID, err := s.api.GetJobID(ctx, int64(handle))
if err != nil {
return nil, err
}
initFunc, ok := s.getInitFunc(handle)
if !ok {
return nil, fmt.Errorf("unrecognized handle")
}
// Create a report for this run.
report := &JobReport{
SourceID: int64(handle),
JobID: jobID,
StartTime: time.Now(),
}
defer func() { report.EndTime = time.Now() }()
// Initialize the source.
source, err := initFunc(ctx, jobID, int64(handle))
if err != nil {
report.AddError(err)
return report, err
}
// Check for the preferred method of tracking source units.
if enumChunker, ok := source.(SourceUnitEnumChunker); ok && s.useSourceUnits {
return s.runWithUnits(ctx, handle, enumChunker, report)
}
return s.runWithoutUnits(ctx, handle, source, report)
}
// runWithoutUnits is a helper method to run a Source. It has coarse-grained
// job reporting.
func (s *SourceManager) runWithoutUnits(ctx context.Context, handle handle, source Source, report *JobReport) (*JobReport, error) {
// Introspect on the chunks we get from the Chunks method.
ch := make(chan *Chunk)
var wg sync.WaitGroup
// Consume chunks and export chunks.
wg.Add(1)
go func() {
defer wg.Done()
for chunk := range ch {
atomic.AddUint64(&report.TotalChunks, 1)
_ = common.CancellableWrite(ctx, s.outputChunks, chunk)
}
}()
// Don't return from this function until the goroutine has finished
// outputting chunks to the downstream channel. Closing the channel
// will stop the goroutine, so that needs to happen first in the defer
// stack.
defer wg.Wait()
defer close(ch)
if err := source.Chunks(ctx, ch); err != nil {
report.AddError(err)
return report, err
}
return report, nil
}
// runWithUnits is a helper method to run a Source that is also a
// SourceUnitEnumChunker. This allows better introspection of what is getting
// scanned and any errors encountered.
func (s *SourceManager) runWithUnits(ctx context.Context, handle handle, source SourceUnitEnumChunker, report *JobReport) (*JobReport, error) {
reporter := &mgrUnitReporter{
unitCh: make(chan SourceUnit),
}
// Produce units.
go func() {
// TODO: Catch panics and add to report.
defer close(reporter.unitCh)
if err := source.Enumerate(ctx, reporter); err != nil {
report.AddError(err)
}
}()
var wg sync.WaitGroup
// TODO: Maybe switch to using a semaphore.Weighted.
var unitPool errgroup.Group
if s.concurrentUnits != 0 {
// Negative values indicated no limit.
unitPool.SetLimit(s.concurrentUnits)
}
for unit := range reporter.unitCh {
reporter := &mgrChunkReporter{
unitID: unit.SourceUnitID(),
chunkCh: make(chan *Chunk),
}
unit := unit
// Consume units and produce chunks.
unitPool.Go(func() error {
// TODO: Catch panics and add to report.
defer close(reporter.chunkCh)
if err := source.ChunkUnit(ctx, unit, reporter); err != nil {
report.AddError(err)
}
return nil
})
// Consume chunks and export chunks.
wg.Add(1)
go func() {
defer wg.Done()
for chunk := range reporter.chunkCh {
// TODO: Introspect on the chunks we got from this unit.
atomic.AddUint64(&report.TotalChunks, 1)
_ = common.CancellableWrite(ctx, s.outputChunks, chunk)
}
}()
}
wg.Wait()
// TODO: Return fatal errors.
return report, nil
}
// getInitFunc is a helper method for safe concurrent access to the
// map[handle]SourceInitFunc map.
func (s *SourceManager) getInitFunc(handle handle) (SourceInitFunc, bool) {
s.handlesLock.Lock()
defer s.handlesLock.Unlock()
f, ok := s.handles[handle]
return f, ok
}
// headlessAPI implements the apiClient interface locally.
type headlessAPI struct {
// Counters for assigning handle and job IDs.
sourceIDCounter int64
jobIDCounter int64
}
func (api *headlessAPI) RegisterSource(ctx context.Context, name string, kind sourcespb.SourceType) (int64, error) {
return atomic.AddInt64(&api.sourceIDCounter, 1), nil
}
func (api *headlessAPI) GetJobID(ctx context.Context, id int64) (int64, error) {
return atomic.AddInt64(&api.jobIDCounter, 1), nil
}
// mgrUnitReporter implements the UnitReporter interface.
type mgrUnitReporter struct {
unitCh chan SourceUnit
unitErrs []error
unitErrsLock sync.Mutex
}
func (s *mgrUnitReporter) UnitOk(ctx context.Context, unit SourceUnit) error {
return common.CancellableWrite(ctx, s.unitCh, unit)
}
func (s *mgrUnitReporter) UnitErr(ctx context.Context, err error) error {
s.unitErrsLock.Lock()
defer s.unitErrsLock.Unlock()
s.unitErrs = append(s.unitErrs, err)
return nil
}
// mgrChunkReporter implements the ChunkReporter interface.
type mgrChunkReporter struct {
unitID string
chunkCh chan *Chunk
chunkErrs []error
chunkErrsLock sync.Mutex
}
func (s *mgrChunkReporter) ChunkOk(ctx context.Context, chunk Chunk) error {
return common.CancellableWrite(ctx, s.chunkCh, &chunk)
}
func (s *mgrChunkReporter) ChunkErr(ctx context.Context, err error) error {
s.chunkErrsLock.Lock()
defer s.chunkErrsLock.Unlock()
s.chunkErrs = append(s.chunkErrs, err)
return nil
}
-194
View File
@@ -1,194 +0,0 @@
package sources
import (
"fmt"
"testing"
"github.com/trufflesecurity/trufflehog/v3/pkg/context"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
"google.golang.org/protobuf/types/known/anypb"
)
// DummySource implements Source and is used for testing a SourceManager.
type DummySource struct {
sourceID int64
jobID int64
chunker
}
func (d *DummySource) Type() sourcespb.SourceType { return 1337 }
func (d *DummySource) SourceID() int64 { return d.sourceID }
func (d *DummySource) JobID() int64 { return d.jobID }
func (d *DummySource) Init(_ context.Context, _ string, jobID, sourceID int64, _ bool, _ *anypb.Any, _ int) error {
d.sourceID = sourceID
d.jobID = jobID
return nil
}
func (d *DummySource) GetProgress() *Progress { return nil }
// Interface to easily test different chunking methods.
type chunker interface {
Chunks(context.Context, chan *Chunk) error
ChunkUnit(ctx context.Context, unit SourceUnit, reporter ChunkReporter) error
Enumerate(ctx context.Context, reporter UnitReporter) error
}
// Chunk method that writes count bytes to the channel before returning.
type counterChunker struct {
chunkCounter byte
count int
}
func (c *counterChunker) Chunks(_ context.Context, ch chan *Chunk) error {
for i := 0; i < c.count; i++ {
ch <- &Chunk{Data: []byte{c.chunkCounter}}
c.chunkCounter++
}
return nil
}
// countChunk implements SourceUnit.
type countChunk byte
func (c countChunk) SourceUnitID() string { return fmt.Sprintf("countChunk(%d)", c) }
func (c *counterChunker) Enumerate(ctx context.Context, reporter UnitReporter) error {
for i := 0; i < c.count; i++ {
if err := reporter.UnitOk(ctx, countChunk(byte(i))); err != nil {
return err
}
}
return nil
}
func (c *counterChunker) ChunkUnit(ctx context.Context, unit SourceUnit, reporter ChunkReporter) error {
return reporter.ChunkOk(ctx, Chunk{Data: []byte{byte(unit.(countChunk))}})
}
// Chunk method that always returns an error.
type errorChunker struct{ error }
func (c errorChunker) Chunks(context.Context, chan *Chunk) error { return c }
func (c errorChunker) Enumerate(context.Context, UnitReporter) error { return c }
func (c errorChunker) ChunkUnit(context.Context, SourceUnit, ChunkReporter) error { return c }
// enrollDummy is a helper function to enroll a DummySource with a SourceManager.
func enrollDummy(mgr *SourceManager, chunkMethod chunker) (handle, error) {
return mgr.Enroll(context.Background(), "dummy", 1337,
func(ctx context.Context, jobID, sourceID int64) (Source, error) {
source := &DummySource{chunker: chunkMethod}
if err := source.Init(ctx, "dummy", jobID, sourceID, true, nil, 42); err != nil {
return nil, err
}
return source, nil
})
}
// tryRead is a helper function that will try to read from a channel and return
// an error if it cannot.
func tryRead(ch <-chan *Chunk) (*Chunk, error) {
select {
case chunk := <-ch:
return chunk, nil
default:
return nil, fmt.Errorf("no chunk available")
}
}
func TestSourceManagerRun(t *testing.T) {
mgr := NewManager(WithBufferedOutput(8))
handle, err := enrollDummy(mgr, &counterChunker{count: 1})
if err != nil {
t.Fatalf("unexpected error enrolling source: %v", err)
}
for i := 0; i < 3; i++ {
if err := mgr.Run(context.Background(), handle); err != nil {
t.Fatalf("unexpected error running source: %v", err)
}
chunk, err := tryRead(mgr.Chunks())
if err != nil {
t.Fatalf("reading chunk failed: %v", err)
}
if chunk.Data[0] != byte(i) {
t.Fatalf("unexpected chunk value, wanted %v, got: %v", chunk.Data[0], i)
}
// The Chunks channel should be empty now.
if chunk, err := tryRead(mgr.Chunks()); err == nil {
t.Fatalf("unexpected chunk found: %+v", chunk)
}
}
}
func TestSourceManagerWait(t *testing.T) {
mgr := NewManager()
handle, err := enrollDummy(mgr, &counterChunker{count: 1})
if err != nil {
t.Fatalf("unexpected error enrolling source: %v", err)
}
// Asynchronously run the source.
if err := mgr.ScheduleRun(context.Background(), handle); err != nil {
t.Fatalf("unexpected error scheduling run: %v", err)
}
// Read the 1 chunk we're expecting so Waiting completes.
<-mgr.Chunks()
// Wait for all resources to complete.
if err := mgr.Wait(); err != nil {
t.Fatalf("unexpected error waiting: %v", err)
}
// Enroll and run should return an error now.
if _, err := enrollDummy(mgr, &counterChunker{count: 1}); err == nil {
t.Fatalf("expected enroll to fail")
}
if err := mgr.ScheduleRun(context.Background(), handle); err == nil {
t.Fatalf("expected scheduling run to fail")
}
}
func TestSourceManagerError(t *testing.T) {
mgr := NewManager()
handle, err := enrollDummy(mgr, errorChunker{fmt.Errorf("oops")})
if err != nil {
t.Fatalf("unexpected error enrolling source: %v", err)
}
// A synchronous run should fail.
if err := mgr.Run(context.Background(), handle); err == nil {
t.Fatalf("expected run to fail")
}
// Scheduling a run should not fail, but the error should surface in
// Wait().
if err := mgr.ScheduleRun(context.Background(), handle); err != nil {
t.Fatalf("unexpected error scheduling run: %v", err)
}
if err := mgr.Wait(); err == nil {
t.Fatalf("expected wait to fail")
}
}
func TestSourceManagerReport(t *testing.T) {
for _, opts := range [][]func(*SourceManager){
{WithBufferedOutput(8)},
{WithBufferedOutput(8), WithSourceUnits()},
{WithBufferedOutput(8), WithSourceUnits(), WithConcurrentUnits(1)},
} {
mgr := NewManager(opts...)
handle, err := enrollDummy(mgr, &counterChunker{count: 4})
if err != nil {
t.Fatalf("unexpected error enrolling source: %v", err)
}
// Synchronously run the source.
if err := mgr.Run(context.Background(), handle); err != nil {
t.Fatalf("unexpected error running source: %v", err)
}
report := mgr.Report(handle)
if report == nil {
t.Fatalf("expected a report")
}
if err := report.Errors(); err != nil {
t.Fatalf("unexpected error in report: %v", err)
}
if report.TotalChunks != 4 {
t.Fatalf("expected report to have 4 chunks, got: %d", report.TotalChunks)
}
}
}
+26 -40
View File
@@ -44,13 +44,6 @@ type Source interface {
GetProgress() *Progress
}
// SourceUnitEnumChunker are the two required interfaces to support enumerating
// and chunking of units.
type SourceUnitEnumChunker interface {
SourceUnitEnumerator
SourceUnitChunker
}
// SourceUnitUnmarshaller defines an optional interface a Source can implement
// to support units coming from an external source.
type SourceUnitUnmarshaller interface {
@@ -60,41 +53,21 @@ type SourceUnitUnmarshaller interface {
// SourceUnitEnumerator defines an optional interface a Source can implement to
// support enumerating an initialized Source into SourceUnits.
type SourceUnitEnumerator interface {
// Enumerate creates 0 or more units from an initialized source,
// reporting them or any errors to the UnitReporter. This method is
// synchronous but can be called in a goroutine to support concurrent
// enumeration and chunking. An error should only be returned from this
// method in the case of context cancellation, fatal source errors, or
// errors returned by the reporter All other errors related to unit
// enumeration are tracked by the UnitReporter.
Enumerate(ctx context.Context, reporter UnitReporter) error
// Enumerate enumerates the initialized Source, outputting units. This
// method is synchronous but can be called in a goroutine to support
// concurrent enumeration and chunking. An error should only be
// returned from this method in the case of context cancellation. All
// other errors related to unit enumeration are tracked in the
// EnumerationResult.
Enumerate(ctx context.Context, units chan<- EnumerationResult) error
}
// UnitReporter defines the interface a source will use to report whether a
// unit was found during enumeration. Either method may be called any number of
// times. Implementors of this interface should allow for concurrent calls.
type UnitReporter interface {
UnitOk(ctx context.Context, unit SourceUnit) error
UnitErr(ctx context.Context, err error) error
}
// SourceUnitChunker defines an optional interface a Source can implement to
// support chunking a single SourceUnit.
type SourceUnitChunker interface {
// ChunkUnit creates 0 or more chunks from a unit, reporting them or
// any errors to the ChunkReporter. An error should only be returned
// from this method in the case of context cancellation, fatal source
// errors, or errors returned by the reporter. All other errors related
// to unit chunking are tracked by the ChunkReporter.
ChunkUnit(ctx context.Context, unit SourceUnit, reporter ChunkReporter) error
}
// ChunkReporter defines the interface a source will use to report whether a
// chunk was found during unit chunking. Either method may be called any number
// of times. Implementors of this interface should allow for concurrent calls.
type ChunkReporter interface {
ChunkOk(ctx context.Context, chunk Chunk) error
ChunkErr(ctx context.Context, err error) error
// EnumerationResult is the result of an enumeration, containing the unit and
// error if any. Unit and Error are mutually exclusive (only one will be
// non-nil).
type EnumerationResult struct {
Unit SourceUnit
Error error
}
// SourceUnit is an object that represents a Source's unit of work. This is
@@ -270,3 +243,16 @@ func (p *Progress) GetProgress() *Progress {
defer p.mut.Unlock()
return p
}
// CommonEnumerationOk is a helper function to construct an EnumerationResult
// using a CommonSourceUnit.
func CommonEnumerationOk(id string) EnumerationResult {
unit := CommonSourceUnit{ID: id}
return EnumerationResult{Unit: unit}
}
// EnumerationErr is a helper function to construct an EnumerationResult from
// an error.
func EnumerationErr(err error) EnumerationResult {
return EnumerationResult{Error: err}
}
+1 -3
View File
@@ -929,9 +929,7 @@ enum DetectorType {
Prefect = 918;
Docusign = 919;
Couchbase = 920;
Dockerhub = 921;
TrufflehogEnterprise = 922;
EnvoyApiKey = 923;
IBMCloud = 923; // 921 and 922 are reserved for Trufflehog Web and Scanner Detectors
}
message Result {
-16
View File
@@ -248,21 +248,6 @@ message GoogleDrive {
string timestamp = 4;
bool shared = 5;
string last_modified_by = 6;
string path = 7;
}
message AzureRepos {
string link = 1;
string username = 2;
string repository = 3;
string commit = 4;
string email = 5;
string file = 6;
string timestamp = 7;
int64 line = 8;
Visibility visibility = 9;
string project = 10;
string organization = 11;
}
message MetaData {
@@ -293,6 +278,5 @@ message MetaData {
PublicEventMonitoring publicEventMonitoring = 24;
SharePoint sharepoint = 25;
GoogleDrive googleDrive = 26;
AzureRepos azureRepos = 27;
}
}
-17
View File
@@ -43,7 +43,6 @@ enum SourceType {
SOURCE_TYPE_GOOGLE_DRIVE = 28;
SOURCE_TYPE_SHAREPOINT = 29;
SOURCE_TYPE_GCS_UNAUTHED = 30;
SOURCE_TYPE_AZURE_REPOS = 31;
}
message LocalSource {
@@ -314,19 +313,3 @@ message Sharepoint {
}
string site_url = 2;
}
message AzureRepos {
string endpoint = 1 [(validate.rules).string.uri_ref = true];
oneof credential {
string token = 2;
credentials.Oauth2 oauth = 3;
}
repeated string repositories = 4;
repeated string organizations = 5;
repeated string projects = 6;
bool includeForks = 7;
repeated string ignoreRepos = 8;
repeated string includeRepos = 9;
repeated string includeProjects = 10;
repeated string ignoreProjects = 11;
}