Compare commits
6 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 065984ee58 | |||
| fe90effec1 | |||
| e5a34a9809 | |||
| ec3c59d025 | |||
| c99e0b93f7 | |||
| b960231155 |
@@ -65,6 +65,7 @@ require (
|
|||||||
github.com/prometheus/client_golang v1.16.0
|
github.com/prometheus/client_golang v1.16.0
|
||||||
github.com/rabbitmq/amqp091-go v1.8.1
|
github.com/rabbitmq/amqp091-go v1.8.1
|
||||||
github.com/sergi/go-diff v1.3.1
|
github.com/sergi/go-diff v1.3.1
|
||||||
|
github.com/shuheiktgw/go-travis v0.3.1
|
||||||
github.com/snowflakedb/gosnowflake v1.6.23
|
github.com/snowflakedb/gosnowflake v1.6.23
|
||||||
github.com/stretchr/testify v1.8.4
|
github.com/stretchr/testify v1.8.4
|
||||||
github.com/tailscale/depaware v0.0.0-20210622194025-720c4b409502
|
github.com/tailscale/depaware v0.0.0-20210622194025-720c4b409502
|
||||||
|
|||||||
@@ -369,6 +369,7 @@ github.com/google/go-github/v42 v42.0.0 h1:YNT0FwjPrEysRkLIiKuEfSvBPCGKphW5aS5Px
|
|||||||
github.com/google/go-github/v42 v42.0.0/go.mod h1:jgg/jvyI0YlDOM1/ps6XYh04HNQ3vKf0CVko62/EhRg=
|
github.com/google/go-github/v42 v42.0.0/go.mod h1:jgg/jvyI0YlDOM1/ps6XYh04HNQ3vKf0CVko62/EhRg=
|
||||||
github.com/google/go-github/v55 v55.0.0 h1:4pp/1tNMB9X/LuAhs5i0KQAE40NmiR/y6prLNb9x9cg=
|
github.com/google/go-github/v55 v55.0.0 h1:4pp/1tNMB9X/LuAhs5i0KQAE40NmiR/y6prLNb9x9cg=
|
||||||
github.com/google/go-github/v55 v55.0.0/go.mod h1:JLahOTA1DnXzhxEymmFF5PP2tSS9JVNj68mSZNDwskA=
|
github.com/google/go-github/v55 v55.0.0/go.mod h1:JLahOTA1DnXzhxEymmFF5PP2tSS9JVNj68mSZNDwskA=
|
||||||
|
github.com/google/go-querystring v0.0.0-20170111101155-53e6ce116135/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck=
|
||||||
github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8=
|
github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8=
|
||||||
github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU=
|
github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU=
|
||||||
github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPgecwXBIDzw5no=
|
github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPgecwXBIDzw5no=
|
||||||
@@ -605,6 +606,8 @@ github.com/sahilm/fuzzy v0.1.0/go.mod h1:VFvziUEIMCrT6A6tw2RFIXPXXmzXbOsSHF0DOI8
|
|||||||
github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
|
github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
|
||||||
github.com/sergi/go-diff v1.3.1 h1:xkr+Oxo4BOQKmkn/B9eMK0g5Kg/983T9DqqPHwYqD+8=
|
github.com/sergi/go-diff v1.3.1 h1:xkr+Oxo4BOQKmkn/B9eMK0g5Kg/983T9DqqPHwYqD+8=
|
||||||
github.com/sergi/go-diff v1.3.1/go.mod h1:aMJSSKb2lpPvRNec0+w3fl7LP9IOFzdc9Pa4NFbPK1I=
|
github.com/sergi/go-diff v1.3.1/go.mod h1:aMJSSKb2lpPvRNec0+w3fl7LP9IOFzdc9Pa4NFbPK1I=
|
||||||
|
github.com/shuheiktgw/go-travis v0.3.1 h1:SAT16mi77ccqogOslnXxBXzXbpeyChaIYUwi2aJpVZY=
|
||||||
|
github.com/shuheiktgw/go-travis v0.3.1/go.mod h1:avnFFDqJDdRHwlF9tgqvYi3asQCm/HGL8aLxYiKa4Yg=
|
||||||
github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
|
github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
|
||||||
github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0=
|
github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0=
|
||||||
github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
|
github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
|
||||||
|
|||||||
@@ -139,6 +139,9 @@ var (
|
|||||||
|
|
||||||
dockerScan = cli.Command("docker", "Scan Docker Image")
|
dockerScan = cli.Command("docker", "Scan Docker Image")
|
||||||
dockerScanImages = dockerScan.Flag("image", "Docker image to scan. Use the file:// prefix to point to a local tarball, otherwise a image registry is assumed.").Required().Strings()
|
dockerScanImages = dockerScan.Flag("image", "Docker image to scan. Use the file:// prefix to point to a local tarball, otherwise a image registry is assumed.").Required().Strings()
|
||||||
|
|
||||||
|
travisCiScan = cli.Command("travisci", "Scan TravisCI")
|
||||||
|
travisCiScanToken = travisCiScan.Flag("token", "TravisCI token. Can also be provided with environment variable").Envar("TRAVISCI_TOKEN").Required().String()
|
||||||
)
|
)
|
||||||
|
|
||||||
func init() {
|
func init() {
|
||||||
@@ -498,6 +501,10 @@ func run(state overseer.State) {
|
|||||||
if err := e.ScanCircleCI(ctx, *circleCiScanToken); err != nil {
|
if err := e.ScanCircleCI(ctx, *circleCiScanToken); err != nil {
|
||||||
logFatal(err, "Failed to scan CircleCI.")
|
logFatal(err, "Failed to scan CircleCI.")
|
||||||
}
|
}
|
||||||
|
case travisCiScan.FullCommand():
|
||||||
|
if err := e.ScanTravisCI(ctx, *travisCiScanToken); err != nil {
|
||||||
|
logFatal(err, "Failed to scan TravisCI.")
|
||||||
|
}
|
||||||
case gcsScan.FullCommand():
|
case gcsScan.FullCommand():
|
||||||
cfg := sources.GCSConfig{
|
cfg := sources.GCSConfig{
|
||||||
ProjectID: *gcsProjectID,
|
ProjectID: *gcsProjectID,
|
||||||
|
|||||||
@@ -306,6 +306,7 @@ func Start(ctx context.Context, options ...EngineOption) (*Engine, error) {
|
|||||||
e.sourceManager = sources.NewManager(
|
e.sourceManager = sources.NewManager(
|
||||||
sources.WithConcurrentSources(int(e.concurrency)),
|
sources.WithConcurrentSources(int(e.concurrency)),
|
||||||
sources.WithConcurrentUnits(int(e.concurrency)),
|
sources.WithConcurrentUnits(int(e.concurrency)),
|
||||||
|
sources.WithSourceUnits(),
|
||||||
)
|
)
|
||||||
|
|
||||||
if len(e.decoders) == 0 {
|
if len(e.decoders) == 0 {
|
||||||
|
|||||||
@@ -0,0 +1,38 @@
|
|||||||
|
package engine
|
||||||
|
|
||||||
|
import (
|
||||||
|
"runtime"
|
||||||
|
|
||||||
|
"google.golang.org/protobuf/proto"
|
||||||
|
"google.golang.org/protobuf/types/known/anypb"
|
||||||
|
|
||||||
|
"github.com/trufflesecurity/trufflehog/v3/pkg/context"
|
||||||
|
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
|
||||||
|
"github.com/trufflesecurity/trufflehog/v3/pkg/sources/travisci"
|
||||||
|
)
|
||||||
|
|
||||||
|
// ScanTravisCI scans TravisCI logs.
|
||||||
|
func (e *Engine) ScanTravisCI(ctx context.Context, token string) error {
|
||||||
|
connection := &sourcespb.TravisCI{
|
||||||
|
Credential: &sourcespb.TravisCI_Token{
|
||||||
|
Token: token,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
var conn anypb.Any
|
||||||
|
err := anypb.MarshalFrom(&conn, connection, proto.MarshalOptions{})
|
||||||
|
if err != nil {
|
||||||
|
ctx.Logger().Error(err, "failed to marshal Travis CI connection")
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
sourceName := "trufflehog - Travis CI"
|
||||||
|
sourceID, jobID, _ := e.sourceManager.GetIDs(ctx, sourceName, travisci.SourceType)
|
||||||
|
|
||||||
|
travisSource := &travisci.Source{}
|
||||||
|
if err := travisSource.Init(ctx, sourceName, jobID, sourceID, true, &conn, runtime.NumCPU()); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
_, err = e.sourceManager.Run(ctx, sourceName, travisSource)
|
||||||
|
return err
|
||||||
|
}
|
||||||
File diff suppressed because it is too large
Load Diff
@@ -486,6 +486,117 @@ var _ interface {
|
|||||||
ErrorName() string
|
ErrorName() string
|
||||||
} = CircleCIValidationError{}
|
} = CircleCIValidationError{}
|
||||||
|
|
||||||
|
// Validate checks the field values on TravisCI with the rules defined in the
|
||||||
|
// proto definition for this message. If any rules are violated, the first
|
||||||
|
// error encountered is returned, or nil if there are no violations.
|
||||||
|
func (m *TravisCI) Validate() error {
|
||||||
|
return m.validate(false)
|
||||||
|
}
|
||||||
|
|
||||||
|
// ValidateAll checks the field values on TravisCI with the rules defined in
|
||||||
|
// the proto definition for this message. If any rules are violated, the
|
||||||
|
// result is a list of violation errors wrapped in TravisCIMultiError, or nil
|
||||||
|
// if none found.
|
||||||
|
func (m *TravisCI) ValidateAll() error {
|
||||||
|
return m.validate(true)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *TravisCI) validate(all bool) error {
|
||||||
|
if m == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
var errors []error
|
||||||
|
|
||||||
|
// no validation rules for Username
|
||||||
|
|
||||||
|
// no validation rules for Repository
|
||||||
|
|
||||||
|
// no validation rules for BuildNumber
|
||||||
|
|
||||||
|
// no validation rules for JobNumber
|
||||||
|
|
||||||
|
// no validation rules for Link
|
||||||
|
|
||||||
|
// no validation rules for Public
|
||||||
|
|
||||||
|
if len(errors) > 0 {
|
||||||
|
return TravisCIMultiError(errors)
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// TravisCIMultiError is an error wrapping multiple validation errors returned
|
||||||
|
// by TravisCI.ValidateAll() if the designated constraints aren't met.
|
||||||
|
type TravisCIMultiError []error
|
||||||
|
|
||||||
|
// Error returns a concatenation of all the error messages it wraps.
|
||||||
|
func (m TravisCIMultiError) Error() string {
|
||||||
|
var msgs []string
|
||||||
|
for _, err := range m {
|
||||||
|
msgs = append(msgs, err.Error())
|
||||||
|
}
|
||||||
|
return strings.Join(msgs, "; ")
|
||||||
|
}
|
||||||
|
|
||||||
|
// AllErrors returns a list of validation violation errors.
|
||||||
|
func (m TravisCIMultiError) AllErrors() []error { return m }
|
||||||
|
|
||||||
|
// TravisCIValidationError is the validation error returned by
|
||||||
|
// TravisCI.Validate if the designated constraints aren't met.
|
||||||
|
type TravisCIValidationError struct {
|
||||||
|
field string
|
||||||
|
reason string
|
||||||
|
cause error
|
||||||
|
key bool
|
||||||
|
}
|
||||||
|
|
||||||
|
// Field function returns field value.
|
||||||
|
func (e TravisCIValidationError) Field() string { return e.field }
|
||||||
|
|
||||||
|
// Reason function returns reason value.
|
||||||
|
func (e TravisCIValidationError) Reason() string { return e.reason }
|
||||||
|
|
||||||
|
// Cause function returns cause value.
|
||||||
|
func (e TravisCIValidationError) Cause() error { return e.cause }
|
||||||
|
|
||||||
|
// Key function returns key value.
|
||||||
|
func (e TravisCIValidationError) Key() bool { return e.key }
|
||||||
|
|
||||||
|
// ErrorName returns error name.
|
||||||
|
func (e TravisCIValidationError) ErrorName() string { return "TravisCIValidationError" }
|
||||||
|
|
||||||
|
// Error satisfies the builtin error interface
|
||||||
|
func (e TravisCIValidationError) Error() string {
|
||||||
|
cause := ""
|
||||||
|
if e.cause != nil {
|
||||||
|
cause = fmt.Sprintf(" | caused by: %v", e.cause)
|
||||||
|
}
|
||||||
|
|
||||||
|
key := ""
|
||||||
|
if e.key {
|
||||||
|
key = "key for "
|
||||||
|
}
|
||||||
|
|
||||||
|
return fmt.Sprintf(
|
||||||
|
"invalid %sTravisCI.%s: %s%s",
|
||||||
|
key,
|
||||||
|
e.field,
|
||||||
|
e.reason,
|
||||||
|
cause)
|
||||||
|
}
|
||||||
|
|
||||||
|
var _ error = TravisCIValidationError{}
|
||||||
|
|
||||||
|
var _ interface {
|
||||||
|
Field() string
|
||||||
|
Reason() string
|
||||||
|
Key() bool
|
||||||
|
Cause() error
|
||||||
|
ErrorName() string
|
||||||
|
} = TravisCIValidationError{}
|
||||||
|
|
||||||
// Validate checks the field values on Confluence with the rules defined in the
|
// Validate checks the field values on Confluence with the rules defined in the
|
||||||
// proto definition for this message. If any rules are violated, the first
|
// proto definition for this message. If any rules are violated, the first
|
||||||
// error encountered is returned, or nil if there are no violations.
|
// error encountered is returned, or nil if there are no violations.
|
||||||
@@ -3926,6 +4037,37 @@ func (m *MetaData) validate(all bool) error {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
case *MetaData_TravisCI:
|
||||||
|
|
||||||
|
if all {
|
||||||
|
switch v := interface{}(m.GetTravisCI()).(type) {
|
||||||
|
case interface{ ValidateAll() error }:
|
||||||
|
if err := v.ValidateAll(); err != nil {
|
||||||
|
errors = append(errors, MetaDataValidationError{
|
||||||
|
field: "TravisCI",
|
||||||
|
reason: "embedded message failed validation",
|
||||||
|
cause: err,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
case interface{ Validate() error }:
|
||||||
|
if err := v.Validate(); err != nil {
|
||||||
|
errors = append(errors, MetaDataValidationError{
|
||||||
|
field: "TravisCI",
|
||||||
|
reason: "embedded message failed validation",
|
||||||
|
cause: err,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} else if v, ok := interface{}(m.GetTravisCI()).(interface{ Validate() error }); ok {
|
||||||
|
if err := v.Validate(); err != nil {
|
||||||
|
return MetaDataValidationError{
|
||||||
|
field: "TravisCI",
|
||||||
|
reason: "embedded message failed validation",
|
||||||
|
cause: err,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
if len(errors) > 0 {
|
if len(errors) > 0 {
|
||||||
|
|||||||
+741
-640
File diff suppressed because it is too large
Load Diff
@@ -667,6 +667,124 @@ var _ interface {
|
|||||||
ErrorName() string
|
ErrorName() string
|
||||||
} = CircleCIValidationError{}
|
} = CircleCIValidationError{}
|
||||||
|
|
||||||
|
// Validate checks the field values on TravisCI with the rules defined in the
|
||||||
|
// proto definition for this message. If any rules are violated, the first
|
||||||
|
// error encountered is returned, or nil if there are no violations.
|
||||||
|
func (m *TravisCI) Validate() error {
|
||||||
|
return m.validate(false)
|
||||||
|
}
|
||||||
|
|
||||||
|
// ValidateAll checks the field values on TravisCI with the rules defined in
|
||||||
|
// the proto definition for this message. If any rules are violated, the
|
||||||
|
// result is a list of violation errors wrapped in TravisCIMultiError, or nil
|
||||||
|
// if none found.
|
||||||
|
func (m *TravisCI) ValidateAll() error {
|
||||||
|
return m.validate(true)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *TravisCI) validate(all bool) error {
|
||||||
|
if m == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
var errors []error
|
||||||
|
|
||||||
|
if _, err := url.Parse(m.GetEndpoint()); err != nil {
|
||||||
|
err = TravisCIValidationError{
|
||||||
|
field: "Endpoint",
|
||||||
|
reason: "value must be a valid URI",
|
||||||
|
cause: err,
|
||||||
|
}
|
||||||
|
if !all {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
errors = append(errors, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
switch m.Credential.(type) {
|
||||||
|
|
||||||
|
case *TravisCI_Token:
|
||||||
|
// no validation rules for Token
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
if len(errors) > 0 {
|
||||||
|
return TravisCIMultiError(errors)
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// TravisCIMultiError is an error wrapping multiple validation errors returned
|
||||||
|
// by TravisCI.ValidateAll() if the designated constraints aren't met.
|
||||||
|
type TravisCIMultiError []error
|
||||||
|
|
||||||
|
// Error returns a concatenation of all the error messages it wraps.
|
||||||
|
func (m TravisCIMultiError) Error() string {
|
||||||
|
var msgs []string
|
||||||
|
for _, err := range m {
|
||||||
|
msgs = append(msgs, err.Error())
|
||||||
|
}
|
||||||
|
return strings.Join(msgs, "; ")
|
||||||
|
}
|
||||||
|
|
||||||
|
// AllErrors returns a list of validation violation errors.
|
||||||
|
func (m TravisCIMultiError) AllErrors() []error { return m }
|
||||||
|
|
||||||
|
// TravisCIValidationError is the validation error returned by
|
||||||
|
// TravisCI.Validate if the designated constraints aren't met.
|
||||||
|
type TravisCIValidationError struct {
|
||||||
|
field string
|
||||||
|
reason string
|
||||||
|
cause error
|
||||||
|
key bool
|
||||||
|
}
|
||||||
|
|
||||||
|
// Field function returns field value.
|
||||||
|
func (e TravisCIValidationError) Field() string { return e.field }
|
||||||
|
|
||||||
|
// Reason function returns reason value.
|
||||||
|
func (e TravisCIValidationError) Reason() string { return e.reason }
|
||||||
|
|
||||||
|
// Cause function returns cause value.
|
||||||
|
func (e TravisCIValidationError) Cause() error { return e.cause }
|
||||||
|
|
||||||
|
// Key function returns key value.
|
||||||
|
func (e TravisCIValidationError) Key() bool { return e.key }
|
||||||
|
|
||||||
|
// ErrorName returns error name.
|
||||||
|
func (e TravisCIValidationError) ErrorName() string { return "TravisCIValidationError" }
|
||||||
|
|
||||||
|
// Error satisfies the builtin error interface
|
||||||
|
func (e TravisCIValidationError) Error() string {
|
||||||
|
cause := ""
|
||||||
|
if e.cause != nil {
|
||||||
|
cause = fmt.Sprintf(" | caused by: %v", e.cause)
|
||||||
|
}
|
||||||
|
|
||||||
|
key := ""
|
||||||
|
if e.key {
|
||||||
|
key = "key for "
|
||||||
|
}
|
||||||
|
|
||||||
|
return fmt.Sprintf(
|
||||||
|
"invalid %sTravisCI.%s: %s%s",
|
||||||
|
key,
|
||||||
|
e.field,
|
||||||
|
e.reason,
|
||||||
|
cause)
|
||||||
|
}
|
||||||
|
|
||||||
|
var _ error = TravisCIValidationError{}
|
||||||
|
|
||||||
|
var _ interface {
|
||||||
|
Field() string
|
||||||
|
Reason() string
|
||||||
|
Key() bool
|
||||||
|
Cause() error
|
||||||
|
ErrorName() string
|
||||||
|
} = TravisCIValidationError{}
|
||||||
|
|
||||||
// Validate checks the field values on Confluence with the rules defined in the
|
// Validate checks the field values on Confluence with the rules defined in the
|
||||||
// proto definition for this message. If any rules are violated, the first
|
// proto definition for this message. If any rules are violated, the first
|
||||||
// error encountered is returned, or nil if there are no violations.
|
// error encountered is returned, or nil if there are no violations.
|
||||||
|
|||||||
@@ -1,8 +1,8 @@
|
|||||||
package sources
|
package sources
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
"reflect"
|
|
||||||
"sync"
|
"sync"
|
||||||
"testing"
|
"testing"
|
||||||
)
|
)
|
||||||
@@ -40,7 +40,7 @@ func TestNewScanErrors(t *testing.T) {
|
|||||||
t.Run(tc.name, func(t *testing.T) {
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
got := NewScanErrors()
|
got := NewScanErrors()
|
||||||
|
|
||||||
if !reflect.DeepEqual(got, tc.want) {
|
if !errors.Is(got.Errors(), tc.want.Errors()) {
|
||||||
t.Errorf("got %+v, want %+v", got, tc.want)
|
t.Errorf("got %+v, want %+v", got, tc.want)
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
|
|||||||
@@ -0,0 +1,223 @@
|
|||||||
|
package travisci
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"strconv"
|
||||||
|
|
||||||
|
"github.com/go-errors/errors"
|
||||||
|
"github.com/shuheiktgw/go-travis"
|
||||||
|
"golang.org/x/sync/errgroup"
|
||||||
|
"google.golang.org/protobuf/proto"
|
||||||
|
"google.golang.org/protobuf/types/known/anypb"
|
||||||
|
|
||||||
|
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||||
|
"github.com/trufflesecurity/trufflehog/v3/pkg/context"
|
||||||
|
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/source_metadatapb"
|
||||||
|
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
|
||||||
|
"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
|
||||||
|
)
|
||||||
|
|
||||||
|
const (
|
||||||
|
SourceType = sourcespb.SourceType_SOURCE_TYPE_TRAVISCI
|
||||||
|
|
||||||
|
baseURL = "https://api.travis-ci.com/"
|
||||||
|
|
||||||
|
pageSize = 100
|
||||||
|
)
|
||||||
|
|
||||||
|
type Source struct {
|
||||||
|
name string
|
||||||
|
sourceId sources.SourceID
|
||||||
|
jobId sources.JobID
|
||||||
|
verify bool
|
||||||
|
jobPool *errgroup.Group
|
||||||
|
sources.Progress
|
||||||
|
client *travis.Client
|
||||||
|
sources.CommonSourceUnitUnmarshaller
|
||||||
|
returnAfterFirstChunk bool
|
||||||
|
}
|
||||||
|
|
||||||
|
// Ensure the Source satisfies the interfaces at compile time.
|
||||||
|
var _ sources.Source = (*Source)(nil)
|
||||||
|
var _ sources.SourceUnitUnmarshaller = (*Source)(nil)
|
||||||
|
|
||||||
|
// Type returns the type of source.
|
||||||
|
// It is used for matching source types in configuration and job input.
|
||||||
|
func (s *Source) Type() sourcespb.SourceType {
|
||||||
|
return SourceType
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *Source) SourceID() sources.SourceID {
|
||||||
|
return s.sourceId
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *Source) JobID() sources.JobID {
|
||||||
|
return s.jobId
|
||||||
|
}
|
||||||
|
|
||||||
|
// Init returns an initialized TravisCI source.
|
||||||
|
func (s *Source) Init(ctx context.Context, name string, jobId sources.JobID, sourceId sources.SourceID, verify bool, connection *anypb.Any, concurrency int) error {
|
||||||
|
s.name = name
|
||||||
|
s.sourceId = sourceId
|
||||||
|
s.jobId = jobId
|
||||||
|
s.verify = verify
|
||||||
|
s.jobPool = &errgroup.Group{}
|
||||||
|
s.jobPool.SetLimit(concurrency)
|
||||||
|
|
||||||
|
var conn sourcespb.TravisCI
|
||||||
|
if err := anypb.UnmarshalTo(connection, &conn, proto.UnmarshalOptions{}); err != nil {
|
||||||
|
return errors.WrapPrefix(err, "error unmarshalling connection", 0)
|
||||||
|
}
|
||||||
|
|
||||||
|
switch conn.Credential.(type) {
|
||||||
|
case *sourcespb.TravisCI_Token:
|
||||||
|
if conn.GetToken() == "" {
|
||||||
|
return errors.New("token is empty")
|
||||||
|
}
|
||||||
|
s.client = travis.NewClient(baseURL, conn.GetToken())
|
||||||
|
s.client.HTTPClient = common.RetryableHttpClientTimeout(3)
|
||||||
|
|
||||||
|
user, _, err := s.client.User.Current(ctx, nil)
|
||||||
|
if err != nil {
|
||||||
|
return errors.WrapPrefix(err, "error getting testing travis client", 0)
|
||||||
|
}
|
||||||
|
ctx.Logger().V(2).Info("authenticated to Travis CI with user", "username", user.Login)
|
||||||
|
default:
|
||||||
|
return errors.New("credential type not implemented for Travis CI")
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *Source) Enumerate(ctx context.Context, reporter sources.UnitReporter) error {
|
||||||
|
for repoPage := 0; ; repoPage++ {
|
||||||
|
repositories, _, err := s.client.Repositories.List(ctx, &travis.RepositoriesOption{
|
||||||
|
Limit: pageSize,
|
||||||
|
Offset: repoPage * pageSize,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
if repoPage == 0 {
|
||||||
|
return fmt.Errorf("error listing repositories: %w", err)
|
||||||
|
}
|
||||||
|
err = reporter.UnitErr(ctx, err)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("error reporting error: %w", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if len(repositories) == 0 {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, repo := range repositories {
|
||||||
|
err = reporter.UnitOk(ctx, sources.CommonSourceUnit{
|
||||||
|
ID: strconv.Itoa(int(*repo.Id)),
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("error reporting unit: %w", err)
|
||||||
|
}
|
||||||
|
ctx.Logger().V(2).Info("enumerated repository", "id", repo.Id, "name", repo.Name)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// ChunkUnit implements SourceUnitChunker interface.
|
||||||
|
func (s *Source) ChunkUnit(ctx context.Context, unit sources.SourceUnit, reporter sources.ChunkReporter) error {
|
||||||
|
repo, _, err := s.client.Repositories.Find(ctx, unit.SourceUnitID(), nil)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("error finding repository: %w", err)
|
||||||
|
}
|
||||||
|
logger := ctx.Logger().WithValues("repo", *repo.Name)
|
||||||
|
logger.V(2).Info("scanning repository")
|
||||||
|
|
||||||
|
// Counts continuous errors from ListByRepoSlug. Used to quit early in
|
||||||
|
// case the API always returns an error.
|
||||||
|
var buildPageErrs int
|
||||||
|
for buildPage := 0; ; buildPage++ {
|
||||||
|
builds, _, err := s.client.Builds.ListByRepoSlug(ctx, *repo.Slug, &travis.BuildsByRepoOption{
|
||||||
|
Limit: pageSize,
|
||||||
|
Offset: buildPage * pageSize,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
if err := reporter.ChunkErr(ctx, err); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
buildPageErrs++
|
||||||
|
if buildPageErrs >= 5 {
|
||||||
|
return fmt.Errorf("encountered too many errors listing builds, aborting")
|
||||||
|
}
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
// Reset the page error counter.
|
||||||
|
buildPageErrs = 0
|
||||||
|
|
||||||
|
if len(builds) == 0 {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, build := range builds {
|
||||||
|
jobs, _, err := s.client.Jobs.ListByBuild(ctx, *build.Id)
|
||||||
|
if err != nil {
|
||||||
|
if err := reporter.ChunkErr(ctx, err); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
if len(jobs) == 0 {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, job := range jobs {
|
||||||
|
log, _, err := s.client.Logs.FindByJobId(ctx, *job.Id)
|
||||||
|
if err != nil {
|
||||||
|
if err := reporter.ChunkErr(ctx, err); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
logger.V(3).Info("scanning job", "id", *job.Id, "number", *job.Number)
|
||||||
|
|
||||||
|
chunk := sources.Chunk{
|
||||||
|
SourceType: s.Type(),
|
||||||
|
SourceName: s.name,
|
||||||
|
SourceID: s.SourceID(),
|
||||||
|
JobID: s.JobID(),
|
||||||
|
Data: []byte(*log.Content),
|
||||||
|
SourceMetadata: &source_metadatapb.MetaData{
|
||||||
|
Data: &source_metadatapb.MetaData_TravisCI{
|
||||||
|
TravisCI: &source_metadatapb.TravisCI{
|
||||||
|
Username: *job.Owner.Login,
|
||||||
|
Repository: *repo.Name,
|
||||||
|
BuildNumber: *build.Number,
|
||||||
|
JobNumber: *job.Number,
|
||||||
|
Link: fmt.Sprintf("https://app.travis-ci.com/github/%s/%s/jobs/%d", *job.Owner.Login, *repo.Name, *job.Id),
|
||||||
|
Public: !*repo.Private,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
Verify: s.verify,
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := reporter.ChunkOk(ctx, chunk); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
if s.returnAfterFirstChunk {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Chunks emits chunks of bytes over a channel.
|
||||||
|
// It's a no-op because we've implemented the SourceUnitChunker interface instead.
|
||||||
|
func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk, _ ...sources.ChunkingTarget) error {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
@@ -0,0 +1,99 @@
|
|||||||
|
package travisci
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/kylelemons/godebug/pretty"
|
||||||
|
"google.golang.org/protobuf/types/known/anypb"
|
||||||
|
|
||||||
|
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||||
|
"github.com/trufflesecurity/trufflehog/v3/pkg/context"
|
||||||
|
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/source_metadatapb"
|
||||||
|
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
|
||||||
|
"github.com/trufflesecurity/trufflehog/v3/pkg/sourcestest"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestSource_Scan(t *testing.T) {
|
||||||
|
ctx := context.Background()
|
||||||
|
|
||||||
|
secret, err := common.GetTestSecret(ctx)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(fmt.Errorf("failed to access secret: %v", err))
|
||||||
|
}
|
||||||
|
token := secret.MustGetField("TRAVISCI_TOKEN")
|
||||||
|
|
||||||
|
type init struct {
|
||||||
|
name string
|
||||||
|
verify bool
|
||||||
|
connection *sourcespb.TravisCI
|
||||||
|
}
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
init init
|
||||||
|
wantSourceMetadata *source_metadatapb.MetaData
|
||||||
|
wantErr bool
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "get a chunk",
|
||||||
|
init: init{
|
||||||
|
name: "example repo",
|
||||||
|
connection: &sourcespb.TravisCI{
|
||||||
|
Credential: &sourcespb.TravisCI_Token{
|
||||||
|
Token: token,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
verify: true,
|
||||||
|
},
|
||||||
|
wantSourceMetadata: &source_metadatapb.MetaData{
|
||||||
|
Data: &source_metadatapb.MetaData_TravisCI{
|
||||||
|
TravisCI: &source_metadatapb.TravisCI{
|
||||||
|
Username: "truffle-sandbox",
|
||||||
|
Repository: "test-repo",
|
||||||
|
BuildNumber: "1",
|
||||||
|
JobNumber: "1.1",
|
||||||
|
Link: "https://app.travis-ci.com/github/truffle-sandbox/test-repo/jobs/611053994",
|
||||||
|
Public: false,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
wantErr: false,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
for _, tt := range tests {
|
||||||
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
|
s := Source{}
|
||||||
|
|
||||||
|
conn, err := anypb.New(tt.init.connection)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
err = s.Init(ctx, tt.init.name, 0, 0, tt.init.verify, conn, 5)
|
||||||
|
if (err != nil) != tt.wantErr {
|
||||||
|
t.Fatalf("Source.Init() error = %v, wantErr %v", err, tt.wantErr)
|
||||||
|
}
|
||||||
|
|
||||||
|
reporter := sourcestest.TestReporter{}
|
||||||
|
s.returnAfterFirstChunk = true
|
||||||
|
|
||||||
|
err = s.Enumerate(ctx, &reporter)
|
||||||
|
if (err != nil) != tt.wantErr {
|
||||||
|
t.Fatalf("Source.Enumerate() error = %v, wantErr %v", err, tt.wantErr)
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, unit := range reporter.Units {
|
||||||
|
err = s.ChunkUnit(ctx, unit, &reporter)
|
||||||
|
if (err != nil) != tt.wantErr {
|
||||||
|
t.Fatalf("Source.ChunkUnit() error = %v, wantErr %v", err, tt.wantErr)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, chunk := range reporter.Chunks {
|
||||||
|
if diff := pretty.Compare(chunk.SourceMetadata, tt.wantSourceMetadata); diff != "" {
|
||||||
|
t.Fatalf("comparing chunks: %s diff: (-got +want)\n%s", tt.name, diff)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -44,6 +44,16 @@ message CircleCI {
|
|||||||
string email = 7;
|
string email = 7;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
message TravisCI {
|
||||||
|
string username = 2;
|
||||||
|
string repository = 3;
|
||||||
|
string build_number = 4;
|
||||||
|
string job_number = 5;
|
||||||
|
string link = 6;
|
||||||
|
bool public = 7;
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
message Confluence {
|
message Confluence {
|
||||||
string page = 1;
|
string page = 1;
|
||||||
string space = 2;
|
string space = 2;
|
||||||
@@ -294,5 +304,6 @@ message MetaData {
|
|||||||
SharePoint sharepoint = 25;
|
SharePoint sharepoint = 25;
|
||||||
GoogleDrive googleDrive = 26;
|
GoogleDrive googleDrive = 26;
|
||||||
AzureRepos azureRepos = 27;
|
AzureRepos azureRepos = 27;
|
||||||
|
TravisCI travisCI = 28;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -44,6 +44,7 @@ enum SourceType {
|
|||||||
SOURCE_TYPE_SHAREPOINT = 29;
|
SOURCE_TYPE_SHAREPOINT = 29;
|
||||||
SOURCE_TYPE_GCS_UNAUTHED = 30;
|
SOURCE_TYPE_GCS_UNAUTHED = 30;
|
||||||
SOURCE_TYPE_AZURE_REPOS = 31;
|
SOURCE_TYPE_AZURE_REPOS = 31;
|
||||||
|
SOURCE_TYPE_TRAVISCI = 32;
|
||||||
}
|
}
|
||||||
|
|
||||||
message LocalSource {
|
message LocalSource {
|
||||||
@@ -83,6 +84,13 @@ message CircleCI {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
message TravisCI {
|
||||||
|
string endpoint = 1 [(validate.rules).string.uri_ref = true];
|
||||||
|
oneof credential {
|
||||||
|
string token = 2;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
message Confluence {
|
message Confluence {
|
||||||
string endpoint = 1 [(validate.rules).string.uri_ref = true];
|
string endpoint = 1 [(validate.rules).string.uri_ref = true];
|
||||||
oneof credential {
|
oneof credential {
|
||||||
|
|||||||
Reference in New Issue
Block a user