add extra data to github detector

go.mod

@@ -65,7 +65,6 @@ require (
 	github.com/prometheus/client_golang v1.16.0
 	github.com/rabbitmq/amqp091-go v1.8.1
 	github.com/sergi/go-diff v1.3.1
-	github.com/shuheiktgw/go-travis v0.3.1
 	github.com/snowflakedb/gosnowflake v1.6.23
 	github.com/stretchr/testify v1.8.4
 	github.com/tailscale/depaware v0.0.0-20210622194025-720c4b409502

go.sum

@@ -369,7 +369,6 @@ github.com/google/go-github/v42 v42.0.0 h1:YNT0FwjPrEysRkLIiKuEfSvBPCGKphW5aS5Px
 github.com/google/go-github/v42 v42.0.0/go.mod h1:jgg/jvyI0YlDOM1/ps6XYh04HNQ3vKf0CVko62/EhRg=
 github.com/google/go-github/v55 v55.0.0 h1:4pp/1tNMB9X/LuAhs5i0KQAE40NmiR/y6prLNb9x9cg=
 github.com/google/go-github/v55 v55.0.0/go.mod h1:JLahOTA1DnXzhxEymmFF5PP2tSS9JVNj68mSZNDwskA=
-github.com/google/go-querystring v0.0.0-20170111101155-53e6ce116135/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck=
 github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8=
 github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU=
 github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPgecwXBIDzw5no=
@@ -606,8 +605,6 @@ github.com/sahilm/fuzzy v0.1.0/go.mod h1:VFvziUEIMCrT6A6tw2RFIXPXXmzXbOsSHF0DOI8
 github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
 github.com/sergi/go-diff v1.3.1 h1:xkr+Oxo4BOQKmkn/B9eMK0g5Kg/983T9DqqPHwYqD+8=
 github.com/sergi/go-diff v1.3.1/go.mod h1:aMJSSKb2lpPvRNec0+w3fl7LP9IOFzdc9Pa4NFbPK1I=
-github.com/shuheiktgw/go-travis v0.3.1 h1:SAT16mi77ccqogOslnXxBXzXbpeyChaIYUwi2aJpVZY=
-github.com/shuheiktgw/go-travis v0.3.1/go.mod h1:avnFFDqJDdRHwlF9tgqvYi3asQCm/HGL8aLxYiKa4Yg=
 github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
 github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0=
 github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=

main.go

@@ -139,9 +139,6 @@ var (
 
 	dockerScan       = cli.Command("docker", "Scan Docker Image")
 	dockerScanImages = dockerScan.Flag("image", "Docker image to scan. Use the file:// prefix to point to a local tarball, otherwise a image registry is assumed.").Required().Strings()
-
-	travisCiScan      = cli.Command("travisci", "Scan TravisCI")
-	travisCiScanToken = travisCiScan.Flag("token", "TravisCI token. Can also be provided with environment variable").Envar("TRAVISCI_TOKEN").Required().String()
 )
 
 func init() {
@@ -155,6 +152,9 @@ func init() {
 
 	cli.Version("trufflehog " + version.BuildVersion)
 
+	//Support -h for help
+	cli.HelpFlag.Short('h')
+
 	if len(os.Args) <= 1 && isatty.IsTerminal(os.Stdout.Fd()) {
 		args := tui.Run()
 		if len(args) == 0 {
@@ -501,10 +501,6 @@ func run(state overseer.State) {
 		if err := e.ScanCircleCI(ctx, *circleCiScanToken); err != nil {
 			logFatal(err, "Failed to scan CircleCI.")
 		}
-	case travisCiScan.FullCommand():
-		if err := e.ScanTravisCI(ctx, *travisCiScanToken); err != nil {
-			logFatal(err, "Failed to scan TravisCI.")
-		}
 	case gcsScan.FullCommand():
 		cfg := sources.GCSConfig{
 			ProjectID:      *gcsProjectID,

pkg/detectors/falsepositives_test.go

@@ -74,6 +74,13 @@ func TestStringShannonEntropy(t *testing.T) {
 			},
 			want: 0.22228483068568816,
 		},
+		{
+			name: "empty",
+			args: args{
+				input: "",
+			},
+			want: 0.0,
+		},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {

pkg/detectors/github/github.go

@@ -97,6 +97,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 							s1.ExtraData["site_admin"] = fmt.Sprintf("%t", userResponse.SiteAdmin)
 							s1.ExtraData["name"] = userResponse.Name
 							s1.ExtraData["company"] = userResponse.Company
+							s1.ExtraData["scopes"] = res.Header.Get("X-OAuth-Scopes")
 						}
 					}
 				}

pkg/detectors/github/github_test.go

@@ -28,6 +28,7 @@ func TestGitHub_FromChunk(t *testing.T) {
 	unverifiedGhu := testSecrets.MustGetField("GITHUB_UNVERIFIED_GHU")
 	unverifiedGhs := testSecrets.MustGetField("GITHUB_UNVERIFIED_GHS")
 	unverifiedGhr := testSecrets.MustGetField("GITHUB_UNVERIFIED_GHR")
+	verifiedGhp := testSecrets.MustGetField("GITHUB_VERIFIED_GHP")
 
 	type args struct {
 		ctx    context.Context
@@ -41,6 +42,32 @@ func TestGitHub_FromChunk(t *testing.T) {
 		want    []detectors.Result
 		wantErr bool
 	}{
+		{
+			name: "found, verified ghp",
+			s:    Scanner{},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte(fmt.Sprintf("You can find a github secret %s within", verifiedGhp)),
+				verify: true,
+			},
+			want: []detectors.Result{
+				{
+					DetectorType: detectorspb.DetectorType_Github,
+					Verified:     true,
+					ExtraData: map[string]string{
+						"account_type":   "User",
+						"company":        "",
+						"name":           "",
+						"rotation_guide": "https://howtorotate.com/docs/tutorials/github/",
+						"scopes":         "notifications",
+						"site_admin":     "false",
+						"url":            "https://github.com/truffle-sandbox",
+						"username":       "truffle-sandbox",
+					},
+				},
+			},
+			wantErr: false,
+		},
 		{
 			name: "found, unverified ghp",
 			s:    Scanner{},
@@ -53,6 +80,9 @@ func TestGitHub_FromChunk(t *testing.T) {
 				{
 					DetectorType: detectorspb.DetectorType_Github,
 					Verified:     false,
+					ExtraData: map[string]string{
+						"rotation_guide": "https://howtorotate.com/docs/tutorials/github/",
+					},
 				},
 			},
 			wantErr: false,
@@ -69,6 +99,9 @@ func TestGitHub_FromChunk(t *testing.T) {
 				{
 					DetectorType: detectorspb.DetectorType_Github,
 					Verified:     false,
+					ExtraData: map[string]string{
+						"rotation_guide": "https://howtorotate.com/docs/tutorials/github/",
+					},
 				},
 			},
 			wantErr: false,
@@ -85,6 +118,9 @@ func TestGitHub_FromChunk(t *testing.T) {
 				{
 					DetectorType: detectorspb.DetectorType_Github,
 					Verified:     false,
+					ExtraData: map[string]string{
+						"rotation_guide": "https://howtorotate.com/docs/tutorials/github/",
+					},
 				},
 			},
 			wantErr: false,
@@ -101,6 +137,9 @@ func TestGitHub_FromChunk(t *testing.T) {
 				{
 					DetectorType: detectorspb.DetectorType_Github,
 					Verified:     false,
+					ExtraData: map[string]string{
+						"rotation_guide": "https://howtorotate.com/docs/tutorials/github/",
+					},
 				},
 			},
 			wantErr: false,
@@ -117,6 +156,9 @@ func TestGitHub_FromChunk(t *testing.T) {
 				{
 					DetectorType: detectorspb.DetectorType_Github,
 					Verified:     false,
+					ExtraData: map[string]string{
+						"rotation_guide": "https://howtorotate.com/docs/tutorials/github/",
+					},
 				},
 			},
 			wantErr: false,
@@ -133,6 +175,9 @@ func TestGitHub_FromChunk(t *testing.T) {
 				{
 					DetectorType: detectorspb.DetectorType_Github,
 					Verified:     false,
+					ExtraData: map[string]string{
+						"rotation_guide": "https://howtorotate.com/docs/tutorials/github/",
+					},
 				},
 			},
 			wantErr: false,

pkg/detectors/github_old/github_old.go

@@ -38,6 +38,7 @@ type userRes struct {
 	SiteAdmin bool   `json:"site_admin"`
 	Name      string `json:"name"`
 	Company   string `json:"company"`
+	UserURL   string `json:"html_url"`
 }
 
 // Keywords are used for efficiently pre-filtering chunks.
@@ -91,6 +92,17 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 						res.Body.Close()
 						if err == nil {
 							s1.Verified = true
+
+							if err == nil {
+								s1.Verified = true
+								s1.ExtraData["username"] = userResponse.Login
+								s1.ExtraData["url"] = userResponse.UserURL
+								s1.ExtraData["account_type"] = userResponse.Type
+								s1.ExtraData["site_admin"] = fmt.Sprintf("%t", userResponse.SiteAdmin)
+								s1.ExtraData["name"] = userResponse.Name
+								s1.ExtraData["company"] = userResponse.Company
+								s1.ExtraData["scopes"] = res.Header.Get("X-OAuth-Scopes")
+							}
 						}
 					}
 				}

pkg/engine/engine.go

@@ -306,7 +306,6 @@ func Start(ctx context.Context, options ...EngineOption) (*Engine, error) {
 	e.sourceManager = sources.NewManager(
 		sources.WithConcurrentSources(int(e.concurrency)),
 		sources.WithConcurrentUnits(int(e.concurrency)),
-		sources.WithSourceUnits(),
 	)
 
 	if len(e.decoders) == 0 {
@@ -636,7 +635,7 @@ func FragmentLineOffset(chunk *sources.Chunk, result *detectors.Result) (int64,
 	if !found {
 		return 0, false
 	}
-	lineNumber := int64(bytes.Count(before, []byte("\n")))
+	lineNumber := int64(bytes.Count(before, []byte("\n")) + 1)
 	// If the line contains the ignore tag, we should ignore the result.
 	endLine := bytes.Index(after, []byte("\n"))
 	if endLine == -1 {

pkg/engine/engine_test.go

@@ -32,7 +32,7 @@ func TestFragmentLineOffset(t *testing.T) {
 			result: &detectors.Result{
 				Raw: []byte("secret here"),
 			},
-			expectedLine: 2,
+			expectedLine: 3,
 			ignore:       true,
 		},
 		{
@@ -43,7 +43,7 @@ func TestFragmentLineOffset(t *testing.T) {
 			result: &detectors.Result{
 				Raw: []byte("secret here"),
 			},
-			expectedLine: 2,
+			expectedLine: 3,
 			ignore:       false,
 		},
 		{
@@ -54,7 +54,7 @@ func TestFragmentLineOffset(t *testing.T) {
 			result: &detectors.Result{
 				Raw: []byte("secret here"),
 			},
-			expectedLine: 4,
+			expectedLine: 5,
 			ignore:       false,
 		},
 		{
@@ -65,7 +65,7 @@ func TestFragmentLineOffset(t *testing.T) {
 			result: &detectors.Result{
 				Raw: []byte("secret\nhere"),
 			},
-			expectedLine: 4,
+			expectedLine: 5,
 			ignore:       false,
 		},
 		{
@@ -76,7 +76,7 @@ func TestFragmentLineOffset(t *testing.T) {
 			result: &detectors.Result{
 				Raw: []byte("secret\nhere"),
 			},
-			expectedLine: 3,
+			expectedLine: 4,
 			ignore:       true,
 		},
 		{
@@ -87,7 +87,7 @@ func TestFragmentLineOffset(t *testing.T) {
 			result: &detectors.Result{
 				Raw: []byte("secret here"),
 			},
-			expectedLine: 3,
+			expectedLine: 4,
 			ignore:       true,
 		},
 	}

pkg/engine/travisci.go

@@ -1,38 +0,0 @@
-package engine
-
-import (
-	"runtime"
-
-	"google.golang.org/protobuf/proto"
-	"google.golang.org/protobuf/types/known/anypb"
-
-	"github.com/trufflesecurity/trufflehog/v3/pkg/context"
-	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
-	"github.com/trufflesecurity/trufflehog/v3/pkg/sources/travisci"
-)
-
-// ScanTravisCI scans TravisCI logs.
-func (e *Engine) ScanTravisCI(ctx context.Context, token string) error {
-	connection := &sourcespb.TravisCI{
-		Credential: &sourcespb.TravisCI_Token{
-			Token: token,
-		},
-	}
-
-	var conn anypb.Any
-	err := anypb.MarshalFrom(&conn, connection, proto.MarshalOptions{})
-	if err != nil {
-		ctx.Logger().Error(err, "failed to marshal Travis CI connection")
-		return err
-	}
-
-	sourceName := "trufflehog - Travis CI"
-	sourceID, jobID, _ := e.sourceManager.GetIDs(ctx, sourceName, travisci.SourceType)
-
-	travisSource := &travisci.Source{}
-	if err := travisSource.Init(ctx, sourceName, jobID, sourceID, true, &conn, runtime.NumCPU()); err != nil {
-		return err
-	}
-	_, err = e.sourceManager.Run(ctx, sourceName, travisSource)
-	return err
-}

pkg/giturl/giturl.go

@@ -158,6 +158,11 @@ func UpdateLinkLineNumber(ctx context.Context, link string, newLine int64) strin
 		return link
 	}
 
+	if newLine <= 0 {
+		// Don't change the link if the line number is 0.
+		return link
+	}
+
 	switch determineProvider(link) {
 	case providerBitbucket:
 		// For Bitbucket, it doesn't support line links (based on the GenerateLink function).

pkg/giturl/giturl_test.go

@@ -251,6 +251,14 @@ func TestUpdateLinkLineNumber(t *testing.T) {
 			},
 			want: "https://onprem.customdomain.com/org/repo/commit/xyz123#L50",
 		},
+		{
+			name: "Don't include line when it's 0",
+			args: args{
+				link:    "https://github.com/coinbase/cbpay-js/issues/181",
+				newLine: int64(0),
+			},
+			want: "https://github.com/coinbase/cbpay-js/issues/181",
+		},
 		{
 			name: "Invalid link",
 			args: args{

pkg/pb/source_metadatapb/source_metadata.pb.validate.go

@@ -486,117 +486,6 @@ var _ interface {
 	ErrorName() string
 } = CircleCIValidationError{}
 
-// Validate checks the field values on TravisCI with the rules defined in the
-// proto definition for this message. If any rules are violated, the first
-// error encountered is returned, or nil if there are no violations.
-func (m *TravisCI) Validate() error {
-	return m.validate(false)
-}
-
-// ValidateAll checks the field values on TravisCI with the rules defined in
-// the proto definition for this message. If any rules are violated, the
-// result is a list of violation errors wrapped in TravisCIMultiError, or nil
-// if none found.
-func (m *TravisCI) ValidateAll() error {
-	return m.validate(true)
-}
-
-func (m *TravisCI) validate(all bool) error {
-	if m == nil {
-		return nil
-	}
-
-	var errors []error
-
-	// no validation rules for Username
-
-	// no validation rules for Repository
-
-	// no validation rules for BuildNumber
-
-	// no validation rules for JobNumber
-
-	// no validation rules for Link
-
-	// no validation rules for Public
-
-	if len(errors) > 0 {
-		return TravisCIMultiError(errors)
-	}
-
-	return nil
-}
-
-// TravisCIMultiError is an error wrapping multiple validation errors returned
-// by TravisCI.ValidateAll() if the designated constraints aren't met.
-type TravisCIMultiError []error
-
-// Error returns a concatenation of all the error messages it wraps.
-func (m TravisCIMultiError) Error() string {
-	var msgs []string
-	for _, err := range m {
-		msgs = append(msgs, err.Error())
-	}
-	return strings.Join(msgs, "; ")
-}
-
-// AllErrors returns a list of validation violation errors.
-func (m TravisCIMultiError) AllErrors() []error { return m }
-
-// TravisCIValidationError is the validation error returned by
-// TravisCI.Validate if the designated constraints aren't met.
-type TravisCIValidationError struct {
-	field  string
-	reason string
-	cause  error
-	key    bool
-}
-
-// Field function returns field value.
-func (e TravisCIValidationError) Field() string { return e.field }
-
-// Reason function returns reason value.
-func (e TravisCIValidationError) Reason() string { return e.reason }
-
-// Cause function returns cause value.
-func (e TravisCIValidationError) Cause() error { return e.cause }
-
-// Key function returns key value.
-func (e TravisCIValidationError) Key() bool { return e.key }
-
-// ErrorName returns error name.
-func (e TravisCIValidationError) ErrorName() string { return "TravisCIValidationError" }
-
-// Error satisfies the builtin error interface
-func (e TravisCIValidationError) Error() string {
-	cause := ""
-	if e.cause != nil {
-		cause = fmt.Sprintf(" | caused by: %v", e.cause)
-	}
-
-	key := ""
-	if e.key {
-		key = "key for "
-	}
-
-	return fmt.Sprintf(
-		"invalid %sTravisCI.%s: %s%s",
-		key,
-		e.field,
-		e.reason,
-		cause)
-}
-
-var _ error = TravisCIValidationError{}
-
-var _ interface {
-	Field() string
-	Reason() string
-	Key() bool
-	Cause() error
-	ErrorName() string
-} = TravisCIValidationError{}
-
 // Validate checks the field values on Confluence with the rules defined in the
 // proto definition for this message. If any rules are violated, the first
 // error encountered is returned, or nil if there are no violations.
@@ -4037,37 +3926,6 @@ func (m *MetaData) validate(all bool) error {
 			}
 		}
 
-	case *MetaData_TravisCI:
-
-		if all {
-			switch v := interface{}(m.GetTravisCI()).(type) {
-			case interface{ ValidateAll() error }:
-				if err := v.ValidateAll(); err != nil {
-					errors = append(errors, MetaDataValidationError{
-						field:  "TravisCI",
-						reason: "embedded message failed validation",
-						cause:  err,
-					})
-				}
-			case interface{ Validate() error }:
-				if err := v.Validate(); err != nil {
-					errors = append(errors, MetaDataValidationError{
-						field:  "TravisCI",
-						reason: "embedded message failed validation",
-						cause:  err,
-					})
-				}
-			}
-		} else if v, ok := interface{}(m.GetTravisCI()).(interface{ Validate() error }); ok {
-			if err := v.Validate(); err != nil {
-				return MetaDataValidationError{
-					field:  "TravisCI",
-					reason: "embedded message failed validation",
-					cause:  err,
-				}
-			}
-		}
-
 	}
 
 	if len(errors) > 0 {

pkg/pb/sourcespb/sources.pb.validate.go

@@ -667,124 +667,6 @@ var _ interface {
 	ErrorName() string
 } = CircleCIValidationError{}
 
-// Validate checks the field values on TravisCI with the rules defined in the
-// proto definition for this message. If any rules are violated, the first
-// error encountered is returned, or nil if there are no violations.
-func (m *TravisCI) Validate() error {
-	return m.validate(false)
-}
-
-// ValidateAll checks the field values on TravisCI with the rules defined in
-// the proto definition for this message. If any rules are violated, the
-// result is a list of violation errors wrapped in TravisCIMultiError, or nil
-// if none found.
-func (m *TravisCI) ValidateAll() error {
-	return m.validate(true)
-}
-
-func (m *TravisCI) validate(all bool) error {
-	if m == nil {
-		return nil
-	}
-
-	var errors []error
-
-	if _, err := url.Parse(m.GetEndpoint()); err != nil {
-		err = TravisCIValidationError{
-			field:  "Endpoint",
-			reason: "value must be a valid URI",
-			cause:  err,
-		}
-		if !all {
-			return err
-		}
-		errors = append(errors, err)
-	}
-
-	switch m.Credential.(type) {
-
-	case *TravisCI_Token:
-		// no validation rules for Token
-
-	}
-
-	if len(errors) > 0 {
-		return TravisCIMultiError(errors)
-	}
-
-	return nil
-}
-
-// TravisCIMultiError is an error wrapping multiple validation errors returned
-// by TravisCI.ValidateAll() if the designated constraints aren't met.
-type TravisCIMultiError []error
-
-// Error returns a concatenation of all the error messages it wraps.
-func (m TravisCIMultiError) Error() string {
-	var msgs []string
-	for _, err := range m {
-		msgs = append(msgs, err.Error())
-	}
-	return strings.Join(msgs, "; ")
-}
-
-// AllErrors returns a list of validation violation errors.
-func (m TravisCIMultiError) AllErrors() []error { return m }
-
-// TravisCIValidationError is the validation error returned by
-// TravisCI.Validate if the designated constraints aren't met.
-type TravisCIValidationError struct {
-	field  string
-	reason string
-	cause  error
-	key    bool
-}
-
-// Field function returns field value.
-func (e TravisCIValidationError) Field() string { return e.field }
-
-// Reason function returns reason value.
-func (e TravisCIValidationError) Reason() string { return e.reason }
-
-// Cause function returns cause value.
-func (e TravisCIValidationError) Cause() error { return e.cause }
-
-// Key function returns key value.
-func (e TravisCIValidationError) Key() bool { return e.key }
-
-// ErrorName returns error name.
-func (e TravisCIValidationError) ErrorName() string { return "TravisCIValidationError" }
-
-// Error satisfies the builtin error interface
-func (e TravisCIValidationError) Error() string {
-	cause := ""
-	if e.cause != nil {
-		cause = fmt.Sprintf(" | caused by: %v", e.cause)
-	}
-
-	key := ""
-	if e.key {
-		key = "key for "
-	}
-
-	return fmt.Sprintf(
-		"invalid %sTravisCI.%s: %s%s",
-		key,
-		e.field,
-		e.reason,
-		cause)
-}
-
-var _ error = TravisCIValidationError{}
-
-var _ interface {
-	Field() string
-	Reason() string
-	Key() bool
-	Cause() error
-	ErrorName() string
-} = TravisCIValidationError{}
-
 // Validate checks the field values on Confluence with the rules defined in the
 // proto definition for this message. If any rules are violated, the first
 // error encountered is returned, or nil if there are no violations.

pkg/sources/errors_test.go

@@ -1,8 +1,8 @@
 package sources
 
 import (
-	"errors"
 	"fmt"
+	"reflect"
 	"sync"
 	"testing"
 )
@@ -40,7 +40,7 @@ func TestNewScanErrors(t *testing.T) {
 		t.Run(tc.name, func(t *testing.T) {
 			got := NewScanErrors()
 
-			if !errors.Is(got.Errors(), tc.want.Errors()) {
+			if !reflect.DeepEqual(got, tc.want) {
 				t.Errorf("got %+v, want %+v", got, tc.want)
 			}
 		})

pkg/sources/github/github.go

@@ -233,7 +233,7 @@ func (s *Source) Init(aCtx context.Context, name string, jobID sources.JobID, so
 			aCtx.Logger().Error(err, "invalid repository", "repo", repo)
 			continue
 		}
-		s.filteredRepoCache.Set(r, r)
+		s.filteredRepoCache.Set(repo, r)
 	}
 
 	s.includeIssueComments = s.conn.IncludeIssueComments

pkg/sources/travisci/travisci.go

@@ -1,223 +0,0 @@
-package travisci
-
-import (
-	"fmt"
-	"strconv"
-
-	"github.com/go-errors/errors"
-	"github.com/shuheiktgw/go-travis"
-	"golang.org/x/sync/errgroup"
-	"google.golang.org/protobuf/proto"
-	"google.golang.org/protobuf/types/known/anypb"
-
-	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
-	"github.com/trufflesecurity/trufflehog/v3/pkg/context"
-	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/source_metadatapb"
-	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
-	"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
-)
-
-const (
-	SourceType = sourcespb.SourceType_SOURCE_TYPE_TRAVISCI
-
-	baseURL = "https://api.travis-ci.com/"
-
-	pageSize = 100
-)
-
-type Source struct {
-	name     string
-	sourceId sources.SourceID
-	jobId    sources.JobID
-	verify   bool
-	jobPool  *errgroup.Group
-	sources.Progress
-	client *travis.Client
-	sources.CommonSourceUnitUnmarshaller
-	returnAfterFirstChunk bool
-}
-
-// Ensure the Source satisfies the interfaces at compile time.
-var _ sources.Source = (*Source)(nil)
-var _ sources.SourceUnitUnmarshaller = (*Source)(nil)
-
-// Type returns the type of source.
-// It is used for matching source types in configuration and job input.
-func (s *Source) Type() sourcespb.SourceType {
-	return SourceType
-}
-
-func (s *Source) SourceID() sources.SourceID {
-	return s.sourceId
-}
-
-func (s *Source) JobID() sources.JobID {
-	return s.jobId
-}
-
-// Init returns an initialized TravisCI source.
-func (s *Source) Init(ctx context.Context, name string, jobId sources.JobID, sourceId sources.SourceID, verify bool, connection *anypb.Any, concurrency int) error {
-	s.name = name
-	s.sourceId = sourceId
-	s.jobId = jobId
-	s.verify = verify
-	s.jobPool = &errgroup.Group{}
-	s.jobPool.SetLimit(concurrency)
-
-	var conn sourcespb.TravisCI
-	if err := anypb.UnmarshalTo(connection, &conn, proto.UnmarshalOptions{}); err != nil {
-		return errors.WrapPrefix(err, "error unmarshalling connection", 0)
-	}
-
-	switch conn.Credential.(type) {
-	case *sourcespb.TravisCI_Token:
-		if conn.GetToken() == "" {
-			return errors.New("token is empty")
-		}
-		s.client = travis.NewClient(baseURL, conn.GetToken())
-		s.client.HTTPClient = common.RetryableHttpClientTimeout(3)
-
-		user, _, err := s.client.User.Current(ctx, nil)
-		if err != nil {
-			return errors.WrapPrefix(err, "error getting testing travis client", 0)
-		}
-		ctx.Logger().V(2).Info("authenticated to Travis CI with user", "username", user.Login)
-	default:
-		return errors.New("credential type not implemented for Travis CI")
-	}
-
-	return nil
-}
-
-func (s *Source) Enumerate(ctx context.Context, reporter sources.UnitReporter) error {
-	for repoPage := 0; ; repoPage++ {
-		repositories, _, err := s.client.Repositories.List(ctx, &travis.RepositoriesOption{
-			Limit:  pageSize,
-			Offset: repoPage * pageSize,
-		})
-		if err != nil {
-			if repoPage == 0 {
-				return fmt.Errorf("error listing repositories: %w", err)
-			}
-			err = reporter.UnitErr(ctx, err)
-			if err != nil {
-				return fmt.Errorf("error reporting error: %w", err)
-			}
-		}
-
-		if len(repositories) == 0 {
-			break
-		}
-
-		for _, repo := range repositories {
-			err = reporter.UnitOk(ctx, sources.CommonSourceUnit{
-				ID: strconv.Itoa(int(*repo.Id)),
-			})
-			if err != nil {
-				return fmt.Errorf("error reporting unit: %w", err)
-			}
-			ctx.Logger().V(2).Info("enumerated repository", "id", repo.Id, "name", repo.Name)
-		}
-	}
-
-	return nil
-}
-
-// ChunkUnit implements SourceUnitChunker interface.
-func (s *Source) ChunkUnit(ctx context.Context, unit sources.SourceUnit, reporter sources.ChunkReporter) error {
-	repo, _, err := s.client.Repositories.Find(ctx, unit.SourceUnitID(), nil)
-	if err != nil {
-		return fmt.Errorf("error finding repository: %w", err)
-	}
-	logger := ctx.Logger().WithValues("repo", *repo.Name)
-	logger.V(2).Info("scanning repository")
-
-	// Counts continuous errors from ListByRepoSlug. Used to quit early in
-	// case the API always returns an error.
-	var buildPageErrs int
-	for buildPage := 0; ; buildPage++ {
-		builds, _, err := s.client.Builds.ListByRepoSlug(ctx, *repo.Slug, &travis.BuildsByRepoOption{
-			Limit:  pageSize,
-			Offset: buildPage * pageSize,
-		})
-		if err != nil {
-			if err := reporter.ChunkErr(ctx, err); err != nil {
-				return err
-			}
-			buildPageErrs++
-			if buildPageErrs >= 5 {
-				return fmt.Errorf("encountered too many errors listing builds, aborting")
-			}
-			continue
-		}
-		// Reset the page error counter.
-		buildPageErrs = 0
-
-		if len(builds) == 0 {
-			break
-		}
-
-		for _, build := range builds {
-			jobs, _, err := s.client.Jobs.ListByBuild(ctx, *build.Id)
-			if err != nil {
-				if err := reporter.ChunkErr(ctx, err); err != nil {
-					return err
-				}
-				continue
-			}
-
-			if len(jobs) == 0 {
-				break
-			}
-
-			for _, job := range jobs {
-				log, _, err := s.client.Logs.FindByJobId(ctx, *job.Id)
-				if err != nil {
-					if err := reporter.ChunkErr(ctx, err); err != nil {
-						return err
-					}
-					continue
-				}
-
-				logger.V(3).Info("scanning job", "id", *job.Id, "number", *job.Number)
-
-				chunk := sources.Chunk{
-					SourceType: s.Type(),
-					SourceName: s.name,
-					SourceID:   s.SourceID(),
-					JobID:      s.JobID(),
-					Data:       []byte(*log.Content),
-					SourceMetadata: &source_metadatapb.MetaData{
-						Data: &source_metadatapb.MetaData_TravisCI{
-							TravisCI: &source_metadatapb.TravisCI{
-								Username:    *job.Owner.Login,
-								Repository:  *repo.Name,
-								BuildNumber: *build.Number,
-								JobNumber:   *job.Number,
-								Link:        fmt.Sprintf("https://app.travis-ci.com/github/%s/%s/jobs/%d", *job.Owner.Login, *repo.Name, *job.Id),
-								Public:      !*repo.Private,
-							},
-						},
-					},
-					Verify: s.verify,
-				}
-
-				if err := reporter.ChunkOk(ctx, chunk); err != nil {
-					return err
-				}
-
-				if s.returnAfterFirstChunk {
-					return nil
-				}
-			}
-		}
-	}
-
-	return nil
-}
-
-// Chunks emits chunks of bytes over a channel.
-// It's a no-op because we've implemented the SourceUnitChunker interface instead.
-func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk, _ ...sources.ChunkingTarget) error {
-	return nil
-}

pkg/sources/travisci/travisci_test.go

@@ -1,99 +0,0 @@
-package travisci
-
-import (
-	"fmt"
-	"testing"
-
-	"github.com/kylelemons/godebug/pretty"
-	"google.golang.org/protobuf/types/known/anypb"
-
-	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
-	"github.com/trufflesecurity/trufflehog/v3/pkg/context"
-	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/source_metadatapb"
-	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
-	"github.com/trufflesecurity/trufflehog/v3/pkg/sourcestest"
-)
-
-func TestSource_Scan(t *testing.T) {
-	ctx := context.Background()
-
-	secret, err := common.GetTestSecret(ctx)
-	if err != nil {
-		t.Fatal(fmt.Errorf("failed to access secret: %v", err))
-	}
-	token := secret.MustGetField("TRAVISCI_TOKEN")
-
-	type init struct {
-		name       string
-		verify     bool
-		connection *sourcespb.TravisCI
-	}
-	tests := []struct {
-		name               string
-		init               init
-		wantSourceMetadata *source_metadatapb.MetaData
-		wantErr            bool
-	}{
-		{
-			name: "get a chunk",
-			init: init{
-				name: "example repo",
-				connection: &sourcespb.TravisCI{
-					Credential: &sourcespb.TravisCI_Token{
-						Token: token,
-					},
-				},
-				verify: true,
-			},
-			wantSourceMetadata: &source_metadatapb.MetaData{
-				Data: &source_metadatapb.MetaData_TravisCI{
-					TravisCI: &source_metadatapb.TravisCI{
-						Username:    "truffle-sandbox",
-						Repository:  "test-repo",
-						BuildNumber: "1",
-						JobNumber:   "1.1",
-						Link:        "https://app.travis-ci.com/github/truffle-sandbox/test-repo/jobs/611053994",
-						Public:      false,
-					},
-				},
-			},
-			wantErr: false,
-		},
-	}
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			s := Source{}
-
-			conn, err := anypb.New(tt.init.connection)
-			if err != nil {
-				t.Fatal(err)
-			}
-
-			err = s.Init(ctx, tt.init.name, 0, 0, tt.init.verify, conn, 5)
-			if (err != nil) != tt.wantErr {
-				t.Fatalf("Source.Init() error = %v, wantErr %v", err, tt.wantErr)
-			}
-
-			reporter := sourcestest.TestReporter{}
-			s.returnAfterFirstChunk = true
-
-			err = s.Enumerate(ctx, &reporter)
-			if (err != nil) != tt.wantErr {
-				t.Fatalf("Source.Enumerate() error = %v, wantErr %v", err, tt.wantErr)
-			}
-
-			for _, unit := range reporter.Units {
-				err = s.ChunkUnit(ctx, unit, &reporter)
-				if (err != nil) != tt.wantErr {
-					t.Fatalf("Source.ChunkUnit() error = %v, wantErr %v", err, tt.wantErr)
-				}
-			}
-
-			for _, chunk := range reporter.Chunks {
-				if diff := pretty.Compare(chunk.SourceMetadata, tt.wantSourceMetadata); diff != "" {
-					t.Fatalf("comparing chunks: %s diff: (-got +want)\n%s", tt.name, diff)
-				}
-			}
-		})
-	}
-}

proto/source_metadata.proto

@@ -44,16 +44,6 @@ message CircleCI {
   string email = 7;
 }
 
-message TravisCI {
-  string username = 2;
-  string repository = 3;
-  string build_number = 4;
-  string job_number  = 5;
-  string link = 6;
-  bool public = 7;
-}
-
-
 message Confluence {
   string page = 1;
   string space = 2;
@@ -304,6 +294,5 @@ message MetaData {
     SharePoint sharepoint = 25;
     GoogleDrive googleDrive = 26;
     AzureRepos azureRepos = 27;
-    TravisCI travisCI = 28;
   }
 }

proto/sources.proto

@@ -44,7 +44,6 @@ enum SourceType {
   SOURCE_TYPE_SHAREPOINT = 29;
   SOURCE_TYPE_GCS_UNAUTHED = 30;
   SOURCE_TYPE_AZURE_REPOS = 31;
-  SOURCE_TYPE_TRAVISCI = 32;
 }
 
 message LocalSource {
@@ -84,13 +83,6 @@ message CircleCI {
   }
 }
 
-message TravisCI {
-  string endpoint = 1 [(validate.rules).string.uri_ref = true];
-  oneof credential {
-    string token = 2;
-  }
-}
-
 message Confluence {
   string endpoint = 1 [(validate.rules).string.uri_ref = true];
   oneof credential {