295 lines
8.3 KiB
Go
295 lines
8.3 KiB
Go
package gitlab
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"net/url"
|
|
"os"
|
|
"runtime"
|
|
|
|
"github.com/go-errors/errors"
|
|
gogit "github.com/go-git/go-git/v5"
|
|
log "github.com/sirupsen/logrus"
|
|
"github.com/xanzy/go-gitlab"
|
|
"google.golang.org/protobuf/proto"
|
|
"google.golang.org/protobuf/types/known/anypb"
|
|
|
|
"github.com/trufflesecurity/trufflehog/pkg/common"
|
|
"github.com/trufflesecurity/trufflehog/pkg/pb/source_metadatapb"
|
|
"github.com/trufflesecurity/trufflehog/pkg/pb/sourcespb"
|
|
|
|
"github.com/trufflesecurity/trufflehog/pkg/giturl"
|
|
"github.com/trufflesecurity/trufflehog/pkg/sanitizer"
|
|
"github.com/trufflesecurity/trufflehog/pkg/sources"
|
|
"github.com/trufflesecurity/trufflehog/pkg/sources/git"
|
|
)
|
|
|
|
type Source struct {
|
|
name string
|
|
sourceId int64
|
|
jobId int64
|
|
verify bool
|
|
authMethod string
|
|
user string
|
|
password string
|
|
token string
|
|
url string
|
|
repos []string
|
|
git *git.Git
|
|
aCtx context.Context
|
|
sources.Progress
|
|
}
|
|
|
|
// Ensure the Source satisfies the interface at compile time
|
|
var _ sources.Source = (*Source)(nil)
|
|
|
|
// Type returns the type of source.
|
|
// It is used for matching source types in configuration and job input.
|
|
func (s *Source) Type() sourcespb.SourceType {
|
|
return sourcespb.SourceType_SOURCE_TYPE_GITLAB
|
|
}
|
|
|
|
func (s *Source) SourceID() int64 {
|
|
return s.sourceId
|
|
}
|
|
|
|
func (s *Source) JobID() int64 {
|
|
return s.jobId
|
|
}
|
|
|
|
// Init returns an initialized Gitlab source.
|
|
func (s *Source) Init(aCtx context.Context, name string, jobId, sourceId int64, verify bool, connection *anypb.Any, concurrency int) error {
|
|
|
|
s.aCtx = aCtx
|
|
s.name = name
|
|
s.sourceId = sourceId
|
|
s.jobId = jobId
|
|
s.verify = verify
|
|
|
|
var conn sourcespb.GitLab
|
|
err := anypb.UnmarshalTo(connection, &conn, proto.UnmarshalOptions{})
|
|
if err != nil {
|
|
errors.WrapPrefix(err, "error unmarshalling connection", 0)
|
|
}
|
|
|
|
s.repos = conn.Repositories
|
|
s.url = conn.Endpoint
|
|
switch cred := conn.GetCredential().(type) {
|
|
case *sourcespb.GitLab_Token:
|
|
s.authMethod = "TOKEN"
|
|
s.token = cred.Token
|
|
case *sourcespb.GitLab_Oauth:
|
|
s.authMethod = "OAUTH"
|
|
s.token = cred.Oauth.RefreshToken
|
|
// TODO: is it okay if there is no client id and secret? Might be an issue when marshalling config to proto
|
|
case *sourcespb.GitLab_BasicAuth:
|
|
s.authMethod = "BASIC_AUTH"
|
|
s.user = cred.BasicAuth.Username
|
|
s.password = cred.BasicAuth.Password
|
|
default:
|
|
return errors.Errorf("Invalid configuration given for source. Name: %s, Type: %s", name, s.Type())
|
|
}
|
|
|
|
if len(s.url) == 0 {
|
|
//assuming not custom gitlab url
|
|
s.url = "https://gitlab.com/"
|
|
}
|
|
|
|
s.git = git.NewGit(s.Type(), s.JobID(), s.SourceID(), s.name, s.verify, runtime.NumCPU(),
|
|
func(file, email, commit, repository string) *source_metadatapb.MetaData {
|
|
return &source_metadatapb.MetaData{
|
|
Data: &source_metadatapb.MetaData_Gitlab{
|
|
Gitlab: &source_metadatapb.Gitlab{
|
|
Commit: sanitizer.UTF8(commit),
|
|
File: sanitizer.UTF8(file),
|
|
Email: sanitizer.UTF8(email),
|
|
Repository: sanitizer.UTF8(repository),
|
|
Link: git.GenerateLink(repository, commit, file),
|
|
},
|
|
},
|
|
}
|
|
})
|
|
|
|
return nil
|
|
}
|
|
|
|
func (s *Source) newClient() (*gitlab.Client, error) {
|
|
|
|
// initialize a new api instance
|
|
switch s.authMethod {
|
|
case "OAUTH":
|
|
apiClient, err := gitlab.NewOAuthClient(s.token, gitlab.WithBaseURL(s.url))
|
|
if err != nil {
|
|
return nil, fmt.Errorf("could not authenticate to Gitlab instance %s via OAUTH. Error: %v", s.url, err)
|
|
}
|
|
return apiClient, nil
|
|
case "BASIC_AUTH":
|
|
apiClient, err := gitlab.NewBasicAuthClient(s.user, s.password, gitlab.WithBaseURL(s.url))
|
|
if err != nil {
|
|
return nil, fmt.Errorf("could not authenticate to Gitlab instance %s via BASICAUTH. Error: %v", s.url, err)
|
|
}
|
|
return apiClient, nil
|
|
|
|
case "TOKEN":
|
|
apiClient, err := gitlab.NewOAuthClient(s.token, gitlab.WithBaseURL(s.url))
|
|
if err != nil {
|
|
return nil, fmt.Errorf("could not authenticate to Gitlab instance %s via TOKEN Auth. Error: %v", s.url, err)
|
|
}
|
|
return apiClient, nil
|
|
|
|
default:
|
|
return nil, errors.New("Could not determine authMethod specified for GitLab")
|
|
}
|
|
|
|
}
|
|
|
|
func (s *Source) getAllProjects(apiClient *gitlab.Client) ([]*gitlab.Project, error) {
|
|
// projects without repo will get user projects, groups projects, and subgroup projects.
|
|
user, _, err := apiClient.Users.CurrentUser()
|
|
//TODO what happens if the user is anonymous
|
|
if err != nil {
|
|
return nil, errors.Errorf("unable to authenticate using: %s", s.authMethod)
|
|
}
|
|
//when bool pointers are req'd
|
|
//yes := true
|
|
no := false
|
|
projectQuery := &gitlab.ListProjectsOptions{}
|
|
projects, _, err := apiClient.Projects.ListUserProjects(user.ID, projectQuery)
|
|
if err != nil {
|
|
return nil, errors.Errorf("received error on listing projects: %s\n", err)
|
|
}
|
|
groups, _, err := apiClient.Groups.ListGroups(&gitlab.ListGroupsOptions{AllAvailable: &no})
|
|
if err != nil {
|
|
return nil, errors.Errorf("received error on listing projects: %s\n", err)
|
|
}
|
|
for _, group := range groups {
|
|
grpPrjs, _, err := apiClient.Groups.ListGroupProjects(group.ID, &gitlab.ListGroupProjectsOptions{})
|
|
if err != nil {
|
|
return nil, errors.Errorf("received error on listing projects: %s\n", err)
|
|
}
|
|
projects = append(projects, grpPrjs...)
|
|
subgroups, _, err := apiClient.Groups.ListSubgroups(group.ID, &gitlab.ListSubgroupsOptions{AllAvailable: &no})
|
|
if err != nil {
|
|
log.Debugf("could not retrieve subgroups from %s", group.Name)
|
|
continue
|
|
}
|
|
for _, subgroup := range subgroups {
|
|
projects = append(projects, subgroup.Projects...)
|
|
}
|
|
}
|
|
return projects, nil
|
|
}
|
|
|
|
func (s *Source) getRepos(apiClient *gitlab.Client) ([]*url.URL, []error) {
|
|
//is repo defined?
|
|
var validRepos []*url.URL
|
|
var errs []error
|
|
if len(s.repos) > 0 {
|
|
for _, prj := range s.repos {
|
|
repo, err := giturl.NormalizeGitlabRepo(prj)
|
|
if err != nil {
|
|
errs = append(errs, errors.WrapPrefix(err, fmt.Sprintf("unable to normalize gitlab repo url %s", prj), 0))
|
|
}
|
|
|
|
// The repo normalization has already successfully parsed the URL at this point, so we can ignore the error.
|
|
u, _ := url.ParseRequestURI(repo)
|
|
validRepos = append(validRepos, u)
|
|
}
|
|
return validRepos, errs
|
|
}
|
|
return nil, nil
|
|
|
|
}
|
|
|
|
func (s *Source) scanRepos(ctx context.Context, chunksChan chan *sources.Chunk, repos []*url.URL) []error {
|
|
var errors []error
|
|
if s.authMethod == "UNAUTHENTICATED" {
|
|
for i, u := range repos {
|
|
s.SetProgressComplete(i, len(repos), fmt.Sprintf("Repo: %s", u))
|
|
|
|
if len(u.String()) == 0 {
|
|
continue
|
|
}
|
|
path, repo, err := git.CloneRepoUsingUnauthenticated(u.String())
|
|
defer os.RemoveAll(path)
|
|
if err != nil {
|
|
errors = append(errors, err)
|
|
continue
|
|
}
|
|
err = s.git.ScanRepo(ctx, repo, &gogit.LogOptions{All: true}, nil, common.FilterEmpty(), chunksChan)
|
|
if err != nil {
|
|
errors = append(errors, err)
|
|
continue
|
|
}
|
|
|
|
}
|
|
|
|
} else {
|
|
for i, u := range repos {
|
|
s.SetProgressComplete(i, len(repos), fmt.Sprintf("Repo: %s", u))
|
|
|
|
if len(u.String()) == 0 {
|
|
continue
|
|
}
|
|
path, repo, err := git.CloneRepoUsingToken(s.token, u.String(), s.user)
|
|
defer os.RemoveAll(path)
|
|
if err != nil {
|
|
errors = append(errors, err)
|
|
continue
|
|
}
|
|
err = s.git.ScanRepo(ctx, repo, &gogit.LogOptions{All: true}, nil, common.FilterEmpty(), chunksChan)
|
|
if err != nil {
|
|
errors = append(errors, err)
|
|
continue
|
|
}
|
|
|
|
}
|
|
}
|
|
return errors
|
|
|
|
}
|
|
|
|
// Chunks emits chunks of bytes over a channel.
|
|
func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) error {
|
|
// start client
|
|
apiClient, err := s.newClient()
|
|
if err != nil {
|
|
return errors.New(err)
|
|
}
|
|
// get repo within target
|
|
repos, errs := s.getRepos(apiClient)
|
|
for _, repoErr := range errs {
|
|
log.WithError(repoErr).Warn("error getting repo")
|
|
}
|
|
// get all repos if not specified
|
|
if repos == nil {
|
|
projects, err := s.getAllProjects(apiClient)
|
|
if err != nil {
|
|
return errors.New(err)
|
|
}
|
|
// turn projects into URLs for Git cloner
|
|
for _, prj := range projects {
|
|
u, err := url.Parse(prj.HTTPURLToRepo)
|
|
if err != nil {
|
|
fmt.Printf("could not parse url given by project: %s", prj.HTTPURLToRepo)
|
|
}
|
|
repos = append(repos, u)
|
|
}
|
|
if repos == nil {
|
|
return errors.Errorf("unable to discover any repos")
|
|
}
|
|
}
|
|
errs = s.scanRepos(ctx, chunksChan, repos)
|
|
for _, err := range errs {
|
|
log.WithError(err).WithFields(
|
|
log.Fields{
|
|
"source_name": s.name,
|
|
"source_type": s.Type(),
|
|
"repos": repos,
|
|
},
|
|
).Error("error scanning repo")
|
|
}
|
|
|
|
return nil
|
|
}
|