package git import ( "bytes" "context" "fmt" "io" "io/ioutil" "net/url" "os" "path/filepath" "runtime" "sort" "strings" "sync" "time" "github.com/go-errors/errors" "github.com/go-git/go-git/v5" "github.com/go-git/go-git/v5/plumbing" "github.com/go-git/go-git/v5/plumbing/format/diff" "github.com/go-git/go-git/v5/plumbing/object" "github.com/go-git/go-git/v5/plumbing/transport/http" log "github.com/sirupsen/logrus" "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/source_metadatapb" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb" "github.com/trufflesecurity/trufflehog/v3/pkg/sanitizer" "github.com/trufflesecurity/trufflehog/v3/pkg/sources" "golang.org/x/sync/semaphore" "google.golang.org/protobuf/proto" "google.golang.org/protobuf/types/known/anypb" ) type Source struct { name string sourceId int64 jobId int64 verify bool git *Git aCtx context.Context sources.Progress conn *sourcespb.Git } type Git struct { sourceType sourcespb.SourceType sourceName string sourceID int64 jobID int64 sourceMetadataFunc func(file, email, commit, timestamp, repository string) *source_metadatapb.MetaData verify bool // sem is used to limit concurrency sem *semaphore.Weighted } func NewGit(sourceType sourcespb.SourceType, jobID, sourceID int64, sourceName string, verify bool, concurrency int, sourceMetadataFunc func(file, email, commit, timestamp, repository string) *source_metadatapb.MetaData, ) *Git { return &Git{ sourceType: sourceType, sourceName: sourceName, sourceID: sourceID, jobID: jobID, sourceMetadataFunc: sourceMetadataFunc, verify: verify, sem: semaphore.NewWeighted(int64(concurrency)), } } // Ensure the Source satisfies the interface at compile time var _ sources.Source = (*Source)(nil) // Type returns the type of source. // It is used for matching source types in configuration and job input. func (s *Source) Type() sourcespb.SourceType { return sourcespb.SourceType_SOURCE_TYPE_GIT } func (s *Source) SourceID() int64 { return s.sourceId } func (s *Source) JobID() int64 { return s.jobId } // Init returns an initialized GitHub source. func (s *Source) Init(aCtx context.Context, name string, jobId, sourceId int64, verify bool, connection *anypb.Any, concurrency int) error { s.aCtx = aCtx s.name = name s.sourceId = sourceId s.jobId = jobId s.verify = verify var conn sourcespb.Git err := anypb.UnmarshalTo(connection, &conn, proto.UnmarshalOptions{}) if err != nil { errors.WrapPrefix(err, "error unmarshalling connection", 0) } s.conn = &conn s.git = NewGit(s.Type(), s.jobId, s.sourceId, s.name, s.verify, runtime.NumCPU(), func(file, email, commit, repository, timestamp string) *source_metadatapb.MetaData { return &source_metadatapb.MetaData{ Data: &source_metadatapb.MetaData_Git{ Git: &source_metadatapb.Git{ Commit: sanitizer.UTF8(commit), File: sanitizer.UTF8(file), Email: sanitizer.UTF8(email), Repository: sanitizer.UTF8(repository), Timestamp: sanitizer.UTF8(timestamp), }, }, } }) return nil } // Chunks emits chunks of bytes over a channel. func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) error { switch cred := s.conn.GetCredential().(type) { case *sourcespb.Git_BasicAuth: user := cred.BasicAuth.Username token := cred.BasicAuth.Password for i, repoURI := range s.conn.Repositories { s.SetProgressComplete(i, len(s.conn.Repositories), fmt.Sprintf("Repo: %s", repoURI)) if len(repoURI) == 0 { continue } path, repo, err := CloneRepoUsingToken(token, repoURI, user) defer os.RemoveAll(path) if err != nil { return err } err = s.git.ScanRepo(ctx, repo, NewScanOptions(), chunksChan) if err != nil { return err } } case *sourcespb.Git_Unauthenticated: for i, repoURI := range s.conn.Repositories { s.SetProgressComplete(i, len(s.conn.Repositories), fmt.Sprintf("Repo: %s", repoURI)) if len(repoURI) == 0 { continue } path, repo, err := CloneRepoUsingUnauthenticated(repoURI) defer os.RemoveAll(path) if err != nil { return err } err = s.git.ScanRepo(ctx, repo, NewScanOptions(), chunksChan) if err != nil { return err } } default: return errors.New("invalid connection type for git source") } for i, u := range s.conn.Directories { s.SetProgressComplete(i, len(s.conn.Repositories), fmt.Sprintf("Repo: %s", u)) if len(u) == 0 { continue } if !strings.HasSuffix(u, "git") { //try paths instead of url repo, err := RepoFromPath(u) if err != nil { return err } if strings.HasPrefix(u, filepath.Join(os.TempDir(), "trufflehog")) { defer os.RemoveAll(u) } err = s.git.ScanRepo(ctx, repo, NewScanOptions(), chunksChan) if err != nil { return err } } } return nil } func RepoFromPath(path string) (*git.Repository, error) { return git.PlainOpen(path) } func CloneRepoUsingToken(token, url, user string) (clonePath string, repo *git.Repository, err error) { log.Debugf("Scanning Repo: %s", url) if user == "" { user = "cloner" } //some git clones require username not just token cloneOptions := &git.CloneOptions{ URL: url, Auth: &http.BasicAuth{ Username: user, Password: token, }, } clonePath, err = ioutil.TempDir(os.TempDir(), "trufflehog") if err != nil { err = errors.New(err) return } repo, err = git.PlainClone(clonePath, false, cloneOptions) if err != nil { err = errors.New(err) return } safeRepo, err := stripPassword(url) if err != nil { err = errors.New(err) return } if _, ok := err.(*os.PathError); ok { log.WithField("repo", safeRepo).WithError(err).Error("error cloning repo") } if err != nil && strings.Contains(err.Error(), "cannot read hash, pkt-line too short") { log.WithField("repo", safeRepo).WithError(err).Error("error cloning repo") } return } func CloneRepoUsingUnauthenticated(url string) (clonePath string, repo *git.Repository, err error) { cloneOptions := &git.CloneOptions{ URL: url, } clonePath, err = ioutil.TempDir(os.TempDir(), "trufflehog") if err != nil { return } repo, err = git.PlainClone(clonePath, false, cloneOptions) if err != nil { err = errors.New(err) return } safeRepo, err := stripPassword(url) if err != nil { err = errors.New(err) return } if _, ok := err.(*os.PathError); ok { log.WithField("repo", safeRepo).WithError(err).Error("error cloning repo") } if err != nil && strings.Contains(err.Error(), "cannot read hash, pkt-line too short") { log.WithField("repo", safeRepo).WithError(err).Error("error cloning repo") } return } func (s *Git) ScanCommits(repo *git.Repository, scanOptions *ScanOptions, chunksChan chan *sources.Chunk) error { commitIter, err := repo.Log(scanOptions.LogOptions) if err != nil { return err } commits := map[int64][]*object.Commit{} seenMap := map[plumbing.Hash]bool{} depth := int64(0) if scanOptions.BaseCommit != nil { var head *object.Commit if scanOptions.HeadCommit != nil { head = scanOptions.HeadCommit } headMap := sync.Map{} headIter, err := head.Files() if err == nil { _ = headIter.ForEach(func(file *object.File) error { headMap.Store(file.Hash, struct{}{}) return nil }) } parentCommits := []*object.Commit{} parentHashes := scanOptions.BaseCommit.ParentHashes for _, parentHash := range parentHashes { parentCommit, err := repo.CommitObject(parentHash) if err != nil { log.WithError(err).WithField("parentHash", parentHash.String()).WithField("commit", scanOptions.BaseCommit.Hash.String()).Debug("could not find parent commit") } parentCommits = append(parentCommits, parentCommit) parentIter, err := parentCommit.Files() if err == nil { _ = parentIter.ForEach(func(file *object.File) error { _, ok := headMap.Load(file.Hash) if ok { seenMap[file.Hash] = true } return nil }) } } for _, parentCommit := range parentCommits { s.scanCommit(repo, parentCommit, &seenMap, scanOptions, true, chunksChan) } } // Create a map of timestamps to commits. commitIter.ForEach(func(commit *object.Commit) error { if scanOptions.BaseCommit != nil && commit.Hash.String() == scanOptions.BaseCommit.Hash.String() { return errors.New("reached base commit") } time := commit.Committer.When.Unix() if _, ok := commits[time]; !ok { commits[time] = []*object.Commit{} } commits[time] = append(commits[time], commit) depth++ if scanOptions.MaxDepth > int64(0) && depth >= scanOptions.MaxDepth { return errors.New("reached max depth") } return nil }) // Make a slice of all the timestamp keys so it can be sorted. keys := make([]int64, len(commits)) i := 0 for key, _ := range commits { keys[i] = key i++ } // Sort the timestamps sort.Slice(keys, func(i, j int) bool { return keys[i] < keys[j] }) for _, commitTime := range keys { commitSlice := commits[commitTime] for _, commit := range commitSlice { // Check to make sure a commit's parents are scanned before the commit. This is done by checking each // commit's timestamp to the later of it's two parents. laterParent := int64(0) for _, parentHash := range commit.ParentHashes { parent, err := repo.CommitObject(parentHash) if err != nil { continue } parentTime := parent.Committer.When.Unix() if commitTime > parentTime { continue } log.WithField("parentTime", parentTime).WithField("commitTime", commitTime).Debugf("commit %s is out of order", commit.Hash) if parentTime > laterParent { laterParent = parentTime } } // If the commit has an earlier timestamp than it's parents, something has gone wrong. Moving the commit // to the end of the timestamp the parent is in will ensure the parents are scanned first. if laterParent > 0 { log.Debugf("commit (%s) is out of order moving after its parent", commit.Hash) commits[laterParent] = append(commits[laterParent], commit) } s.scanCommit(repo, commit, &seenMap, scanOptions, false, chunksChan) } } return nil } func (s *Git) scanCommit(repo *git.Repository, commit *object.Commit, seenMap *map[plumbing.Hash]bool, scanOptions *ScanOptions, ignoreResult bool, chunksChan chan *sources.Chunk) { remote, err := repo.Remote("origin") if err != nil { log.Errorf("error getting repo name: %s", err) return } safeRepo, err := stripPassword(remote.Config().URLs[0]) if err != nil { log.WithError(err).Errorf("couldn't get repo name") return } fileIter, err := commit.Files() if err != nil { log.WithError(err).WithField("commit", commit.Hash.String()).Errorf("unable to read files") return } fileIter.ForEach(func(file *object.File) error { if _, ok := (*seenMap)[file.Hash]; ok { return nil } if !scanOptions.Filter.Pass(file.Name) { return nil } reader, err := file.Reader() if err != nil { log.WithError(err).WithField("file", file.Name).Debugf("failed to get reader from file") return nil } defer reader.Close() bytes, err := ioutil.ReadAll(reader) if err != nil { log.WithError(err).WithField("file", file.Name).Debugf("failed to read from file") return nil } metadata := s.sourceMetadataFunc(file.Name, commit.Committer.Email, commit.Hash.String(), commit.Committer.When.String(), safeRepo) chunksChan <- &sources.Chunk{ SourceName: s.sourceName, SourceID: s.sourceID, SourceType: s.sourceType, SourceMetadata: metadata, Data: bytes, Verify: s.verify, IgnoreResult: ignoreResult, } (*seenMap)[file.Hash] = true return nil }) } func (s *Git) ScanUnstaged(repo *git.Repository, scanOptions *ScanOptions, chunksChan chan *sources.Chunk) error { remote, err := repo.Remote("origin") if err != nil { return errors.New(err) } safeRepo, err := stripPassword(remote.Config().URLs[0]) if err != nil { return errors.New(err) } // Also scan any unstaged changes in the working tree of the repo _, err = repo.Head() if err == nil || err == plumbing.ErrReferenceNotFound { wt, err := repo.Worktree() if err != nil { log.WithError(err).Error("error obtaining repo worktree") return err } status, err := wt.Status() if err != nil { log.WithError(err).Error("error obtaining worktree status") return err } for fh := range status { if !scanOptions.Filter.Pass(fh) { continue } metadata := s.sourceMetadataFunc( fh, "unstaged", "unstaged", time.Now().String(), safeRepo, ) fileBuf := bytes.NewBuffer(nil) fileHandle, err := wt.Filesystem.Open(fh) if err != nil { continue } defer fileHandle.Close() _, err = io.Copy(fileBuf, fileHandle) if err != nil { continue } chunksChan <- &sources.Chunk{ SourceType: s.sourceType, SourceName: s.sourceName, SourceID: s.sourceID, Data: fileBuf.Bytes(), SourceMetadata: metadata, Verify: s.verify, } } } return nil } func (s *Git) ScanRepo(ctx context.Context, repo *git.Repository, scanOptions *ScanOptions, chunksChan chan *sources.Chunk) error { if err := verifyOptions(scanOptions); err != nil { return err } start := time.Now().UnixNano() if err := s.ScanCommits(repo, scanOptions, chunksChan); err != nil { return err } if err := s.ScanUnstaged(repo, scanOptions, chunksChan); err != nil { // https://github.com/src-d/go-git/issues/879 if strings.Contains(err.Error(), "object not found") { log.WithError(err).Error("known issue: probably caused by a dangling reference in the repo") } else { return errors.New(err) } return err } scanTime := time.Now().UnixNano() - start log.Debugf("Scanning complete. Scan time: %f", time.Duration(scanTime).Seconds()) return nil } func verifyOptions(scanOptions *ScanOptions) error { base := scanOptions.BaseCommit head := scanOptions.HeadCommit if base != nil && head != nil { if ok, _ := base.IsAncestor(head); !ok { return fmt.Errorf("unable to scan from requested head to end commit. %s is not an ancestor of %s", base.Hash.String(), head.Hash.String()) } } return nil } //GenerateLink crafts a link to the specific file from a commit. This works in most major git providers (Github/Gitlab) func GenerateLink(repo, commit, file string) string { //bitbucket links are commits not commit... if strings.Contains(repo, "bitbucket.org/") { return repo[:len(repo)-4] + "/commits/" + commit } link := repo[:len(repo)-4] + "/blob/" + commit + "/" + file if file == "" { link = repo[:len(repo)-4] + "/commit/" + commit } return link } func (s *Git) scanCommitPatches(ctx context.Context, repo *git.Repository, commit *object.Commit, chunksChan chan *sources.Chunk, filter *common.Filter) error { defer func() { if err := recover(); err != nil { return } }() // If there are no parents, just scan all files present in the commit //log.Debugf("scanning: %v : %s", repo, commit.Hash) if len(commit.ParentHashes) == 0 { err := s.scanFilesForCommit(ctx, repo, commit, chunksChan) if err != nil { return errors.New(err) } return nil } parent, err := commit.Parent(0) if err != nil { return errors.New(err) } remote, err := repo.Remote("origin") if err != nil { return errors.New(err) } safeRepo, err := stripPassword(remote.Config().URLs[0]) if err != nil { return errors.New(err) } patchCtx, cancel := context.WithTimeout(ctx, time.Second*10) defer cancel() patch, err := parent.PatchContext(patchCtx, commit) if err != nil { if errors.Is(context.DeadlineExceeded, err) { return nil } return errors.New(err) } email := commit.Author.Email commitHash := commit.Hash.String() for _, file := range patch.FilePatches() { bf, f := file.Files() filename := f.Path() if !filter.Pass(filename) { continue } if filename == "" { filename = bf.Path() } metadata := s.sourceMetadataFunc( filename, email, commitHash, commit.Committer.When.String(), safeRepo, ) chunk := bytes.NewBuffer(nil) // This makes a chunk for every section of the diff, so lots of little changes in a file can produce a lot of chunks. for _, patchChunk := range file.Chunks() { if patchChunk.Type() != diff.Add { continue } // I wonder if we can eliminate this string conversion chunk.Write([]byte(patchChunk.Content())) } chunksChan <- &sources.Chunk{ SourceType: s.sourceType, SourceName: s.sourceName, SourceID: s.sourceID, Data: chunk.Bytes(), SourceMetadata: metadata, Verify: s.verify, } } return nil } func (s *Git) scanFilesForCommit(ctx context.Context, repo *git.Repository, commit *object.Commit, chunksChan chan *sources.Chunk) error { fileIter, err := commit.Files() if err != nil { return errors.New(err) } remote, err := repo.Remote("origin") if err != nil { return errors.New(err) } safeRepo, err := stripPassword(remote.Config().URLs[0]) if err != nil { return errors.New(err) } err = fileIter.ForEach(func(f *object.File) error { isBinary, err := f.IsBinary() if isBinary { return nil } if err != nil { return errors.New(err) } chunkStr, err := f.Contents() if err != nil { return errors.New(err) } chunksChan <- &sources.Chunk{ SourceType: s.sourceType, SourceName: s.sourceName, SourceID: s.sourceID, Data: []byte(chunkStr), SourceMetadata: s.sourceMetadataFunc( f.Name, commit.Author.Email, commit.Hash.String(), commit.Committer.When.String(), safeRepo, ), Verify: s.verify, } return nil }) return err } func stripPassword(u string) (string, error) { if strings.HasPrefix(u, "git@") { return u, nil } repoURL, err := url.Parse(u) if err != nil { return "", errors.WrapPrefix(err, "repo remote cannot be sanitized as URI", 0) } _, passSet := repoURL.User.Password() if passSet { return strings.Replace(repoURL.String(), repoURL.User.String()+"@", repoURL.User.Username()+":***@", 1), nil } return repoURL.String(), nil } func TryAdditionalBaseRefs(repo *git.Repository, base string) (*plumbing.Hash, error) { revisionPrefixes := []string{ "", "refs/heads/", "refs/remotes/origin/", } for _, prefix := range revisionPrefixes { outHash, err := repo.ResolveRevision(plumbing.Revision(prefix + base)) if err == plumbing.ErrReferenceNotFound { continue } if err != nil { return nil, err } return outHash, nil } return nil, fmt.Errorf("no base refs succeeded for base: %q", base) } func PrepareRepo(uriString string) (string, bool, error) { var path string uri, err := url.Parse(uriString) if err != nil { return "", false, fmt.Errorf("unable to parse Git URI: %s", err) } remote := false switch uri.Scheme { case "file": path = fmt.Sprintf("%s%s", uri.Host, uri.Path) case "https": remotePath := fmt.Sprintf("%s://%s%s", uri.Scheme, uri.Host, uri.Path) remote = true switch { case uri.User != nil: log.Debugf("Cloning remote Git repo with authentication") password, ok := uri.User.Password() if !ok { return "", remote, fmt.Errorf("password must be included in Git repo URL when username is provided") } path, _, err = CloneRepoUsingToken(password, remotePath, uri.User.Username()) if err != nil { return path, remote, fmt.Errorf("failed to clone authenticated Git repo (%s): %s", remotePath, err) } default: log.Debugf("Cloning remote Git repo without authentication") path, _, err = CloneRepoUsingUnauthenticated(remotePath) if err != nil { return path, remote, fmt.Errorf("failed to clone unauthenticated Git repo (%s): %s", remotePath, err) } } default: return "", remote, fmt.Errorf("unsupported Git URI: %s", uriString) } log.Debugf("Git repo local path: %s", path) return path, remote, nil } func FilterCommitByFiles(commit *object.Commit, filter *common.Filter) bool { fileCount := 0 fileIter, err := commit.Files() if err != nil { return true } foundErr := errors.New("file found") err = fileIter.ForEach(func(file *object.File) error { fileCount++ if filter.Pass(file.Name) { return foundErr } return nil }) // fileIter will return a "file found" "error" if any files match the filter. If there are no files, there will // not be a match, but the commit should pass anyway in case we want to do something with the commit message. if !errors.Is(err, foundErr) && fileCount > 0 { return true } return false }