diff --git a/pkg/sources/github/github.go b/pkg/sources/github/github.go index 454bb5d7..1aa9064f 100644 --- a/pkg/sources/github/github.go +++ b/pkg/sources/github/github.go @@ -401,15 +401,15 @@ func (s *Source) scan(ctx context.Context, installationClient *github.Client, ch if common.IsDone(ctx) { return nil } + cnt := s.Counter.IncTotal() var err error - // Only increement the progress counter if the scan was successful. + // Only increement the success progress counter if the scan was successful. defer func() { if err == nil { - s.Counter.Inc() + s.Counter.IncSuccess() } }() - cnt := s.Counter.Get() s.setProgressCompleteWithRepo(progressIndexOffset, repoURL) // Ensure the repo is removed from the resume info after being scanned. defer func(s *Source, repoURL string) { @@ -424,7 +424,7 @@ func (s *Source) scan(ctx context.Context, installationClient *github.Client, ch return nil } - s.log.WithField("repo", repoURL).Debugf("attempting to clone repo #%d of %d", cnt+1, repoCnt) + s.log.WithField("repo", repoURL).Debugf("attempting to clone repo #%d of %d", cnt, repoCnt) var path string var repo *gogit.Repository @@ -443,7 +443,7 @@ func (s *Source) scan(ctx context.Context, installationClient *github.Client, ch git.ScanOptionHeadCommit(s.conn.Head), ) - log.Debugf("Starting to scan repo #%d of %d: %s", cnt+1, repoCnt, repo) + log.Debugf("Starting to scan repo #%d of %d: %s", cnt, repoCnt, repo) if err = s.git.ScanRepo(ctx, repo, path, scanOptions, chunksChan); err != nil { log.WithError(err).Errorf("unable to scan repo, continuing") return nil diff --git a/pkg/sources/github/github_test.go b/pkg/sources/github/github_test.go index b1f4b9ed..570e2b35 100644 --- a/pkg/sources/github/github_test.go +++ b/pkg/sources/github/github_test.go @@ -425,7 +425,7 @@ func Test_setProgressCompleteWithRepo_Progress(t *testing.T) { // Increment the progress counter by number of repos already processed. for i := 1; i <= tt.offset; i++ { - s.Counter.Inc() + s.Counter.IncSuccess() } s.setProgressCompleteWithRepo(tt.offset, "") diff --git a/pkg/sources/gitlab/gitlab.go b/pkg/sources/gitlab/gitlab.go index ba133c06..b33eecd2 100644 --- a/pkg/sources/gitlab/gitlab.go +++ b/pkg/sources/gitlab/gitlab.go @@ -291,14 +291,14 @@ func (s *Source) scanRepos(ctx context.Context, chunksChan chan *sources.Chunk) return nil } + cnt := s.Counter.IncTotal() var err error - // Only increment the progress counter if the scan was successful. + // Only increment the success progress counter if the scan was successful. defer func() { if err == nil { - s.Counter.Inc() + s.Counter.IncSuccess() } }() - cnt := s.Counter.Get() s.setProgressCompleteWithRepo(progressIndexOffset, repoURL) // Ensure the repoURL is removed from the resume info after being scanned. defer func(s *Source) { @@ -326,7 +326,7 @@ func (s *Source) scanRepos(ctx context.Context, chunksChan chan *sources.Chunk) return nil } - log.Debugf("Starting to scan repoURL #%d of %d: %s", cnt+1, repoCnt, repo) + log.Debugf("Starting to scan repoURL #%d of %d: %s", cnt, repoCnt, repo) err = s.git.ScanRepo(ctx, repo, path, git.NewScanOptions(), chunksChan) if err != nil { scanErrs = append(scanErrs, err) @@ -334,7 +334,7 @@ func (s *Source) scanRepos(ctx context.Context, chunksChan chan *sources.Chunk) } atomic.AddUint64(&scanned, 1) log.Debugf("scanned %d/%d repos", scanned, repoCnt) - log.Debugf("Completed scanning repoURL # %d of %d: %s", cnt+1, repoCnt, repo) + log.Debugf("Completed scanning repoURL # %d of %d: %s", cnt, repoCnt, repo) return nil }) @@ -355,7 +355,7 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) err if err != nil { return errors.New(err) } - // Get repo within target. + // GetSuccess repo within target. repos, errs := s.getRepos() for _, repoErr := range errs { log.WithError(repoErr).Warn("error getting repo") @@ -366,7 +366,7 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) err return errors.New("All specified repos had validation issues, ending scan") } - // Get all repos if not specified. + // GetSuccess all repos if not specified. if repos == nil { projects, err := s.getAllProjects(apiClient) if err != nil { diff --git a/pkg/sources/gitlab/gitlab_test.go b/pkg/sources/gitlab/gitlab_test.go index ee94ff3c..fefc6670 100644 --- a/pkg/sources/gitlab/gitlab_test.go +++ b/pkg/sources/gitlab/gitlab_test.go @@ -235,7 +235,7 @@ func Test_setProgressCompleteWithRepo_Progress(t *testing.T) { // Increment the progress counter by number of repos already processed. for i := 1; i <= tt.offset; i++ { - s.Counter.Inc() + s.Counter.IncSuccess() } s.setProgressCompleteWithRepo(tt.offset, "") diff --git a/pkg/sources/s3/s3.go b/pkg/sources/s3/s3.go index 0b3230fc..12b1acf0 100644 --- a/pkg/sources/s3/s3.go +++ b/pkg/sources/s3/s3.go @@ -142,7 +142,7 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) err return nil } - cnt := s.Counter.Inc() + cnt := s.Counter.IncSuccess() s.Update(len(bucketsToScan), fmt.Sprintf("Bucket: %s", bucket), "") s.log.Debugf("Scanning bucket: %s", bucket) diff --git a/pkg/sources/sources.go b/pkg/sources/sources.go index 71f9227f..23b4fdc0 100644 --- a/pkg/sources/sources.go +++ b/pkg/sources/sources.go @@ -105,24 +105,37 @@ func NewConfig(opts ...func(*Config)) Config { } // Counter is a simple monotonically-increasing counter for tracking progress. +// It includes a successCnt that records the number of successful scans. +// It also includes a totalCnt that records the total number of scans. +// This is necessary in order to display the number of concurrent scans +// regardless of whether they are successful or not. +// successCnt is the only field used when calculating the percentage complete. type Counter struct { - count uint32 - mu sync.Mutex + successCnt, totalCnt uint32 + mu sync.Mutex } -// Inc increments the counter and returns the new value. -func (c *Counter) Inc() int { +// IncTotal increments the total field of counter and returns the new value. +func (c *Counter) IncTotal() int { c.mu.Lock() defer c.mu.Unlock() - c.count++ - return int(c.count) + c.totalCnt++ + return int(c.totalCnt) } -// Get returns the current count. -func (c *Counter) Get() uint32 { +// IncSuccess increments the success field of counter and returns the new value. +func (c *Counter) IncSuccess() int { c.mu.Lock() defer c.mu.Unlock() - return c.count + c.successCnt++ + return int(c.successCnt) +} + +// GetSuccess returns the current successCnt. +func (c *Counter) GetSuccess() uint32 { + c.mu.Lock() + defer c.mu.Unlock() + return c.successCnt } // Progress is used to update job completion progress across sources. @@ -144,7 +157,7 @@ func (p *Progress) Update(scope int, message, encodedResumeInfo string) { p.mut.Lock() defer p.mut.Unlock() - i := p.Counter.Get() + i := p.Counter.GetSuccess() p.Message = message p.EncodedResumeInfo = encodedResumeInfo @@ -160,7 +173,7 @@ func (p *Progress) Complete(msg string) { defer p.mut.Unlock() p.SectionsRemaining = 0 - p.SectionsCompleted = int32(p.Counter.Get()) + p.SectionsCompleted = int32(p.Counter.GetSuccess()) p.Message = msg p.EncodedResumeInfo = "" p.PercentComplete = 100