fetcher controls fetch interval, prevent fetching loop

This commit is contained in:
Dustin Decker
2022-04-03 21:13:39 -07:00
parent fda044631b
commit 8d2dd624e4
3 changed files with 31 additions and 19 deletions
+17 -15
View File
@@ -14,15 +14,14 @@ import (
"syscall"
"time"
"github.com/felixge/fgprof"
"github.com/gorilla/mux"
"github.com/jpillora/overseer"
"github.com/sirupsen/logrus"
"github.com/trufflesecurity/trufflehog/v3/pkg/updater"
"github.com/trufflesecurity/trufflehog/v3/pkg/version"
"gopkg.in/alecthomas/kingpin.v2"
"github.com/felixge/fgprof"
"github.com/gorilla/mux"
"github.com/sirupsen/logrus"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/decoders"
"github.com/trufflesecurity/trufflehog/v3/pkg/engine"
@@ -91,6 +90,16 @@ func init() {
}
cmd = kingpin.MustParse(cli.Parse(os.Args[1:]))
if *jsonOut {
logrus.SetFormatter(&logrus.JSONFormatter{})
}
if *debug {
logrus.SetLevel(logrus.DebugLevel)
logrus.Debugf("running version %s", version.BuildVersion)
} else {
logrus.SetLevel(logrus.InfoLevel)
}
}
func main() {
@@ -112,9 +121,11 @@ func main() {
}
func run(state overseer.State) {
if *versionFlag {
if *debug || *versionFlag {
fmt.Println("trufflehog " + version.BuildVersion)
return
if *versionFlag {
return
}
}
// When setting a base commit, chunks must be scanned in order.
@@ -122,15 +133,6 @@ func run(state overseer.State) {
*concurrency = 1
}
if *jsonOut {
logrus.SetFormatter(&logrus.JSONFormatter{})
}
if *debug {
logrus.SetLevel(logrus.DebugLevel)
} else {
logrus.SetLevel(logrus.InfoLevel)
}
if *debug {
go func() {
router := mux.NewRouter()
+3 -3
View File
@@ -3,8 +3,6 @@ package engine
import (
"bytes"
"context"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/source_metadatapb"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
"runtime"
"strings"
"sync"
@@ -15,6 +13,8 @@ import (
"github.com/trufflesecurity/trufflehog/v3/pkg/decoders"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/source_metadatapb"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
)
@@ -73,7 +73,7 @@ func Start(ctx context.Context, options ...EngineOption) *Engine {
logrus.Warn("No concurrency specified, defaulting to ", numCPU)
e.concurrency = numCPU
}
logrus.Debugf("running with up to %d workers", e.concurrency)
logrus.Debugf("running with up to %d workers", e.concurrency)
var workerWg sync.WaitGroup
for i := 0; i < e.concurrency; i++ {
+11 -1
View File
@@ -29,6 +29,7 @@ func Fetcher(version string) fetcher.Interface {
type OSS struct {
Interval time.Duration
CurrentVersion string
Updated bool
}
// Init validates the provided config
@@ -49,7 +50,10 @@ type FormData struct {
// Fetch binary from URL via OSS client
func (g *OSS) Fetch() (io.Reader, error) {
log.Debug("fetching trufflehog update")
if g.Updated {
select {} // block until exit
}
g.Updated = true
zone, _ := time.Now().Zone()
data := &FormData{
@@ -71,6 +75,12 @@ func (g *OSS) Fetch() (io.Reader, error) {
}
defer resp.Body.Close()
if resp.StatusCode == http.StatusNoContent {
return nil, nil
}
log.Debug("fetching trufflehog update")
newBinBytes, err := ioutil.ReadAll(resp.Body)
if err != nil {
return nil, err