fetcher controls fetch interval, prevent fetching loop

This commit is contained in:
Dustin Decker
2022-04-03 21:13:39 -07:00
parent fda044631b
commit 8d2dd624e4
3 changed files with 31 additions and 19 deletions
+17 -15
View File
@@ -14,15 +14,14 @@ import (
"syscall" "syscall"
"time" "time"
"github.com/felixge/fgprof"
"github.com/gorilla/mux"
"github.com/jpillora/overseer" "github.com/jpillora/overseer"
"github.com/sirupsen/logrus"
"github.com/trufflesecurity/trufflehog/v3/pkg/updater" "github.com/trufflesecurity/trufflehog/v3/pkg/updater"
"github.com/trufflesecurity/trufflehog/v3/pkg/version" "github.com/trufflesecurity/trufflehog/v3/pkg/version"
"gopkg.in/alecthomas/kingpin.v2" "gopkg.in/alecthomas/kingpin.v2"
"github.com/felixge/fgprof"
"github.com/gorilla/mux"
"github.com/sirupsen/logrus"
"github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/decoders" "github.com/trufflesecurity/trufflehog/v3/pkg/decoders"
"github.com/trufflesecurity/trufflehog/v3/pkg/engine" "github.com/trufflesecurity/trufflehog/v3/pkg/engine"
@@ -91,6 +90,16 @@ func init() {
} }
cmd = kingpin.MustParse(cli.Parse(os.Args[1:])) cmd = kingpin.MustParse(cli.Parse(os.Args[1:]))
if *jsonOut {
logrus.SetFormatter(&logrus.JSONFormatter{})
}
if *debug {
logrus.SetLevel(logrus.DebugLevel)
logrus.Debugf("running version %s", version.BuildVersion)
} else {
logrus.SetLevel(logrus.InfoLevel)
}
} }
func main() { func main() {
@@ -112,9 +121,11 @@ func main() {
} }
func run(state overseer.State) { func run(state overseer.State) {
if *versionFlag { if *debug || *versionFlag {
fmt.Println("trufflehog " + version.BuildVersion) fmt.Println("trufflehog " + version.BuildVersion)
return if *versionFlag {
return
}
} }
// When setting a base commit, chunks must be scanned in order. // When setting a base commit, chunks must be scanned in order.
@@ -122,15 +133,6 @@ func run(state overseer.State) {
*concurrency = 1 *concurrency = 1
} }
if *jsonOut {
logrus.SetFormatter(&logrus.JSONFormatter{})
}
if *debug {
logrus.SetLevel(logrus.DebugLevel)
} else {
logrus.SetLevel(logrus.InfoLevel)
}
if *debug { if *debug {
go func() { go func() {
router := mux.NewRouter() router := mux.NewRouter()
+3 -3
View File
@@ -3,8 +3,6 @@ package engine
import ( import (
"bytes" "bytes"
"context" "context"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/source_metadatapb"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
"runtime" "runtime"
"strings" "strings"
"sync" "sync"
@@ -15,6 +13,8 @@ import (
"github.com/trufflesecurity/trufflehog/v3/pkg/decoders" "github.com/trufflesecurity/trufflehog/v3/pkg/decoders"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/source_metadatapb"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
"github.com/trufflesecurity/trufflehog/v3/pkg/sources" "github.com/trufflesecurity/trufflehog/v3/pkg/sources"
) )
@@ -73,7 +73,7 @@ func Start(ctx context.Context, options ...EngineOption) *Engine {
logrus.Warn("No concurrency specified, defaulting to ", numCPU) logrus.Warn("No concurrency specified, defaulting to ", numCPU)
e.concurrency = numCPU e.concurrency = numCPU
} }
logrus.Debugf("running with up to %d workers", e.concurrency) logrus.Debugf("running with up to %d workers", e.concurrency)
var workerWg sync.WaitGroup var workerWg sync.WaitGroup
for i := 0; i < e.concurrency; i++ { for i := 0; i < e.concurrency; i++ {
+11 -1
View File
@@ -29,6 +29,7 @@ func Fetcher(version string) fetcher.Interface {
type OSS struct { type OSS struct {
Interval time.Duration Interval time.Duration
CurrentVersion string CurrentVersion string
Updated bool
} }
// Init validates the provided config // Init validates the provided config
@@ -49,7 +50,10 @@ type FormData struct {
// Fetch binary from URL via OSS client // Fetch binary from URL via OSS client
func (g *OSS) Fetch() (io.Reader, error) { func (g *OSS) Fetch() (io.Reader, error) {
log.Debug("fetching trufflehog update") if g.Updated {
select {} // block until exit
}
g.Updated = true
zone, _ := time.Now().Zone() zone, _ := time.Now().Zone()
data := &FormData{ data := &FormData{
@@ -71,6 +75,12 @@ func (g *OSS) Fetch() (io.Reader, error) {
} }
defer resp.Body.Close() defer resp.Body.Close()
if resp.StatusCode == http.StatusNoContent {
return nil, nil
}
log.Debug("fetching trufflehog update")
newBinBytes, err := ioutil.ReadAll(resp.Body) newBinBytes, err := ioutil.ReadAll(resp.Body)
if err != nil { if err != nil {
return nil, err return nil, err