fetcher controls fetch interval, prevent fetching loop
This commit is contained in:
@@ -14,15 +14,14 @@ import (
|
|||||||
"syscall"
|
"syscall"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
|
"github.com/felixge/fgprof"
|
||||||
|
"github.com/gorilla/mux"
|
||||||
"github.com/jpillora/overseer"
|
"github.com/jpillora/overseer"
|
||||||
|
"github.com/sirupsen/logrus"
|
||||||
"github.com/trufflesecurity/trufflehog/v3/pkg/updater"
|
"github.com/trufflesecurity/trufflehog/v3/pkg/updater"
|
||||||
"github.com/trufflesecurity/trufflehog/v3/pkg/version"
|
"github.com/trufflesecurity/trufflehog/v3/pkg/version"
|
||||||
"gopkg.in/alecthomas/kingpin.v2"
|
"gopkg.in/alecthomas/kingpin.v2"
|
||||||
|
|
||||||
"github.com/felixge/fgprof"
|
|
||||||
|
|
||||||
"github.com/gorilla/mux"
|
|
||||||
"github.com/sirupsen/logrus"
|
|
||||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||||
"github.com/trufflesecurity/trufflehog/v3/pkg/decoders"
|
"github.com/trufflesecurity/trufflehog/v3/pkg/decoders"
|
||||||
"github.com/trufflesecurity/trufflehog/v3/pkg/engine"
|
"github.com/trufflesecurity/trufflehog/v3/pkg/engine"
|
||||||
@@ -91,6 +90,16 @@ func init() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
cmd = kingpin.MustParse(cli.Parse(os.Args[1:]))
|
cmd = kingpin.MustParse(cli.Parse(os.Args[1:]))
|
||||||
|
|
||||||
|
if *jsonOut {
|
||||||
|
logrus.SetFormatter(&logrus.JSONFormatter{})
|
||||||
|
}
|
||||||
|
if *debug {
|
||||||
|
logrus.SetLevel(logrus.DebugLevel)
|
||||||
|
logrus.Debugf("running version %s", version.BuildVersion)
|
||||||
|
} else {
|
||||||
|
logrus.SetLevel(logrus.InfoLevel)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func main() {
|
func main() {
|
||||||
@@ -112,9 +121,11 @@ func main() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func run(state overseer.State) {
|
func run(state overseer.State) {
|
||||||
if *versionFlag {
|
if *debug || *versionFlag {
|
||||||
fmt.Println("trufflehog " + version.BuildVersion)
|
fmt.Println("trufflehog " + version.BuildVersion)
|
||||||
return
|
if *versionFlag {
|
||||||
|
return
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// When setting a base commit, chunks must be scanned in order.
|
// When setting a base commit, chunks must be scanned in order.
|
||||||
@@ -122,15 +133,6 @@ func run(state overseer.State) {
|
|||||||
*concurrency = 1
|
*concurrency = 1
|
||||||
}
|
}
|
||||||
|
|
||||||
if *jsonOut {
|
|
||||||
logrus.SetFormatter(&logrus.JSONFormatter{})
|
|
||||||
}
|
|
||||||
if *debug {
|
|
||||||
logrus.SetLevel(logrus.DebugLevel)
|
|
||||||
} else {
|
|
||||||
logrus.SetLevel(logrus.InfoLevel)
|
|
||||||
}
|
|
||||||
|
|
||||||
if *debug {
|
if *debug {
|
||||||
go func() {
|
go func() {
|
||||||
router := mux.NewRouter()
|
router := mux.NewRouter()
|
||||||
|
|||||||
@@ -3,8 +3,6 @@ package engine
|
|||||||
import (
|
import (
|
||||||
"bytes"
|
"bytes"
|
||||||
"context"
|
"context"
|
||||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/source_metadatapb"
|
|
||||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
|
|
||||||
"runtime"
|
"runtime"
|
||||||
"strings"
|
"strings"
|
||||||
"sync"
|
"sync"
|
||||||
@@ -15,6 +13,8 @@ import (
|
|||||||
|
|
||||||
"github.com/trufflesecurity/trufflehog/v3/pkg/decoders"
|
"github.com/trufflesecurity/trufflehog/v3/pkg/decoders"
|
||||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||||
|
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/source_metadatapb"
|
||||||
|
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
|
||||||
"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
|
"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -73,7 +73,7 @@ func Start(ctx context.Context, options ...EngineOption) *Engine {
|
|||||||
logrus.Warn("No concurrency specified, defaulting to ", numCPU)
|
logrus.Warn("No concurrency specified, defaulting to ", numCPU)
|
||||||
e.concurrency = numCPU
|
e.concurrency = numCPU
|
||||||
}
|
}
|
||||||
logrus.Debugf("running with up to %d workers", e.concurrency)
|
logrus.Debugf("running with up to %d workers", e.concurrency)
|
||||||
|
|
||||||
var workerWg sync.WaitGroup
|
var workerWg sync.WaitGroup
|
||||||
for i := 0; i < e.concurrency; i++ {
|
for i := 0; i < e.concurrency; i++ {
|
||||||
|
|||||||
+11
-1
@@ -29,6 +29,7 @@ func Fetcher(version string) fetcher.Interface {
|
|||||||
type OSS struct {
|
type OSS struct {
|
||||||
Interval time.Duration
|
Interval time.Duration
|
||||||
CurrentVersion string
|
CurrentVersion string
|
||||||
|
Updated bool
|
||||||
}
|
}
|
||||||
|
|
||||||
// Init validates the provided config
|
// Init validates the provided config
|
||||||
@@ -49,7 +50,10 @@ type FormData struct {
|
|||||||
|
|
||||||
// Fetch binary from URL via OSS client
|
// Fetch binary from URL via OSS client
|
||||||
func (g *OSS) Fetch() (io.Reader, error) {
|
func (g *OSS) Fetch() (io.Reader, error) {
|
||||||
log.Debug("fetching trufflehog update")
|
if g.Updated {
|
||||||
|
select {} // block until exit
|
||||||
|
}
|
||||||
|
g.Updated = true
|
||||||
|
|
||||||
zone, _ := time.Now().Zone()
|
zone, _ := time.Now().Zone()
|
||||||
data := &FormData{
|
data := &FormData{
|
||||||
@@ -71,6 +75,12 @@ func (g *OSS) Fetch() (io.Reader, error) {
|
|||||||
}
|
}
|
||||||
defer resp.Body.Close()
|
defer resp.Body.Close()
|
||||||
|
|
||||||
|
if resp.StatusCode == http.StatusNoContent {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
log.Debug("fetching trufflehog update")
|
||||||
|
|
||||||
newBinBytes, err := ioutil.ReadAll(resp.Body)
|
newBinBytes, err := ioutil.ReadAll(resp.Body)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
|
|||||||
Reference in New Issue
Block a user