Compare commits

...

20 Commits

Author SHA1 Message Date
joeleonjr f623c0686f Merge branch 'main' into docker-tar-identification-from-filesystem-2 2023-08-30 15:03:48 -04:00
Miccah 522b2fab29 Add a cancel cause to job cancellation (#1728) 2023-08-30 12:00:44 -07:00
Miccah c77c1172c8 Add the 'Cause' family of functions to the context wrapper library (#1725)
Go 1.20 introduced `WithCancelCause`, `WithTimeoutCause`, and
`WithDeadlineCause` to allow adding a reason to context cancellations.
Adding it to our wrapper will allow us to use these features.
2023-08-30 07:57:45 -07:00
joeleonjr b8468af367 updating secret string for docker detection 2023-08-24 13:59:51 -04:00
joeleonjr b1a1fa393f Update archive_integration_test.go 2023-08-24 13:54:07 -04:00
joeleonjr 7077f69552 Merge branch 'main' into docker-tar-identification-from-filesystem-2 2023-08-24 13:50:33 -04:00
joeleonjr b38bb0d110 Merge branch 'main' into docker-tar-identification-from-filesystem-2 2023-08-24 08:46:44 -04:00
Joe Leon 51aadf96e1 added ahrav's comments 2023-08-24 08:46:30 -04:00
joeleonjr f7cf80ee85 Merge branch 'main' into docker-tar-identification-from-filesystem-2 2023-08-23 15:49:10 -04:00
Joe Leon e523b69a4c reverted size 2023-08-23 15:47:27 -04:00
Joe Leon f4119799d9 cleaned up code 2023-08-23 15:46:02 -04:00
Joe Leon 6b8ce2fba2 saving test case progress 2023-08-23 15:43:37 -04:00
Joe Leon 95f65281ea moved docker base/img hint to sources.go 2023-08-23 11:46:04 -04:00
Joe Leon 5f4f513014 refactored to call docker chunk scanner directly 2023-08-23 10:49:58 -04:00
Joe Leon 2d46cf1f74 temp increase file size in maxreader, handler improvements 2023-08-21 12:15:34 -04:00
Joe Leon aa461ae806 attempting to fix archive test 2023-08-18 21:12:54 -04:00
joeleonjr a55237376a Merge branch 'main' into docker-tar-identification-from-filesystem-2 2023-08-18 21:10:54 -04:00
Joe Leon 5b9a862ffa adjusted contexts in archive tests 2023-08-18 21:01:37 -04:00
Joe Leon e01f6d0257 switched back log context naming 2023-08-18 16:48:07 -04:00
Joe Leon af83970f77 reworked tarred docker img scanning 2023-08-18 13:07:14 -04:00
11 changed files with 387 additions and 104 deletions
+40 -2
View File
@@ -26,9 +26,10 @@ type Context interface {
Logger() logr.Logger
}
// CancelFunc is a type alias to context.CancelFunc to allow use as if they are
// the same types.
// CancelFunc and CancelCauseFunc are type aliases to allow use as if they are
// the same types as the standard library variants.
type CancelFunc = context.CancelFunc
type CancelCauseFunc = context.CancelCauseFunc
// logCtx implements Context.
type logCtx struct {
@@ -68,6 +69,16 @@ func WithCancel(parent Context) (Context, context.CancelFunc) {
return lCtx, cancel
}
// WithCancelCause returns context.WithCancelCause with the log object propagated.
func WithCancelCause(parent Context) (Context, context.CancelCauseFunc) {
ctx, cancel := context.WithCancelCause(parent)
lCtx := logCtx{
log: parent.Logger(),
Context: ctx,
}
return lCtx, cancel
}
// WithDeadline returns context.WithDeadline with the log object propagated and
// the deadline added to the structured log values.
func WithDeadline(parent Context, d time.Time) (Context, context.CancelFunc) {
@@ -79,6 +90,17 @@ func WithDeadline(parent Context, d time.Time) (Context, context.CancelFunc) {
return lCtx, cancel
}
// WithDeadlineCause returns context.WithDeadlineCause with the log object
// propagated and the deadline added to the structured log values.
func WithDeadlineCause(parent Context, d time.Time, cause error) (Context, context.CancelFunc) {
ctx, cancel := context.WithDeadlineCause(parent, d, cause)
lCtx := logCtx{
log: parent.Logger().WithValues("deadline", d),
Context: ctx,
}
return lCtx, cancel
}
// WithTimeout returns context.WithTimeout with the log object propagated and
// the timeout added to the structured log values.
func WithTimeout(parent Context, timeout time.Duration) (Context, context.CancelFunc) {
@@ -90,6 +112,22 @@ func WithTimeout(parent Context, timeout time.Duration) (Context, context.Cancel
return lCtx, cancel
}
// WithTimeoutCause returns context.WithTimeoutCause with the log object
// propagated and the timeout added to the structured log values.
func WithTimeoutCause(parent Context, timeout time.Duration, cause error) (Context, context.CancelFunc) {
ctx, cancel := context.WithTimeoutCause(parent, timeout, cause)
lCtx := logCtx{
log: parent.Logger().WithValues("timeout", timeout),
Context: ctx,
}
return lCtx, cancel
}
// Cause returns the context.Cause of the context.
func Cause(ctx context.Context) error {
return context.Cause(ctx)
}
// WithValue returns context.WithValue with the log object propagated and
// the value added to the structured log values (if the key is a string).
func WithValue(parent Context, key, val any) Context {
+8
View File
@@ -3,6 +3,7 @@ package context
import (
"bytes"
"context"
"fmt"
"strings"
"testing"
"time"
@@ -177,3 +178,10 @@ func TestRace(t *testing.T) {
cancel()
_ = ctx.Err()
}
func TestCause(t *testing.T) {
ctx, cancel := WithCancelCause(Background())
err := fmt.Errorf("oh no")
cancel(err)
assert.Equal(t, err, Cause(ctx))
}
+60 -8
View File
@@ -12,6 +12,8 @@ import (
"strings"
"time"
v1 "github.com/google/go-containerregistry/pkg/v1"
"github.com/google/go-containerregistry/pkg/v1/tarball"
"github.com/h2non/filetype"
"github.com/mholt/archiver/v4"
@@ -36,12 +38,33 @@ var (
// Ensure the Archive satisfies the interfaces at compile time.
var _ SpecializedHandler = (*Archive)(nil)
// tempEnv contains the temporary file and directory information for the archive.
type tempEnv struct {
tempFile *os.File
tempFileName string
extractPath string
}
// dockerImageDetails contains the details of the Docker image and the temporary environment.
type dockerImageDetails struct {
image v1.Image
temp tempEnv
}
// Archive is a handler for extracting and decompressing archives.
type Archive struct {
size int
currentDepth int
}
// DockerTarReader is a wrapper for io.Reader that also stores the temp filepath and image of the tar file.
// Temporary file path information needed for inner-workings of go-containerregistry Image methods.
type DockerTarReader struct {
io.Reader
img v1.Image
tmpEnv tempEnv
}
// New sets a default maximum size and current size counter.
func (a *Archive) New() {
a.size = 0
@@ -213,6 +236,7 @@ func (a *Archive) ReadToMax(ctx context.Context, reader io.Reader) (data []byte,
const (
arMimeType = "application/x-unix-archive"
rpmMimeType = "application/x-rpm"
tarMimeType = "application/x-tar"
)
// Define a map of mime types to corresponding command-line tools
@@ -266,6 +290,21 @@ func (a *Archive) HandleSpecialized(ctx logContext.Context, reader io.Reader) (i
return nil, false, err
}
reader, err = a.extractRpmContent(ctx, reader)
case tarMimeType:
//check if tar is a docker image
isImg, details, err := a.isDockerImage(ctx, reader)
if err != nil {
return nil, false, err
}
if !isImg {
return nil, false, nil
}
// Build dockerReader containing docker image
reader = DockerTarReader{
Reader: reader,
img: details.image,
tmpEnv: details.temp,
}
default:
return reader, false, nil
}
@@ -290,8 +329,8 @@ func (a *Archive) extractDebContent(ctx logContext.Context, file io.Reader) (io.
if err != nil {
return nil, err
}
defer os.Remove(tmpEnv.tempFileName)
defer os.RemoveAll(tmpEnv.extractPath)
defer os.Remove(tmpEnv.tempFileName)
cmd := exec.Command("ar", "x", tmpEnv.tempFile.Name())
cmd.Dir = tmpEnv.extractPath
@@ -328,8 +367,8 @@ func (a *Archive) extractRpmContent(ctx logContext.Context, file io.Reader) (io.
if err != nil {
return nil, err
}
defer os.Remove(tmpEnv.tempFileName)
defer os.RemoveAll(tmpEnv.extractPath)
defer os.Remove(tmpEnv.tempFileName)
// Use rpm2cpio to convert the RPM file to a cpio archive and then extract it using cpio command.
cmd := exec.Command("sh", "-c", "rpm2cpio "+tmpEnv.tempFile.Name()+" | cpio -id")
@@ -428,12 +467,6 @@ func (a *Archive) handleExtractedFiles(ctx logContext.Context, env tempEnv, hand
return dataArchiveName, nil
}
type tempEnv struct {
tempFile *os.File
tempFileName string
extractPath string
}
// createTempEnv creates a temporary file and a temporary directory for extracting archives.
// The caller is responsible for removing these temporary resources
// (both the file and directory) when they are no longer needed.
@@ -477,3 +510,22 @@ func openDataArchive(extractPath string, dataArchiveName string) (io.ReadCloser,
}
return dataFile, nil
}
// isDockerImage checks if a reader object is a docker image.
// Returns true if the tar file is a docker image, dockerImageDetails, and error.
// Caller is responsible for removing temporary files and directories IF the file is a docker image.
// The docker scanner requires a file path to the image, so we must leave the temporary file on disk.
func (a *Archive) isDockerImage(ctx context.Context, file io.Reader) (isImg bool, details dockerImageDetails, err error) {
tmpEnv, err := a.createTempEnv(ctx, file)
if err != nil {
return false, dockerImageDetails{}, err
}
img, err := tarball.ImageFromPath(tmpEnv.tempFileName, nil)
if err != nil {
os.Remove(tmpEnv.tempFileName)
os.RemoveAll(tmpEnv.extractPath)
return false, dockerImageDetails{}, err
}
return true, dockerImageDetails{image: img, temp: tmpEnv}, nil
}
+102
View File
@@ -0,0 +1,102 @@
//go:build integration
// +build integration
package handlers
import (
"context"
"os"
"testing"
diskbufferreader "github.com/bill-rich/disk-buffer-reader"
"github.com/google/go-containerregistry/pkg/name"
"github.com/google/go-containerregistry/pkg/v1/remote"
"github.com/google/go-containerregistry/pkg/v1/tarball"
"github.com/stretchr/testify/assert"
logContext "github.com/trufflesecurity/trufflehog/v3/pkg/context"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/source_metadatapb"
"github.com/trufflesecurity/trufflehog/v3/pkg/sanitizer"
"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
)
func TestFoundKeyInDockerTar(t *testing.T) {
// URI of the Docker image to clone
// imageURI := "ghcr.io/joeleonjr/getting-started-app-with-canary-token:main"
imageURI := "trufflesecurity/secrets"
var imageName name.Reference
imageName, err := name.NewTag(imageURI)
if err != nil {
t.Errorf("%s: error creating image name: %s", imageURI, err)
}
// Pull the image
img, err := remote.Image(imageName)
if err != nil {
t.Errorf("%s: error pulling image: %s", imageURI, err)
}
tempFile, err := os.CreateTemp("", "archive_test_docker_img.tar")
if err != nil {
t.Errorf("%s: error creating temporary file: %s", imageURI, err)
return
}
defer os.Remove(tempFile.Name()) // Clean up the temporary file
// Save the image as a tar file
err = tarball.WriteToFile(tempFile.Name(), imageName, img)
if err != nil {
t.Errorf("%s: error saving image as tar file: %s", imageURI, err)
return
}
ctx := logContext.AddLogger(context.Background())
inputFile, err := os.Open(tempFile.Name())
if err != nil {
t.Errorf("%s: error opening tar file: %s", imageURI, err)
return
}
defer inputFile.Close()
reReader, err := diskbufferreader.New(inputFile)
if err != nil {
t.Errorf("%s: error creating re-readable reader: %s", imageURI, err)
return
}
defer reReader.Close()
chunkSkel := &sources.Chunk{
SourceType: 1,
SourceName: "filesystem",
SourceID: 1,
SourceMetadata: &source_metadatapb.MetaData{
Data: &source_metadatapb.MetaData_Filesystem{
Filesystem: &source_metadatapb.Filesystem{
File: sanitizer.UTF8(tempFile.Name()),
},
},
},
Verify: true,
}
chunksChan := make(chan *sources.Chunk, 1)
go func() {
defer close(chunksChan)
HandleFile(ctx, reReader, chunkSkel, chunksChan)
}()
secret := "AKIAXYZDQCEN4B6JSJQI"
// Read from the channel and validate the secrets.
foundSecret := ""
for chunkCh := range chunksChan {
foundSecret += string(chunkCh.Data)
}
assert.Contains(t, foundSecret, secret)
}
+4 -3
View File
@@ -108,7 +108,8 @@ func TestHandleFile(t *testing.T) {
ch := make(chan *sources.Chunk, 2)
// Context cancels the operation.
canceledCtx, cancel := context.WithCancel(context.Background())
ctx := logContext.AddLogger(context.Background())
canceledCtx, cancel := logContext.WithCancel(ctx)
cancel()
assert.False(t, HandleFile(canceledCtx, strings.NewReader("file"), &sources.Chunk{}, ch))
@@ -123,7 +124,7 @@ func TestHandleFile(t *testing.T) {
assert.NoError(t, err)
assert.Equal(t, 0, len(ch))
assert.True(t, HandleFile(context.Background(), reader, &sources.Chunk{}, ch))
assert.True(t, HandleFile(ctx, reader, &sources.Chunk{}, ch))
assert.Equal(t, 1, len(ch))
}
@@ -202,7 +203,7 @@ func TestExtractTarContent(t *testing.T) {
assert.Nil(t, err)
defer file.Close()
ctx := context.Background()
ctx := logContext.AddLogger(context.Background())
chunkCh := make(chan *sources.Chunk)
go func() {
+10 -2
View File
@@ -3,11 +3,13 @@ package handlers
import (
"context"
"io"
"os"
diskbufferreader "github.com/bill-rich/disk-buffer-reader"
logContext "github.com/trufflesecurity/trufflehog/v3/pkg/context"
"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
"github.com/trufflesecurity/trufflehog/v3/pkg/sources/docker"
)
func DefaultHandlers() []Handler {
@@ -37,7 +39,7 @@ type Handler interface {
// packages them in the provided chunk skeleton, and sends them to chunksChan.
// The function returns true if processing was successful and false otherwise.
// Context is used for cancellation, and the caller is responsible for canceling it if needed.
func HandleFile(ctx context.Context, file io.Reader, chunkSkel *sources.Chunk, chunksChan chan *sources.Chunk) bool {
func HandleFile(ctx logContext.Context, file io.Reader, chunkSkel *sources.Chunk, chunksChan chan *sources.Chunk) bool {
aCtx := logContext.AddLogger(ctx)
for _, h := range DefaultHandlers() {
h.New()
@@ -56,9 +58,15 @@ func HandleFile(ctx context.Context, file io.Reader, chunkSkel *sources.Chunk, c
if specialHandler, ok := h.(SpecializedHandler); ok {
file, isSpecial, err := specialHandler.HandleSpecialized(aCtx, reReader)
if isSpecial {
if dockerTarReader, ok := file.(DockerTarReader); ok {
// Clean up temporary files when done.
defer os.RemoveAll(dockerTarReader.tmpEnv.extractPath)
defer os.Remove(dockerTarReader.tmpEnv.tempFileName)
err = docker.ScanDockerImg(ctx, dockerTarReader.img, chunksChan, chunkSkel)
return err == nil
}
return handleChunks(aCtx, h.FromFile(ctx, file), chunkSkel, chunksChan)
}
if err != nil {
aCtx.Logger().Error(err, "error handling file")
}
+106 -82
View File
@@ -78,7 +78,7 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) err
return err
}
logger := ctx.Logger().WithValues("source_type", s.Type(), "source_name", s.name)
ctx.Logger().WithValues("source_type", s.Type(), "source_name", s.name)
for _, image := range s.conn.GetImages() {
var img v1.Image
@@ -115,96 +115,120 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) err
}
}
logger = logger.WithValues("image", base, "tag", tag)
logger.V(2).Info("scanning image")
layers, err := img.Layers()
chunkSkel := &sources.Chunk{
SourceType: s.Type(),
SourceName: s.name,
SourceID: s.SourceID(),
JobID: s.JobID(),
SourceMetadata: &source_metadatapb.MetaData{
Data: &source_metadatapb.MetaData_Docker{
Docker: &source_metadatapb.Docker{
Image: base,
Tag: tag,
},
},
},
Verify: s.verify,
}
err = ScanDockerImg(ctx, img, chunksChan, chunkSkel)
if err != nil {
return err
}
for _, layer := range layers {
digest, err := layer.Digest()
if err != nil {
return err
}
logger = logger.WithValues("layer", digest.String())
logger.V(2).Info("scanning layer")
rc, err := layer.Compressed()
if err != nil {
return err
}
defer rc.Close()
gzipReader, err := gzip.NewReader(rc)
if err != nil {
return err
}
defer gzipReader.Close()
tarReader := tar.NewReader(gzipReader)
for {
header, err := tarReader.Next()
if errors.Is(err, io.EOF) {
break // End of archive
}
if err != nil {
return err
}
// Skip files larger than FilesizeLimitBytes
if header.Size > FilesizeLimitBytes {
continue
}
file := bytes.NewBuffer(make([]byte, 0, header.Size))
_, err = io.Copy(file, tarReader)
if err != nil {
return err
}
chunk := &sources.Chunk{
SourceType: s.Type(),
SourceName: s.name,
SourceID: s.SourceID(),
JobID: s.JobID(),
Data: file.Bytes(),
SourceMetadata: &source_metadatapb.MetaData{
Data: &source_metadatapb.MetaData_Docker{
Docker: &source_metadatapb.Docker{
File: "/" + header.Name, // Prepend /, because the file is relative to the root of the image
Image: base,
Tag: tag,
Layer: digest.String(),
},
},
},
Verify: s.verify,
}
select {
case chunksChan <- chunk:
case <-ctx.Done():
return ctx.Err()
}
}
dockerLayersScanned.WithLabelValues(s.name).Inc()
}
dockerImagesScanned.WithLabelValues(s.name).Inc()
}
return nil
}
func ScanDockerImg(ctx context.Context, img v1.Image, chunksChan chan *sources.Chunk, chunkSkel *sources.Chunk) error {
base, tag := chunkSkel.BaseAndTagForDockerImg()
logger := ctx.Logger().WithValues("image", base, "tag", tag)
logger.V(2).Info("scanning image")
layers, err := img.Layers()
if err != nil {
return err
}
for _, layer := range layers {
digest, err := layer.Digest()
if err != nil {
return err
}
logger = logger.WithValues("layer", digest.String())
logger.V(2).Info("scanning layer")
rc, err := layer.Compressed()
if err != nil {
return err
}
defer rc.Close()
gzipReader, err := gzip.NewReader(rc)
if err != nil {
return err
}
defer gzipReader.Close()
tarReader := tar.NewReader(gzipReader)
for {
header, err := tarReader.Next()
if errors.Is(err, io.EOF) {
break // End of archive
}
if err != nil {
return err
}
// Skip files larger than FilesizeLimitBytes
if header.Size > FilesizeLimitBytes {
continue
}
file := bytes.NewBuffer(make([]byte, 0, header.Size))
_, err = io.Copy(file, tarReader)
if err != nil {
return err
}
chunk := &sources.Chunk{
// This must be TYPE_DOCKER or SetResultLineNumber could panic
SourceType: sourcespb.SourceType_SOURCE_TYPE_DOCKER,
SourceName: chunkSkel.SourceName,
SourceID: chunkSkel.SourceID,
Data: file.Bytes(),
SourceMetadata: &source_metadatapb.MetaData{
Data: &source_metadatapb.MetaData_Docker{
Docker: &source_metadatapb.Docker{
File: "/" + header.Name, // Prepend /, because the file is relative to the root of the image
Image: base,
Tag: tag,
Layer: digest.String(),
},
},
},
Verify: chunkSkel.Verify,
}
select {
case chunksChan <- chunk:
case <-ctx.Done():
return ctx.Err()
}
}
dockerLayersScanned.WithLabelValues(chunkSkel.SourceName).Inc()
}
dockerImagesScanned.WithLabelValues(chunkSkel.SourceName).Inc()
return nil
}
func baseAndTagFromImage(image string) (base, tag string, hasDigest bool) {
regRepoDelimiter := "/"
tagDelim := ":"
+4 -4
View File
@@ -71,11 +71,11 @@ func (r *JobProgressRef) Done() <-chan struct{} {
// CancelRun requests that the job this is referencing is cancelled and stops
// running. This method will have no effect if the job does not allow
// cancellation.
func (r *JobProgressRef) CancelRun() {
func (r *JobProgressRef) CancelRun(cause error) {
if r.jobProgress == nil || r.jobProgress.jobCancel == nil {
return
}
r.jobProgress.jobCancel()
r.jobProgress.jobCancel(cause)
}
// Fatal is a wrapper around error to differentiate non-fatal errors from fatal
@@ -108,7 +108,7 @@ type JobProgress struct {
ctx context.Context
cancel context.CancelFunc
// Requests to cancel the job.
jobCancel context.CancelFunc
jobCancel context.CancelCauseFunc
// Metrics.
metrics JobProgressMetrics
metricsLock sync.Mutex
@@ -150,7 +150,7 @@ func WithHooks(hooks ...JobProgressHook) func(*JobProgress) {
}
// WithCancel allows cancelling the job by the JobProgressRef.
func WithCancel(cancel context.CancelFunc) func(*JobProgress) {
func WithCancel(cancel context.CancelCauseFunc) func(*JobProgress) {
return func(jp *JobProgress) { jp.jobCancel = cancel }
}
+8 -2
View File
@@ -170,7 +170,7 @@ func (s *SourceManager) asyncRun(ctx context.Context, handle handle) (JobProgres
return JobProgressRef{SourceID: int64(handle), SourceName: sourceName}, err
}
// Create a JobProgress object for tracking progress.
ctx, cancel := context.WithCancel(ctx)
ctx, cancel := context.WithCancelCause(ctx)
progress := NewJobProgress(jobID, int64(handle), sourceName, WithHooks(s.hooks...), WithCancel(cancel))
s.pool.Go(func() error {
atomic.AddInt32(&s.currentRunningCount, 1)
@@ -180,7 +180,7 @@ func (s *SourceManager) asyncRun(ctx context.Context, handle handle) (JobProgres
"source_manager_worker_id", common.RandomID(5),
)
defer common.Recover(ctx)
defer cancel()
defer cancel(nil)
return s.run(ctx, handle, jobID, progress)
})
return progress.Ref(), nil
@@ -247,6 +247,12 @@ func (s *SourceManager) run(ctx context.Context, handle handle, jobID int64, rep
report.Start(time.Now())
defer func() { report.End(time.Now()) }()
defer func() {
if err := context.Cause(ctx); err != nil {
report.ReportError(Fatal{err})
}
}()
// Initialize the source.
sourceInfo, ok := s.getSourceInfo(handle)
if !ok {
+3 -1
View File
@@ -312,10 +312,12 @@ func TestSourceManagerCancelRun(t *testing.T) {
ref, err := mgr.ScheduleRun(context.Background(), handle)
assert.NoError(t, err)
ref.CancelRun()
cancelErr := fmt.Errorf("abort! abort!")
ref.CancelRun(cancelErr)
<-ref.Done()
assert.Error(t, ref.Snapshot().FatalError())
assert.True(t, errors.Is(ref.Snapshot().FatalError(), returnedErr))
assert.True(t, errors.Is(ref.Snapshot().FatalErrors(), cancelErr))
}
func TestSourceManagerAvailableCapacity(t *testing.T) {
+42
View File
@@ -30,6 +30,48 @@ type Chunk struct {
Verify bool
}
// BaseAndTagForDockerImg returns a base and tag value for a
// docker image regardless of the source type. If the source type
// is not a docker image, the base will be a filename/path from the source metadata.
// The default is the entire source metadata string. Update for new source types.
func (c *Chunk) BaseAndTagForDockerImg() (base string, tag string) {
switch c.SourceType {
case sourcespb.SourceType_SOURCE_TYPE_DOCKER:
base = c.SourceMetadata.GetDocker().GetImage()
tag = c.SourceMetadata.GetDocker().GetTag()
case sourcespb.SourceType_SOURCE_TYPE_FILESYSTEM:
base = c.SourceMetadata.GetFilesystem().GetFile()
case sourcespb.SourceType_SOURCE_TYPE_S3:
bucket := c.SourceMetadata.GetS3().GetBucket()
file := c.SourceMetadata.GetS3().GetFile()
base = bucket + "/" + file
case sourcespb.SourceType_SOURCE_TYPE_GCS:
bucket := c.SourceMetadata.GetGcs().GetBucket()
file := c.SourceMetadata.GetGcs().GetFilename()
base = bucket + "/" + file
case sourcespb.SourceType_SOURCE_TYPE_SYSLOG:
host := c.SourceMetadata.GetSyslog().GetHostname()
app := c.SourceMetadata.GetSyslog().GetAppname()
base = host + "/" + app
case sourcespb.SourceType_SOURCE_TYPE_GIT:
repo := c.SourceMetadata.GetGit().GetRepository()
file := c.SourceMetadata.GetGit().GetFile()
base = file + " (repo: " + repo + ")"
case sourcespb.SourceType_SOURCE_TYPE_GITLAB:
repo := c.SourceMetadata.GetGitlab().GetRepository()
file := c.SourceMetadata.GetGitlab().GetFile()
base = file + " (repo: " + repo + ")"
case sourcespb.SourceType_SOURCE_TYPE_GITHUB:
repo := c.SourceMetadata.GetGithub().GetRepository()
file := c.SourceMetadata.GetGithub().GetFile()
base = file + " (repo: " + repo + ")"
default:
base = c.SourceMetadata.String()
tag = ""
}
return base, tag
}
// Source defines the interface required to implement a source chunker.
type Source interface {
// Type returns the source type, used for matching against configuration and jobs.