cli: Introduce glooctl export report

pkg/cliutil/export/cmd.go

@@ -0,0 +1,33 @@
+package export
+
+import (
+	"github.com/solo-io/gloo/pkg/utils/cmdutils"
+	"os"
+	"path/filepath"
+)
+
+func RunCommandOutputToFile(cmd cmdutils.Cmd, path string) func() error {
+	return func() error {
+		f, err := fileOnHost(path)
+		if err != nil {
+			return err
+		}
+		defer func() {
+			_ = f.Close()
+		}()
+		// We intentionally do not output stderr to the output file
+		// Otherwise, the output file will contain errors that may be misleading to users
+		// For example, if a curl request failed first, and then succeeded, we do not want to
+		// populate the failures in the output file
+		return cmd.WithStdout(f).Run().Cause()
+	}
+}
+
+// FileOnHost is a helper to create a file at path even if the parent directory doesn't exist
+// in which case it will be created with ModePerm
+func fileOnHost(path string) (*os.File, error) {
+	if err := os.MkdirAll(filepath.Dir(path), os.ModePerm); err != nil {
+		return nil, err
+	}
+	return os.Create(path)
+}

pkg/cliutil/export/default.go

@@ -0,0 +1,21 @@
+package export
+
+import (
+	"context"
+	"io"
+)
+
+// ToLocalFile is the entrypoint for our exporter today
+// It is used by the CLI and tests, and provides a standard mechanism to capture an export
+// and write it to a zipped file on the local filesystem
+func ToLocalFile(ctx context.Context, zippedFilePath string, progressReporter io.Writer) error {
+	archiveWriter := NewLocalArchiveWriter(zippedFilePath)
+
+	exporter := NewReportExporter(progressReporter)
+
+	exportOptions := Options{
+		// todo: create this object from user-defined parameters
+	}
+
+	return exporter.Export(ctx, exportOptions, archiveWriter)
+}

pkg/cliutil/export/envoy.go

@@ -0,0 +1,44 @@
+package export
+
+import (
+	"context"
+	"github.com/solo-io/gloo/pkg/utils/envoyutils/adminctl"
+	"github.com/solo-io/gloo/pkg/utils/errutils"
+	"github.com/solo-io/gloo/pkg/utils/kubeutils"
+	"github.com/solo-io/gloo/pkg/utils/kubeutils/kubectl"
+	"github.com/solo-io/gloo/pkg/utils/kubeutils/portforward"
+	"github.com/solo-io/gloo/projects/gloo/pkg/defaults"
+	"os"
+	"path/filepath"
+)
+
+func CollectEnvoyData(ctx context.Context, namespace, envoyDataDir string) error {
+	// 1. Open a port-forward to the Kubernetes Deployment, so that we can query the Envoy Admin API directly
+	portForwarder, err := kubectl.NewCli(os.Stdout).StartPortForward(ctx,
+		portforward.WithDeployment(kubeutils.GatewayProxyDeploymentName, namespace),
+		portforward.WithRemotePort(int(defaults.EnvoyAdminPort)))
+	if err != nil {
+		return err
+	}
+
+	// 2. Close the port-forward when we're done accessing data
+	defer func() {
+		portForwarder.Close()
+		portForwarder.WaitForStop()
+	}()
+
+	// 3. Create a CLI that connects to the Envoy Admin API
+	adminCli := adminctl.NewCli(os.Stdout, portForwarder.Address())
+
+	// 4. Execute parallel requests, emitting output to defined files
+	return errutils.AggregateConcurrent([]func() error{
+		RunCommandOutputToFile(
+			adminCli.ConfigDumpCmd(ctx),
+			filepath.Join(envoyDataDir, "config_dump.log"),
+		),
+		RunCommandOutputToFile(
+			adminCli.StatsCmd(ctx),
+			filepath.Join(envoyDataDir, "stats.log"),
+		),
+	})
+}

pkg/cliutil/export/exporter.go

@@ -0,0 +1,88 @@
+package export
+
+import (
+	"context"
+	"fmt"
+	"github.com/solo-io/gloo/pkg/utils/errutils"
+	"github.com/solo-io/gloo/projects/gloo/pkg/defaults"
+	"io"
+	"os"
+	"path/filepath"
+)
+
+var _ ReportExporter = new(reportExporter)
+
+type ReportExporter interface {
+	// Export generates a report archive, and then relies on the ArchiveWriter to export that archive to a destination
+	Export(ctx context.Context, options Options, writer ArchiveWriter) error
+}
+
+// Options is the set of parameters that effect what is captured in an export
+type Options struct {
+	// TODO: This should include more intelligent options for filtering data, namespaces..etc
+}
+
+// NewReportExporter returns an implementation of a ReportExporter
+func NewReportExporter(progressReporter io.Writer) ReportExporter {
+	return &reportExporter{
+		progressReporter: progressReporter,
+	}
+}
+
+type reportExporter struct {
+	// progressReporter is the io.Writer used to write progress updates during the export process
+	// This is intended to be used so there is feedback to callers, if they want it
+	// HELP-WANTED: We rely on an io.Writer, but perhaps a better implementation would be to have a more
+	// intelligent component that can write progress updates (percentages) as well
+	progressReporter io.Writer
+
+	// tmpArchiveDir is the directory where the report archive will be persisted, while it is
+	// being generated. This is created when Export is invoked, and will be cleaned up by the reportExporter
+	tmpArchiveDir string
+}
+
+// Export generates a report archive, and then relies on the ArchiveWriter to export that archive to a destination
+// This implementation relies on a tmp directory to aggregate the report archive
+func (r *reportExporter) Export(ctx context.Context, options Options, writer ArchiveWriter) error {
+	r.reportProgress("starting report export")
+	if err := r.setTmpArchiveDir(); err != nil {
+		return err
+	}
+	defer func() {
+		r.reportProgress("finishing report export")
+		_ = os.RemoveAll(r.tmpArchiveDir)
+	}()
+
+	if err := r.doExport(ctx, options); err != nil {
+		return err
+	}
+
+	r.reportProgress("Export completed. Uploading report")
+	return writer.Write(ctx, r.tmpArchiveDir)
+}
+
+func (r *reportExporter) setTmpArchiveDir() error {
+	tmpDir, err := os.MkdirTemp("", "gloo-report-export")
+	if err != nil {
+		return err
+	}
+	r.reportProgress(fmt.Sprintf("using %s to store export temporarily", tmpDir))
+	r.tmpArchiveDir = tmpDir
+	return nil
+}
+
+func (r *reportExporter) reportProgress(progressUpdate string) {
+	_, _ = r.progressReporter.Write([]byte(fmt.Sprintf("%s\n", progressUpdate)))
+}
+
+func (r *reportExporter) doExport(ctx context.Context, options Options) error {
+
+	// todo: pull this from options
+	envoyNamespace := defaults.GlooSystem
+
+	return errutils.AggregateConcurrent([]func() error{
+		func() error {
+			return CollectEnvoyData(ctx, envoyNamespace, filepath.Join(r.tmpArchiveDir, "envoy"))
+		},
+	})
+}

pkg/cliutil/export/writer.go

@@ -0,0 +1,81 @@
+package export
+
+import (
+	"archive/tar"
+	"compress/gzip"
+	"context"
+	"io"
+	"os"
+	"path/filepath"
+	"strings"
+)
+
+var _ ArchiveWriter = new(localArchiveWriter)
+
+type ArchiveWriter interface {
+	Write(ctx context.Context, artifactDir string) error
+}
+
+func NewLocalArchiveWriter(targetPath string) ArchiveWriter {
+	return &localArchiveWriter{
+		targetPath: targetPath,
+	}
+}
+
+type localArchiveWriter struct {
+	// targetDir is the destination directory where the artifact will be written
+	targetPath string
+}
+
+func (l localArchiveWriter) Write(ctx context.Context, artifactDir string) error {
+	if err := os.MkdirAll(filepath.Dir(l.targetPath), os.ModePerm); err != nil {
+		return err
+	}
+
+	return CreateTarFile(artifactDir, l.targetPath)
+}
+
+// CreateTarFile creates a gzipped tar file from srcDir and writes it to outPath.
+func CreateTarFile(srcDir, outPath string) error {
+	mw, err := os.Create(outPath)
+	if err != nil {
+		return err
+	}
+	defer mw.Close()
+	gzw := gzip.NewWriter(mw)
+	defer gzw.Close()
+
+	tw := tar.NewWriter(gzw)
+	defer tw.Close()
+
+	return filepath.Walk(srcDir, func(file string, fi os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+		if !fi.Mode().IsRegular() {
+			return nil
+		}
+		header, err := tar.FileInfoHeader(fi, fi.Name())
+		if err != nil {
+			return err
+		}
+		header.Name = strings.TrimPrefix(strings.Replace(file, srcDir, "", -1), string(filepath.Separator))
+		header.Size = fi.Size()
+		header.Mode = int64(fi.Mode())
+		header.ModTime = fi.ModTime()
+		if err := tw.WriteHeader(header); err != nil {
+			return err
+		}
+
+		f, err := os.Open(file)
+		if err != nil {
+			return err
+		}
+		defer f.Close()
+
+		if _, err := io.Copy(tw, f); err != nil {
+			return err
+		}
+		return nil
+	})
+}

pkg/utils/envoyutils/adminctl/cli.go

@@ -0,0 +1,86 @@
+package adminctl
+
+import (
+	"context"
+	"github.com/solo-io/gloo/pkg/utils/cmdutils"
+	"github.com/solo-io/gloo/test/testutils"
+	"github.com/solo-io/go-utils/contextutils"
+	"io"
+	"os"
+	"strconv"
+	"strings"
+)
+
+const (
+	ConfigDumpPath = "config_dump"
+	StatsPath      = "stats"
+	ClustersPath   = "clusters"
+	ListenersPath  = "listeners"
+)
+
+// Cli is a utility for executing `kubectl` commands
+type Cli struct {
+	// receiver is the default destination for the curl stdout and stderr
+	receiver io.Writer
+
+	// requestBuilder is the set of default request properties for the Envoy Admin API
+	requestBuilder *testutils.CurlRequestBuilder
+}
+
+// NewCli returns an implementation of the adminctl.Cli
+func NewCli(receiver io.Writer, address string) *Cli {
+	addressParts := strings.Split(address, ":")
+	service := addressParts[0]
+	port, _ := strconv.Atoi(addressParts[1])
+
+	requestBuilder := testutils.DefaultCurlRequestBuilder().
+		WithScheme("http").
+		WithService(service).
+		WithPort(port).
+		// 5 retries, exponential back-off, 10 second max
+		WithRetries(5, 0, 10)
+
+	return &Cli{
+		receiver:       receiver,
+		requestBuilder: requestBuilder,
+	}
+}
+
+func (c *Cli) Command(ctx context.Context, builder *testutils.CurlRequestBuilder) cmdutils.Cmd {
+	args, err := builder.BuildArgs()
+	if err != nil {
+		// An error is returned here to improve the dev experience, as the CurlRequest that was
+		// constructed is known to be invalid
+		// Therefore, for developers we error loudly using a DPanic
+		contextutils.LoggerFrom(ctx).DPanic(err)
+	}
+
+	cmd := cmdutils.Command(ctx, "curl", args...)
+	cmd.WithEnv(os.Environ()...)
+
+	// For convenience, we set the stdout and stderr to the receiver
+	// This can still be overwritten by consumers who use the commands
+	cmd.WithStdout(c.receiver)
+	cmd.WithStderr(c.receiver)
+	return cmd
+}
+
+func (c *Cli) RequestPathCmd(ctx context.Context, path string) cmdutils.Cmd {
+	return c.Command(ctx, c.requestBuilder.WithPath(path))
+}
+
+func (c *Cli) StatsCmd(ctx context.Context) cmdutils.Cmd {
+	return c.RequestPathCmd(ctx, StatsPath)
+}
+
+func (c *Cli) ClustersCmd(ctx context.Context) cmdutils.Cmd {
+	return c.RequestPathCmd(ctx, ClustersPath)
+}
+
+func (c *Cli) ListenersCmd(ctx context.Context) cmdutils.Cmd {
+	return c.RequestPathCmd(ctx, ListenersPath)
+}
+
+func (c *Cli) ConfigDumpCmd(ctx context.Context) cmdutils.Cmd {
+	return c.RequestPathCmd(ctx, ConfigDumpPath)
+}

pkg/utils/kubeutils/names.go

@@ -4,6 +4,8 @@ const (
 	GlooDeploymentName = "gloo"
 	GlooServiceName    = "gloo"
 
+	GatewayProxyDeploymentName = "gateway-proxy"
+
 	// The name of the port in the gloo control plane Kubernetes Service that serves xDS config.
 	GlooXdsPortName = "grpc-xds"
 )