From 5e1be102736be50de3456bd4815016e4352c6f3c Mon Sep 17 00:00:00 2001 From: Sam Bukowski Date: Wed, 14 Aug 2024 12:25:08 -0600 Subject: [PATCH] Add export of service logs to `dev run` command (#148) * migrate log handler from old pr * purge logs command * remove unneeded comment * function comments * update Infof to Info * update go lint action version * Revert "update go lint action version" This reverts commit 5b2adc183d3a67fbde6165422476ffb395ceb4b2. --- modules/cli/cmd/devrunner/config/constants.go | 1 + modules/cli/cmd/devrunner/init.go | 4 ++ modules/cli/cmd/devrunner/purge.go | 30 ++++++++ modules/cli/cmd/devrunner/run.go | 16 +++++ .../cli/internal/processrunner/logHandler.go | 70 +++++++++++++++++++ .../internal/processrunner/processrunner.go | 31 +++++++- modules/cli/internal/testutils/mocks.go | 9 +++ modules/cli/internal/ui/processpane.go | 8 +++ 8 files changed, 168 insertions(+), 1 deletion(-) create mode 100644 modules/cli/internal/processrunner/logHandler.go diff --git a/modules/cli/cmd/devrunner/config/constants.go b/modules/cli/cmd/devrunner/config/constants.go index da055a0f..47f42c41 100644 --- a/modules/cli/cmd/devrunner/config/constants.go +++ b/modules/cli/cmd/devrunner/config/constants.go @@ -4,6 +4,7 @@ const ( duskNum = "9" dawnNum = "0" BinariesDirName = "bin" + LogsDirName = "logs" DataDirName = "data" DefaultBaseConfigName = "base-config.toml" DefaultCometbftGenesisFilename = "genesis.json" diff --git a/modules/cli/cmd/devrunner/init.go b/modules/cli/cmd/devrunner/init.go index 1557e104..00f45c34 100644 --- a/modules/cli/cmd/devrunner/init.go +++ b/modules/cli/cmd/devrunner/init.go @@ -52,6 +52,10 @@ func runInitialization(c *cobra.Command, _ []string) { log.Info("Creating new instance in:", instanceDir) cmd.CreateDirOrPanic(instanceDir) + // create a directory for all log files + logsDir := filepath.Join(instanceDir, config.LogsDirName) + cmd.CreateDirOrPanic(logsDir) + // create the local bin directory for downloaded binaries localBinPath := filepath.Join(instanceDir, config.BinariesDirName) log.Info("Binary files for locally running a sequencer placed in: ", localBinPath) diff --git a/modules/cli/cmd/devrunner/purge.go b/modules/cli/cmd/devrunner/purge.go index 1d79440c..c7317dd7 100644 --- a/modules/cli/cmd/devrunner/purge.go +++ b/modules/cli/cmd/devrunner/purge.go @@ -78,6 +78,35 @@ func purgeAllCmdHandler(c *cobra.Command, _ []string) { log.Infof("Successfully deleted instance '%s'", instance) } +// purgeLogsCmd represents the 'purge logs' command +var purgeLogsCmd = &cobra.Command{ + Use: "logs", + Short: "Delete all logs for a given instance. Re-initializing is NOT required after using this command.", + Run: purgeLogsCmdHandler, +} + +func purgeLogsCmdHandler(c *cobra.Command, _ []string) { + flagHandler := cmd.CreateCliFlagHandler(c, cmd.EnvPrefix) + + instance := flagHandler.GetValue("instance") + config.IsInstanceNameValidOrPanic(instance) + + homeDir := cmd.GetUserHomeDirOrPanic() + logDir := filepath.Join(homeDir, ".astria", instance, config.LogsDirName) + + log.Infof("Deleting logs for instance '%s'", instance) + + err := os.RemoveAll(logDir) + if err != nil { + fmt.Println("Error removing file:", err) + return + } + cmd.CreateDirOrPanic(logDir) + + log.Infof("Successfully deleted logs for instance '%s'", instance) + +} + func init() { // top level command devCmd.AddCommand(purgeCmd) @@ -85,4 +114,5 @@ func init() { // subcommands purgeCmd.AddCommand(purgeBinariesCmd) purgeCmd.AddCommand(purgeAllCmd) + purgeCmd.AddCommand(purgeLogsCmd) } diff --git a/modules/cli/cmd/devrunner/run.go b/modules/cli/cmd/devrunner/run.go index 2ef14f0d..57f04989 100644 --- a/modules/cli/cmd/devrunner/run.go +++ b/modules/cli/cmd/devrunner/run.go @@ -36,6 +36,7 @@ func init() { flagHandler.BindStringFlag("cometbft-path", "", "Provide an override path to a specific cometbft binary.") flagHandler.BindStringFlag("composer-path", "", "Provide an override path to a specific composer binary.") flagHandler.BindStringFlag("sequencer-path", "", "Provide an override path to a specific sequencer binary.") + flagHandler.BindBoolFlag("export-logs", false, "Export logs to files.") } func runCmdHandler(c *cobra.Command, _ []string) { @@ -49,6 +50,11 @@ func runCmdHandler(c *cobra.Command, _ []string) { instance := flagHandler.GetValue("instance") config.IsInstanceNameValidOrPanic(instance) + exportLogs := flagHandler.GetValue("export-logs") == "true" + logsDir := filepath.Join(astriaDir, instance, config.LogsDirName) + currentTime := time.Now() + appStartTime := currentTime.Format("20060102-150405") // YYYYMMDD-HHMMSS + cmd.CreateUILog(filepath.Join(astriaDir, instance)) network := flagHandler.GetValue("network") @@ -106,6 +112,8 @@ func runCmdHandler(c *cobra.Command, _ []string) { Env: environment, Args: nil, ReadyCheck: &seqReadinessCheck, + LogPath: filepath.Join(logsDir, appStartTime+"-astria-sequencer.log"), + ExportLogs: exportLogs, } seqRunner = processrunner.NewProcessRunner(ctx, seqOpts) case "composer": @@ -116,6 +124,8 @@ func runCmdHandler(c *cobra.Command, _ []string) { Env: environment, Args: nil, ReadyCheck: nil, + LogPath: filepath.Join(logsDir, appStartTime+"-astria-composer.log"), + ExportLogs: exportLogs, } compRunner = processrunner.NewProcessRunner(ctx, composerOpts) case "conductor": @@ -126,6 +136,8 @@ func runCmdHandler(c *cobra.Command, _ []string) { Env: environment, Args: nil, ReadyCheck: nil, + LogPath: filepath.Join(logsDir, appStartTime+"-astria-conductor.log"), + ExportLogs: exportLogs, } condRunner = processrunner.NewProcessRunner(ctx, conductorOpts) case "cometbft": @@ -146,6 +158,8 @@ func runCmdHandler(c *cobra.Command, _ []string) { Env: environment, Args: []string{"node", "--home", cometDataPath, "--log_level", serviceLogLevel}, ReadyCheck: &cometReadinessCheck, + LogPath: filepath.Join(logsDir, appStartTime+"-cometbft.log"), + ExportLogs: exportLogs, } cometRunner = processrunner.NewProcessRunner(ctx, cometOpts) default: @@ -155,6 +169,8 @@ func runCmdHandler(c *cobra.Command, _ []string) { Env: environment, Args: nil, // TODO: implement generic args? ReadyCheck: nil, + LogPath: filepath.Join(logsDir, appStartTime+"-"+service.Name+".log"), + ExportLogs: exportLogs, } genericRunner := processrunner.NewProcessRunner(ctx, genericOpts) genericRunners = append(genericRunners, genericRunner) diff --git a/modules/cli/internal/processrunner/logHandler.go b/modules/cli/internal/processrunner/logHandler.go new file mode 100644 index 00000000..dc208a81 --- /dev/null +++ b/modules/cli/internal/processrunner/logHandler.go @@ -0,0 +1,70 @@ +package processrunner + +import ( + "os" + "regexp" + + log "github.com/sirupsen/logrus" +) + +// LogHandler is a struct that manages the writing of process logs to a file. +type LogHandler struct { + logPath string + exportLogs bool + fileDescriptor *os.File +} + +// NewLogHandler creates a new LogHandler to be used by a ProcessRunner to write +// the process logs to a file. +func NewLogHandler(logPath string, exportLogs bool) *LogHandler { + var fileDescriptor *os.File + + // conditionally create the log file + if exportLogs { + // Open the log file + logFile, err := os.OpenFile(logPath, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644) + if err != nil { + log.Fatalf("error opening file: %v", err) + } + log.Info("New log file created successfully:", logPath) + fileDescriptor = logFile + } else { + fileDescriptor = nil + } + + return &LogHandler{ + logPath: logPath, + exportLogs: exportLogs, + fileDescriptor: fileDescriptor, + } +} + +// Writeable reports if the data sent to the LogHandler.Write() function will be +// written to the log file. If Writeable() returns false, the data will not be +// written to a log file. If Writeable() returns true, the data will be written +// to the log file when the Write() function is called. +func (lh *LogHandler) Writeable() bool { + return lh.exportLogs +} + +// Write writes the data to the log file managed by the LogHandler. +func (lh *LogHandler) Write(data string) error { + // Remove ANSI escape codes from data + ansiRegex := regexp.MustCompile(`\x1b\[[0-9;]*[mGKH]`) + cleanData := ansiRegex.ReplaceAllString(data, "") + + _, err := lh.fileDescriptor.Write([]byte(cleanData)) + if err != nil { + log.Fatalf("error writing to logfile %s: %v", lh.logPath, err) + return err + } + return nil +} + +// Close closes the log file within the LogHandler. +func (lh *LogHandler) Close() error { + if lh.fileDescriptor != nil { + return lh.fileDescriptor.Close() + } + return nil +} diff --git a/modules/cli/internal/processrunner/processrunner.go b/modules/cli/internal/processrunner/processrunner.go index 8cb191ec..13528cf4 100644 --- a/modules/cli/internal/processrunner/processrunner.go +++ b/modules/cli/internal/processrunner/processrunner.go @@ -21,6 +21,8 @@ type ProcessRunner interface { GetOutputAndClearBuf() string GetInfo() string GetEnvironment() []string + CanWriteToLog() bool + WriteToLog(data string) error } // ProcessRunner is a struct that represents a process to be run. @@ -41,6 +43,7 @@ type processRunner struct { outputBuf *safebuffer.SafeBuffer readyChecker *ReadyChecker + logHandler *LogHandler } type NewProcessRunnerOpts struct { @@ -49,6 +52,8 @@ type NewProcessRunnerOpts struct { Env []string Args []string ReadyCheck *ReadyChecker + LogPath string + ExportLogs bool } // NewProcessRunner creates a new ProcessRunner. @@ -58,6 +63,7 @@ func NewProcessRunner(ctx context.Context, opts NewProcessRunnerOpts) ProcessRun // using exec.CommandContext to allow for cancellation from caller cmd := exec.CommandContext(ctx, opts.BinPath, opts.Args...) cmd.Env = opts.Env + logHandler := NewLogHandler(opts.LogPath, opts.ExportLogs) return &processRunner{ ctx: ctx, cmd: cmd, @@ -67,6 +73,7 @@ func NewProcessRunner(ctx context.Context, opts NewProcessRunnerOpts) ProcessRun opts: opts, env: opts.Env, readyChecker: opts.ReadyCheck, + logHandler: logHandler, } } @@ -175,7 +182,7 @@ func (pr *processRunner) Start(ctx context.Context, depStarted <-chan bool) erro } else { exitStatusMessage := fmt.Sprintf("%s process exited cleanly", pr.title) outputStatusMessage := fmt.Sprintf("[black:white][astria-go] %s[-:-]", exitStatusMessage) - log.Infof(exitStatusMessage) + log.Info(exitStatusMessage) _, err := pr.outputBuf.WriteString(outputStatusMessage) if err != nil { return @@ -188,6 +195,9 @@ func (pr *processRunner) Start(ctx context.Context, depStarted <-chan bool) erro // Stop stops the process. func (pr *processRunner) Stop() { + if err := pr.logHandler.Close(); err != nil { + log.WithError(err).Errorf("Error closing log file for process %s", pr.title) + } // send SIGINT to the process if err := pr.cmd.Process.Signal(syscall.SIGINT); err != nil { log.WithError(err).Errorf("Error sending SIGINT for process %s", pr.title) @@ -224,3 +234,22 @@ func (pr *processRunner) GetInfo() string { func (pr *processRunner) GetEnvironment() []string { return pr.env } + +// CanWriteToLog returns whether the service terminal outputs can be written to +// a log file. If CanWriteToLog() returns false, the data will not be written to +// a log file. If CanWriteToLog() returns true, a log file exists and the data +// can be written to the log file when the WriteToLog() function is called. +func (pr *processRunner) CanWriteToLog() bool { + return pr.logHandler.Writeable() +} + +// WriteToLog writes the data to the log file managed by the LogHandler within +// the ProcessRunner. +func (pr *processRunner) WriteToLog(data string) error { + err := pr.logHandler.Write(data) + if err != nil { + return err + } + + return nil +} diff --git a/modules/cli/internal/testutils/mocks.go b/modules/cli/internal/testutils/mocks.go index 6e3554d6..89d93f1a 100644 --- a/modules/cli/internal/testutils/mocks.go +++ b/modules/cli/internal/testutils/mocks.go @@ -59,3 +59,12 @@ func (m *MockProcessRunner) GetInfo() string { args := m.Called() return args.String(0) } + +func (m *MockProcessRunner) CanWriteToLog() bool { + return false +} + +func (m *MockProcessRunner) WriteToLog(data string) error { + args := m.Called(data) + return args.Error(0) +} diff --git a/modules/cli/internal/ui/processpane.go b/modules/cli/internal/ui/processpane.go index 1119234d..e9762667 100644 --- a/modules/cli/internal/ui/processpane.go +++ b/modules/cli/internal/ui/processpane.go @@ -60,6 +60,14 @@ func (pp *ProcessPane) StartScan() { // new, unprocessed data. pp.tApp.QueueUpdateDraw(func() { + // write output data to logs if possible + if pp.pr.CanWriteToLog() { + err := pp.pr.WriteToLog(currentOutput) + if err != nil { + log.WithError(err).Error("Error writing to log") + } + } + // write output data to ui element _, err := pp.ansiWriter.Write([]byte(currentOutput)) if err != nil { log.WithError(err).Error("Error writing to textView")