use dataDir/config.yaml as the default config path

cmd/cli/config/set.go

@@ -11,6 +11,7 @@ import (
 	"github.com/spf13/viper"
 
 	"github.com/bacalhau-project/bacalhau/cmd/util"
+	"github.com/bacalhau-project/bacalhau/cmd/util/flags/cliflags"
 	"github.com/bacalhau-project/bacalhau/cmd/util/hook"
 	"github.com/bacalhau-project/bacalhau/pkg/config"
 	"github.com/bacalhau-project/bacalhau/pkg/config/types"
@@ -26,40 +27,58 @@ func newSetCmd() *cobra.Command {
 		PostRunE:     hook.ClientPostRunHooks,
 		SilenceUsage: true,
 		RunE: func(cmd *cobra.Command, args []string) error {
-			// initialize a new or open an existing repo. We need to ensure a repo
-			// exists before we can create or modify a config file in it.
-			_, err := util.SetupRepoConfig(cmd)
+			// Get the value of the --config flag
+			configFlag, err := cmd.Flags().GetString("config")
 			if err != nil {
-				return fmt.Errorf("failed to setup repo: %w", err)
+				return fmt.Errorf("failed to get config flag: %w", err)
 			}
-			return setConfig(cmd.PersistentFlags().Lookup("config").Value.String(), args[0], args[1:]...)
-		},
-		// provide auto completion for arguments to the `set` command
-		ValidArgsFunction: setAutoComplete,
-	}
 
-	bacalhauCfgDir := "bacalhau"
-	bacalhauCfgFile := config.DefaultFileName
+			// If --config flag is set, use it to set the Viper key
+			if configFlag != "" {
+				viper.Set(cliflags.RootCommandConfigFiles, []string{configFlag})
+			}
 
-	usrCfgDir, err := os.UserConfigDir()
-	if err != nil {
-		log.Warn().Err(err).Msg("Failed to find user-specific configuration directory. Using current directory to write config.")
-	} else {
-		bacalhauCfgDir = filepath.Join(usrCfgDir, bacalhauCfgDir)
-		if err := os.MkdirAll(bacalhauCfgDir, util2.OS_USER_RWX); err != nil {
-			// This means we failed to create a directory either in the current directory, or the user config dir
-			// indicating a some-what serious misconfiguration of the system. We panic here to provide as much
-			// detail as possible.
-			log.Panic().Err(err).Msgf("Failed to create bacalhau configuration directory: %s", bacalhauCfgDir)
-		}
-		bacalhauCfgFile = filepath.Join(bacalhauCfgDir, bacalhauCfgFile)
+			var configPath string
+			// load configs to get the config file path
+			rawConfig, err := util.SetupConfigType(cmd)
+			if err != nil {
+				return fmt.Errorf("failed to setup config: %w", err)
+			}
+
+			configPath = rawConfig.ConfigFileUsed()
+			if configPath == "" {
+				// we fall back to the default config file path $BACALHAU_DIR/config.yaml
+				// this requires initializing a new or opening an existing data-dir
+				bacalhauConfig, err := util.DecodeBacalhauConfig(rawConfig)
+				if err != nil {
+					return fmt.Errorf("failed to decode bacalhau config: %w", err)
+				}
+				_, err = util.SetupRepo(bacalhauConfig)
+				if err != nil {
+					return fmt.Errorf("failed to setup repo: %w", err)
+				}
+				configPath = filepath.Join(bacalhauConfig.DataDir, config.DefaultFileName)
+
+				// create the config file if it doesn't exist
+				if _, err := os.Stat(configPath); os.IsNotExist(err) {
+					if err := os.WriteFile(configPath, []byte{}, util2.OS_USER_RWX); err != nil {
+						return fmt.Errorf("failed to create default config file %s: %w", configPath, err)
+					}
+				}
+			}
+
+			return setConfig(configPath, args[0], args[1:]...)
+		},
+		// Provide auto completion for arguments to the `set` command
+		ValidArgsFunction: setAutoComplete,
 	}
 
-	setCmd.PersistentFlags().String("config", bacalhauCfgFile, "Path to the config file")
+	setCmd.PersistentFlags().String("config", "", "Path to the config file (default is $BACALHAU_DIR/config.yaml)")
 	return setCmd
 }
 
 func setConfig(cfgFilePath, key string, value ...string) error {
+	log.Info().Msgf("Writing config to %s", cfgFilePath)
 	v := viper.New()
 	v.SetConfigFile(cfgFilePath)
 	if err := v.ReadInConfig(); err != nil {

cmd/cli/config/set_test.go

@@ -1,6 +1,6 @@
 //go:build unit || !integration
 
-package config
+package config_test
 
 import (
 	"io"
@@ -11,61 +11,108 @@ import (
 	"github.com/stretchr/testify/require"
 	"sigs.k8s.io/yaml"
 
+	cmd2 "github.com/bacalhau-project/bacalhau/cmd/cli"
 	"github.com/bacalhau-project/bacalhau/pkg/config/types"
 )
 
-// TestAdditiveSet calls sets on the config system sequentially with different values
-// and asserts that after each set only the newly set value was added to the config.
-// Essentially we are testing two different things:
-func TestAdditiveSet(t *testing.T) {
-	cfgFilePath := filepath.Join(t.TempDir(), "config.yaml")
+func TestSetWithExplicitConfigPath(t *testing.T) {
+	tempDir := t.TempDir()
+	explicitConfigPath := filepath.Join(tempDir, "explicit_config.yaml")
 
-	err := setConfig(cfgFilePath, "api.host", "127.0.0.1")
-	require.NoError(t, err)
-	err = setConfig(cfgFilePath, "api.port", "1234")
+	// Create an empty config file
+	_, err := os.Create(explicitConfigPath)
 	require.NoError(t, err)
 
-	expected := types.Bacalhau{API: types.API{
-		Host: "127.0.0.1",
-		Port: 1234,
-	}}
-	actual := unmarshalConfigFile(t, cfgFilePath)
+	cmd := cmd2.NewRootCmd()
+	cmd.SetArgs([]string{"config", "set", "--config", explicitConfigPath, "api.host", "2.2.2.2"})
 
+	err = cmd.Execute()
+	require.NoError(t, err)
+
+	actual := unmarshalConfigFile(t, explicitConfigPath)
+	expected := types.Bacalhau{API: types.API{Host: "2.2.2.2"}}
 	require.Equal(t, expected, actual)
+}
+
+func TestSetWithDefaultConfigPath(t *testing.T) {
+	tempDir := t.TempDir()
+	t.Setenv("BACALHAU_DIR", tempDir)
+
+	cmd := cmd2.NewRootCmd()
+	cmd.SetArgs([]string{"config", "set", "api.port", "5678"})
 
-	err = setConfig(cfgFilePath, "compute.enabled", "true")
+	err := cmd.Execute()
 	require.NoError(t, err)
-	err = setConfig(cfgFilePath, "compute.orchestrators", "http://127.0.0.1:1234", "http://1.1.1.1:1234")
+
+	defaultConfigPath := filepath.Join(tempDir, "config.yaml")
+	actual := unmarshalConfigFile(t, defaultConfigPath)
+	expected := types.Bacalhau{API: types.API{Port: 5678}}
+	require.Equal(t, expected, actual)
+}
+
+func TestSetMultipleValues(t *testing.T) {
+	tempDir := t.TempDir()
+	os.Setenv("BACALHAU_DIR", tempDir)
+	defer os.Unsetenv("BACALHAU_DIR")
+
+	cmd := cmd2.NewRootCmd()
+	cmd.SetArgs([]string{"config", "set", "compute.orchestrators", "http://127.0.0.1:1234", "http://1.1.1.1:1234"})
+
+	err := cmd.Execute()
 	require.NoError(t, err)
 
-	expected = types.Bacalhau{
-		API: types.API{
-			Host: "127.0.0.1",
-			Port: 1234,
-		},
-		Compute: types.Compute{
-			Enabled: true,
-			Orchestrators: []string{
-				"http://127.0.0.1:1234",
-				"http://1.1.1.1:1234",
-			},
+	defaultConfigPath := filepath.Join(tempDir, "config.yaml")
+	actual := unmarshalConfigFile(t, defaultConfigPath)
+	expected := types.Bacalhau{Compute: types.Compute{
+		Orchestrators: []string{
+			"http://127.0.0.1:1234",
+			"http://1.1.1.1:1234",
 		},
-	}
-	actual = unmarshalConfigFile(t, cfgFilePath)
-
+	}}
 	require.Equal(t, expected, actual)
-
 }
 
-func TestSetFailure(t *testing.T) {
-	cfgFilePath := filepath.Join(t.TempDir(), "config.yaml")
-	// fails as the key isn't a valid config key
-	err := setConfig(cfgFilePath, "not.a.config.key", "porkchop sandwiches")
+func TestSetInvalidKey(t *testing.T) {
+	tempDir := t.TempDir()
+	t.Setenv("BACALHAU_DIR", tempDir)
+
+	cmd := cmd2.NewRootCmd()
+	cmd.SetArgs([]string{"config", "set", "invalid.key", "value"})
+
+	err := cmd.Execute()
 	require.Error(t, err)
+	require.Contains(t, err.Error(), "not a valid config key")
 }
 
-func unmarshalConfigFile(t testing.TB, path string) types.Bacalhau {
+func TestSetAdditiveChanges(t *testing.T) {
+	tempDir := t.TempDir()
+	configPath := filepath.Join(tempDir, "config.yaml")
+
+	// Create an empty config file
+	_, err := os.Create(configPath)
+	require.NoError(t, err)
+
+	// First set
+	cmd1 := cmd2.NewRootCmd()
+	cmd1.SetArgs([]string{"config", "set", "--config", configPath, "api.host", "127.0.0.1"})
+	err = cmd1.Execute()
+	require.NoError(t, err)
+
+	// Second set
+	cmd2 := cmd2.NewRootCmd()
+	cmd2.SetArgs([]string{"config", "set", "--config", configPath, "api.port", "1234"})
+	err = cmd2.Execute()
+	require.NoError(t, err)
 
+	actual := unmarshalConfigFile(t, configPath)
+	expected := types.Bacalhau{API: types.API{
+		Host: "127.0.0.1",
+		Port: 1234,
+	}}
+	require.Equal(t, expected, actual)
+}
+
+func unmarshalConfigFile(t testing.TB, path string) types.Bacalhau {
 	configFile, err := os.Open(path)
 	require.NoError(t, err)
 	t.Cleanup(func() {

cmd/cli/root.go

@@ -24,6 +24,8 @@ import (
 	"github.com/bacalhau-project/bacalhau/cmd/util"
 	"github.com/bacalhau-project/bacalhau/cmd/util/flags/cliflags"
 	"github.com/bacalhau-project/bacalhau/cmd/util/flags/configflags"
+	"github.com/bacalhau-project/bacalhau/pkg/config/types"
+	"github.com/bacalhau-project/bacalhau/pkg/logger"
 	"github.com/bacalhau-project/bacalhau/pkg/system"
 )
 
@@ -69,6 +71,22 @@ func NewRootCmd() *cobra.Command {
 			return err
 		}
 
+		// Configure logging
+		// While we allow users to configure logging via the config file, they are applied
+		// and will override this configuration at a later stage when the config is loaded.
+		// This is needed to ensure any logs before the config is loaded are captured.
+		logMode := viper.GetString(types.LoggingModeKey)
+		if logMode == "" {
+			logMode = string(logger.LogModeDefault)
+		}
+		logLevel := viper.GetString(types.LoggingLevelKey)
+		if logLevel == "" {
+			logLevel = "Info"
+		}
+		if err := logger.ConfigureLogging(logMode, logLevel); err != nil {
+			return fmt.Errorf("failed to configure logging: %w", err)
+		}
+
 		return nil
 	}
 
@@ -110,6 +128,8 @@ func NewRootCmd() *cobra.Command {
 func Execute(ctx context.Context) {
 	rootCmd := NewRootCmd()
 	rootCmd.SetContext(ctx)
+	rootCmd.SilenceErrors = true
+	rootCmd.SilenceUsage = true
 	if err := rootCmd.Execute(); err != nil {
 		util.Fatal(rootCmd, err, 1)
 	}

cmd/util/repo.go

@@ -2,8 +2,6 @@ package util
 
 import (
 	"fmt"
-	"os"
-	"path/filepath"
 
 	"github.com/spf13/cobra"
 	"github.com/spf13/pflag"
@@ -14,7 +12,6 @@ import (
 	"github.com/bacalhau-project/bacalhau/cmd/util/hook"
 	"github.com/bacalhau-project/bacalhau/pkg/config"
 	"github.com/bacalhau-project/bacalhau/pkg/config/types"
-	"github.com/bacalhau-project/bacalhau/pkg/config_legacy"
 	"github.com/bacalhau-project/bacalhau/pkg/repo"
 	"github.com/bacalhau-project/bacalhau/pkg/setup"
 )
@@ -51,26 +48,9 @@ func SetupConfigType(cmd *cobra.Command) (*config.Config, error) {
 	// check if the user specified config files via the --config flag
 	configFiles := getConfigFiles(v)
 
-	// if none were provided look in $XDG_CONFIG_HOME/bacalhau/config.yaml
-	if len(configFiles) == 0 {
-		xdgPath, err := os.UserConfigDir()
-		if err == nil {
-			path := filepath.Join(xdgPath, "bacalhau", config_legacy.FileName)
-			if _, err := os.Stat(path); err != nil {
-				// if the file exists and could not be read, return an error
-				if !os.IsNotExist(err) {
-					return nil, fmt.Errorf("loading config file at %q: %w", path, err)
-				}
-			} else {
-				// the file exists, use it.
-				configFiles = append(configFiles, path)
-			}
-		}
-	}
-	// if a config file is present, apply it to the config
+	// apply user specified config files via the --config flag, if any
 	if len(configFiles) > 0 {
-		cmd.Printf("Config file(s) found at path(s): %s\n", configFiles)
-		opts = append(opts, config.WithPaths(configFiles...))
+		opts = append(opts, config.WithPaths(getConfigFiles(v)...))
 	}
 
 	configFlags, err := getConfigFlags(v, cmd)
@@ -103,6 +83,10 @@ func SetupConfig(cmd *cobra.Command) (types.Bacalhau, error) {
 	if err != nil {
 		return types.Bacalhau{}, err
 	}
+	return DecodeBacalhauConfig(cfg)
+}
+
+func DecodeBacalhauConfig(cfg *config.Config) (types.Bacalhau, error) {
 	var out types.Bacalhau
 	if err := cfg.Unmarshal(&out); err != nil {
 		return types.Bacalhau{}, err

pkg/config/config.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"fmt"
 	"os"
+	"path/filepath"
 	"strings"
 
 	"github.com/mitchellh/mapstructure"
@@ -188,6 +189,26 @@ func New(opts ...Option) (*Config, error) {
 		c.base.Set(name, value)
 	}
 
+	// if no config file was provided, we look for a config.yaml under the resolved data directory,
+	// and if it exists, we create and return a new config with the resolved path.
+	// we attempt this last to ensure the data-dir is resolved correctly from all config sources.
+	if len(c.paths) == 0 {
+		configFile := filepath.Join(c.base.GetString(types.DataDirKey), DefaultFileName)
+		if _, err := os.Stat(configFile); err == nil {
+			opts = append(opts, WithPaths(configFile))
+			return New(opts...)
+		}
+	}
+
+	// log the resolved config paths
+	absoluteConfigPaths := make([]string, len(c.paths))
+	for i, path := range c.paths {
+		absoluteConfigPaths[i], err = filepath.Abs(path)
+		if err != nil {
+			absoluteConfigPaths[i] = path
+		}
+	}
+	log.Info().Msgf("Config loaded from: %s, and with data-dir %s", absoluteConfigPaths, c.base.Get(types.DataDirKey))
 	return c, nil
 }
 
@@ -218,7 +239,6 @@ func getNodeType(input string) (requester, compute bool, err error) {
 // from the read config file.
 // Load returns an error if the file cannot be read.
 func (c *Config) Load(path string) error {
-	log.Info().Msgf("loading config file: %q", path)
 	c.base.SetConfigFile(path)
 	if err := c.base.ReadInConfig(); err != nil {
 		return err
@@ -229,7 +249,6 @@ func (c *Config) Load(path string) error {
 // Merge merges a new configuration file specified by `path` with the existing config.
 // Merge returns an error if the file cannot be read
 func (c *Config) Merge(path string) error {
-	log.Info().Msgf("merging config file: %q", path)
 	c.base.SetConfigFile(path)
 	if err := c.base.MergeInConfig(); err != nil {
 		return err