legacy: Builder refactorization (part 5...)

arduino/builder/build_options_manager.go

@@ -0,0 +1,182 @@
+// This file is part of arduino-cli.
+//
+// Copyright 2020 ARDUINO SA (http://www.arduino.cc/)
+//
+// This software is released under the GNU General Public License version 3,
+// which covers the main part of arduino-cli.
+// The terms of this license can be found at:
+// https://www.gnu.org/licenses/gpl-3.0.en.html
+//
+// You can be released from the requirements of the above licenses by purchasing
+// a commercial license. Buying such a license is mandatory if you want to
+// modify or otherwise use the software for commercial activities involving the
+// Arduino software without disclosing the source code of your own applications.
+// To purchase a commercial license, send an email to [email protected].
+
+package builder
+
+import (
+	"encoding/json"
+	"path/filepath"
+	"strings"
+
+	"github.com/arduino/arduino-cli/arduino/builder/logger"
+	"github.com/arduino/arduino-cli/arduino/builder/utils"
+	"github.com/arduino/arduino-cli/arduino/cores"
+	"github.com/arduino/arduino-cli/arduino/sketch"
+	"github.com/arduino/go-paths-helper"
+	properties "github.com/arduino/go-properties-orderedmap"
+	"github.com/pkg/errors"
+)
+
+// BuildOptionsManager fixdoc
+type BuildOptionsManager struct {
+	currentOptions          *properties.Map
+	currentBuildOptionsJSON []byte
+
+	hardwareDirs              paths.PathList
+	builtInToolsDirs          paths.PathList
+	otherLibrariesDirs        paths.PathList
+	builtInLibrariesDirs      *paths.Path
+	buildPath                 *paths.Path
+	runtimePlatformPath       *paths.Path
+	buildCorePath             *paths.Path
+	sketch                    *sketch.Sketch
+	customBuildProperties     []string
+	compilerOptimizationFlags string
+	clean                     bool
+	builderLogger             *logger.BuilderLogger
+}
+
+// NewBuildOptionsManager fixdoc
+func NewBuildOptionsManager(
+	hardwareDirs, builtInToolsDirs, otherLibrariesDirs paths.PathList,
+	builtInLibrariesDirs, buildPath *paths.Path,
+	sketch *sketch.Sketch,
+	customBuildProperties []string,
+	fqbn *cores.FQBN,
+	clean bool,
+	compilerOptimizationFlags string,
+	runtimePlatformPath, buildCorePath *paths.Path,
+	buildLogger *logger.BuilderLogger,
+) *BuildOptionsManager {
+	opts := properties.NewMap()
+
+	opts.Set("hardwareFolders", strings.Join(hardwareDirs.AsStrings(), ","))
+	opts.Set("builtInToolsFolders", strings.Join(builtInToolsDirs.AsStrings(), ","))
+	opts.Set("otherLibrariesFolders", strings.Join(otherLibrariesDirs.AsStrings(), ","))
+	opts.SetPath("sketchLocation", sketch.FullPath)
+	opts.Set("fqbn", fqbn.String())
+	opts.Set("customBuildProperties", strings.Join(customBuildProperties, ","))
+	opts.Set("compiler.optimization_flags", compilerOptimizationFlags)
+
+	if builtInLibrariesDirs != nil {
+		opts.Set("builtInLibrariesFolders", builtInLibrariesDirs.String())
+	}
+
+	absPath := sketch.FullPath.Parent()
+	var additionalFilesRelative []string
+	for _, f := range sketch.AdditionalFiles {
+		relPath, err := f.RelTo(absPath)
+		if err != nil {
+			continue // ignore
+		}
+		additionalFilesRelative = append(additionalFilesRelative, relPath.String())
+	}
+	opts.Set("additionalFiles", strings.Join(additionalFilesRelative, ","))
+
+	return &BuildOptionsManager{
+		currentOptions:            opts,
+		hardwareDirs:              hardwareDirs,
+		builtInToolsDirs:          builtInToolsDirs,
+		otherLibrariesDirs:        otherLibrariesDirs,
+		builtInLibrariesDirs:      builtInLibrariesDirs,
+		buildPath:                 buildPath,
+		runtimePlatformPath:       runtimePlatformPath,
+		buildCorePath:             buildCorePath,
+		sketch:                    sketch,
+		customBuildProperties:     customBuildProperties,
+		compilerOptimizationFlags: compilerOptimizationFlags,
+		clean:                     clean,
+		builderLogger:             buildLogger,
+	}
+}
+
+// WipeBuildPath fixdoc
+func (m *BuildOptionsManager) WipeBuildPath() error {
+	buildOptionsJSON, err := json.MarshalIndent(m.currentOptions, "", "  ")
+	if err != nil {
+		return errors.WithStack(err)
+	}
+	m.currentBuildOptionsJSON = buildOptionsJSON
+
+	if err := m.wipeBuildPath(); err != nil {
+		return errors.WithStack(err)
+	}
+	return m.buildPath.Join("build.options.json").WriteFile(buildOptionsJSON)
+}
+
+func (m *BuildOptionsManager) wipeBuildPath() error {
+	wipe := func() error {
+		// FIXME: this should go outside legacy and behind a `logrus` call so users can
+		// control when this should be printed.
+		// logger.Println(constants.LOG_LEVEL_INFO, constants.MSG_BUILD_OPTIONS_CHANGED + constants.MSG_REBUILD_ALL)
+		if err := m.buildPath.RemoveAll(); err != nil {
+			return errors.WithMessage(err, tr("cleaning build path"))
+		}
+		if err := m.buildPath.MkdirAll(); err != nil {
+			return errors.WithMessage(err, tr("cleaning build path"))
+		}
+		return nil
+	}
+
+	if m.clean {
+		return wipe()
+	}
+
+	// Load previous build options map
+	var buildOptionsJSONPrevious []byte
+	var _err error
+	if buildOptionsFile := m.buildPath.Join("build.options.json"); buildOptionsFile.Exist() {
+		buildOptionsJSONPrevious, _err = buildOptionsFile.ReadFile()
+		if _err != nil {
+			return errors.WithStack(_err)
+		}
+	}
+
+	if len(buildOptionsJSONPrevious) == 0 {
+		return nil
+	}
+
+	var prevOpts *properties.Map
+	if err := json.Unmarshal(buildOptionsJSONPrevious, &prevOpts); err != nil || prevOpts == nil {
+		m.builderLogger.Info(tr("%[1]s invalid, rebuilding all", "build.options.json"))
+		return wipe()
+	}
+
+	// If SketchLocation path is different but filename is the same, consider it equal
+	if filepath.Base(m.currentOptions.Get("sketchLocation")) == filepath.Base(prevOpts.Get("sketchLocation")) {
+		m.currentOptions.Remove("sketchLocation")
+		prevOpts.Remove("sketchLocation")
+	}
+
+	// If options are not changed check if core has
+	if m.currentOptions.Equals(prevOpts) {
+		// check if any of the files contained in the core folders has changed
+		// since the json was generated - like platform.txt or similar
+		// if so, trigger a "safety" wipe
+		targetCoreFolder := m.runtimePlatformPath
+		coreFolder := m.buildCorePath
+		realCoreFolder := coreFolder.Parent().Parent()
+		jsonPath := m.buildPath.Join("build.options.json")
+		coreUnchanged, _ := utils.DirContentIsOlderThan(realCoreFolder, jsonPath, ".txt")
+		if coreUnchanged && targetCoreFolder != nil && !realCoreFolder.EqualsTo(targetCoreFolder) {
+			coreUnchanged, _ = utils.DirContentIsOlderThan(targetCoreFolder, jsonPath, ".txt")
+		}
+		if coreUnchanged {
+			return nil
+		}
+	}
+
+	return wipe()
+}

arduino/builder/builder.go

@@ -16,21 +16,81 @@
 package builder
 
 import (
+	"errors"
+	"fmt"
+
+	"github.com/arduino/arduino-cli/arduino/builder/compilation"
+	"github.com/arduino/arduino-cli/arduino/builder/logger"
+	"github.com/arduino/arduino-cli/arduino/builder/progress"
+	"github.com/arduino/arduino-cli/arduino/cores"
 	"github.com/arduino/arduino-cli/arduino/sketch"
 	"github.com/arduino/go-paths-helper"
 	"github.com/arduino/go-properties-orderedmap"
 )
 
+// ErrSketchCannotBeLocatedInBuildPath fixdoc
+var ErrSketchCannotBeLocatedInBuildPath = errors.New("sketch cannot be located in build path")
+
 // Builder is a Sketch builder.
 type Builder struct {
 	sketch          *sketch.Sketch
 	buildProperties *properties.Map
 
+	buildPath          *paths.Path
+	sketchBuildPath    *paths.Path
+	coreBuildPath      *paths.Path
+	librariesBuildPath *paths.Path
+
 	// Parallel processes
 	jobs int
 
+	// Custom build properties defined by user (line by line as "key=value" pairs)
+	customBuildProperties []string
+
 	// core related
 	coreBuildCachePath *paths.Path
+
+	logger *logger.BuilderLogger
+	clean  bool
+
+	// Source code overrides (filename -> content map).
+	// The provided source data is used instead of reading it from disk.
+	// The keys of the map are paths relative to sketch folder.
+	sourceOverrides map[string]string
+
+	// Set to true to skip build and produce only Compilation Database
+	onlyUpdateCompilationDatabase bool
+	// Compilation Database to build/update
+	compilationDatabase *compilation.Database
+
+	// Progress of all various steps
+	Progress *progress.Struct
+
+	// Sizer results
+	executableSectionsSize ExecutablesFileSections
+
+	// C++ Parsing
+	lineOffset int
+
+	targetPlatform *cores.PlatformRelease
+	actualPlatform *cores.PlatformRelease
+
+	buildArtifacts *BuildArtifacts
+
+	*BuildOptionsManager
+}
+
+// BuildArtifacts contains the result of various build
+type BuildArtifacts struct {
+	// populated by BuildCore
+	coreArchiveFilePath *paths.Path
+	coreObjectsFiles    paths.PathList
+
+	// populated by BuildLibraries
+	librariesObjectFiles paths.PathList
+
+	// populated by BuildSketch
+	sketchObjectFiles paths.PathList
 }
 
 // NewBuilder creates a sketch Builder.
@@ -41,7 +101,17 @@ func NewBuilder(
 	optimizeForDebug bool,
 	coreBuildCachePath *paths.Path,
 	jobs int,
-) *Builder {
+	requestBuildProperties []string,
+	hardwareDirs, builtInToolsDirs, otherLibrariesDirs paths.PathList,
+	builtInLibrariesDirs *paths.Path,
+	fqbn *cores.FQBN,
+	clean bool,
+	sourceOverrides map[string]string,
+	onlyUpdateCompilationDatabase bool,
+	targetPlatform, actualPlatform *cores.PlatformRelease,
+	logger *logger.BuilderLogger,
+	progressStats *progress.Struct,
+) (*Builder, error) {
 	buildProperties := properties.NewMap()
 	if boardBuildProperties != nil {
 		buildProperties.Merge(boardBuildProperties)
@@ -64,20 +134,103 @@ func NewBuilder(
 		}
 	}
 
-	return &Builder{
-		sketch:             sk,
-		buildProperties:    buildProperties,
-		coreBuildCachePath: coreBuildCachePath,
-		jobs:               jobs,
+	// Add user provided custom build properties
+	customBuildProperties, err := properties.LoadFromSlice(requestBuildProperties)
+	if err != nil {
+		return nil, fmt.Errorf("invalid build properties: %w", err)
+	}
+	buildProperties.Merge(customBuildProperties)
+	customBuildPropertiesArgs := append(requestBuildProperties, "build.warn_data_percentage=75")
+
+	sketchBuildPath, err := buildPath.Join("sketch").Abs()
+	if err != nil {
+		return nil, err
 	}
+	librariesBuildPath, err := buildPath.Join("libraries").Abs()
+	if err != nil {
+		return nil, err
+	}
+	coreBuildPath, err := buildPath.Join("core").Abs()
+	if err != nil {
+		return nil, err
+	}
+
+	if buildPath.Canonical().EqualsTo(sk.FullPath.Canonical()) {
+		return nil, ErrSketchCannotBeLocatedInBuildPath
+	}
+
+	if progressStats == nil {
+		progressStats = progress.New(nil)
+	}
+
+	return &Builder{
+		sketch:                        sk,
+		buildProperties:               buildProperties,
+		buildPath:                     buildPath,
+		sketchBuildPath:               sketchBuildPath,
+		coreBuildPath:                 coreBuildPath,
+		librariesBuildPath:            librariesBuildPath,
+		jobs:                          jobs,
+		customBuildProperties:         customBuildPropertiesArgs,
+		coreBuildCachePath:            coreBuildCachePath,
+		logger:                        logger,
+		clean:                         clean,
+		sourceOverrides:               sourceOverrides,
+		onlyUpdateCompilationDatabase: onlyUpdateCompilationDatabase,
+		compilationDatabase:           compilation.NewDatabase(buildPath.Join("compile_commands.json")),
+		Progress:                      progressStats,
+		executableSectionsSize:        []ExecutableSectionSize{},
+		buildArtifacts:                &BuildArtifacts{},
+		targetPlatform:                targetPlatform,
+		actualPlatform:                actualPlatform,
+		BuildOptionsManager: NewBuildOptionsManager(
+			hardwareDirs, builtInToolsDirs, otherLibrariesDirs,
+			builtInLibrariesDirs, buildPath,
+			sk,
+			customBuildPropertiesArgs,
+			fqbn,
+			clean,
+			buildProperties.Get("compiler.optimization_flags"),
+			buildProperties.GetPath("runtime.platform.path"),
+			buildProperties.GetPath("build.core.path"), // TODO can we buildCorePath ?
+			logger,
+		),
+	}, nil
 }
 
 // GetBuildProperties returns the build properties for running this build
 func (b *Builder) GetBuildProperties() *properties.Map {
 	return b.buildProperties
 }
 
-// Jobs number of parallel processes
-func (b *Builder) Jobs() int {
-	return b.jobs
+// GetBuildPath returns the build path
+func (b *Builder) GetBuildPath() *paths.Path {
+	return b.buildPath
+}
+
+// GetSketchBuildPath returns the sketch build path
+func (b *Builder) GetSketchBuildPath() *paths.Path {
+	return b.sketchBuildPath
+}
+
+// GetLibrariesBuildPath returns the libraries build path
+func (b *Builder) GetLibrariesBuildPath() *paths.Path {
+	return b.librariesBuildPath
+}
+
+// ExecutableSectionsSize fixdoc
+func (b *Builder) ExecutableSectionsSize() ExecutablesFileSections {
+	return b.executableSectionsSize
+}
+
+// SaveCompilationDatabase fixdoc
+func (b *Builder) SaveCompilationDatabase() {
+	if b.compilationDatabase != nil {
+		b.compilationDatabase.SaveToFile()
+	}
+}
+
+// TargetPlatform fixdoc
+func (b *Builder) TargetPlatform() *cores.PlatformRelease {
+	return b.targetPlatform
 }