feat: lunchpail "needs" support for installing dependencies

Signed-off-by: aavarghese <[email protected]>

cmd/subcommands/needs.go

@@ -0,0 +1,20 @@
+package subcommands
+
+import (
+	"github.com/spf13/cobra"
+
+	"lunchpail.io/cmd/subcommands/needs"
+)
+
+func init() {
+	var cmd = &cobra.Command{
+		Use:     "needs",
+		GroupID: internalGroup.ID,
+		Short:   "Commands for installing dependencies to run the application",
+		Long:    "Commands for installing dependencies to run the application",
+	}
+
+	rootCmd.AddCommand(cmd)
+	cmd.AddCommand(needs.Minio())
+	cmd.AddCommand(needs.Python())
+}

cmd/subcommands/needs/minio.go

@@ -0,0 +1,32 @@
+package needs
+
+import (
+	"context"
+
+	"github.com/spf13/cobra"
+
+	"lunchpail.io/cmd/options"
+	"lunchpail.io/pkg/runtime/needs"
+)
+
+func Minio() *cobra.Command {
+	cmd := &cobra.Command{
+		Use:   "minio <version>",
+		Short: "Install minio",
+		Long:  "Install minio",
+		Args:  cobra.MatchAll(cobra.MaximumNArgs(1), cobra.OnlyValidArgs),
+	}
+
+	logOpts := options.AddLogOptions(cmd)
+
+	cmd.RunE = func(cmd *cobra.Command, args []string) error {
+		version := "latest"
+		if len(args) > 0 {
+			version = args[0]
+		}
+
+		return needs.InstallMinio(context.Background(), version, needs.Options{LogOptions: *logOpts})
+	}
+
+	return cmd
+}

cmd/subcommands/needs/python.go

@@ -0,0 +1,36 @@
+package needs
+
+import (
+	"context"
+
+	"github.com/spf13/cobra"
+
+	"lunchpail.io/cmd/options"
+	"lunchpail.io/pkg/runtime/needs"
+)
+
+func Python() *cobra.Command {
+	var requirementsPath string
+	var virtualEnvPath string
+	cmd := &cobra.Command{
+		Use:   "python <version> [-r /path/to/requirements.txt] [-v /path/to/.venv]",
+		Short: "Install python environment",
+		Long:  "Install python environment",
+		Args:  cobra.MatchAll(cobra.MaximumNArgs(1), cobra.OnlyValidArgs),
+	}
+
+	logOpts := options.AddLogOptions(cmd)
+	cmd.Flags().StringVarP(&requirementsPath, "requirements", "r", requirementsPath, "Install from the given requirements file")
+	cmd.Flags().StringVarP(&virtualEnvPath, "venv", "d", virtualEnvPath, "Path to virtual environment dir")
+
+	cmd.RunE = func(cmd *cobra.Command, args []string) error {
+		version := "latest"
+		if len(args) >= 1 {
+			version = args[0]
+		}
+
+		return needs.InstallPython(context.Background(), version, virtualEnvPath, requirementsPath, needs.Options{LogOptions: *logOpts})
+	}
+
+	return cmd
+}

pkg/be/kubernetes/shell/chart/templates/containers/main.yaml

@@ -19,6 +19,8 @@
       valueFrom:
         fieldRef:
           fieldPath: metadata.name
+    - name: VIRTUAL_ENV
+      value: {{ .Values.venvPath }}
   {{- if .Values.env }}
     {{ .Values.env | b64dec | fromJsonArray | toYaml | nindent 4 }}
   {{- end }}

pkg/be/local/shell/ok.go

@@ -13,7 +13,8 @@ import (
 func isCompatibleImage(image string) bool {
 	return strings.HasPrefix(image, lunchpail.ImageRegistry+"/"+lunchpail.ImageRepo+"/lunchpail") ||
 		strings.Contains(image, "alpine") ||
-		strings.Contains(image, "minio/minio")
+		strings.Contains(image, "minio/minio") ||
+		strings.Contains(image, "python")
 }
 
 func IsCompatible(c llir.ShellComponent) error {

pkg/fe/transformer/api/minio/transpile.go

@@ -20,6 +20,8 @@ func transpile(runname string, ir llir.LLIR) (hlir.Application, error) {
 	app.Spec.Expose = []string{fmt.Sprintf("%d:%d", ir.Queue.Port, ir.Queue.Port)}
 	app.Spec.Command = fmt.Sprintf("$LUNCHPAIL_EXE component minio server --port %d", ir.Queue.Port)
 
+	/*app.Spec.Needs = []hlir.Needs{
+	{Name: "minio", Version: "latest"}}*/
 	prefixIncludingBucket := api.QueuePrefixPath(ir.Queue, runname)
 	A := strings.Split(prefixIncludingBucket, "/")
 	prefixExcludingBucket := filepath.Join(A[1:]...)

pkg/fe/transformer/api/shell/lower.go

@@ -2,6 +2,7 @@ package shell
 
 import (
 	"fmt"
+	"os"
 	"path/filepath"
 
 	"lunchpail.io/pkg/build"
@@ -36,6 +37,31 @@ func LowerAsComponent(buildName, runname string, app hlir.Application, ir llir.L
 		component.InstanceName = runname
 	}
 
+	for _, needs := range app.Spec.Needs {
+		var file *os.File
+		var err error
+		var req string
+
+		if needs.Requirements != "" {
+			file, err = os.CreateTemp("", "requirements.txt")
+			if err != nil {
+				return nil, err
+			}
+
+			if err := os.WriteFile(file.Name(), []byte(needs.Requirements), 0644); err != nil {
+				return nil, err
+			}
+			req = "--requirements " + file.Name()
+			if opts.Log.Verbose {
+				fmt.Printf("Setting requirements %s in %s \n", needs.Requirements, file.Name())
+			}
+		}
+		component.Spec.Command = fmt.Sprintf(`$LUNCHPAIL_EXE needs %s %s %s --verbose=%v
+%s
+sleep 100000`, needs.Name, needs.Version, req, opts.Log.Verbose, component.Spec.Command)
+
+	}
+
 	for _, dataset := range app.Spec.Datasets {
 		if dataset.S3.Rclone.RemoteName != "" && dataset.S3.CopyIn.Path != "" {
 			// We were asked to copy data in from s3, so

pkg/ir/hlir/application.go

@@ -7,6 +7,12 @@ type Code struct {
 	Source string
 }
 
+type Needs struct {
+	Name         string
+	Version      string
+	Requirements string
+}
+
 type Application struct {
 	ApiVersion string `yaml:"apiVersion"`
 	Kind       string
@@ -25,6 +31,7 @@ type Application struct {
 		Datasets                 []Dataset                `yaml:"datasets,omitempty"`
 		SecurityContext          SecurityContext          `yaml:"securityContext,omitempty"`
 		ContainerSecurityContext ContainerSecurityContext `yaml:"containerSecurityContext,omitempty"`
+		Needs                    []Needs                  `yaml:"needs,omitempty"`
 	}
 }
 

pkg/runtime/needs/install_darwin.go

@@ -0,0 +1,81 @@
+package needs
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"os/exec"
+	"os/user"
+	"path/filepath"
+)
+
+func homedir() (string, error) {
+	currentUser, err := user.Current()
+	if err != nil {
+		return "", err
+	}
+
+	return currentUser.HomeDir, nil
+}
+
+func installMinio(ctx context.Context, version string, verbose bool) error {
+	if err := setenv(); err != nil { //$HOME must be set for brew
+		return err
+	}
+
+	return brewInstall(ctx, "minio/stable/minio", version, verbose) //Todo: versions other than latest
+}
+
+func installPython(ctx context.Context, version string, verbose bool) error {
+	if err := setenv(); err != nil { //$HOME must be set for brew
+		return err
+	}
+
+	return brewInstall(ctx, "python3", version, verbose) //Todo: versions other than latest
+}
+
+func brewInstall(ctx context.Context, pkg string, version string, verbose bool) error {
+	var cmd *exec.Cmd
+	if verbose {
+		fmt.Fprintf(os.Stdout, "Installing %s release of %s \n", version, pkg)
+		cmd = exec.CommandContext(ctx, "brew", "install", "--verbose", "--debug", pkg)
+		cmd.Stdout = os.Stdout
+	} else {
+		cmd = exec.Command("brew", "install", pkg)
+	}
+	cmd.Stderr = os.Stderr
+	return cmd.Run()
+}
+
+func requirementsInstall(ctx context.Context, venvPath string, requirementsPath string, verbose bool) error {
+	var cmd *exec.Cmd
+	var verboseFlag string
+	dir := filepath.Dir(venvPath)
+
+	if verbose {
+		verboseFlag = "--verbose"
+	}
+
+	venvRequirementsPath := filepath.Join(venvPath, filepath.Base(requirementsPath))
+	cmds := fmt.Sprintf(`python3 -m venv %s
+cp %s %s
+source %s/bin/activate
+python3 -m pip install --upgrade pip %s
+pip3 install -r %s %s 1>&2`, venvPath, requirementsPath, venvPath, venvPath, verboseFlag, venvRequirementsPath, verboseFlag)
+
+	cmd = exec.CommandContext(ctx, "/bin/bash", "-c", cmds)
+	cmd.Dir = dir
+	if verbose {
+		cmd.Stdout = os.Stdout
+	}
+	cmd.Stderr = os.Stderr
+	return cmd.Run()
+}
+
+func setenv() error {
+	dir, err := homedir()
+	if err != nil {
+		return err
+	}
+	return os.Setenv("HOME", dir)
+}

pkg/runtime/needs/install_linux.go

@@ -0,0 +1,125 @@
+package needs
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"os/exec"
+	"path/filepath"
+)
+
+func bindir() (string, error) {
+	cachedir, err := os.UserCacheDir()
+	if err != nil {
+		return "", err
+	}
+
+	return filepath.Join(cachedir, "lunchpail", "bin"), nil
+}
+
+func installMinio(ctx context.Context, version string, verbose bool) error {
+	if verbose {
+		fmt.Printf("Installing %s release of minio \n", version)
+	}
+
+	dir, err := bindir()
+	if err != nil {
+		return err
+	}
+
+	if err := os.MkdirAll(dir, 0755); err != nil {
+		return err
+	}
+
+	//Todo: versions other than latest
+	cmd := exec.CommandContext(ctx, "/bin/sh", "-c", "apt update; apt -y install wget; wget https://dl.min.io/server/minio/release/linux-amd64/minio")
+	cmd.Dir = dir
+	if verbose {
+		cmd.Stdout = os.Stdout
+	}
+	cmd.Stderr = os.Stderr
+	if err := cmd.Run(); err != nil {
+		return err
+	}
+
+	if err := setenv(dir); err != nil { //setting $PATH
+		return err
+	}
+
+	return os.Chmod(filepath.Join(dir, "minio"), 0755)
+}
+
+func installPython(ctx context.Context, version string, verbose bool) error {
+	/*
+			if verbose {
+			fmt.Fprintf(os.Stdout, "Installing %s release of python \n", version)
+		}
+
+			dir, err := bindir()
+			if err != nil {
+				return err
+			}
+
+			if err := os.MkdirAll(dir, 0755); err != nil {
+				return err
+			}
+
+			//Todo: versions other than latest
+			cmd := exec.Command("wget", "https://www.python.org/ftp/python/3.12.7/Python-3.12.7.tgz")
+			cmd.Dir = dir
+			if verbose {
+				cmd.Stdout = os.Stdout
+			}
+			cmd.Stderr = os.Stderr
+			if err := cmd.Run(); err != nil {
+				return err
+			}
+
+			cmd = exec.Command("tar", "xf", "Python-3.12.7.tgz")
+			cmd.Dir = dir
+			if verbose {
+				cmd.Stdout = os.Stdout
+			}
+			cmd.Stderr = os.Stderr
+			if err := cmd.Run(); err != nil {
+				return err
+			}
+
+			if err := setenv(dir); err != nil { //setting $PATH
+				return err
+			}
+
+			os.Chmod(filepath.Join(dir, "python"), 0755)
+	*/
+
+	return nil
+}
+
+func requirementsInstall(ctx context.Context, venvPath string, requirementsPath string, verbose bool) error {
+	var cmd *exec.Cmd
+	var verboseFlag string
+	dir := filepath.Dir(venvPath)
+
+	if verbose {
+		verboseFlag = "--verbose"
+	}
+
+	venvRequirementsPath := filepath.Join(venvPath, filepath.Base(requirementsPath))
+	cmds := fmt.Sprintf(`python3 -m venv %s
+cp %s %s
+source %s/bin/activate
+python3 -m pip install --upgrade pip %s
+pip3 install -r %s %s 1>&2`, venvPath, requirementsPath, venvPath, venvPath, verboseFlag, venvRequirementsPath, verboseFlag)
+
+	cmd = exec.CommandContext(ctx, "/bin/bash", "-c", cmds)
+	cmd.Dir = dir
+	if verbose {
+		cmd.Stdout = os.Stdout
+	}
+	cmd.Stderr = os.Stderr
+	return cmd.Run()
+}
+
+func setenv(dir string) error {
+	return os.Setenv("PATH", os.Getenv("PATH")+":"+dir)
+}