From 590ca66088ca43fc4d2e4f5aaf0cce91cf7859f7 Mon Sep 17 00:00:00 2001 From: aavarghese Date: Thu, 19 Sep 2024 12:37:55 -0400 Subject: [PATCH] feat: lunchpail "needs" support for installing dependencies Signed-off-by: aavarghese --- cmd/subcommands/needs.go | 20 +++ cmd/subcommands/needs/minio.go | 32 +++++ cmd/subcommands/needs/python.go | 36 +++++ .../chart/templates/containers/main.yaml | 2 + pkg/be/local/shell/ok.go | 3 +- pkg/fe/transformer/api/minio/transpile.go | 2 + pkg/fe/transformer/api/shell/lower.go | 26 ++++ pkg/ir/hlir/application.go | 7 + pkg/runtime/needs/install_darwin.go | 81 ++++++++++++ pkg/runtime/needs/install_linux.go | 125 ++++++++++++++++++ pkg/runtime/needs/minio.go | 18 +++ pkg/runtime/needs/needs.go | 7 + pkg/runtime/needs/python.go | 29 ++++ tests/bin/run.sh | 11 +- tests/bin/up.sh | 1 + tests/tests/python-basic/pail/app.yaml | 1 + tests/tests/python-basic/target | 1 - tests/tests/python-doc-chunk/pail/app.yaml | 1 + tests/tests/python-doc-quality/pail/app.yaml | 1 + tests/tests/python-html2parquet/pail/app.yaml | 1 + tests/tests/python-lang-id/pail/app.yaml | 1 + tests/tests/python-pdf2parquet/pail/app.yaml | 1 + tests/tests/python-pii-redactor/pail/app.yaml | 1 + tests/tests/python-pii-redactor/preinit.sh | 14 +- tests/tests/python-text-encoder/pail/app.yaml | 1 + 25 files changed, 404 insertions(+), 19 deletions(-) create mode 100644 cmd/subcommands/needs.go create mode 100644 cmd/subcommands/needs/minio.go create mode 100644 cmd/subcommands/needs/python.go create mode 100644 pkg/runtime/needs/install_darwin.go create mode 100644 pkg/runtime/needs/install_linux.go create mode 100644 pkg/runtime/needs/minio.go create mode 100644 pkg/runtime/needs/needs.go create mode 100644 pkg/runtime/needs/python.go delete mode 100644 tests/tests/python-basic/target diff --git a/cmd/subcommands/needs.go b/cmd/subcommands/needs.go new file mode 100644 index 000000000..03bc75098 --- /dev/null +++ b/cmd/subcommands/needs.go @@ -0,0 +1,20 @@ +package subcommands + +import ( + "github.com/spf13/cobra" + + "lunchpail.io/cmd/subcommands/needs" +) + +func init() { + var cmd = &cobra.Command{ + Use: "needs", + GroupID: internalGroup.ID, + Short: "Commands for installing dependencies to run the application", + Long: "Commands for installing dependencies to run the application", + } + + rootCmd.AddCommand(cmd) + cmd.AddCommand(needs.Minio()) + cmd.AddCommand(needs.Python()) +} diff --git a/cmd/subcommands/needs/minio.go b/cmd/subcommands/needs/minio.go new file mode 100644 index 000000000..774d9158e --- /dev/null +++ b/cmd/subcommands/needs/minio.go @@ -0,0 +1,32 @@ +package needs + +import ( + "context" + + "github.com/spf13/cobra" + + "lunchpail.io/cmd/options" + "lunchpail.io/pkg/runtime/needs" +) + +func Minio() *cobra.Command { + cmd := &cobra.Command{ + Use: "minio ", + Short: "Install minio", + Long: "Install minio", + Args: cobra.MatchAll(cobra.MaximumNArgs(1), cobra.OnlyValidArgs), + } + + logOpts := options.AddLogOptions(cmd) + + cmd.RunE = func(cmd *cobra.Command, args []string) error { + version := "latest" + if len(args) > 0 { + version = args[0] + } + + return needs.InstallMinio(context.Background(), version, needs.Options{LogOptions: *logOpts}) + } + + return cmd +} diff --git a/cmd/subcommands/needs/python.go b/cmd/subcommands/needs/python.go new file mode 100644 index 000000000..dc5cdbca8 --- /dev/null +++ b/cmd/subcommands/needs/python.go @@ -0,0 +1,36 @@ +package needs + +import ( + "context" + + "github.com/spf13/cobra" + + "lunchpail.io/cmd/options" + "lunchpail.io/pkg/runtime/needs" +) + +func Python() *cobra.Command { + var requirementsPath string + var virtualEnvPath string + cmd := &cobra.Command{ + Use: "python [-r /path/to/requirements.txt] [-v /path/to/.venv]", + Short: "Install python environment", + Long: "Install python environment", + Args: cobra.MatchAll(cobra.MaximumNArgs(1), cobra.OnlyValidArgs), + } + + logOpts := options.AddLogOptions(cmd) + cmd.Flags().StringVarP(&requirementsPath, "requirements", "r", requirementsPath, "Install from the given requirements file") + cmd.Flags().StringVarP(&virtualEnvPath, "venv", "d", virtualEnvPath, "Path to virtual environment dir") + + cmd.RunE = func(cmd *cobra.Command, args []string) error { + version := "latest" + if len(args) >= 1 { + version = args[0] + } + + return needs.InstallPython(context.Background(), version, virtualEnvPath, requirementsPath, needs.Options{LogOptions: *logOpts}) + } + + return cmd +} diff --git a/pkg/be/kubernetes/shell/chart/templates/containers/main.yaml b/pkg/be/kubernetes/shell/chart/templates/containers/main.yaml index dca122c4d..11bf102b6 100644 --- a/pkg/be/kubernetes/shell/chart/templates/containers/main.yaml +++ b/pkg/be/kubernetes/shell/chart/templates/containers/main.yaml @@ -19,6 +19,8 @@ valueFrom: fieldRef: fieldPath: metadata.name + - name: VIRTUAL_ENV + value: {{ .Values.venvPath }} {{- if .Values.env }} {{ .Values.env | b64dec | fromJsonArray | toYaml | nindent 4 }} {{- end }} diff --git a/pkg/be/local/shell/ok.go b/pkg/be/local/shell/ok.go index 5ebdaba4e..118e5b1cd 100644 --- a/pkg/be/local/shell/ok.go +++ b/pkg/be/local/shell/ok.go @@ -13,7 +13,8 @@ import ( func isCompatibleImage(image string) bool { return strings.HasPrefix(image, lunchpail.ImageRegistry+"/"+lunchpail.ImageRepo+"/lunchpail") || strings.Contains(image, "alpine") || - strings.Contains(image, "minio/minio") + strings.Contains(image, "minio/minio") || + strings.Contains(image, "python") } func IsCompatible(c llir.ShellComponent) error { diff --git a/pkg/fe/transformer/api/minio/transpile.go b/pkg/fe/transformer/api/minio/transpile.go index c1cc40dd3..82aaf666d 100644 --- a/pkg/fe/transformer/api/minio/transpile.go +++ b/pkg/fe/transformer/api/minio/transpile.go @@ -20,6 +20,8 @@ func transpile(runname string, ir llir.LLIR) (hlir.Application, error) { app.Spec.Expose = []string{fmt.Sprintf("%d:%d", ir.Queue.Port, ir.Queue.Port)} app.Spec.Command = fmt.Sprintf("$LUNCHPAIL_EXE component minio server --port %d", ir.Queue.Port) + /*app.Spec.Needs = []hlir.Needs{ + {Name: "minio", Version: "latest"}}*/ prefixIncludingBucket := api.QueuePrefixPath(ir.Queue, runname) A := strings.Split(prefixIncludingBucket, "/") prefixExcludingBucket := filepath.Join(A[1:]...) diff --git a/pkg/fe/transformer/api/shell/lower.go b/pkg/fe/transformer/api/shell/lower.go index 76f48cdb8..ef10b2daa 100644 --- a/pkg/fe/transformer/api/shell/lower.go +++ b/pkg/fe/transformer/api/shell/lower.go @@ -2,6 +2,7 @@ package shell import ( "fmt" + "os" "path/filepath" "lunchpail.io/pkg/build" @@ -36,6 +37,31 @@ func LowerAsComponent(buildName, runname string, app hlir.Application, ir llir.L component.InstanceName = runname } + for _, needs := range app.Spec.Needs { + var file *os.File + var err error + var req string + + if needs.Requirements != "" { + file, err = os.CreateTemp("", "requirements.txt") + if err != nil { + return nil, err + } + + if err := os.WriteFile(file.Name(), []byte(needs.Requirements), 0644); err != nil { + return nil, err + } + req = "--requirements " + file.Name() + if opts.Log.Verbose { + fmt.Printf("Setting requirements %s in %s \n", needs.Requirements, file.Name()) + } + } + component.Spec.Command = fmt.Sprintf(`$LUNCHPAIL_EXE needs %s %s %s --verbose=%v +%s +sleep 100000`, needs.Name, needs.Version, req, opts.Log.Verbose, component.Spec.Command) + + } + for _, dataset := range app.Spec.Datasets { if dataset.S3.Rclone.RemoteName != "" && dataset.S3.CopyIn.Path != "" { // We were asked to copy data in from s3, so diff --git a/pkg/ir/hlir/application.go b/pkg/ir/hlir/application.go index 8a1e7008f..161188b0b 100644 --- a/pkg/ir/hlir/application.go +++ b/pkg/ir/hlir/application.go @@ -7,6 +7,12 @@ type Code struct { Source string } +type Needs struct { + Name string + Version string + Requirements string +} + type Application struct { ApiVersion string `yaml:"apiVersion"` Kind string @@ -25,6 +31,7 @@ type Application struct { Datasets []Dataset `yaml:"datasets,omitempty"` SecurityContext SecurityContext `yaml:"securityContext,omitempty"` ContainerSecurityContext ContainerSecurityContext `yaml:"containerSecurityContext,omitempty"` + Needs []Needs `yaml:"needs,omitempty"` } } diff --git a/pkg/runtime/needs/install_darwin.go b/pkg/runtime/needs/install_darwin.go new file mode 100644 index 000000000..9aba63a6b --- /dev/null +++ b/pkg/runtime/needs/install_darwin.go @@ -0,0 +1,81 @@ +package needs + +import ( + "context" + "fmt" + "os" + "os/exec" + "os/user" + "path/filepath" +) + +func homedir() (string, error) { + currentUser, err := user.Current() + if err != nil { + return "", err + } + + return currentUser.HomeDir, nil +} + +func installMinio(ctx context.Context, version string, verbose bool) error { + if err := setenv(); err != nil { //$HOME must be set for brew + return err + } + + return brewInstall(ctx, "minio/stable/minio", version, verbose) //Todo: versions other than latest +} + +func installPython(ctx context.Context, version string, verbose bool) error { + if err := setenv(); err != nil { //$HOME must be set for brew + return err + } + + return brewInstall(ctx, "python3", version, verbose) //Todo: versions other than latest +} + +func brewInstall(ctx context.Context, pkg string, version string, verbose bool) error { + var cmd *exec.Cmd + if verbose { + fmt.Fprintf(os.Stdout, "Installing %s release of %s \n", version, pkg) + cmd = exec.CommandContext(ctx, "brew", "install", "--verbose", "--debug", pkg) + cmd.Stdout = os.Stdout + } else { + cmd = exec.Command("brew", "install", pkg) + } + cmd.Stderr = os.Stderr + return cmd.Run() +} + +func requirementsInstall(ctx context.Context, venvPath string, requirementsPath string, verbose bool) error { + var cmd *exec.Cmd + var verboseFlag string + dir := filepath.Dir(venvPath) + + if verbose { + verboseFlag = "--verbose" + } + + venvRequirementsPath := filepath.Join(venvPath, filepath.Base(requirementsPath)) + cmds := fmt.Sprintf(`python3 -m venv %s +cp %s %s +source %s/bin/activate +python3 -m pip install --upgrade pip %s +pip3 install -r %s %s 1>&2`, venvPath, requirementsPath, venvPath, venvPath, verboseFlag, venvRequirementsPath, verboseFlag) + + cmd = exec.CommandContext(ctx, "/bin/bash", "-c", cmds) + cmd.Dir = dir + if verbose { + cmd.Stdout = os.Stdout + } + cmd.Stderr = os.Stderr + return cmd.Run() +} + +func setenv() error { + dir, err := homedir() + if err != nil { + return err + } + return os.Setenv("HOME", dir) +} diff --git a/pkg/runtime/needs/install_linux.go b/pkg/runtime/needs/install_linux.go new file mode 100644 index 000000000..502c94ad4 --- /dev/null +++ b/pkg/runtime/needs/install_linux.go @@ -0,0 +1,125 @@ +package needs + +import ( + "context" + "fmt" + "os" + "os/exec" + "path/filepath" +) + +func bindir() (string, error) { + cachedir, err := os.UserCacheDir() + if err != nil { + return "", err + } + + return filepath.Join(cachedir, "lunchpail", "bin"), nil +} + +func installMinio(ctx context.Context, version string, verbose bool) error { + if verbose { + fmt.Printf("Installing %s release of minio \n", version) + } + + dir, err := bindir() + if err != nil { + return err + } + + if err := os.MkdirAll(dir, 0755); err != nil { + return err + } + + //Todo: versions other than latest + cmd := exec.CommandContext(ctx, "/bin/sh", "-c", "apt update; apt -y install wget; wget https://dl.min.io/server/minio/release/linux-amd64/minio") + cmd.Dir = dir + if verbose { + cmd.Stdout = os.Stdout + } + cmd.Stderr = os.Stderr + if err := cmd.Run(); err != nil { + return err + } + + if err := setenv(dir); err != nil { //setting $PATH + return err + } + + return os.Chmod(filepath.Join(dir, "minio"), 0755) +} + +func installPython(ctx context.Context, version string, verbose bool) error { + /* + if verbose { + fmt.Fprintf(os.Stdout, "Installing %s release of python \n", version) + } + + dir, err := bindir() + if err != nil { + return err + } + + if err := os.MkdirAll(dir, 0755); err != nil { + return err + } + + //Todo: versions other than latest + cmd := exec.Command("wget", "https://www.python.org/ftp/python/3.12.7/Python-3.12.7.tgz") + cmd.Dir = dir + if verbose { + cmd.Stdout = os.Stdout + } + cmd.Stderr = os.Stderr + if err := cmd.Run(); err != nil { + return err + } + + cmd = exec.Command("tar", "xf", "Python-3.12.7.tgz") + cmd.Dir = dir + if verbose { + cmd.Stdout = os.Stdout + } + cmd.Stderr = os.Stderr + if err := cmd.Run(); err != nil { + return err + } + + if err := setenv(dir); err != nil { //setting $PATH + return err + } + + os.Chmod(filepath.Join(dir, "python"), 0755) + */ + + return nil +} + +func requirementsInstall(ctx context.Context, venvPath string, requirementsPath string, verbose bool) error { + var cmd *exec.Cmd + var verboseFlag string + dir := filepath.Dir(venvPath) + + if verbose { + verboseFlag = "--verbose" + } + + venvRequirementsPath := filepath.Join(venvPath, filepath.Base(requirementsPath)) + cmds := fmt.Sprintf(`python3 -m venv %s +cp %s %s +source %s/bin/activate +python3 -m pip install --upgrade pip %s +pip3 install -r %s %s 1>&2`, venvPath, requirementsPath, venvPath, venvPath, verboseFlag, venvRequirementsPath, verboseFlag) + + cmd = exec.CommandContext(ctx, "/bin/bash", "-c", cmds) + cmd.Dir = dir + if verbose { + cmd.Stdout = os.Stdout + } + cmd.Stderr = os.Stderr + return cmd.Run() +} + +func setenv(dir string) error { + return os.Setenv("PATH", os.Getenv("PATH")+":"+dir) +} diff --git a/pkg/runtime/needs/minio.go b/pkg/runtime/needs/minio.go new file mode 100644 index 000000000..7b7a1e2ae --- /dev/null +++ b/pkg/runtime/needs/minio.go @@ -0,0 +1,18 @@ +package needs + +import ( + "context" + "errors" + "os/exec" +) + +func InstallMinio(ctx context.Context, version string, opts Options) error { + if _, err := exec.LookPath("minio"); err != nil { + if errors.Is(err, exec.ErrNotFound) { + //Todo: use context? + return installMinio(ctx, version, opts.Verbose) + } + return err + } + return nil +} diff --git a/pkg/runtime/needs/needs.go b/pkg/runtime/needs/needs.go new file mode 100644 index 000000000..ea79f866a --- /dev/null +++ b/pkg/runtime/needs/needs.go @@ -0,0 +1,7 @@ +package needs + +import "lunchpail.io/pkg/build" + +type Options struct { + build.LogOptions +} diff --git a/pkg/runtime/needs/python.go b/pkg/runtime/needs/python.go new file mode 100644 index 000000000..633a17571 --- /dev/null +++ b/pkg/runtime/needs/python.go @@ -0,0 +1,29 @@ +package needs + +import ( + "context" + "errors" + "os" + "os/exec" +) + +func InstallPython(ctx context.Context, version string, venvPath string, requirementsPath string, opts Options) error { + if _, err := exec.LookPath("python3"); err != nil { + if errors.Is(err, exec.ErrNotFound) { + if err := installPython(ctx, version, opts.Verbose); err != nil { + return err + } + } + return err + } + if requirementsPath != "" { + if venvPath == "" { + venvPath = ".venv" + } + if err := os.MkdirAll(venvPath, os.ModePerm); err != nil { + return err + } + return requirementsInstall(ctx, venvPath, requirementsPath, opts.Verbose) + } + return nil +} diff --git a/tests/bin/run.sh b/tests/bin/run.sh index 0c0dfee86..6426b314d 100755 --- a/tests/bin/run.sh +++ b/tests/bin/run.sh @@ -84,16 +84,15 @@ then fi fi + build $testname $app $branch $deployname if [[ -e "$1"/preinit.sh ]]; then - # the preinit.sh may return a PATH it wants us to use (e.g. for a python venv) - P=$("$1"/preinit.sh) - if [[ -n "$P" ]] - then export PATH="$P" + # the preinit.sh may return path to python venv it wants us to use in up + V=$("$1"/preinit.sh) + if [[ -n "$V" ]] + then export VIRTUAL_ENV="$V" fi fi - - build $testname $app $branch $deployname if [[ -n "$expectBuildFailure" ]] then diff --git a/tests/bin/up.sh b/tests/bin/up.sh index c381faa0d..c9fec1d2f 100755 --- a/tests/bin/up.sh +++ b/tests/bin/up.sh @@ -29,6 +29,7 @@ eval $testapp up \ --create-cluster \ --target=${LUNCHPAIL_TARGET:-kubernetes} \ --watch=false \ + --set venvPath=$VIRTUAL_ENV \ --set kubernetes.context=kind-lunchpail \ --set cosAccessKey=$COS_ACCESS_KEY \ --set cosSecretKey=$COS_SECRET_KEY diff --git a/tests/tests/python-basic/pail/app.yaml b/tests/tests/python-basic/pail/app.yaml index 76166e4fa..4946c081c 100644 --- a/tests/tests/python-basic/pail/app.yaml +++ b/tests/tests/python-basic/pail/app.yaml @@ -4,6 +4,7 @@ metadata: name: python-basic spec: role: worker + image: docker.io/python:3.12 code: - name: main.py source: | diff --git a/tests/tests/python-basic/target b/tests/tests/python-basic/target deleted file mode 100644 index c2c027fec..000000000 --- a/tests/tests/python-basic/target +++ /dev/null @@ -1 +0,0 @@ -local \ No newline at end of file diff --git a/tests/tests/python-doc-chunk/pail/app.yaml b/tests/tests/python-doc-chunk/pail/app.yaml index 37e2c30bb..bd5768a90 100644 --- a/tests/tests/python-doc-chunk/pail/app.yaml +++ b/tests/tests/python-doc-chunk/pail/app.yaml @@ -4,6 +4,7 @@ metadata: name: doc_chunk spec: role: worker + image: docker.io/python:3.12 command: python3 ./main.py code: - name: main.py diff --git a/tests/tests/python-doc-quality/pail/app.yaml b/tests/tests/python-doc-quality/pail/app.yaml index e5ccc5e92..617734170 100644 --- a/tests/tests/python-doc-quality/pail/app.yaml +++ b/tests/tests/python-doc-quality/pail/app.yaml @@ -4,6 +4,7 @@ metadata: name: doc_quality spec: role: worker + image: docker.io/python:3.12 command: python3 ./main.py datasets: {{ range $path, $_ := .Files.Glob "data/ldnoobw/*" }} diff --git a/tests/tests/python-html2parquet/pail/app.yaml b/tests/tests/python-html2parquet/pail/app.yaml index 37e2c30bb..bd5768a90 100644 --- a/tests/tests/python-html2parquet/pail/app.yaml +++ b/tests/tests/python-html2parquet/pail/app.yaml @@ -4,6 +4,7 @@ metadata: name: doc_chunk spec: role: worker + image: docker.io/python:3.12 command: python3 ./main.py code: - name: main.py diff --git a/tests/tests/python-lang-id/pail/app.yaml b/tests/tests/python-lang-id/pail/app.yaml index b60293847..eac558f66 100644 --- a/tests/tests/python-lang-id/pail/app.yaml +++ b/tests/tests/python-lang-id/pail/app.yaml @@ -4,6 +4,7 @@ metadata: name: lang_id spec: role: worker + image: docker.io/python:3.12 command: python3 ./main.py code: - name: main.py diff --git a/tests/tests/python-pdf2parquet/pail/app.yaml b/tests/tests/python-pdf2parquet/pail/app.yaml index 37e2c30bb..bd5768a90 100644 --- a/tests/tests/python-pdf2parquet/pail/app.yaml +++ b/tests/tests/python-pdf2parquet/pail/app.yaml @@ -4,6 +4,7 @@ metadata: name: doc_chunk spec: role: worker + image: docker.io/python:3.12 command: python3 ./main.py code: - name: main.py diff --git a/tests/tests/python-pii-redactor/pail/app.yaml b/tests/tests/python-pii-redactor/pail/app.yaml index d9407f2ce..a984ca94d 100644 --- a/tests/tests/python-pii-redactor/pail/app.yaml +++ b/tests/tests/python-pii-redactor/pail/app.yaml @@ -4,6 +4,7 @@ metadata: name: pii_redactor spec: role: worker + image: docker.io/python:3.12 command: python3 ./main.py code: - name: main.py diff --git a/tests/tests/python-pii-redactor/preinit.sh b/tests/tests/python-pii-redactor/preinit.sh index f71f5bccd..f17f508e4 100755 --- a/tests/tests/python-pii-redactor/preinit.sh +++ b/tests/tests/python-pii-redactor/preinit.sh @@ -3,17 +3,9 @@ set -e venv="$TEST_PATH"/.venv +reqFile="$TEST_PATH"/pail/requirements.txt -if [ ! -d "$venv" ] -then python3 -m venv "$venv" 1>&2 -fi - +$testapp needs python latest --requirements $reqFile --venv $venv source "$TEST_PATH"/.venv/bin/activate -if [ ! -f "$venv"/requirements.txt ] || ! diff -q "$venv"/requirements.txt "$TEST_PATH"/pail/requirements.txt -then - pip3 install -r "$TEST_PATH"/pail/requirements.txt 1>&2 - cp "$TEST_PATH"/pail/requirements.txt "$TEST_PATH"/.venv -fi - -echo "$PATH" +echo "$venv" diff --git a/tests/tests/python-text-encoder/pail/app.yaml b/tests/tests/python-text-encoder/pail/app.yaml index 8772a0bc6..20b2e3472 100644 --- a/tests/tests/python-text-encoder/pail/app.yaml +++ b/tests/tests/python-text-encoder/pail/app.yaml @@ -4,6 +4,7 @@ metadata: name: text_encoder spec: role: worker + image: docker.io/python:3.12 command: python3 ./main.py code: - name: main.py