diff --git a/assets/tests/jerm.json b/assets/tests/jerm.json index 890eb40..56c609f 100644 --- a/assets/tests/jerm.json +++ b/assets/tests/jerm.json @@ -11,6 +11,5 @@ "memory": 512, "keep_warm": false }, - "dir": "/home/ubuntu/bodystats", - "entry": "bodyie" + "dir": "/home/ubuntu/bodystats" } \ No newline at end of file diff --git a/cloud/aws/lambda.go b/cloud/aws/lambda.go index 0ff7ab7..8a94cc0 100644 --- a/cloud/aws/lambda.go +++ b/cloud/aws/lambda.go @@ -10,7 +10,6 @@ import ( "path/filepath" "sort" "strconv" - "strings" "time" "github.com/aws/aws-sdk-go-v2/aws" @@ -56,13 +55,15 @@ func NewLambda(cfg *config.Config) (*Lambda, error) { timeout: DefaultTimeout, } - lambdaConfig := config.Platform{Name: config.Lambda} - err := lambdaConfig.Defaults() - if err != nil { - return nil, err + if l.config.Platform.Name == "" { + lambdaConfig := config.Platform{Name: config.Lambda} + err := lambdaConfig.Defaults() + if err != nil { + return nil, err + } + l.config.Platform = lambdaConfig } - l.config.Platform = lambdaConfig awsConfig, err := l.getAwsConfig() if err != nil { return nil, err @@ -98,9 +99,6 @@ func (l *Lambda) Build() (string, error) { log.Debug("building Jerm project for Lambda...") r := config.NewRuntime() - if l.config.Entry == "" { - l.config.Entry = r.Entry() - } go func() { err := l.config.ToJson(jerm.DefaultConfigFile) @@ -109,18 +107,14 @@ func (l *Lambda) Build() (string, error) { } }() - handler, err := r.Build(l.config) + packageDir, function, err := r.Build(l.config) if err != nil { return "", err } - dir := filepath.Dir(handler) + l.functionHandler = function - if l.config.Platform.Handler == "" { - err := l.CreateFunctionEntry(handler) - return dir, err - } - return dir, err + return packageDir, nil } func (l *Lambda) Invoke(command string) error { @@ -433,24 +427,6 @@ func (l *Lambda) listLambdaVersions() ([]lambdaTypes.FunctionConfiguration, erro return response.Versions, err } -// CreateFunctionEntry creates a Lambda function handler file -func (l *Lambda) CreateFunctionEntry(file string) error { - log.Debug("creating lambda handler...") - f, err := os.Create(file) - if err != nil { - return err - } - defer f.Close() - - handler := strings.ReplaceAll(awsLambdaHandler, ".wsgi", l.config.Entry+".wsgi") - _, err = f.Write([]byte(handler)) - if err != nil { - return err - } - l.functionHandler = "handler.lambda_handler" - return nil -} - func (l *Lambda) isAlreadyDeployed() (bool, error) { log.Debug("fetching function code location...") versions, err := l.listLambdaVersions() diff --git a/cmd/manage.go b/cmd/manage.go index bd08746..981111a 100644 --- a/cmd/manage.go +++ b/cmd/manage.go @@ -12,6 +12,7 @@ import ( "github.com/spatocode/jerm/cloud/aws" "github.com/spatocode/jerm/config" "github.com/spatocode/jerm/internal/log" + "github.com/spatocode/jerm/internal/utils" ) // manageCmd represents the manage command @@ -37,7 +38,7 @@ var manageCmd = &cobra.Command{ return } - runtime := config.NewPythonRuntime() + runtime := config.NewPythonRuntime(utils.Command()) python, ok := runtime.(*config.Python) if !ok || !python.IsDjango() { log.PrintError("manage command is for Django projects only") diff --git a/config/config.go b/config/config.go index 73e2107..c0ee357 100644 --- a/config/config.go +++ b/config/config.go @@ -35,7 +35,6 @@ type Config struct { Region string `json:"region"` Platform Platform `json:"platform"` Dir string `json:"dir"` - Entry string `json:"entry"` } func (c *Config) GetFunctionName() string { diff --git a/config/config_test.go b/config/config_test.go index fdb134a..e2e6a11 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -29,6 +29,19 @@ func TestConfigDefaults(t *testing.T) { assert.NotNil(cfg.Region) } +func TestConfigToJson(t *testing.T) { + assert := assert.New(t) + testfile := "../assets/test.json" + cfg := &Config{} + + assert.False(utils.FileExists(testfile)) + err := cfg.ToJson(testfile) + assert.Nil(err) + assert.True(utils.FileExists(testfile)) + + helperCleanup(t, []string{testfile}) +} + func TestReadConfig(t *testing.T) { assert := assert.New(t) c, err := ReadConfig("../assets/tests/jerm.json") @@ -44,7 +57,6 @@ func TestReadConfig(t *testing.T) { assert.Equal(512, c.Platform.Memory) assert.Equal(false, c.Platform.KeepWarm) assert.Equal("/home/ubuntu/bodystats", c.Dir) - assert.Equal("bodyie", c.Entry) } func TestIgnoredFiles(t *testing.T) { diff --git a/config/golang.go b/config/golang.go index 52fc2ff..b527c96 100644 --- a/config/golang.go +++ b/config/golang.go @@ -15,14 +15,12 @@ type Go struct { } // NewGoConfig instantiates a new Go runtime -func NewGoRuntime() RuntimeInterface { - runtime := &Runtime{} +func NewGoRuntime(cmd utils.ShellCommand) RuntimeInterface { + runtime := &Runtime{cmd, RuntimeGo, DefaultGoVersion, ""} g := &Go{runtime} - g.Name = RuntimeGo version, err := g.getVersion() if err != nil { log.Debug(fmt.Sprintf("encountered an error while getting go version. Default to %s", DefaultGoVersion)) - g.Version = DefaultGoVersion return g } g.Version = version @@ -32,27 +30,33 @@ func NewGoRuntime() RuntimeInterface { // Gets the go version func (g *Go) getVersion() (string, error) { log.Debug("getting go version...") - goVersion, err := utils.GetShellCommandOutput("go", "version") + goVersion, err := g.RunCommand("go", "version") if err != nil { return "", err } s := strings.Split(goVersion, " ") if len(s) > 1 { version := strings.Split(s[2], "go") - return version[1], nil + return strings.TrimSpace(version[1]), nil } return "", errors.New("encountered error on go version") } -// Builds the go deployment package -func (g *Go) Build(config *Config) (string, error) { - return "", nil -} +// Build builds the go deployment package +// It returns the executable path, the function name and error if any +func (g *Go) Build(config *Config) (string, string, error) { + _, err := g.RunCommand("go", "mod", "tidy") + if err != nil { + return "", "", err + } + + env := []string{"GOOS=linux", "GOARCH=amd64", "CGO_ENABLED=0"} + _, err = g.RunCommandWithEnv(env, "go", "build", "main.go") + if err != nil { + return "", "", err + } -// Entry is the directory where the cloud function handler resides. -// The directory can be a file. -func (g *Go) Entry() string { - return "main.go" + return "main", "main", nil } // lambdaRuntime is the name of the go runtime as specified by AWS Lambda diff --git a/config/golang_test.go b/config/golang_test.go new file mode 100644 index 0000000..9b4b564 --- /dev/null +++ b/config/golang_test.go @@ -0,0 +1,79 @@ +package config + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestNewGolangRuntime(t *testing.T) { + assert := assert.New(t) + fakeOutput = "go version go1.21.0 linux/amd64" + r := NewGoRuntime(fakeCommandExecutor{}) + g := r.(*Go) + assert.Equal(RuntimeGo, g.Name) + assert.Equal("1.21.0", g.Version) +} + +func TestNewGolangRuntimeDefaultVersion(t *testing.T) { + assert := assert.New(t) + fakeOutput = "" + r := NewGoRuntime(fakeCommandExecutor{}) + g := r.(*Go) + assert.Equal(RuntimeGo, g.Name) + assert.Equal(DefaultGoVersion, g.Version) +} + +func TestGoGetVersion(t *testing.T) { + assert := assert.New(t) + fakeOutput = "go version go1.21.0 linux/amd64" + r := NewGoRuntime(fakeCommandExecutor{}) + g := r.(*Go) + v, err := g.getVersion() + assert.Nil(err) + assert.Equal(RuntimeGo, g.Name) + assert.Equal("1.21.0", v) +} + +func TestGoGetVersionError(t *testing.T) { + assert := assert.New(t) + fakeOutput = "" + r := NewGoRuntime(fakeCommandExecutor{}) + g := r.(*Go) + v, err := g.getVersion() + assert.NotNil(err) + assert.Equal(RuntimeGo, g.Name) + assert.Equal("", v) +} + +func TestGoLambdaRuntime(t *testing.T) { + assert := assert.New(t) + fakeOutput = "go version go1.21.0 linux/amd64" + r := NewGoRuntime(fakeCommandExecutor{}) + g := r.(*Go) + v, err := g.lambdaRuntime() + assert.Nil(err) + assert.Equal("go1.x", v) +} + +func TestGoBuild(t *testing.T) { + assert := assert.New(t) + fakeOutput = "go version go1.21.0 linux/amd64" + r := NewGoRuntime(fakeCommandExecutor{}) + cfg := &Config{Name: "test", Stage: "env"} + p, f, err := r.Build(cfg) + assert.Nil(err) + assert.Equal("main", p) + assert.Equal("main", f) +} + +func TestGoBuildError(t *testing.T) { + assert := assert.New(t) + fakeOutput = "" + r := NewGoRuntime(fakeCommandExecutor{}) + cfg := &Config{Name: "test", Stage: "env"} + p, f, err := r.Build(cfg) + assert.NotNil(err) + assert.Equal("", p) + assert.Equal("", f) +} diff --git a/cloud/aws/handler.go b/config/handlers/django.go similarity index 93% rename from cloud/aws/handler.go rename to config/handlers/django.go index 56fc52a..1ebefb2 100644 --- a/cloud/aws/handler.go +++ b/config/handlers/django.go @@ -1,7 +1,7 @@ -package aws +package handlers const ( - awsLambdaHandler = ` + AwsLambdaHandlerDjango = ` import sys import json import io @@ -20,7 +20,7 @@ logger = logging.getLogger() logger.setLevel(logging.INFO) -def lambda_handler(event, context): +def handler(event, context): if settings.DEBUG: logger.debug("Jerm Event: {}".format(event)) diff --git a/config/handlers/statichtml.go b/config/handlers/statichtml.go new file mode 100644 index 0000000..5cd06b8 --- /dev/null +++ b/config/handlers/statichtml.go @@ -0,0 +1,19 @@ +package handlers + +const ( + AwsLambdaHandlerStaticPage = ` +const fs = require('fs'); +const html = fs.readFileSync('index.html', { encoding:'utf8' }); + +exports.handler = async (event) => { + const response = { + statusCode: 200, + headers: { + 'Content-Type': 'text/html', + }, + body: html, + }; + return response; +}; + ` +) diff --git a/config/node.go b/config/node.go index d7c9754..c628b92 100644 --- a/config/node.go +++ b/config/node.go @@ -14,14 +14,12 @@ type Node struct { } // NewNodeConfig instantiates a new NodeJS runtime -func NewNodeRuntime() RuntimeInterface { - runtime := &Runtime{} +func NewNodeRuntime(cmd utils.ShellCommand) RuntimeInterface { + runtime := &Runtime{cmd, RuntimeNode, DefaultNodeVersion, ""} n := &Node{runtime} - n.Name = RuntimeNode version, err := n.getVersion() if err != nil { - log.Debug(fmt.Sprintf("encountered an error while getting nodejs version. Default to %s", DefaultNodeVersion)) - n.Version = DefaultNodeVersion + log.Debug(fmt.Sprintf("encountered an error while getting nodejs version. Default to v%s", DefaultNodeVersion)) return n } n.Version = version @@ -31,19 +29,14 @@ func NewNodeRuntime() RuntimeInterface { // Gets the nodejs version func (n *Node) getVersion() (string, error) { log.Debug("getting nodejs version...") - nodeVersion, err := utils.GetShellCommandOutput("node", "-v") + nodeVersion, err := n.RunCommand("node", "-v") if err != nil { return "", err } - nodeVersion = nodeVersion[1:] + nodeVersion = strings.TrimSpace(nodeVersion[1:]) return nodeVersion, nil } -// Builds the nodejs deployment package -func (n *Node) Build(config *Config) (string, error) { - return "", nil -} - // lambdaRuntime is the name of the nodejs runtime as specified by AWS Lambda func (n *Node) lambdaRuntime() (string, error) { v := strings.Split(n.Version, ".") diff --git a/config/node_test.go b/config/node_test.go new file mode 100644 index 0000000..1278390 --- /dev/null +++ b/config/node_test.go @@ -0,0 +1,37 @@ +package config + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestNewNodeRuntime(t *testing.T) { + assert := assert.New(t) + fakeOutput = "v13.2.0" + r := NewNodeRuntime(fakeCommandExecutor{}) + n := r.(*Node) + assert.Equal(RuntimeNode, n.Name) + assert.Equal("13.2.0", n.Version) +} + +func TestNodeGetVersion(t *testing.T) { + assert := assert.New(t) + fakeOutput = "v13.2.0" + r := NewNodeRuntime(fakeCommandExecutor{}) + n := r.(*Node) + v, err := n.getVersion() + assert.Nil(err) + assert.Equal(RuntimeNode, n.Name) + assert.Equal("13.2.0", v) +} + +func TestNodeLambdaRuntime(t *testing.T) { + assert := assert.New(t) + fakeOutput = "v13.2.0" + r := NewNodeRuntime(fakeCommandExecutor{}) + n := r.(*Node) + v, err := n.lambdaRuntime() + assert.Nil(err) + assert.Equal("nodejs13.x", v) +} diff --git a/config/platform.go b/config/platform.go index affcf9d..01bc5e3 100644 --- a/config/platform.go +++ b/config/platform.go @@ -1,5 +1,7 @@ package config +import "fmt" + const ( DefaultTimeout = 30 DefaultMemory = 512 @@ -37,6 +39,9 @@ func (l *Platform) Defaults() error { if err != nil { return err } + if l.Runtime == RuntimeStatic { + l.Runtime = fmt.Sprintf("nodejs%s.x", DefaultNodeVersion[0:2]) + } } } diff --git a/config/platform_test.go b/config/platform_test.go index 7325d32..f999bb7 100644 --- a/config/platform_test.go +++ b/config/platform_test.go @@ -8,10 +8,36 @@ import ( func TestPlaformDefaults(t *testing.T) { assert := assert.New(t) + indexHtml := "index.html" + requirementsTxt := "requirements.txt" + p := &Platform{Name: Lambda} assert.Equal(0, p.Memory) assert.Equal(0, p.Timeout) - p.Defaults() + assert.Equal("", p.Runtime) + err := p.Defaults() + assert.ErrorContains(err, "cannot detect runtime. please specify runtime in your Jerm.json file") + assert.Equal("", p.Runtime) assert.Equal(DefaultMemory, p.Memory) assert.Equal(DefaultTimeout, p.Timeout) + + helperCreateFile(t, indexHtml) + err = p.Defaults() + assert.Nil(err) + assert.Equal("nodejs18.x", p.Runtime) + helperCleanup(t, []string{indexHtml}) + + helperCreateFile(t, requirementsTxt) + err = p.Defaults() + assert.Nil(err) + assert.Equal("nodejs18.x", p.Runtime) + helperCleanup(t, []string{requirementsTxt}) + + p.Runtime = "" + helperCreateFile(t, requirementsTxt) + err = p.Defaults() + assert.Nil(err) + assert.NotEqual("nodejs18.x", p.Runtime) + assert.Contains(p.Runtime, "python") + helperCleanup(t, []string{requirementsTxt}) } diff --git a/config/python.go b/config/python.go index 4794e4e..77093de 100644 --- a/config/python.go +++ b/config/python.go @@ -14,55 +14,49 @@ import ( "golang.org/x/sync/errgroup" - "github.com/otiai10/copy" + "github.com/spatocode/jerm/config/handlers" "github.com/spatocode/jerm/internal/log" "github.com/spatocode/jerm/internal/utils" ) +const ( + DefaultPythonFunctionFile = "handler" +) + type Python struct { *Runtime } // NewPythonConfig instantiates a new Python runtime -func NewPythonRuntime() RuntimeInterface { - runtime := &Runtime{} +func NewPythonRuntime(cmd utils.ShellCommand) RuntimeInterface { + runtime := &Runtime{ + cmd, + RuntimePython, + DefaultPythonVersion, + handlers.AwsLambdaHandlerDjango, + } p := &Python{runtime} - p.Name = RuntimePython version, err := p.getVersion() if err != nil { log.Debug(fmt.Sprintf("encountered an error while getting python version. Default to %s", DefaultPythonVersion)) - p.Version = DefaultPythonVersion return p } p.Version = version return p } -// Entry is the directory where the cloud function handler resides. -// The directory can be a file. -func (p *Python) Entry() string { - switch { - case p.IsDjango(): - workDir, _ := os.Getwd() - entry, err := p.getDjangoProject(workDir) - if err != nil { - log.Debug(err.Error()) - } - return entry - default: - return "" - } -} - // Gets the python version func (p *Python) getVersion() (string, error) { log.Debug("getting python version...") - pythonVersion, err := utils.GetShellCommandOutput("python", "-V") + pythonVersion, err := p.RunCommand("python", "-V") if err != nil || strings.Contains(pythonVersion, " 2.") { - pythonVersion, err = utils.GetShellCommandOutput("python3", "-V") + pythonVersion, err = p.RunCommand("python3", "-V") + if err != nil { + return "", err + } } s := strings.Split(pythonVersion, " ") - version := s[len(s)-1] + version := strings.TrimSpace(s[len(s)-1]) return version, err } @@ -73,38 +67,37 @@ func (p *Python) getVirtualEnvironment() (string, error) { return strings.TrimSpace(venv), nil } - _, err := utils.GetShellCommandOutput("pyenv") + pyenvRoot, err := p.RunCommand("pyenv", "root") if err != nil { - pyenvRoot, err := utils.GetShellCommandOutput("pyenv", "root") - if err != nil { - return "", err - } - pyenvRoot = strings.TrimSpace(pyenvRoot) + return "", err + } + pyenvRoot = strings.TrimSpace(pyenvRoot) - pyenvVersionName, err := utils.GetShellCommandOutput("pyenv", "version-name") - if err != nil { - return "", err - } - pyenvVersionName = strings.TrimSpace(pyenvVersionName) - venv = path.Join(pyenvRoot, "versions", pyenvVersionName) - return venv, nil + pyenvVersionName, err := p.RunCommand("pyenv", "version-name") + if err != nil { + return "", err } - return "", nil + pyenvVersionName = strings.TrimSpace(pyenvVersionName) + venv = path.Join(pyenvRoot, "versions", pyenvVersionName) + + return venv, nil } -// Builds the Python deployment package -func (p *Python) Build(config *Config) (string, error) { - tempDir, err := os.MkdirTemp(os.TempDir(), "jerm-python") +// Build builds the Python deployment package +// It returns the package path, the function name and error if any +func (p *Python) Build(config *Config) (string, string, error) { + function := config.Platform.Handler + tempDir, err := os.MkdirTemp(os.TempDir(), "jerm-package") if err != nil { - return "", err + return "", "", err } - handlerPath := filepath.Join(tempDir, "handler.py") + handlerFilepath := filepath.Join(tempDir, "handler.py") venv, err := p.getVirtualEnvironment() if err != nil { //TODO: installs requirements listed in requirements.txt file - return "", fmt.Errorf("cannot find a virtual env. Please ensure you're running in a virtual env") + return "", "", fmt.Errorf("cannot find a virtual env. Please ensure you're running in a virtual env") } version := strings.Split(p.Version, ".") @@ -120,56 +113,53 @@ func (p *Python) Build(config *Config) (string, error) { dependencies["werkzeug"] = "0.16.1" } - err = p.installNecessaryDependencies(tempDir, sitePackages, dependencies) + err = p.installNecessaryDependencies(tempDir, dependencies) if err != nil { - return "", err + return "", "", err } - err = p.copyNecessaryFilesToTempDir(config.Dir, tempDir, jermIgnoreFile) + err = p.copyNecessaryFilesToPackageDir(config.Dir, tempDir, jermIgnoreFile) if err != nil { - return "", err + return "", "", err } - err = p.copyNecessaryFilesToTempDir(sitePackages, tempDir, jermIgnoreFile) + err = p.copyNecessaryFilesToPackageDir(sitePackages, tempDir, jermIgnoreFile) if err != nil { - return "", err + return "", "", err } log.Debug(fmt.Sprintf("built Python deployment package at %s", tempDir)) - return handlerPath, err -} + if function == "" && p.IsDjango() { // for now it works for Django projects only + workDir, _ := os.Getwd() + djangoProject, err := p.getDjangoProject(workDir) + if err != nil { + log.Debug(err.Error()) + } + handler := strings.ReplaceAll(p.handlerTemplate, ".wsgi", djangoProject+".wsgi") + function, err = p.createFunctionHandler(config, handlerFilepath, handler) + if err != nil { + return "", "", err + } + } -// Copies files from src to dest -func (p *Python) copyNecessaryFilesToTempDir(src, dest, ignoreFile string) error { - log.Debug("copying necessary Python files...") + return tempDir, function, err +} - ignoredFiles := defaultIgnoredGlobs - files, err := ReadIgnoredFiles(ignoreFile) - if err == nil { - ignoredFiles = append(ignoredFiles, files...) +// createFunctionHandler creates a serverless function handler file +func (p *Python) createFunctionHandler(config *Config, file, handler string) (string, error) { + log.Debug("creating lambda handler...") + f, err := os.Create(file) + if err != nil { + return "", err } + defer f.Close() - opt := copy.Options{ - Skip: func(srcinfo os.FileInfo, src, dest string) (bool, error) { - for _, ignoredFile := range ignoredFiles { - match, _ := filepath.Match(ignoredFile, srcinfo.Name()) - matchedFile := srcinfo.Name() == ignoredFile || match || - strings.HasSuffix(srcinfo.Name(), ignoredFile) || - strings.HasPrefix(srcinfo.Name(), ignoredFile) - if matchedFile { - return matchedFile, nil - } - } - return false, nil - }, - } - err = copy.Copy(src, dest, opt) + _, err = f.Write([]byte(handler)) if err != nil { - return err + return "", err } - - return nil + return fmt.Sprintf("%s.%s", DefaultPythonFunctionFile, DefaultServerlessFunction), nil } // installRequirements installs requirements listed in requirements.txt file @@ -178,7 +168,7 @@ func (p *Python) copyNecessaryFilesToTempDir(src, dest, ignoreFile string) error // } // Installs dependencies needed to run serverless Python -func (p *Python) installNecessaryDependencies(dir, sitePackages string, dependencies map[string]string) error { +func (p *Python) installNecessaryDependencies(dir string, dependencies map[string]string) error { log.Debug("installing necessary Python dependencies...") var eg errgroup.Group diff --git a/config/python_test.go b/config/python_test.go index d79769f..d3f928e 100644 --- a/config/python_test.go +++ b/config/python_test.go @@ -1,37 +1,81 @@ package config import ( - "os" + "fmt" "testing" - "github.com/spatocode/jerm/internal/utils" "github.com/stretchr/testify/assert" ) -func TestIgnoredFilesWhileCopying(t *testing.T) { +func TestNewPythonRuntime(t *testing.T) { assert := assert.New(t) - jermJson := "../assets/jerm.json" - jermIgnore := "../assets/.jermignore" + fakeOutput = "Python 3.9.0" + r := NewPythonRuntime(fakeCommandExecutor{}) + p := r.(*Python) + assert.Equal(RuntimePython, p.Name) + assert.Equal("3.9.0", p.Version) +} + +func TestNewPythonRuntimeDefaultVersion(t *testing.T) { + assert := assert.New(t) + fakeOutput = "" + r := NewPythonRuntime(fakeCommandExecutor{}) + p := r.(*Python) + assert.Equal(RuntimePython, p.Name) + assert.Equal(DefaultPythonVersion, p.Version) +} + +func TestPythonGetVersion(t *testing.T) { + assert := assert.New(t) + fakeOutput = "Python 3.9.0" + r := NewPythonRuntime(fakeCommandExecutor{}) + p := r.(*Python) + v, err := p.getVersion() + assert.Nil(err) + assert.Equal(RuntimePython, p.Name) + assert.Equal("3.9.0", v) +} - pr := NewPythonRuntime() - p := pr.(*Python) - err := p.copyNecessaryFilesToTempDir("../assets/tests", "../assets", "../assets/tests/.jermignore") - testfile1Exists := utils.FileExists("../assets/testfile1") - testfile2Exists := utils.FileExists("../assets/testfile2") - jermJsonExists := utils.FileExists(jermJson) - jermIgnoreExists := utils.FileExists(jermIgnore) +func TestPythonGetVersionError(t *testing.T) { + assert := assert.New(t) + fakeOutput = "" + r := NewPythonRuntime(fakeCommandExecutor{}) + p := r.(*Python) + v, err := p.getVersion() + assert.NotNil(err) + assert.Equal(RuntimePython, p.Name) + assert.Equal("", v) +} +func TestPythonGetVirtualEnvironment(t *testing.T) { + assert := assert.New(t) + fakeOutput = "/usr/fake" + r := NewPythonRuntime(fakeCommandExecutor{}) + p := r.(*Python) + venv, err := p.getVirtualEnvironment() assert.Nil(err) - assert.False(testfile1Exists) - assert.False(testfile2Exists) - assert.True(jermJsonExists) - assert.True(jermIgnoreExists) + assert.Equal(fmt.Sprintf("%s/versions%s", fakeOutput, fakeOutput), venv) +} - cleanup([]string{jermJson, jermIgnore}) +func TestPythonLambdaRuntime(t *testing.T) { + assert := assert.New(t) + fakeOutput = "Python 3.9.0" + r := NewPythonRuntime(fakeCommandExecutor{}) + p := r.(*Python) + v, err := p.lambdaRuntime() + assert.Nil(err) + assert.Equal("python3.9", v) } -func cleanup(files []string) { - for _, file := range files { - os.Remove(file) - } +func TestPythonIsDjango(t *testing.T) { + assert := assert.New(t) + fakeOutput = "Python 3.9.0" + r := NewPythonRuntime(fakeCommandExecutor{}) + p := r.(*Python) + + managePy := "manage.py" + helperCreateFile(t, managePy) + is := p.IsDjango() + assert.True(is) + helperCleanup(t, []string{managePy}) } diff --git a/config/runtime.go b/config/runtime.go index d35f301..2bbd802 100644 --- a/config/runtime.go +++ b/config/runtime.go @@ -2,18 +2,27 @@ package config import ( "errors" + "fmt" + "os" + "path/filepath" + "strings" + "github.com/otiai10/copy" + "github.com/spatocode/jerm/config/handlers" + "github.com/spatocode/jerm/internal/log" "github.com/spatocode/jerm/internal/utils" ) const ( - RuntimeUnknown = "unknown" - RuntimePython = "python" - RuntimeGo = "go" - RuntimeNode = "nodejs" - DefaultNodeVersion = "18.13.0" - DefaultPythonVersion = "3.9.0" - DefaultGoVersion = "1.19.0" + RuntimeUnknown = "unknown" + RuntimePython = "python" + RuntimeGo = "go" + RuntimeNode = "nodejs" + RuntimeStatic = "static" + DefaultNodeVersion = "18.13.0" + DefaultPythonVersion = "3.9.0" + DefaultGoVersion = "1.19.0" + DefaultServerlessFunction = "handler" ) var ( @@ -37,46 +46,123 @@ var ( type RuntimeInterface interface { // Builds the deployment package for the underlying runtime - Build(*Config) (string, error) + // It returns the package path, the function name and error if any. + // The package path can be an executable for runtimes that compiles + // to standalone executable. + Build(*Config) (string, string, error) - // Entry is the directory where the cloud function handler resides. - // The directory can be a file. - Entry() string - - // lambdaRuntime is the name of runtime as specified by AWS Lambda + // lambdaRuntime returns the name of runtime as specified by AWS Lambda lambdaRuntime() (string, error) } // Base Runtime type Runtime struct { - Name string - Version string + utils.ShellCommand + Name string + Version string + handlerTemplate string } // NewRuntime instantiates a new runtime func NewRuntime() RuntimeInterface { + command := utils.Command() r := &Runtime{} switch { case utils.FileExists("requirements.txt"): - return NewPythonRuntime() + return NewPythonRuntime(command) case utils.FileExists("main.go"): - return NewGoRuntime() + return NewGoRuntime(command) case utils.FileExists("package.json"): - return NewNodeRuntime() + return NewNodeRuntime(command) + case utils.FileExists("index.html"): + r.Name = RuntimeStatic + r.handlerTemplate = handlers.AwsLambdaHandlerStaticPage default: r.Name = RuntimeUnknown - return r } + r.ShellCommand = command + return r } -func (r *Runtime) Build(*Config) (string, error) { - return "", nil +// Build builds the project for deployment +func (r *Runtime) Build(config *Config) (string, string, error) { + function := config.Platform.Handler + tempDir, err := os.MkdirTemp(os.TempDir(), "jerm-package") + if err != nil { + return "", "", err + } + + err = r.copyNecessaryFilesToPackageDir(config.Dir, tempDir, jermIgnoreFile) + if err != nil { + return "", "", err + } + + if r.Name == RuntimeStatic && function == "" { + handlerFilepath := filepath.Join(tempDir, "index.js") + function, err = r.createFunctionHandler(config, handlerFilepath) + if err != nil { + return "", "", err + } + } + + return tempDir, function, nil } -func (r *Runtime) Entry() string { - return "" +// createFunctionHandler creates a serverless function handler file +func (r *Runtime) createFunctionHandler(config *Config, file string) (string, error) { + log.Debug("creating lambda handler...") + f, err := os.Create(file) + if err != nil { + return "", err + } + defer f.Close() + + _, err = f.Write([]byte(r.handlerTemplate)) + if err != nil { + return "", err + } + filename := strings.Split(filepath.Base(f.Name()), ".")[0] + return fmt.Sprintf("%s.%s", filename, DefaultServerlessFunction), nil +} + +// Copies files from src to dest ignoring file names listed in ignoreFile +func (r *Runtime) copyNecessaryFilesToPackageDir(src, dest, ignoreFile string) error { + log.Debug(fmt.Sprintf("copying necessary files to package dir %s...", dest)) + + ignoredFiles := defaultIgnoredGlobs + files, err := ReadIgnoredFiles(ignoreFile) + if err == nil { + ignoredFiles = append(ignoredFiles, files...) + } + + opt := copy.Options{ + Skip: func(srcinfo os.FileInfo, src, dest string) (bool, error) { + for _, ignoredFile := range ignoredFiles { + match, _ := filepath.Match(ignoredFile, srcinfo.Name()) + matchedFile := srcinfo.Name() == ignoredFile || match || + strings.HasSuffix(srcinfo.Name(), ignoredFile) || + strings.HasPrefix(srcinfo.Name(), ignoredFile) + if matchedFile { + return matchedFile, nil + } + } + return false, nil + }, + } + err = copy.Copy(src, dest, opt) + if err != nil { + return err + } + + return nil } func (r *Runtime) lambdaRuntime() (string, error) { - return "", errors.New("cannot detect runtime. please specify runtime in your Jerm.json file") + if r.Name == RuntimeUnknown { + return "", errors.New("cannot detect runtime. please specify runtime in your Jerm.json file") + } + // TODO: Some AWS runtime id doesn't tally with this format. + // Need to support as needed + v := strings.Split(r.Version, ".") + return fmt.Sprintf("%s%s", r.Name, v[0]), nil } diff --git a/config/runtime_test.go b/config/runtime_test.go new file mode 100644 index 0000000..17d7709 --- /dev/null +++ b/config/runtime_test.go @@ -0,0 +1,143 @@ +package config + +import ( + "fmt" + "os" + "path/filepath" + "testing" + + "github.com/spatocode/jerm/internal/utils" + "github.com/stretchr/testify/assert" +) + +var ( + fakeOutput = "fakeoutput" +) + +type fakeCommandExecutor struct { +} + +func (c fakeCommandExecutor) RunCommand(command string, args ...string) (string, error) { + if fakeOutput == "" { + return "", fmt.Errorf("fake err") + } + return fakeOutput, nil +} + +func (c fakeCommandExecutor) RunCommandWithEnv(env []string, command string, args ...string) (string, error) { + if fakeOutput == "" { + return "", fmt.Errorf("fake err") + } + return fakeOutput, nil +} + +func TestIgnoredFilesWhileCopying(t *testing.T) { + assert := assert.New(t) + jermJson := "../assets/jerm.json" + jermIgnore := "../assets/.jermignore" + + pr := NewPythonRuntime(fakeCommandExecutor{}) + p := pr.(*Python) + err := p.copyNecessaryFilesToPackageDir("../assets/tests", "../assets", "../assets/tests/.jermignore") + testfile1Exists := utils.FileExists("../assets/testfile1") + testfile2Exists := utils.FileExists("../assets/testfile2") + jermJsonExists := utils.FileExists(jermJson) + jermIgnoreExists := utils.FileExists(jermIgnore) + + assert.Nil(err) + assert.False(testfile1Exists) + assert.False(testfile2Exists) + assert.True(jermJsonExists) + assert.True(jermIgnoreExists) + + helperCleanup(t, []string{jermJson, jermIgnore}) +} + +func TestRuntimeBuild(t *testing.T) { + assert := assert.New(t) + + cfg := &Config{Name: "test", Stage: "env", Dir: "../assets/tests"} + ri := NewRuntime() + r := ri.(*Runtime) + pkgDir, f, err := r.Build(cfg) + + testfile1 := fmt.Sprintf("%s/testfile1", pkgDir) + testfile2 := fmt.Sprintf("%s/testfile2", pkgDir) + jermIgnore := fmt.Sprintf("%s/.jermignore", pkgDir) + jermJson := fmt.Sprintf("%s/jerm.json", pkgDir) + + assert.Nil(err) + assert.Contains(pkgDir, "jerm-package") + assert.Equal("", f) + assert.True(utils.FileExists(testfile1)) + assert.True(utils.FileExists(testfile2)) + assert.True(utils.FileExists(jermJson)) + assert.True(utils.FileExists(jermIgnore)) +} + +func TestRuntimeCreateFunctionHandler(t *testing.T) { + assert := assert.New(t) + + cfg := &Config{Name: "test", Stage: "env", Dir: "../assets/tests"} + ri := NewRuntime() + r := ri.(*Runtime) + + handlerFile := filepath.Join("../assets/tests", "index.js") + handler, err := r.createFunctionHandler(cfg, handlerFile) + + assert.Nil(err) + assert.Equal("index.handler", handler) + helperCleanup(t, []string{handlerFile}) +} + +func TestNewRuntime(t *testing.T) { + assert := assert.New(t) + requirementsTxt := "requirements.txt" + mainGo := "main.go" + indexHtml := "index.html" + packageJson := "package.json" + + ri := NewRuntime() + r := ri.(*Runtime) + assert.Equal(RuntimeUnknown, r.Name) + + helperCreateFile(t, requirementsTxt) + ri = NewRuntime() + p := ri.(*Python) + assert.Equal(RuntimePython, p.Name) + helperCleanup(t, []string{requirementsTxt}) + + helperCreateFile(t, packageJson) + ri = NewRuntime() + n := ri.(*Node) + assert.Equal(RuntimeNode, n.Name) + helperCleanup(t, []string{packageJson}) + + helperCreateFile(t, mainGo) + ri = NewRuntime() + g := ri.(*Go) + assert.Equal(RuntimeGo, g.Name) + helperCleanup(t, []string{mainGo}) + + helperCreateFile(t, indexHtml) + ri = NewRuntime() + r = ri.(*Runtime) + assert.Equal(RuntimeStatic, r.Name) + helperCleanup(t, []string{indexHtml}) +} + +func helperCleanup(t *testing.T, files []string) { + for _, file := range files { + err := os.Remove(file) + if err != nil { + t.Fatal(err) + } + } +} + +func helperCreateFile(t *testing.T, file string) { + _, err := os.Create(file) + if err != nil { + t.Fatal(err) + } +} diff --git a/internal/utils/utils.go b/internal/utils/utils.go index 78892f0..3214185 100644 --- a/internal/utils/utils.go +++ b/internal/utils/utils.go @@ -2,7 +2,9 @@ package utils import ( "bufio" + "bytes" "context" + "errors" "io" "net/http" "os" @@ -12,6 +14,42 @@ import ( "github.com/spatocode/jerm/internal/log" ) +type ShellCommand interface { + RunCommand(command string, args ...string) (string, error) + RunCommandWithEnv(env []string, command string, args ...string) (string, error) +} + +type cmdExecutor struct { + cmd func(name string, arg ...string) *exec.Cmd +} + +func Command() ShellCommand { + return cmdExecutor{cmd: exec.Command} +} + +func (c cmdExecutor) RunCommand(command string, args ...string) (string, error) { + cmd := c.cmd(command, args...) + var stderr bytes.Buffer + cmd.Stderr = &stderr + out, err := cmd.Output() + if err != nil && stderr.Available() != 0 { + return string(out), errors.New(stderr.String()) + } + return string(out), err +} + +func (c cmdExecutor) RunCommandWithEnv(env []string, command string, args ...string) (string, error) { + cmd := c.cmd(command, args...) + cmd.Env = append(os.Environ(), env...) + var stderr bytes.Buffer + cmd.Stderr = &stderr + out, err := cmd.Output() + if err != nil && stderr.Available() != 0 { + return string(out), errors.New(stderr.String()) + } + return string(out), err +} + func RemoveLocalFile(zipPath string) error { err := os.Remove(zipPath) if err != nil { @@ -37,12 +75,6 @@ func Request(location string) (*http.Response, error) { return res, err } -func GetShellCommandOutput(command string, args ...string) (string, error) { - cmd := exec.Command(command, args...) - out, err := cmd.Output() - return string(out), err -} - // GetStdIn gets a stdin prompt from user func ReadPromptInput(prompt string, input io.Reader) (string, error) { if prompt != "" { diff --git a/jerm.go b/jerm.go index 979c20d..ef7c064 100644 --- a/jerm.go +++ b/jerm.go @@ -168,7 +168,7 @@ func (p *Project) packageProject() (*string, int64, error) { } // archivePackage creates an archive file from a project -func (p *Project) archivePackage(archivePath, dir string) (int64, error) { +func (p *Project) archivePackage(archivePath, project string) (int64, error) { log.Debug("archiving package...") archive, err := os.Create(archivePath) @@ -180,6 +180,35 @@ func (p *Project) archivePackage(archivePath, dir string) (int64, error) { writer := zip.NewWriter(archive) defer writer.Close() + file, err := os.Open(project) + if err != nil { + return 0, err + } + defer file.Close() + + fileInfo, err := file.Stat() + if err != nil { + return 0, err + } + + if !fileInfo.IsDir() { + // project is probably a standalone executable + w, err := writer.Create(project) + if err != nil { + return 0, err + } + if _, err := io.Copy(w, file); err != nil { + return 0, err + } + + info, err := archive.Stat() + if err != nil { + return 0, err + } + + return info.Size(), nil + } + walker := func(path string, d fs.DirEntry, err error) error { if err != nil { return err @@ -194,7 +223,7 @@ func (p *Project) archivePackage(archivePath, dir string) (int64, error) { } defer f.Close() - sPath := strings.Split(path, dir) + sPath := strings.Split(path, project) zipContentPath := sPath[len(sPath)-1] w, err := writer.Create(zipContentPath) if err != nil { @@ -207,7 +236,7 @@ func (p *Project) archivePackage(archivePath, dir string) (int64, error) { return nil } - err = filepath.WalkDir(dir, walker) + err = filepath.WalkDir(project, walker) if err != nil { return 0, err } diff --git a/jerm_test.go b/jerm_test.go index d7ac920..1ba0cb7 100644 --- a/jerm_test.go +++ b/jerm_test.go @@ -21,5 +21,4 @@ func TestJermConfigure(t *testing.T) { assert.Equal(false, c.Platform.KeepWarm) assert.Equal("/home/ubuntu/bodystats", c.Dir) assert.Equal(30, c.Platform.Timeout) - assert.Equal("bodyie", c.Entry) }