feat: clean up & refactor majorly

MANIFEST.in

@@ -1 +1,2 @@
+# TODO: Remove and add in `pyproject.toml`
 include README.md

Makefile

@@ -0,0 +1,75 @@
+# TODO: Add commands to be compatible for windows
+# TODO: Add commands for tesk_api
+
+# Define variables
+PYTHON_CMD := $(shell command -v python3 2> /dev/null)
+BUILDAH_CMD := $(shell command -v buildah 2> /dev/null)
+DOCKER_CMD := $(shell command -v docker 2> /dev/null)
+ELIXIR_CLOUD_REGISTRY := docker.io/elixircloud
+DOCKER_FILE_PATH := deployment/containers
+
+# Define arguments
+IMAGE ?= filer
+TAG ?= testing
+
+# Default target, build image if `make` is called without any arguments
+default: build
+
+# Create and activate virtual environment
+.PHONY: venv
+venv:
+	@if [ -x "$(PYTHON_CMD)" ]; then \
+		$(PYTHON_CMD) -m venv .venv; \
+		echo "Virtual environment created. To activate, run: source .venv/bin/activate"; \
+	else \
+		echo "Please install python3 to create virtual environment."; \
+		exit 1; \
+	fi
+
+# Remove virtual environment
+.PHONY: clean-venv
+clean-venv:
+	rm -rf .venv
+
+# TODO: Install package manages and change the below commands
+# Install dependencies
+.PHONY: install
+install:
+	@if [ -f .venv/bin/activate ]; then \
+		pip install .; \
+	else \
+		echo "Virtual environment not found. Please create it first using 'make venv'."; \
+	fi
+
+# Build image
+.PHONY: build
+build:
+	@if [ -x "$(BUILDAH_CMD)" ]; then \
+		$(BUILDAH_CMD) bud \
+			-t $(ELIXIR_CLOUD_REGISTRY)/tesk-core-$(IMAGE):$(TAG) \
+			--format=docker \
+			--no-cache \
+			-f $(DOCKER_FILE_PATH)/$(IMAGE).Dockerfile; \
+	elif [ -x "$(DOCKER_CMD)" ]; then \
+		$(DOCKER_CMD) build \
+			-t $(ELIXIR_CLOUD_REGISTRY)/tesk-core-$(IMAGE):$(TAG) \
+			-f $(DOCKER_FILE_PATH)/$(IMAGE).Dockerfile .; \
+	else \
+		echo "Please install buildah or docker to build images."; \
+		exit 1; \
+	fi
+
+# Run image
+.PHONY: run
+run:
+	@if [ -x "$(DOCKER_CMD)" ]; then \
+		$(DOCKER_CMD) run \
+			-it --rm $(ELIXIR_CLOUD_REGISTRY)/tesk-core-$(IMAGE):$(TAG); \
+	else \
+		echo "Please install docker to run images."; \
+		exit 1; \
+	fi
+
+# Clean up built images or other generated artifacts
+clean:
+	docker rmi $(ELIXIR_CLOUD_REGISTRY)/tesk-core-$(IMAGE):$(TAG)

README.md

@@ -1,7 +1,7 @@
-<img src="documentation/img/TESKlogowfont.png" height="200">
+<img src="deployment/documentation/img/TESKlogowfont.png" height="200">
 
 
-An implementation of a task execution engine based on the [TES standard](https://github.com/ga4gh/task-execution-schemas) running on `Kubernetes`. For more details on `TES`, see the (very) brief [introduction to TES](documentation/tesintro.md).
+An implementation of a task execution engine based on the [TES standard](https://github.com/ga4gh/task-execution-schemas) running on `Kubernetes`. For more details on `TES`, see the (very) brief [introduction to TES](deployment/documentation/tesintro.md).
 
 For organisational reasons, this project is split into 3 repositories:
 - This one, which contains documentation and deployment files
@@ -10,15 +10,16 @@ For organisational reasons, this project is split into 3 repositories:
 
 If the API is running on your cluster it will pull the images from our `docker.io` repository automatically.
 
-`TESK` is designed with the goal to support any `Kubernetes` cluster, for its deployment please refer to the [deployment](documentation/deployment.md) page.
+`TESK` is designed with the goal to support any `Kubernetes` cluster, for its deployment please refer to the [deployment](deployment/documentation/deployment.md) page.
 
-The technical documentation is available in the [documentation](documentation) folder.
+The technical documentation is available in the [documentation](deployment/documentation) folder.
 
 
 ## Architecture
 As a diagram:
 
-![TESK architecture](documentation/img/architecture.png)
+<!-- TODO: Change the image remove tomcat, change naming etc -->
+![TESK architecture](deployment/documentation/img/architecture.png)
 
 **Description**: The first pod in the task lifecycle is the API pod, a pod which runs a web server (`Tomcat`) and exposes the `TES` specified endpoints. It consumes `TES` requests, validates them and translates them to `Kubernetes` jobs. The API pod then creates a `task controller` pod, or `taskmaster`. 
 
@@ -30,4 +31,4 @@ After the last executor, the `filer` is called once more to process the outputs
 -   A working [Kubernetes](https://kubernetes.io/) cluster version 1.9 and later.
 -   If you want TESK to handle tasks with I/O (and you probably want), you additionally need:
 -   A default storage class, which TESK will use to create temporary PVCs. It is enough that the storage class supports the RWO mode.
--   And, if you want TESK to integrate with workflow managers, you additionally need either an FTP account or a PVC that can be accessed from within or from outside of the cluster by the workflow manager (more in the [deployment](documentation/deployment.md) page).
+-   And, if you want TESK to integrate with workflow managers, you additionally need either an FTP account or a PVC that can be accessed from within or from outside of the cluster by the workflow manager (more in the [deployment](deployment/documentation/deployment.md) page).

pytest.ini

@@ -1,3 +1,4 @@
+# TODO: Remove and add in `pyproject.toml`
 [pytest]
 env =
     FTP_USER=user

scripts/run

@@ -1,3 +1,5 @@
+# TODO: Remove once API is stable and tested on cluster.
+#       This script can also be added to Makefile
 
 # 
 # Usage:

setup.cfg

@@ -1,3 +1,4 @@
+# TODO: Remove and add in `pyproject.toml`
 [metadata]
 description-file=README.md
 [aliases]

setup.py

@@ -1,3 +1,4 @@
+# TODO: Remove and add in `pyproject.toml`
 import codecs
 from os import path
 from setuptools import setup, find_packages
@@ -59,13 +60,13 @@
     # What does your project relate to?
     keywords='tes kubernetes ebi',
 
-    packages = find_packages('src'),
-    package_dir = {'': 'src'},
+    packages = find_packages('tesk'),
+    package_dir = {'': 'tesk'},
 
     entry_points={
         'console_scripts' : [
-            'filer = tesk_core.filer:main',
-            'taskmaster = tesk_core.taskmaster:main'
+            'filer = service.filer:main',
+            'taskmaster = service.taskmaster:main'
         ]
     },
     test_suite='tests',

taskmaster

@@ -1,3 +1,3 @@
+# TODO: Add to makefile after undertanding what its doing 😕
 
-
-PYTHONPATH="src" python src/tesk_core/taskmaster.py "$@"
+PYTHONPATH="tesk" python tesk/service/taskmaster.py "$@"

src/tesk_core/filer.py → tesk/service/filer.py

@@ -12,12 +12,12 @@
 import netrc
 import requests
 import gzip
-from tesk_core.exception import UnknownProtocol, FileProtocolDisabled
+from service.exception import UnknownProtocol, FileProtocolDisabled
 import shutil
 from glob import glob
-from tesk_core.path import containerPath, getPath, fileEnabled
-from tesk_core.transput import Type, Transput, urlparse
-from tesk_core.filer_s3 import S3Transput
+from service.path import containerPath, getPath, fileEnabled
+from service.transput import Type, Transput, urlparse
+from service.filer_s3 import S3Transput
 
 
 

src/tesk_core/filer_class.py → tesk/service/filer_class.py

@@ -1,6 +1,6 @@
 import json
-from tesk_core import path
-from tesk_core.path import fileEnabled
+from service import path
+from service.path import fileEnabled
 
 
 class Filer:

src/tesk_core/filer_s3.py → tesk/service/filer_s3.py

@@ -4,7 +4,7 @@
 import re
 import botocore
 import boto3
-from tesk_core.transput import Transput, Type
+from service.transput import Transput, Type
 
 class S3Transput(Transput):
     def __init__(self, path, url, ftype):

src/tesk_core/job.py → tesk/service/job.py

@@ -3,7 +3,7 @@
 from datetime import datetime, timezone
 from kubernetes import client, config
 from kubernetes.client.rest import ApiException
-from tesk_core.Util import pprint
+from service.Util import pprint
 
 
 logging.basicConfig(format='%(message)s', level=logging.INFO)

src/tesk_core/path.py → tesk/service/path.py

@@ -1,6 +1,6 @@
 import os
 from os.path import relpath
-from tesk_core.exception import InvalidHostPath
+from service.exception import InvalidHostPath
 try:
     from urllib.parse import urlparse
 except ImportError:

src/tesk_core/pvc.py → tesk/service/pvc.py

@@ -1,6 +1,6 @@
 from kubernetes import client, config
 from kubernetes.client.rest import ApiException
-from tesk_core.Util import pprint
+from service.Util import pprint
 import os
 import logging
 

src/tesk_core/taskmaster.py → tesk/service/taskmaster.py

@@ -8,9 +8,9 @@
 import logging
 import gzip
 from kubernetes import client, config
-from tesk_core.job import Job
-from tesk_core.pvc import PVC
-from tesk_core.filer_class import Filer
+from service.job import Job
+from service.pvc import PVC
+from service.filer_class import Filer
 
 created_jobs = []
 poll_interval = 5

tests/FilerClassTest.py → tests/unit_test/service/FilerClassTest.py

@@ -2,9 +2,9 @@
 
 import unittest
 import os
-from tesk_core.filer_class import Filer
-from tesk_core import path
-from tesk_core.Util import pprint
+from service.filer_class import Filer
+from service import path
+from service.Util import pprint
 
 try:
     from unittest.mock import patch  # Python 3 @UnresolvedImport
@@ -14,9 +14,9 @@
 
 
 
-@patch('tesk_core.path.HOST_BASE_PATH'          , '/home/tfga/workspace/cwl-tes')
-@patch('tesk_core.path.CONTAINER_BASE_PATH'     , '/transfer')
-@patch('tesk_core.path.TRANSFER_PVC_NAME'       , 'transfer-pvc')
+@patch('service.path.HOST_BASE_PATH'          , '/home/tfga/workspace/cwl-tes')
+@patch('service.path.CONTAINER_BASE_PATH'     , '/transfer')
+@patch('service.path.TRANSFER_PVC_NAME'       , 'transfer-pvc')
 @patch.dict(os.environ,
             {
                 "AWS_SHARED_CREDENTIALS_FILE": "/aws/credentials",

tests/TaskMasterTest.py → tests/unit_test/service/TaskMasterTest.py

@@ -1,5 +1,5 @@
 import unittest
-from tesk_core.taskmaster import newParser, run_task, newLogger
+from service.taskmaster import newParser, run_task, newLogger
 from argparse import Namespace
 import json
 import logging
@@ -64,11 +64,11 @@ def test_pullPolicyAlways(self):
 
 
 
-    @patch('tesk_core.taskmaster.args'                  , Namespace(debug=True, namespace='default', pull_policy_always=True))
-    @patch('tesk_core.taskmaster.logger'                , newLogger(logging.DEBUG))
-    @patch('tesk_core.taskmaster.PVC.create'            , pvcCreateMock)
-    @patch('tesk_core.taskmaster.PVC.delete'            , pvcDeleteMock)
-    @patch('tesk_core.taskmaster.Job.run_to_completion' , jobRunToCompletionMock)
+    @patch('service.taskmaster.args'                  , Namespace(debug=True, namespace='default', pull_policy_always=True))
+    @patch('service.taskmaster.logger'                , newLogger(logging.DEBUG))
+    @patch('service.taskmaster.PVC.create'            , pvcCreateMock)
+    @patch('service.taskmaster.PVC.delete'            , pvcDeleteMock)
+    @patch('service.taskmaster.Job.run_to_completion' , jobRunToCompletionMock)
     def test_run_task(self):
 
         with open('tests/resources/inputFile.json') as fh: