Rename package (#18)

* missing changes for introducing dep on eqt

* rename package to brem

* build works on my machine!

* shuffle code around

* created RemoteRunControl base class and DVCRemoteRunControl

* created RemoteRunControl in brem.py

* use remotepath

* use port parameter

* add docstring

* add slurm* to gitignore

* massive changes

* update version

* add changelog

* renamed tests directory to examples

Co-authored-by: Edoardo Pasca <[email protected]>

.github/workflows/ci.yml

@@ -0,0 +1,33 @@
+# This is a basic workflow to help you get started with Actions
+
+name: CI
+
+# Controls when the action will run. 
+on:
+  # Triggers the workflow on push or pull request events but only for the main branch
+  push:
+    branches: [ main ]
+  pull_request:
+    branches: [ main ]
+
+  # Allows you to run this workflow manually from the Actions tab
+  workflow_dispatch:
+
+# A workflow run is made up of one or more jobs that can run sequentially or in parallel
+jobs:
+  # This workflow contains a single job called "build"
+  build:
+    # The type of runner that the job will run on
+    runs-on: ubuntu-latest
+
+    # Steps represent a sequence of tasks that will be executed as part of the job
+    steps:
+      # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
+      - uses: actions/checkout@v2
+      - uses: actions/setup-python@v2
+      - uses: casperdcl/deploy-pypi@v2
+        with:
+          password: ${{ secrets.PYPI_SECRET_TOKEN }}
+          pip: wheel -w dist/ --no-deps .
+          # only upload if a tag is pushed (otherwise just build & check)
+          upload: ${{ github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags') }}

.gitignore

@@ -127,3 +127,5 @@ dmypy.json
 
 # Pyre type checker
 .pyre/
+
+slurm*

CHANGELOG.md

@@ -0,0 +1,2 @@
+## v1.0.0
+- First release as brem

dvc_x/__init__.py → brem/__init__.py

@@ -2,7 +2,7 @@
 
 typical usage
 
->>>   from dvc_x import DVCRem
+>>>   from brem import BasicRemoteExecutionManager
 
 """
 
@@ -17,19 +17,21 @@
 
 if LooseVersion(pk.__version__) < '2.7.2':
     raise ImportError(
-        'dvc_x needs paramiko-2.7.2 or later. You have: {:s}'.format(pk.__version__))
+        'brem needs paramiko-2.7.2 or later. You have: {:s}'.format(pk.__version__))
 
 
 # from https://stackoverflow.com/questions/1057431
 modules = glob.glob(join(dirname(__file__), "*.py"))
 __all__ = [basename(f)[:-3] for f in modules if isfile(f)
            and not f.endswith('__init__.py')]
 
-__version__ = '0.0.1'
+__version__ = '1.0.0'
 
 try:
-    from .dvc_x import DVCRem
-    __version__ = DVCRem.__version__
+    from .brem import BasicRemoteExecutionManager
+    from .brem import RemoteRunControl
+    from .brem import RemoteRunControlSignals
+
 
 except ImportError:
-    raise ImportError('error importing dvc_x')
+    raise ImportError('error importing brem')

dvc_x/dvc_x.py → brem/brem.py

@@ -1,12 +1,14 @@
 import paramiko as ssh
 import posixpath, ntpath, os
 from getpass import getpass
+from PySide2 import QtCore
+from eqt.threading import Worker
 
 remotepath = os.path
 
-class DVCRem:
-    """ Main class of a dvcrem"""
-    __version__ = "0.0.1"  #
+class BasicRemoteExecutionManager(object):
+    """ Main class of a brem"""
+
 
     def __init__(self,logfile=None, port=None, host=None,username=None,\
         private_key=None, remote_os=None):
@@ -191,11 +193,177 @@ def job_cancel(self,jid):
          stdout, stderr = self.run("scancel  {}".format(jid))
          return stdout
 
+import functools
+
+class RemoteRunControlSignals(QtCore.QObject):
+    status = QtCore.Signal(tuple)
+    job_id = QtCore.Signal(int)
+
+class RemoteRunControl(object):
+    '''RemoteRunControl base class to handle asynchronous interaction with a remote running job
+
+    :param connection_details: required parameters passed in a dictionary server_name, server_port, username, private_key
+    :param type: dict 
+    '''
+
+    def __init__(self, connection_details=None):
+        self.connection_details = connection_details
+        self.conn                = None
+        self._jobid              = None
+        self._job_status         = None
+        self.jobs                = {}
+
+        self.internalsignals = RemoteRunControlSignals()
+        self.internalsignals.job_id.connect(self.set_job_id)
+        self.internalsignals.status.connect(self.set_job_status)
+
+        self.threadpool = QtCore.QThreadPool()
+        self._Worker = None
+
+
+    def set_job_id(self, value):
+        self.job_id = value
+        self.jobs[value] = None
+        # attach finished signal
+        # self.dvcWorker.signals.finished.connect(lambda: self.job_finished())
+    def set_job_status(self, value):
+        self.job_status = value[1]
+        self.jobs[value[0]] = value[1]
+    def get_job_status(self, jobid):
+        try:
+            return self.jobs[jobid]
+        except KeyError as ke:
+            return 'Job not found.'
 
+    @property
+    def Worker(self):
+        return self._Worker
+
+    def create_job(self, fn, **kwargs):
+        '''Creates a job to run function fn'''
+        if not self.check_configuration():
+            raise ValueError('Connection details are not specified or complete. Got', \
+                        self.connection_details)
+
+        kwargs['username']    = self.connection_details['username']
+        kwargs['port']        = self.connection_details['server_port']
+        kwargs['host']        = self.connection_details['server_name']
+        kwargs['private_key'] = self.connection_details['private_key']
+        self._Worker_kwargs = kwargs
+        self._Worker = Worker(fn, **kwargs)
+
+
+        # other signal/slots should be connected from outside
+
+    @property
+    def signals(self):
+        if self.Worker is not None:
+            return self.Worker.signals
+        else:
+            raise ValueError('Worker function is not defined')
+            # return self.Worker.signals
+    def run_job(self):
+        self.threadpool.start(self.Worker)
+
+    @property
+    def job_id(self):
+        return self._jobid
+    @job_id.setter
+    def job_id(self, value):
+        print ("setting job_id", value)
+        self._jobid = value
+    @property
+    def connection_details(self):
+        return self._connection_details
+    @connection_details.setter
+    def connection_details(self, value):
+        if value is not None:
+            self._connection_details = dict(value)
+        else:
+            self._connection_details = None
+    @property
+    def job_status(self):
+        return self._job_status
+    @job_status.setter
+    def job_status(self, value):
+        print("setting job_status", value)
+        if self.job_id is not None:
+            self._job_status = value
+
+
+    def check_configuration(self):
+        def f (a,x,y):
+            return x in a and y
+        ff = functools.partial(f, self.connection_details.keys())
+        # return functools.reduce(ff, ['username','server_port', 'server_name','private_key'], True)
+        required = ['username','server_port', 'server_name','private_key']
+        available = self.connection_details.keys()
+        ret = True
+        for x in required:
+            ret = ret and (x in available)
+        return ret
+
+    def cancel_job(self, job_id):
+        host = self.connection_details['server_name']
+        username = self.connection_details['username']
+        port = self.connection_details['server_port']
+        private_key = self.connection_details['private_key']
+
+        a = BasicRemoteExecutionManager( host=host, 
+                                              username=username,
+                                              port=port,
+                                              private_key=private_key)
+        a.login(passphrase=False)
+        self.internalsignals.status.emit((job_id, "CANCELLING"))
+        a.job_cancel(job_id)
+        self.internalsignals.status.emit((job_id, "CANCELLED"))
+        a.logout()
+
+    def pytail(self, connection, logfile, start_at):
+
+        tail = '''
+import os, sys, functools
+def pytail(filename, start_at):
+    with open(filename, 'r') as f:
+        # skip to line start_at
+        ret = []
+        i = 0
+        while True:
+            line = f.readline()
+            if line == '':
+                break
+            if i > start_at:
+                ret.append(line)
+            i += 1
+    return ret
+
+if __name__ == '__main__':
+    stdout = pytail(sys.argv[1],int(sys.argv[2]))
+    msg = functools.reduce(lambda x,y: x+y, stdout, '')
+    print (msg)
+'''             
+        remotehomedir = connection.remote_home_dir
+
+        with open("pytail.py", 'w') as pytail:
+            print (tail, file=pytail)
+        connection.put_file("pytail.py", remotepath.join(remotehomedir, 'pytail.py'))
+
+        stdout, stderr = connection.run('python pytail.py {} {}'.format(logfile, start_at))
+
+        # remove pytail.py from the server.
+        # connection.remove_file(dpath.join(connection.remote_home_dir, 'pytail.py'))
+
+        # print ("logfile", logfile)
+        # print ("stdout", stdout.decode('utf-8'))
+        # print ("stdout", stderr)
+
+        # expand tabs and newlines
+        return stdout
+
 
 def main():
     """ Run the main program """
-    t=DVCRem()
+    t=BasicRemoteExecutionManager()
     print(t.info())
 
 if __name__ == "__main__":

dvc_x/ui/RemoteFileDialog.py → brem/ui/RemoteFileDialog.py

@@ -6,7 +6,7 @@
 from PySide2.QtCore import QRegExp
 import glob
 from functools import partial
-import dvc_x as drx
+import brem
 import stat
 from eqt.threading import Worker
 
@@ -111,7 +111,11 @@ def createButtonBoxSave(self):
     def setupConnection(self, logfile=None, port=None, host=None, \
             username=None, private_key=None, remote_os=None):
 
-        a=drx.DVCRem(logfile=logfile, port=port, host=host, username=username, private_key=private_key)
+        a = brem.RemoteExecutionManager(logfile=logfile, 
+                                        port=port, 
+                                        host=host, 
+                                        username=username, 
+                                        private_key=private_key)
         global dpath 
         dpath = posixpath 
         if remote_os == 'Windows':
@@ -339,9 +343,11 @@ def asyncStatRemotePath(self, **kwargs):#path, row, table, host, port, username,
         remote_os = kwargs.get('remote_os')
 
         logfile = 'logfile.log'
-        conn = drx.DVCRem(logfile=logfile, 
-                          port=port, host=host, username=username, 
-                          private_key=private_key)
+        conn = brem.RemoteExecutionManager(logfile=logfile,
+                                           port=port, 
+                                           host=host, 
+                                           username=username,
+                                           private_key=private_key)
         error = None
         try:
             conn.login(passphrase=False)

dvc_x/ui/RemoteServerSettingDialog.py → brem/ui/RemoteServerSettingDialog.py

@@ -2,7 +2,7 @@
 from eqt.ui.UIFormWidget import UIFormFactory
 import os
 import configparser
-import dvc_x as drx
+import brem
 from eqt.threading.QtThreading import Worker, WorkerSignals, ErrorObserver
 
 
@@ -421,8 +421,11 @@ def setInfo(self, text):
 
     def authorize_key_worker(self, host, username, port, 
                              progress_callback, message_callback):
-        a=drx.DVCRem(logfile='generatekey.log', port=port, 
-                         host=host, username=username, private_key=None)
+        a = brem.RemoteExecutionManager( logfile='generatekey.log', 
+                                         port=port, 
+                                         host=host, 
+                                         username=username, 
+                                         private_key=None)
 
         a.login_pw(self.formWidget.widgets['server_password_field'].text())
         message_callback.emit("Generating Key...")

tests/dialog_example.py → examples/dialog_example.py

@@ -3,8 +3,8 @@
 from PySide2.QtCore import QRegExp
 import glob, sys, os
 from functools import partial
-from dvc_x.ui import RemoteFileDialog
-from dvc_x.ui import RemoteServerSettingDialog, GenerateKeygenDialog
+from brem.ui import RemoteFileDialog
+from brem.ui import RemoteServerSettingDialog, GenerateKeygenDialog
 
 
 class MainUI(QtWidgets.QMainWindow):

tests/run_dvc.py → examples/run_dvc.py

@@ -1,10 +1,10 @@
 
-import dvc_x as drx
+import brem
 
 from time import sleep
 #a=drx.DVCRem(private_key="/home/drFaustroll/.ssh/id_routers")
 
-a=drx.DVCRem(host="scarf.rl.ac.uk",username="scarf595",port=22,private_key="C:/Apps/cygwin64/home/ofn77899/.ssh/id_rsa")
+a = brem.BasicRemoteExecutionManager(host="scarf.rl.ac.uk",username="scarf595",port=22,private_key="C:/Apps/cygwin64/home/ofn77899/.ssh/id_rsa")
 
 a.login(passphrase=False)