Merge pull request #44 from brain-bican/advanced_git

Merge command

setup.py

@@ -8,7 +8,7 @@
 
 setup(
     name="tdta",
-    version="0.1.0.dev14",
+    version="0.1.0.dev15",
     description="The aim of this project is to provide taxonomy development tools custom actions.",
     long_description=README,
     long_description_content_type="text/markdown",

src/tdta/__main__.py

@@ -3,6 +3,7 @@
 from tdta.purl_publish import publish_to_purl
 from tdta.tdt_export import export_cas_data
 from tdta.anndata_export import export_anndata
+from tdta.version_control import git_update_local
 
 
 def main():
@@ -12,6 +13,7 @@ def main():
     create_purl_operation_parser(subparsers)
     create_save_operation_parser(subparsers)
     create_anndata_operation_parser(subparsers)
+    create_merge_operation_parser(subparsers)
 
     args = parser.parse_args()
 
@@ -27,6 +29,8 @@ def main():
         if "cache" in args and args.cache:
             cache_folder_path = args.cache
         export_anndata(args.database, args.json, args.output, cache_folder_path)
+    elif args.action == "merge":
+        git_update_local(str(args.project), str(args.message))
 
 
 def create_purl_operation_parser(subparsers):
@@ -64,5 +68,13 @@ def create_anndata_operation_parser(subparsers):
                                help="Dataset cache folder path.")
 
 
+def create_merge_operation_parser(subparsers):
+    parser_purl = subparsers.add_parser("merge",
+                                        description="The version control merge operation parser",
+                                        help="Pulls remote changes and merges with local.")
+    parser_purl.add_argument('-p', '--project', action='store', type=pathlib.Path, required=True, help="Project folder path.")
+    parser_purl.add_argument('-m', '--message', required=True, help="Commit message.")
+
+
 if __name__ == "__main__":
     main()

src/tdta/command_line_utils.py

@@ -0,0 +1,41 @@
+import subprocess
+import logging
+
+
+def runcmd(cmd, supress_exceptions=False, supress_logs=False):
+    """
+    Runs the given command in the command line.
+    :param cmd: command to run
+    :param supress_exceptions: flag to suppress the exception on failure
+    :param supress_logs: flag to suppress the logs in the output
+    :return: output of the command
+    """
+    log_info("RUNNING: {}".format(cmd), supress_logs)
+    p = subprocess.Popen([cmd], stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True, universal_newlines=True)
+    (out, err) = p.communicate()
+    log_info('OUT: {}'.format(out), supress_logs)
+    if err:
+        log_error("Error: " + err, supress_logs)
+    if not supress_exceptions and p.returncode != 0:
+        raise Exception('Failed: {}: {}'.format(cmd, err))
+    return out
+
+
+def log_info(msg, supress_logs=False):
+    """
+    Logs the given message as info.
+    :param msg: message to log
+    :param supress_logs: flag to suppress the logs in the output
+    """
+    if not supress_logs:
+        logging.info(msg)
+
+
+def log_error(msg, supress_logs=False):
+    """
+    Logs the given message as error.
+    :param msg: error message to log
+    :param supress_logs: flag to suppress the logs in the output
+    """
+    if not supress_logs:
+        logging.error(msg)

src/tdta/purl_publish.py

@@ -1,10 +1,10 @@
 import os
 import requests
 import shutil
-import subprocess
-import logging
 
 from typing import Optional
+from tdta.command_line_utils import runcmd
+
 
 GITHUB_TOKEN_ENV = 'GITHUB_AUTH_TOKEN'
 GITHUB_USER_ENV = 'GITHUB_USER'
@@ -210,21 +210,3 @@ def clone_project(purl_folder, user_name):
     # runcmd("cd {dir} && gh repo clone {repo}".format(dir=purl_folder, repo=PURL_REPO))
 
     return os.path.join(purl_folder, PURL_REPO_NAME)
-
-
-def runcmd(cmd):
-    """
-    Runs the given command in the command line.
-    :param cmd: command to run
-    :return: output of the command
-    """
-    logging.info("RUNNING: {}".format(cmd))
-    p = subprocess.Popen([cmd], stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True, universal_newlines=True)
-    (out, err) = p.communicate()
-    logging.info('OUT: {}'.format(out))
-    if err:
-        logging.error(err)
-    if p.returncode != 0:
-        raise Exception('Failed: {}'.format(cmd))
-    return out
-

src/tdta/version_control.py

@@ -0,0 +1,80 @@
+import os
+import glob
+
+from tdta.command_line_utils import runcmd
+
+
+def git_update_local(project_folder: str, commit_message: str):
+    """
+    Composite git merge action to pull remote changes and merge them into the local branch. Instruct users about how to
+    solve conflicts if they arise.
+    :param project_folder: path to the project root folder
+    :param commit_message: commit message for the merge action
+
+    """
+    print("Updating the project from the remote repository...")
+    work_dir = os.path.abspath(project_folder)
+    check_all_files_for_conflict(work_dir)
+    current_branch = runcmd("cd {dir} && git branch --show-current".format(dir=work_dir), supress_logs=True).strip()
+    if not current_branch:
+        print("Git branch couldn't be identified in the project folder. Probably a previous rebase operation is in "
+              "progress. Continuing the rebase operation.")
+        runcmd("cd {dir} && git commit -a --message \"{msg}\"".format(dir=work_dir, msg=commit_message), supress_logs=True)
+        runcmd("cd {dir} && git rebase --continue".format(dir=work_dir), supress_logs=True)
+    try:
+        runcmd("cd {dir} && git commit -a --message \"{msg}\"".format(dir=work_dir, msg=commit_message), supress_logs=True)
+    except Exception as e:
+        print("Error occurred during commit: " + str(e))
+
+    pull_output = runcmd("cd {dir} && git pull --rebase=merges".format(dir=work_dir, branch=current_branch), supress_exceptions=True, supress_logs=True)
+    if "CONFLICT" in pull_output:
+        print("Conflicts occurred during the update process. Please resolve them manually and run the action again.")
+        for message in pull_output.split("\n"):
+            if "Merge conflict in" in message:
+                print(">>>>  " + message)
+        raise Exception("Conflicts occurred during the merge process. "
+                        "Please resolve them manually and run the action again.")
+
+    print("Project updated successfully.")
+
+
+def check_all_files_for_conflict(work_dir: str):
+    """
+    Checks all files in the project folder for unresolved conflicts.
+    """
+    text_file_formats = ["json", "yml", "yaml", "txt", "md", "csv", "tsv", "html", "xml", "ts", "js", "py", "sh", "bat", "toml", "css"]
+    managed_files_out = runcmd("cd {dir} && git ls-tree --full-tree --name-only -r HEAD".format(dir=work_dir),
+                               supress_exceptions=True, supress_logs=True)
+    managed_files = managed_files_out.split("\n")
+    if not managed_files or len(managed_files) < 5:
+        managed_files = get_all_files(work_dir)
+    files_with_conflict = []
+    for managed_file in managed_files:
+        if managed_file.split(".")[-1] in text_file_formats:
+            local_file = os.path.abspath(os.path.join(work_dir, managed_file))
+            with open(local_file, 'r') as file:
+                data = file.read()
+                if "<<<<<<<" in data and "=======" in data:
+                    files_with_conflict.append(local_file)
+
+    for file in files_with_conflict:
+        print(">>> Unresolved conflicts in file: " + file)
+    if files_with_conflict:
+        raise Exception("There are unresolved conflicts in the project. Please manually resolve them before continuing."
+                        " See conflict handling instructions at https://brain-bican.github.io/"
+                        "taxonomy-development-tools/Collaboration/ for more information.")
+
+
+def get_all_files(work_dir: str):
+    """
+    Get all files in the project folder.
+    :param work_dir: path to the project root folder
+    :return: list of all files' path in the project folder relative to the work_dir
+    """
+    all_files = list()
+    if not work_dir.endswith(os.sep):
+        work_dir += os.sep
+    for filename in glob.iglob(work_dir + '**/**', recursive=True):
+        if not os.path.isdir(filename):
+            all_files.append(os.path.relpath(filename, work_dir))
+    return all_files