From d80a9b426be435d19b8998e18bee6f4e33743f3f Mon Sep 17 00:00:00 2001 From: hkir-dev Date: Mon, 1 Jul 2024 13:23:21 +0100 Subject: [PATCH] fixes https://github.com/brain-bican/taxonomy-development-tools/issues/34 --- setup.py | 2 +- src/tdta/__main__.py | 12 +++++ src/tdta/command_line_utils.py | 41 +++++++++++++++++ src/tdta/purl_publish.py | 22 +--------- src/tdta/version_control.py | 80 ++++++++++++++++++++++++++++++++++ 5 files changed, 136 insertions(+), 21 deletions(-) create mode 100644 src/tdta/command_line_utils.py create mode 100644 src/tdta/version_control.py diff --git a/setup.py b/setup.py index 489a1e1..97536a9 100644 --- a/setup.py +++ b/setup.py @@ -8,7 +8,7 @@ setup( name="tdta", - version="0.1.0.dev14", + version="0.1.0.dev15", description="The aim of this project is to provide taxonomy development tools custom actions.", long_description=README, long_description_content_type="text/markdown", diff --git a/src/tdta/__main__.py b/src/tdta/__main__.py index 77b17b9..37abbee 100644 --- a/src/tdta/__main__.py +++ b/src/tdta/__main__.py @@ -3,6 +3,7 @@ from tdta.purl_publish import publish_to_purl from tdta.tdt_export import export_cas_data from tdta.anndata_export import export_anndata +from tdta.version_control import git_update_local def main(): @@ -12,6 +13,7 @@ def main(): create_purl_operation_parser(subparsers) create_save_operation_parser(subparsers) create_anndata_operation_parser(subparsers) + create_merge_operation_parser(subparsers) args = parser.parse_args() @@ -27,6 +29,8 @@ def main(): if "cache" in args and args.cache: cache_folder_path = args.cache export_anndata(args.database, args.json, args.output, cache_folder_path) + elif args.action == "merge": + git_update_local(str(args.project), str(args.message)) def create_purl_operation_parser(subparsers): @@ -64,5 +68,13 @@ def create_anndata_operation_parser(subparsers): help="Dataset cache folder path.") +def create_merge_operation_parser(subparsers): + parser_purl = subparsers.add_parser("merge", + description="The version control merge operation parser", + help="Pulls remote changes and merges with local.") + parser_purl.add_argument('-p', '--project', action='store', type=pathlib.Path, required=True, help="Project folder path.") + parser_purl.add_argument('-m', '--message', required=True, help="Commit message.") + + if __name__ == "__main__": main() diff --git a/src/tdta/command_line_utils.py b/src/tdta/command_line_utils.py new file mode 100644 index 0000000..59d115b --- /dev/null +++ b/src/tdta/command_line_utils.py @@ -0,0 +1,41 @@ +import subprocess +import logging + + +def runcmd(cmd, supress_exceptions=False, supress_logs=False): + """ + Runs the given command in the command line. + :param cmd: command to run + :param supress_exceptions: flag to suppress the exception on failure + :param supress_logs: flag to suppress the logs in the output + :return: output of the command + """ + log_info("RUNNING: {}".format(cmd), supress_logs) + p = subprocess.Popen([cmd], stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True, universal_newlines=True) + (out, err) = p.communicate() + log_info('OUT: {}'.format(out), supress_logs) + if err: + log_error("Error: " + err, supress_logs) + if not supress_exceptions and p.returncode != 0: + raise Exception('Failed: {}: {}'.format(cmd, err)) + return out + + +def log_info(msg, supress_logs=False): + """ + Logs the given message as info. + :param msg: message to log + :param supress_logs: flag to suppress the logs in the output + """ + if not supress_logs: + logging.info(msg) + + +def log_error(msg, supress_logs=False): + """ + Logs the given message as error. + :param msg: error message to log + :param supress_logs: flag to suppress the logs in the output + """ + if not supress_logs: + logging.error(msg) diff --git a/src/tdta/purl_publish.py b/src/tdta/purl_publish.py index 6d2bd1f..14b139e 100644 --- a/src/tdta/purl_publish.py +++ b/src/tdta/purl_publish.py @@ -1,10 +1,10 @@ import os import requests import shutil -import subprocess -import logging from typing import Optional +from tdta.command_line_utils import runcmd + GITHUB_TOKEN_ENV = 'GITHUB_AUTH_TOKEN' GITHUB_USER_ENV = 'GITHUB_USER' @@ -210,21 +210,3 @@ def clone_project(purl_folder, user_name): # runcmd("cd {dir} && gh repo clone {repo}".format(dir=purl_folder, repo=PURL_REPO)) return os.path.join(purl_folder, PURL_REPO_NAME) - - -def runcmd(cmd): - """ - Runs the given command in the command line. - :param cmd: command to run - :return: output of the command - """ - logging.info("RUNNING: {}".format(cmd)) - p = subprocess.Popen([cmd], stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True, universal_newlines=True) - (out, err) = p.communicate() - logging.info('OUT: {}'.format(out)) - if err: - logging.error(err) - if p.returncode != 0: - raise Exception('Failed: {}'.format(cmd)) - return out - diff --git a/src/tdta/version_control.py b/src/tdta/version_control.py new file mode 100644 index 0000000..b4190a7 --- /dev/null +++ b/src/tdta/version_control.py @@ -0,0 +1,80 @@ +import os +import glob + +from tdta.command_line_utils import runcmd + + +def git_update_local(project_folder: str, commit_message: str): + """ + Composite git merge action to pull remote changes and merge them into the local branch. Instruct users about how to + solve conflicts if they arise. + :param project_folder: path to the project root folder + :param commit_message: commit message for the merge action + + """ + print("Updating the project from the remote repository...") + work_dir = os.path.abspath(project_folder) + check_all_files_for_conflict(work_dir) + current_branch = runcmd("cd {dir} && git branch --show-current".format(dir=work_dir), supress_logs=True).strip() + if not current_branch: + print("Git branch couldn't be identified in the project folder. Probably a previous rebase operation is in " + "progress. Continuing the rebase operation.") + runcmd("cd {dir} && git commit -a --message \"{msg}\"".format(dir=work_dir, msg=commit_message), supress_logs=True) + runcmd("cd {dir} && git rebase --continue".format(dir=work_dir), supress_logs=True) + try: + runcmd("cd {dir} && git commit -a --message \"{msg}\"".format(dir=work_dir, msg=commit_message), supress_logs=True) + except Exception as e: + print("Error occurred during commit: " + str(e)) + + pull_output = runcmd("cd {dir} && git pull --rebase=merges".format(dir=work_dir, branch=current_branch), supress_exceptions=True, supress_logs=True) + if "CONFLICT" in pull_output: + print("Conflicts occurred during the update process. Please resolve them manually and run the action again.") + for message in pull_output.split("\n"): + if "Merge conflict in" in message: + print(">>>> " + message) + raise Exception("Conflicts occurred during the merge process. " + "Please resolve them manually and run the action again.") + + print("Project updated successfully.") + + +def check_all_files_for_conflict(work_dir: str): + """ + Checks all files in the project folder for unresolved conflicts. + """ + text_file_formats = ["json", "yml", "yaml", "txt", "md", "csv", "tsv", "html", "xml", "ts", "js", "py", "sh", "bat", "toml", "css"] + managed_files_out = runcmd("cd {dir} && git ls-tree --full-tree --name-only -r HEAD".format(dir=work_dir), + supress_exceptions=True, supress_logs=True) + managed_files = managed_files_out.split("\n") + if not managed_files or len(managed_files) < 5: + managed_files = get_all_files(work_dir) + files_with_conflict = [] + for managed_file in managed_files: + if managed_file.split(".")[-1] in text_file_formats: + local_file = os.path.abspath(os.path.join(work_dir, managed_file)) + with open(local_file, 'r') as file: + data = file.read() + if "<<<<<<<" in data and "=======" in data: + files_with_conflict.append(local_file) + + for file in files_with_conflict: + print(">>> Unresolved conflicts in file: " + file) + if files_with_conflict: + raise Exception("There are unresolved conflicts in the project. Please manually resolve them before continuing." + " See conflict handling instructions at https://brain-bican.github.io/" + "taxonomy-development-tools/Collaboration/ for more information.") + + +def get_all_files(work_dir: str): + """ + Get all files in the project folder. + :param work_dir: path to the project root folder + :return: list of all files' path in the project folder relative to the work_dir + """ + all_files = list() + if not work_dir.endswith(os.sep): + work_dir += os.sep + for filename in glob.iglob(work_dir + '**/**', recursive=True): + if not os.path.isdir(filename): + all_files.append(os.path.relpath(filename, work_dir)) + return all_files