From 3970bb3cb98b3734814c920b895c740a49e9ad14 Mon Sep 17 00:00:00 2001 From: Clare Macrae Date: Fri, 10 Mar 2023 18:49:59 +0000 Subject: [PATCH 1/8] prevent case-name collision for theme files Co-Authored-By: Andrew Nutter-Upham <59033+andynu@users.noreply.github.com> --- .github/scripts/update_releases.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/.github/scripts/update_releases.py b/.github/scripts/update_releases.py index 5692b7ea5..5a2beadfe 100755 --- a/.github/scripts/update_releases.py +++ b/.github/scripts/update_releases.py @@ -2,6 +2,7 @@ import os import sys import argparse +from pathlib import Path from typing import Sequence from authors import AllAuthors @@ -67,11 +68,25 @@ def process_released_themes(overwrite: bool = False, verbose: bool = False) -> T theme_downloads = ThemeDownloadCount.get_theme_downloads() + themes_dir = os.path.dirname(get_output_dir(Theme.template, 'nonsense')) + file_case_lookup = dict() + for filename in os.listdir(themes_dir): + base_name = Path(filename).stem + file_case_lookup[base_name.lower()] = base_name + for index, theme in enumerate(theme_list): current_name, valid = theme.collect_data_for_theme(theme_downloads, file_groups) if not valid: continue + # Prefer the existing capitalisation of any existing filename, + # to prevent case-conflicts. + if current_name.lower() in file_case_lookup: + existing_case = file_case_lookup[current_name.lower()] + if existing_case != current_name: + print(f'Overriding filename {current_name} to {existing_case}') + current_name = existing_case + group = write_template_file( Theme.template, current_name, overwrite=overwrite, verbose=verbose, **theme.data() ) From 69c5f19d7e70574ccb22fca7fdf9fa4293cf74e9 Mon Sep 17 00:00:00 2001 From: Clare Macrae Date: Tue, 21 Mar 2023 18:06:14 +0000 Subject: [PATCH 2/8] Extract class FileNameCaseCollisionsPreventer Co-Authored-By: Andrew Nutter-Upham <59033+andynu@users.noreply.github.com> --- .github/scripts/update_releases.py | 34 ++++++++++++++++++++++-------- 1 file changed, 25 insertions(+), 9 deletions(-) diff --git a/.github/scripts/update_releases.py b/.github/scripts/update_releases.py index 5a2beadfe..ce66fabc6 100755 --- a/.github/scripts/update_releases.py +++ b/.github/scripts/update_releases.py @@ -56,6 +56,29 @@ def process_released_plugins(overwrite: bool = False, verbose: bool = False) -> return valid_plugins +class FileNameCaseCollisionsPreventer: + def __init__(self, directory: str) -> None: + self.file_case_lookup = dict() + for filename in os.listdir(directory): + base_name = Path(filename).stem + self.file_case_lookup[base_name.lower()] = base_name + + def get_name(self, current_name: str) -> str: + """ + Prefer the existing capitalisation of any existing filename, + to prevent case-conflicts. + + :param current_name: + :return: + """ + if current_name.lower() in self.file_case_lookup: + existing_case = self.file_case_lookup[current_name.lower()] + if existing_case != current_name: + print(f'Overriding filename {current_name} to {existing_case}') + current_name = existing_case + return current_name + + def process_released_themes(overwrite: bool = False, verbose: bool = False) -> ThemeList: print("-----\nProcessing themes....\n") theme_list = get_community_themes() @@ -69,10 +92,7 @@ def process_released_themes(overwrite: bool = False, verbose: bool = False) -> T theme_downloads = ThemeDownloadCount.get_theme_downloads() themes_dir = os.path.dirname(get_output_dir(Theme.template, 'nonsense')) - file_case_lookup = dict() - for filename in os.listdir(themes_dir): - base_name = Path(filename).stem - file_case_lookup[base_name.lower()] = base_name + collision_preventer = FileNameCaseCollisionsPreventer(themes_dir) for index, theme in enumerate(theme_list): current_name, valid = theme.collect_data_for_theme(theme_downloads, file_groups) @@ -81,11 +101,7 @@ def process_released_themes(overwrite: bool = False, verbose: bool = False) -> T # Prefer the existing capitalisation of any existing filename, # to prevent case-conflicts. - if current_name.lower() in file_case_lookup: - existing_case = file_case_lookup[current_name.lower()] - if existing_case != current_name: - print(f'Overriding filename {current_name} to {existing_case}') - current_name = existing_case + current_name = collision_preventer.get_name(current_name) group = write_template_file( Theme.template, current_name, overwrite=overwrite, verbose=verbose, **theme.data() From fcbd2cb19aeda8e6afa1b6d9a970c4bcc61ca0e1 Mon Sep 17 00:00:00 2001 From: Clare Macrae Date: Tue, 21 Mar 2023 18:20:59 +0000 Subject: [PATCH 3/8] Prevent case-name collisions for author files Co-Authored-By: Andrew Nutter-Upham <59033+andynu@users.noreply.github.com> --- .github/scripts/update_releases.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/scripts/update_releases.py b/.github/scripts/update_releases.py index ce66fabc6..b96204f14 100755 --- a/.github/scripts/update_releases.py +++ b/.github/scripts/update_releases.py @@ -165,13 +165,15 @@ def process_authors(themes: ThemeList, verbose: bool = False) -> None: print("-----\nProcessing authors....\n") template = get_template("author") + authors_dir = os.path.dirname(get_output_dir(template, 'nonsense')) + collision_preventer = FileNameCaseCollisionsPreventer(authors_dir) all_authors = collate_authors(themes, plugins) print("\nCreating author notes....\n") file_groups: FileGroups = dict() for user, author_info in all_authors.items(): group = write_template_file( - template, user, overwrite=overwrite, verbose=verbose, **author_info + template, collision_preventer.get_name(user), overwrite=overwrite, verbose=verbose, **author_info ) add_file_group(file_groups, group, user) From 92c56c4e36f91424c60b05ef3c12e7bd95d56030 Mon Sep 17 00:00:00 2001 From: Clare Macrae Date: Tue, 21 Mar 2023 18:23:08 +0000 Subject: [PATCH 4/8] Rename get_output_dir() to get_output_path() Co-Authored-By: Andrew Nutter-Upham <59033+andynu@users.noreply.github.com> --- .github/scripts/themes.py | 6 +++--- .github/scripts/update_releases.py | 8 ++++---- .github/scripts/utils.py | 4 ++-- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/scripts/themes.py b/.github/scripts/themes.py index 3aec97d44..b867b684e 100644 --- a/.github/scripts/themes.py +++ b/.github/scripts/themes.py @@ -11,7 +11,7 @@ from core_plugins import CORE_PLUGINS from utils import ( THEME_CSS_FILE, - get_output_dir, + get_output_path, get_theme_css, FileGroups, add_file_group, get_template, get_json_from_github @@ -245,7 +245,7 @@ def get_theme_previous_download_count_or_none(template: Template, current_name: Read the theme file from disk, and return the previously-saved download count :return: The saved theme download count, or None if this could not be obtained """ - file_name = get_output_dir(template, current_name) + file_name = get_output_path(template, current_name) if not os.path.exists(file_name): # This is a new theme, so we don't yet have a previous download count: return None @@ -261,7 +261,7 @@ def get_theme_previous_download_count_or_none(template: Template, current_name: @staticmethod def set_theme_download_count(template: Template, current_name: str, new_download_count: int, verbose: bool) -> None: - file_name = get_output_dir(template, current_name) + file_name = get_output_path(template, current_name) if not os.path.exists(file_name): if verbose: diff --git a/.github/scripts/update_releases.py b/.github/scripts/update_releases.py index b96204f14..da39bece4 100755 --- a/.github/scripts/update_releases.py +++ b/.github/scripts/update_releases.py @@ -19,7 +19,7 @@ print_progress_bar, write_template_file, add_file_group, - get_output_dir, + get_output_path, ) from themes import ThemeList, Theme, ThemeDownloadCount, get_community_themes @@ -91,7 +91,7 @@ def process_released_themes(overwrite: bool = False, verbose: bool = False) -> T theme_downloads = ThemeDownloadCount.get_theme_downloads() - themes_dir = os.path.dirname(get_output_dir(Theme.template, 'nonsense')) + themes_dir = os.path.dirname(get_output_path(Theme.template, 'nonsense')) collision_preventer = FileNameCaseCollisionsPreventer(themes_dir) for index, theme in enumerate(theme_list): @@ -154,7 +154,7 @@ def update_uncategorized_plugins(valid_plugins: PluginList, overwrite: bool = Tr ) # Alphabetize the plugin list - file_path = get_output_dir(template, UNCATEGORIZED) + file_path = get_output_path(template, UNCATEGORIZED) absolute_file_path = os.path.abspath(file_path) sort_links_under_heading(absolute_file_path) @@ -165,7 +165,7 @@ def process_authors(themes: ThemeList, verbose: bool = False) -> None: print("-----\nProcessing authors....\n") template = get_template("author") - authors_dir = os.path.dirname(get_output_dir(template, 'nonsense')) + authors_dir = os.path.dirname(get_output_path(template, 'nonsense')) collision_preventer = FileNameCaseCollisionsPreventer(authors_dir) all_authors = collate_authors(themes, plugins) diff --git a/.github/scripts/utils.py b/.github/scripts/utils.py index c694bfce6..83c15a531 100755 --- a/.github/scripts/utils.py +++ b/.github/scripts/utils.py @@ -63,7 +63,7 @@ def get_template_from_directory(directory: str, template_name_with_extensions: s return env.get_template(template_name_with_extensions) -def get_output_dir(template: Template, file_name: str) -> str: +def get_output_path(template: Template, file_name: str) -> str: template_name, _, _ = template.name.split(".") return os.path.join( "../..", @@ -77,7 +77,7 @@ def write_template_file(template: Template, overwrite: bool = False, verbose: bool = False, **kwargs: Any) -> str: - file_path = get_output_dir(template, file_name) + file_path = get_output_path(template, file_name) absolute_file_path = os.path.abspath(file_path) file_content = render_template_for_file(template, absolute_file_path, **kwargs) From 82bd5a0fe79772fc1f8f7e05b8b1489a22054dc9 Mon Sep 17 00:00:00 2001 From: Clare Macrae Date: Tue, 21 Mar 2023 18:31:56 +0000 Subject: [PATCH 5/8] Add get_output_dir() to utils.py Co-Authored-By: Andrew Nutter-Upham <59033+andynu@users.noreply.github.com> --- .github/scripts/update_releases.py | 5 +++-- .github/scripts/utils.py | 4 ++++ 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/.github/scripts/update_releases.py b/.github/scripts/update_releases.py index da39bece4..fbe8c18a6 100755 --- a/.github/scripts/update_releases.py +++ b/.github/scripts/update_releases.py @@ -20,6 +20,7 @@ write_template_file, add_file_group, get_output_path, + get_output_dir, ) from themes import ThemeList, Theme, ThemeDownloadCount, get_community_themes @@ -91,7 +92,7 @@ def process_released_themes(overwrite: bool = False, verbose: bool = False) -> T theme_downloads = ThemeDownloadCount.get_theme_downloads() - themes_dir = os.path.dirname(get_output_path(Theme.template, 'nonsense')) + themes_dir = get_output_dir(Theme.template) collision_preventer = FileNameCaseCollisionsPreventer(themes_dir) for index, theme in enumerate(theme_list): @@ -165,7 +166,7 @@ def process_authors(themes: ThemeList, verbose: bool = False) -> None: print("-----\nProcessing authors....\n") template = get_template("author") - authors_dir = os.path.dirname(get_output_path(template, 'nonsense')) + authors_dir = get_output_dir(template) collision_preventer = FileNameCaseCollisionsPreventer(authors_dir) all_authors = collate_authors(themes, plugins) diff --git a/.github/scripts/utils.py b/.github/scripts/utils.py index 83c15a531..515d45723 100755 --- a/.github/scripts/utils.py +++ b/.github/scripts/utils.py @@ -72,6 +72,10 @@ def get_output_path(template: Template, file_name: str) -> str: ) +def get_output_dir(template: Template) -> str: + return os.path.dirname(get_output_path(template, 'nonsense')) + + def write_template_file(template: Template, file_name: str, overwrite: bool = False, From c58cc1a4c6d91bdba30c0944e4121db27c8898d0 Mon Sep 17 00:00:00 2001 From: Clare Macrae Date: Tue, 21 Mar 2023 18:41:19 +0000 Subject: [PATCH 6/8] Cleaner code in get_output_path() and get_output_dir() Co-Authored-By: Andrew Nutter-Upham <59033+andynu@users.noreply.github.com> --- .github/scripts/tests/test_utils.py | 9 +++++++++ .github/scripts/utils.py | 10 ++++++---- 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/.github/scripts/tests/test_utils.py b/.github/scripts/tests/test_utils.py index 0265ec3e7..b507ae7d5 100644 --- a/.github/scripts/tests/test_utils.py +++ b/.github/scripts/tests/test_utils.py @@ -1,9 +1,18 @@ import os.path import utils +from themes import Theme def test_get_root_of_vault() -> None: root = utils.get_root_of_vault() contributing = os.path.join(root, 'CONTRIBUTING.md') assert os.path.exists(contributing), contributing + + +def test_get_output_path() -> None: + assert utils.get_output_path(Theme.template, 'CONTRIBUTING') == '../../02 - Community Expansions/02.05 All Community Expansions/Themes/CONTRIBUTING.md' + + +def test_get_output_dir() -> None: + assert utils.get_output_dir(Theme.template) == '../../02 - Community Expansions/02.05 All Community Expansions/Themes' diff --git a/.github/scripts/utils.py b/.github/scripts/utils.py index 515d45723..f7a0de5ea 100755 --- a/.github/scripts/utils.py +++ b/.github/scripts/utils.py @@ -64,16 +64,18 @@ def get_template_from_directory(directory: str, template_name_with_extensions: s def get_output_path(template: Template, file_name: str) -> str: - template_name, _, _ = template.name.split(".") return os.path.join( - "../..", - OUTPUT_DIR.get(template_name, "08 - Seedbox"), + get_output_dir(template), "{}.md".format(file_name), ) def get_output_dir(template: Template) -> str: - return os.path.dirname(get_output_path(template, 'nonsense')) + template_name, _, _ = template.name.split(".") + return os.path.join( + "../..", + OUTPUT_DIR.get(template_name, "08 - Seedbox"), + ) def write_template_file(template: Template, From 0c8919133197c27803e98ddce4d16619dac404a9 Mon Sep 17 00:00:00 2001 From: Clare Macrae Date: Tue, 21 Mar 2023 19:05:17 +0000 Subject: [PATCH 7/8] Prevent case-name collisions for plugin files Co-Authored-By: Andrew Nutter-Upham <59033+andynu@users.noreply.github.com> --- .github/scripts/update_releases.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/scripts/update_releases.py b/.github/scripts/update_releases.py index fbe8c18a6..806c0caf6 100755 --- a/.github/scripts/update_releases.py +++ b/.github/scripts/update_releases.py @@ -28,6 +28,8 @@ def process_released_plugins(overwrite: bool = False, verbose: bool = False) -> PluginList: print("-----\nProcessing plugins....\n") template = get_template("plugin") + plugins_dir = get_output_dir(template) + collision_preventer = FileNameCaseCollisionsPreventer(plugins_dir) plugin_list: PluginList = get_community_plugins() valid_plugins: PluginList = list() @@ -43,7 +45,7 @@ def process_released_plugins(overwrite: bool = False, verbose: bool = False) -> continue group = write_template_file( - template, plugin.id(), overwrite=overwrite, verbose=verbose, **plugin.data() + template, collision_preventer.get_name(plugin.id()), overwrite=overwrite, verbose=verbose, **plugin.data() ) valid_plugins.append(plugin) From cbd3ee5b832f5d364a8115687180928d1ae4a4c7 Mon Sep 17 00:00:00 2001 From: Clare Macrae Date: Tue, 21 Mar 2023 19:08:16 +0000 Subject: [PATCH 8/8] Move FileNameCaseCollisionsPreventer to utils.py --- .github/scripts/update_releases.py | 25 +------------------------ .github/scripts/utils.py | 22 ++++++++++++++++++++++ 2 files changed, 23 insertions(+), 24 deletions(-) diff --git a/.github/scripts/update_releases.py b/.github/scripts/update_releases.py index 806c0caf6..6e9c4beae 100755 --- a/.github/scripts/update_releases.py +++ b/.github/scripts/update_releases.py @@ -2,7 +2,6 @@ import os import sys import argparse -from pathlib import Path from typing import Sequence from authors import AllAuthors @@ -21,6 +20,7 @@ add_file_group, get_output_path, get_output_dir, + FileNameCaseCollisionsPreventer, ) from themes import ThemeList, Theme, ThemeDownloadCount, get_community_themes @@ -59,29 +59,6 @@ def process_released_plugins(overwrite: bool = False, verbose: bool = False) -> return valid_plugins -class FileNameCaseCollisionsPreventer: - def __init__(self, directory: str) -> None: - self.file_case_lookup = dict() - for filename in os.listdir(directory): - base_name = Path(filename).stem - self.file_case_lookup[base_name.lower()] = base_name - - def get_name(self, current_name: str) -> str: - """ - Prefer the existing capitalisation of any existing filename, - to prevent case-conflicts. - - :param current_name: - :return: - """ - if current_name.lower() in self.file_case_lookup: - existing_case = self.file_case_lookup[current_name.lower()] - if existing_case != current_name: - print(f'Overriding filename {current_name} to {existing_case}') - current_name = existing_case - return current_name - - def process_released_themes(overwrite: bool = False, verbose: bool = False) -> ThemeList: print("-----\nProcessing themes....\n") theme_list = get_community_themes() diff --git a/.github/scripts/utils.py b/.github/scripts/utils.py index f7a0de5ea..9847a8807 100755 --- a/.github/scripts/utils.py +++ b/.github/scripts/utils.py @@ -2,6 +2,7 @@ import json import glob import typing +from pathlib import Path from re import sub, search from typing import Dict, List, Union, Any @@ -302,3 +303,24 @@ def write_file(absolute_path: str, replacement: str) -> None: f.write(replacement) +class FileNameCaseCollisionsPreventer: + def __init__(self, directory: str) -> None: + self.file_case_lookup = dict() + for filename in os.listdir(directory): + base_name = Path(filename).stem + self.file_case_lookup[base_name.lower()] = base_name + + def get_name(self, current_name: str) -> str: + """ + Prefer the existing capitalisation of any existing filename, + to prevent case-conflicts. + + :param current_name: + :return: + """ + if current_name.lower() in self.file_case_lookup: + existing_case = self.file_case_lookup[current_name.lower()] + if existing_case != current_name: + print(f'Overriding filename {current_name} to {existing_case}') + current_name = existing_case + return current_name