From 7ef0be03a622ad5828bf0b94a813b283afbbb219 Mon Sep 17 00:00:00 2001 From: per1234 Date: Thu, 30 Nov 2023 03:26:02 -0800 Subject: [PATCH 1/2] Simplify file discovery code in `markdown:check-links` task Since projects often contain many Markdown files and new files may be added or the paths of the existing files changed frequently, the best approach for validating Markdown files is to search the project file tree recursively for all Markdown files, with exclusions configured for the paths of any externally maintained files. The `markdown:check-links` task uses the markdown-link-check tool. This tool does not have a capability for discovering Markdown files so it is necessary to use the `find` command to discover the files, then pass their paths to the markdown-link-check tool. Previously the discovery code used `find` to generate an array of paths, which was iterated over passed individually to markdown-link-check in a `for` loop. The `for` loop is unnecessary because `find` has an `-exec` flag that can be used to execute commands using the discovered paths. Although the syntax and behavior of this flag is unintuitive, these disadvantages that come from its use are outweighed by the benefits of the significant amount of code that can be replaced by it. Since the `-exec`/`-execdir` flags are already in use in the assets and project infrastructure, the maintainer will be forced to work with them regardless. --- Taskfile.yml | 65 ++++++++----------- .../assets/check-markdown-task/Taskfile.yml | 57 +++++++--------- 2 files changed, 50 insertions(+), 72 deletions(-) diff --git a/Taskfile.yml b/Taskfile.yml index 9032c1ec..ec80b13a 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -638,50 +638,39 @@ tasks: echo "Please install: https://github.com/tcort/markdown-link-check#readme" exit 1 fi - # Default behavior of the task on Windows is to exit the task when the first broken link causes a non-zero - # exit status, but it's better to check all links before exiting. - set +o errexit - STATUS=0 # Using -regex instead of -name to avoid Task's behavior of globbing even when quoted on Windows # The odd method for escaping . in the regex is required for windows compatibility because mvdan.cc/sh gives # \ characters special treatment on Windows in an attempt to support them as path separators. - for file in $( - find . \ - -type d -name '.git' -prune -o \ - -type d -name '.licenses' -prune -o \ - -type d -name '__pycache__' -prune -o \ - -type d -name 'node_modules' -prune -o \ - -path './{{.CLANG_FORMAT_GOLDEN_TEST_DATA_FOLDER}}/samples' -prune -o \ - -path './{{.CLANG_FORMAT_INPUT_TEST_DATA_FOLDER}}/samples' -prune -o \ - -regex ".*[.]md" -print - ); do - markdown-link-check \ - --quiet \ - --config "./.markdown-link-check.json" \ - "$file" - STATUS=$(( $STATUS + $? )) - done - exit $STATUS - else - npx --package=markdown-link-check --call=' - STATUS=0 - for file in $( - find . \ - -type d -name '.git' -prune -o \ - -type d -name '.licenses' -prune -o \ - -type d -name '__pycache__' -prune -o \ - -type d -name 'node_modules' -prune -o \ - -path './{{.CLANG_FORMAT_GOLDEN_TEST_DATA_FOLDER}}/samples' -prune -o \ - -path './{{.CLANG_FORMAT_INPUT_TEST_DATA_FOLDER}}/samples' -prune -o \ - -regex ".*[.]md" -print - ); do + find . \ + -type d -name ".git" -prune -o \ + -type d -name ".licenses" -prune -o \ + -type d -name "__pycache__" -prune -o \ + -type d -name "node_modules" -prune -o \ + -path "./{{.CLANG_FORMAT_GOLDEN_TEST_DATA_FOLDER}}/samples" -prune -o \ + -path "./{{.CLANG_FORMAT_INPUT_TEST_DATA_FOLDER}}/samples" -prune -o \ + -regex ".*[.]md" \ + -exec \ markdown-link-check \ --quiet \ --config "./.markdown-link-check.json" \ - "$file" - STATUS=$(( $STATUS + $? )) - done - exit $STATUS + \{\} \ + + + else + npx --package=markdown-link-check --call=' + find . \ + -type d -name ".git" -prune -o \ + -type d -name ".licenses" -prune -o \ + -type d -name "__pycache__" -prune -o \ + -type d -name "node_modules" -prune -o \ + -path "./{{.CLANG_FORMAT_GOLDEN_TEST_DATA_FOLDER}}/samples" -prune -o \ + -path "./{{.CLANG_FORMAT_INPUT_TEST_DATA_FOLDER}}/samples" -prune -o \ + -regex ".*[.]md" \ + -exec \ + markdown-link-check \ + --quiet \ + --config "./.markdown-link-check.json" \ + \{\} \ + + ' fi diff --git a/workflow-templates/assets/check-markdown-task/Taskfile.yml b/workflow-templates/assets/check-markdown-task/Taskfile.yml index 8860d17d..dc0fe598 100644 --- a/workflow-templates/assets/check-markdown-task/Taskfile.yml +++ b/workflow-templates/assets/check-markdown-task/Taskfile.yml @@ -23,46 +23,35 @@ tasks: echo "Please install: https://github.com/tcort/markdown-link-check#readme" exit 1 fi - # Default behavior of the task on Windows is to exit the task when the first broken link causes a non-zero - # exit status, but it's better to check all links before exiting. - set +o errexit - STATUS=0 # Using -regex instead of -name to avoid Task's behavior of globbing even when quoted on Windows # The odd method for escaping . in the regex is required for windows compatibility because mvdan.cc/sh gives # \ characters special treatment on Windows in an attempt to support them as path separators. - for file in $( - find . \ - -type d -name '.git' -prune -o \ - -type d -name '.licenses' -prune -o \ - -type d -name '__pycache__' -prune -o \ - -type d -name 'node_modules' -prune -o \ - -regex ".*[.]md" -print - ); do - markdown-link-check \ - --quiet \ - --config "./.markdown-link-check.json" \ - "$file" - STATUS=$(( $STATUS + $? )) - done - exit $STATUS - else - npx --package=markdown-link-check --call=' - STATUS=0 - for file in $( - find . \ - -type d -name '.git' -prune -o \ - -type d -name '.licenses' -prune -o \ - -type d -name '__pycache__' -prune -o \ - -type d -name 'node_modules' -prune -o \ - -regex ".*[.]md" -print - ); do + find . \ + -type d -name ".git" -prune -o \ + -type d -name ".licenses" -prune -o \ + -type d -name "__pycache__" -prune -o \ + -type d -name "node_modules" -prune -o \ + -regex ".*[.]md" \ + -exec \ markdown-link-check \ --quiet \ --config "./.markdown-link-check.json" \ - "$file" - STATUS=$(( $STATUS + $? )) - done - exit $STATUS + \{\} \ + + + else + npx --package=markdown-link-check --call=' + find . \ + -type d -name ".git" -prune -o \ + -type d -name ".licenses" -prune -o \ + -type d -name "__pycache__" -prune -o \ + -type d -name "node_modules" -prune -o \ + -regex ".*[.]md" \ + -exec \ + markdown-link-check \ + --quiet \ + --config "./.markdown-link-check.json" \ + \{\} \ + + ' fi From fa253b2ecc316cd9820338b704c3719322b2eeea Mon Sep 17 00:00:00 2001 From: per1234 Date: Thu, 30 Nov 2023 04:10:34 -0800 Subject: [PATCH 2/2] Avoid platform-specific code in `markdown:check-links` task The `markdown:check-links` task uses the markdown-link-check tool. This tool does not have a capability for discovering Markdown files so it is necessary to use the `find` command to discover the files, then pass their paths to the markdown-link-check tool. Since it is managed as a project dependency using npm, the markdown-link-check tool is invoked using npx. Since the `find` command must be ran in combination with markdown-link-check, it is necessary to use the `--call` flag of npx. Even though Windows contributors are required to use a POSIX-compliant shell such as Git Bash when working with the assets, the commands ran via the `--call` flag are executed using the native shell, which means the Windows command interpreter on a Windows machine even if the task was invoked via a different shell. This causes commands completely valid for use on a Linux or macOS machine to fail to run on a Windows machine due to the significant differences in the Windows command interpreter syntax. During the original development of the task, a reasonably maintainable cross-platform command could not be found. Lacking a better option the hacky approach was taken of using a conditional to run a different command depending on whether the task was running on Windows or not, and not using npx for the Windows command. This resulted in a degraded experience for Windows contributors because they were forced to manually manage the markdown-link-check tool dependency and make it available in the system path. It also resulted in duplication of the fairly complex code contained in the task. Following the elimination of unnecessary complexity in the task code, it became possible to use a single command on all platforms. The Windows command interpreter syntax still posed a difficulty even for the simplified command: A beneficial practice, used throughout the assets, is to break commands into multiple lines to make them and the diffs of their development easier to read. With a POSIX-compliant shell this is accomplished by escaping the introduced newlines with a backslash. However, the Windows command interpreter does not recognize this syntax, making the commands formatted in that manner invalid when the task was ran on a Windows machine. The identified solution was to define the command via a Taskfile variable. The YAML syntax was carefully chosen to support the use of the familiar backslash escaping syntax, while also producing in a string that did not contain this non-portable escaping syntax after passing through the YAML parser. --- Taskfile.yml | 73 ++++++++----------- .../assets/check-markdown-task/Taskfile.yml | 67 +++++++---------- 2 files changed, 58 insertions(+), 82 deletions(-) diff --git a/Taskfile.yml b/Taskfile.yml index ec80b13a..1ec7e8ea 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -625,54 +625,41 @@ tasks: # Source: https://github.com/arduino/tooling-project-assets/blob/main/workflow-templates/assets/check-markdown-task/Taskfile.yml markdown:check-links: desc: Check for broken links - deps: - - task: docs:generate - - task: npm:install-deps - cmds: - - | - if [[ "{{.OS}}" == "Windows_NT" ]]; then - # npx --call uses the native shell, which makes it too difficult to use npx for this application on Windows, - # so the Windows user is required to have markdown-link-check installed and in PATH. - if ! which markdown-link-check &>/dev/null; then - echo "markdown-link-check not found or not in PATH." - echo "Please install: https://github.com/tcort/markdown-link-check#readme" - exit 1 - fi - # Using -regex instead of -name to avoid Task's behavior of globbing even when quoted on Windows - # The odd method for escaping . in the regex is required for windows compatibility because mvdan.cc/sh gives - # \ characters special treatment on Windows in an attempt to support them as path separators. + vars: + # The command is defined in a Taskfile variable to allow it to be broken into multiple lines for readability. + # This can't be done in the `cmd` object of the Taskfile because `npx --call` uses the native shell, which causes + # standard newline escaping syntax to not work when the task is run on Windows. + # + # Using -regex instead of -name to avoid Task's behavior of globbing even when quoted on Windows + # The odd method for escaping . in the regex is required for windows compatibility because mvdan.cc/sh gives + # \ characters special treatment on Windows in an attempt to support them as path separators. + # + # prettier-ignore + CHECK_LINKS_COMMAND: + " find . \ - -type d -name ".git" -prune -o \ - -type d -name ".licenses" -prune -o \ - -type d -name "__pycache__" -prune -o \ - -type d -name "node_modules" -prune -o \ - -path "./{{.CLANG_FORMAT_GOLDEN_TEST_DATA_FOLDER}}/samples" -prune -o \ - -path "./{{.CLANG_FORMAT_INPUT_TEST_DATA_FOLDER}}/samples" -prune -o \ - -regex ".*[.]md" \ + -type d -name \".git\" -prune -o \ + -type d -name \".licenses\" -prune -o \ + -type d -name \"__pycache__\" -prune -o \ + -type d -name \"node_modules\" -prune -o \ + -path \"./{{.CLANG_FORMAT_GOLDEN_TEST_DATA_FOLDER}}/samples\" -prune -o \ + -path \"./{{.CLANG_FORMAT_INPUT_TEST_DATA_FOLDER}}/samples\" -prune -o \ + -regex \".*[.]md\" \ -exec \ markdown-link-check \ --quiet \ - --config "./.markdown-link-check.json" \ - \{\} \ + --config \"./.markdown-link-check.json\" \ + \\{\\} \ + - else - npx --package=markdown-link-check --call=' - find . \ - -type d -name ".git" -prune -o \ - -type d -name ".licenses" -prune -o \ - -type d -name "__pycache__" -prune -o \ - -type d -name "node_modules" -prune -o \ - -path "./{{.CLANG_FORMAT_GOLDEN_TEST_DATA_FOLDER}}/samples" -prune -o \ - -path "./{{.CLANG_FORMAT_INPUT_TEST_DATA_FOLDER}}/samples" -prune -o \ - -regex ".*[.]md" \ - -exec \ - markdown-link-check \ - --quiet \ - --config "./.markdown-link-check.json" \ - \{\} \ - + - ' - fi + " + deps: + - task: docs:generate + - task: npm:install-deps + cmds: + - | + npx \ + --package=markdown-link-check \ + --call='{{.CHECK_LINKS_COMMAND}}' # Source: https://github.com/arduino/tooling-project-assets/blob/main/workflow-templates/assets/check-markdown-task/Taskfile.yml markdown:fix: diff --git a/workflow-templates/assets/check-markdown-task/Taskfile.yml b/workflow-templates/assets/check-markdown-task/Taskfile.yml index dc0fe598..a968b975 100644 --- a/workflow-templates/assets/check-markdown-task/Taskfile.yml +++ b/workflow-templates/assets/check-markdown-task/Taskfile.yml @@ -10,50 +10,39 @@ tasks: # Source: https://github.com/arduino/tooling-project-assets/blob/main/workflow-templates/assets/check-markdown-task/Taskfile.yml markdown:check-links: desc: Check for broken links - deps: - - task: docs:generate - - task: npm:install-deps - cmds: - - | - if [[ "{{.OS}}" == "Windows_NT" ]]; then - # npx --call uses the native shell, which makes it too difficult to use npx for this application on Windows, - # so the Windows user is required to have markdown-link-check installed and in PATH. - if ! which markdown-link-check &>/dev/null; then - echo "markdown-link-check not found or not in PATH." - echo "Please install: https://github.com/tcort/markdown-link-check#readme" - exit 1 - fi - # Using -regex instead of -name to avoid Task's behavior of globbing even when quoted on Windows - # The odd method for escaping . in the regex is required for windows compatibility because mvdan.cc/sh gives - # \ characters special treatment on Windows in an attempt to support them as path separators. + vars: + # The command is defined in a Taskfile variable to allow it to be broken into multiple lines for readability. + # This can't be done in the `cmd` object of the Taskfile because `npx --call` uses the native shell, which causes + # standard newline escaping syntax to not work when the task is run on Windows. + # + # Using -regex instead of -name to avoid Task's behavior of globbing even when quoted on Windows + # The odd method for escaping . in the regex is required for windows compatibility because mvdan.cc/sh gives + # \ characters special treatment on Windows in an attempt to support them as path separators. + # + # prettier-ignore + CHECK_LINKS_COMMAND: + " find . \ - -type d -name ".git" -prune -o \ - -type d -name ".licenses" -prune -o \ - -type d -name "__pycache__" -prune -o \ - -type d -name "node_modules" -prune -o \ - -regex ".*[.]md" \ + -type d -name \".git\" -prune -o \ + -type d -name \".licenses\" -prune -o \ + -type d -name \"__pycache__\" -prune -o \ + -type d -name \"node_modules\" -prune -o \ + -regex \".*[.]md\" \ -exec \ markdown-link-check \ --quiet \ - --config "./.markdown-link-check.json" \ - \{\} \ + --config \"./.markdown-link-check.json\" \ + \\{\\} \ + - else - npx --package=markdown-link-check --call=' - find . \ - -type d -name ".git" -prune -o \ - -type d -name ".licenses" -prune -o \ - -type d -name "__pycache__" -prune -o \ - -type d -name "node_modules" -prune -o \ - -regex ".*[.]md" \ - -exec \ - markdown-link-check \ - --quiet \ - --config "./.markdown-link-check.json" \ - \{\} \ - + - ' - fi + " + deps: + - task: docs:generate + - task: npm:install-deps + cmds: + - | + npx \ + --package=markdown-link-check \ + --call='{{.CHECK_LINKS_COMMAND}}' # Source: https://github.com/arduino/tooling-project-assets/blob/main/workflow-templates/assets/check-markdown-task/Taskfile.yml markdown:fix: