From 65f6a144dc50678cf72ad8067e2f6a74356a4d9c Mon Sep 17 00:00:00 2001 From: Audric Ackermann Date: Thu, 19 Sep 2024 17:04:19 +1000 Subject: [PATCH] chore: test breaking workflow into separate jobs --- .../workflows/check_for_crowdin_updates.yml | 280 ++++++++++++------ actions/checkout_android/action.yml | 16 + actions/checkout_desktop/action.yml | 15 + actions/checkout_ios/action.yml | 11 + actions/setup_shared/action.yml | 23 ++ crowdin/generate_android_strings.py | 36 ++- 6 files changed, 271 insertions(+), 110 deletions(-) create mode 100644 actions/checkout_android/action.yml create mode 100644 actions/checkout_desktop/action.yml create mode 100644 actions/checkout_ios/action.yml create mode 100644 actions/setup_shared/action.yml diff --git a/.github/workflows/check_for_crowdin_updates.yml b/.github/workflows/check_for_crowdin_updates.yml index 25a8877..a48e875 100644 --- a/.github/workflows/check_for_crowdin_updates.yml +++ b/.github/workflows/check_for_crowdin_updates.yml @@ -1,12 +1,29 @@ name: Check for Crowdin Updates +# Not sure why yet, but uploading artefacts after creating the pull requests +# seems to only include a part of what should be in. +# As a dirty fix, we upload the artefacts first, and then make the pull request + on: schedule: - cron: '0 0 * * 1' # Every Monday at 12:00 AM UTC, which is 10:00 AM Melbourne time (AEST) workflow_dispatch: +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +env: + PR_TITLE: "[Automated] Update translations from Crowdin" + PR_DESCRIPTION: | + [Automated] + This PR includes the latest translations from Crowdin + + Session uses the community-driven translation platform Crowdin for localization, anyone can contribute at https://getsession.org/translate + jobs: - fetch_convert_and_diff_translations: + + fetch_translations: runs-on: ubuntu-latest steps: - name: Checkout Repo Content @@ -14,41 +31,13 @@ jobs: with: path: 'scripts' # don't provide a branch (ref) so it uses the default for that event - - name: Checkout Android - uses: actions/checkout@v4 - with: - repository: 'oxen-io/session-android' - path: 'android' - submodules: recursive - ref: 'release/1.20.0' - - name: Checkout Desktop - uses: actions/checkout@v4 - with: - repository: 'oxen-io/session-desktop' - path: 'desktop' - ref: 'standardised_strings_qa_2' - - name: Checkout iOS - uses: actions/checkout@v4 - with: - repository: 'oxen-io/session-ios' - path: 'ios' - ref: 'dev' - name: Setup Python uses: actions/setup-python@v5 with: python-version: 3.8 cache: 'pip' # caching pip dependencies - - name: Setup Java - uses: actions/setup-java@v4 - with: - distribution: 'temurin' - java-version: 17 - cache: gradle - - - name: Setup Gradle - uses: gradle/actions/setup-gradle@v4 - - name: Install Dependencies + shell: bash run: | pip install -r ${{ github.workspace }}/scripts/crowdin/requirements.txt - name: Download Translations @@ -62,77 +51,84 @@ jobs: --glossary_id 407522 \ --concept_id 36 \ --skip-untranslated-strings - - - name: Upload xliff artefacts + - name: Upload download artefacts uses: actions/upload-artifact@v4 with: - name: session-xliff-artefact - path: "{{ github.workspace }}/raw_translations/*" + name: session-download + path: | + ${{ github.workspace }}/raw_translations/*.xliff + ${{ github.workspace }}/raw_translations/_non_translatable_strings.json + ${{ github.workspace }}/raw_translations/_project_info.json overwrite: true if-no-files-found: warn retention-days: 7 - - name: Prepare Android Strings - run: | - python "${{ github.workspace }}/scripts/crowdin/generate_android_strings.py" \ - "${{ github.workspace }}/raw_translations" \ - "${{ github.workspace }}/android/libsession/src/main/res" \ - "${{ github.workspace }}/android/libsession/src/main/java/org/session/libsession/utilities/NonTranslatableStringConstants.kt" - - + build_ios: + runs-on: ubuntu-latest + needs: [fetch_translations] + steps: + - name: Checkout Repo Content + uses: actions/checkout@v4 + with: + path: 'scripts' + # don't provide a branch (ref) so it uses the default for that event + - name: Checkout iOS + uses: ./scripts/actions/checkout_ios - - name: Print Android Strings - run: | - ls -l "${{ github.workspace }}/android/libsession/src/main/res/" + - name: Setup shared + uses: ./scripts/actions/setup_shared - - name: Prepare Desktop Strings - run: | - rm -rf ${{ github.workspace }}/desktop/_locales/* - python "${{ github.workspace }}/scripts/crowdin/generate_desktop_strings.py" \ - "${{ github.workspace }}/raw_translations" \ - "${{ github.workspace }}/desktop/_locales" \ - "${{ github.workspace }}/desktop/ts/localization/constants.ts" - name: Prepare iOS Strings run: | python "${{ github.workspace }}/scripts/crowdin/generate_ios_strings.py" \ "${{ github.workspace }}/raw_translations" \ "${{ github.workspace }}/ios/Session/Meta" \ "${{ github.workspace }}/ios/SessionUtilitiesKit/General/Constants.swift" - - - name: Upload Android artefacts + - name: Upload iOS artefacts uses: actions/upload-artifact@v4 with: - name: session-android-artefact + name: session-ios path: | - ${{ github.workspace }}/android/libsession/src/main/res/values*/strings.xml - ${{ github.workspace }}/android/libsession/src/main/java/org/session/libsession/utilities/NonTranslatableStringConstants.kt + ${{ github.workspace }}/ios/Session/Meta/Localizable.xcstrings + ${{ github.workspace }}/ios/SessionUtilitiesKit/General/Constants.swift overwrite: true if-no-files-found: warn retention-days: 7 + + + + build_desktop: + needs: [fetch_translations] + runs-on: ubuntu-latest + steps: + - name: Checkout Repo Content + uses: actions/checkout@v4 + with: + path: 'scripts' + # don't provide a branch (ref) so it uses the default for that event + - name: Setup shared + uses: ./scripts/actions/setup_shared + + - name: Checkout Desktop + uses: ./scripts/actions/checkout_desktop + + + - name: Prepare Desktop Strings + run: | + python "${{ github.workspace }}/scripts/crowdin/generate_desktop_strings.py" \ + "${{ github.workspace }}/raw_translations" \ + "${{ github.workspace }}/desktop/_locales" \ + "${{ github.workspace }}/desktop/ts/localization/constants.ts" - name: Upload Desktop artifacts uses: actions/upload-artifact@v4 with: - name: session-desktop-artifact + name: session-desktop path: | ${{ github.workspace }}/desktop/_locales ${{ github.workspace }}/desktop/ts/localization/constants.ts overwrite: true if-no-files-found: warn retention-days: 7 - - name: Upload iOS artefacts - uses: actions/upload-artifact@v4 - with: - name: session-ios-artifact - path: | - ${{ github.workspace }}/ios/Session/Meta/Localizable.xcstrings - ${{ github.workspace }}/ios/SessionUtilitiesKit/General/Constants.swift - overwrite: true - if-no-files-found: warn - retention-days: 7 - - - name: Validate strings for Android - run: ${{ github.workspace }}/android/gradlew :libsession:packageDebugResources - # It's easier to find what went wrong with some strings if we can get the files from the artefact upload step above. # The job will still be marked as failed and no Pull Requests will be made. - name: Prepare QA strings @@ -143,7 +139,7 @@ jobs: - name: Upload QA artefacts uses: actions/upload-artifact@v4 with: - name: session-qa-artifact + name: session-qa path: | ${{ github.workspace }}/desktop/ts/localization/locales.ts ${{ github.workspace }}/desktop/ts/localization/constants.ts @@ -151,52 +147,148 @@ jobs: if-no-files-found: warn retention-days: 7 + build_android: + runs-on: ubuntu-latest + needs: [fetch_translations] + + steps: + - name: Checkout Repo Content + uses: actions/checkout@v4 + with: + path: 'scripts' + # don't provide a branch (ref) so it uses the default for that event + - name: Checkout Android + uses: ./scripts/actions/checkout_android + - name: Setup shared + uses: ./scripts/actions/setup_shared + - name: Setup Java + uses: actions/setup-java@v4 + with: + distribution: 'temurin' + java-version: 17 + cache: gradle + - name: Setup Gradle + uses: gradle/actions/setup-gradle@v4 + + - name: Prepare Android Strings + run: | + rm -rf ${{ github.workspace }}/android/libsession/src/main/res/values*/strings.xml + python "${{ github.workspace }}/scripts/crowdin/generate_android_strings.py" \ + "${{ github.workspace }}/raw_translations" \ + "${{ github.workspace }}/android/libsession/src/main/res" \ + "${{ github.workspace }}/android/libsession/src/main/java/org/session/libsession/utilities/NonTranslatableStringConstants.kt" + - name: Upload Android artefacts + uses: actions/upload-artifact@v4 + with: + name: session-android + path: | + ${{ github.workspace }}/android/libsession/src/main/res/values*/strings.xml + ${{ github.workspace }}/android/libsession/src/main/java/org/session/libsession/utilities/NonTranslatableStringConstants.kt + overwrite: true + if-no-files-found: warn + retention-days: 7 + + - name: Validate strings for Android + run: cd ${{ github.workspace }}/android && ${{ github.workspace }}/android/gradlew app:mergePlayDebugResources + + + make_prs: + needs: [build_android, build_ios, build_desktop] + runs-on: ubuntu-latest + + steps: + - name: Nothing to do + shell: bash + run: echo "Nothing to do here" + + make_android_pr: + needs: [make_prs] + runs-on: ubuntu-latest + steps: + - name: Checkout Repo Content + uses: actions/checkout@v4 + with: + path: 'scripts' + # don't provide a branch (ref) so it uses the default for that event + - name: Checkout Android + uses: ./scripts/actions/checkout_android + + - uses: actions/download-artifact@v4 + with: + name: session-android + # this has to be the first shared parent on the upload artefact task for Android + path: "${{ github.workspace }}/android/libsession/src/main" - # Not sure why yet, but uploading artefacts after creating the pull requests - # seems to only include a part of what should be in. - # As a dirty fix we upload the artefacts first, and then make the pull request - name: Create Android Pull Request uses: peter-evans/create-pull-request@v6 with: path: 'android' token: ${{ secrets.CROWDIN_PR_TOKEN }} - title: "[Automated] Update translations from Crowdin" - body: | - [Automated] - This PR includes the latest translations from Crowdin - - Session uses the community-driven translation platform Crowdin for localization, anyone can contribute at https://getsession.org/translate + title: "{{env.PR_TITLE}}" + body: "{{env.PR_DESCRIPTION}}" branch: feature/update-crowdin-translations commit-message: "[Automated] Update translations from Crowdin" delete-branch: true + + make_desktop_pr: + needs: [make_prs] + runs-on: ubuntu-latest + steps: + - name: Checkout Repo Content + uses: actions/checkout@v4 + with: + path: 'scripts' + # don't provide a branch (ref) so it uses the default for that event + - name: Checkout Desktop + uses: ./scripts/actions/checkout_desktop + + - uses: actions/download-artifact@v4 + with: + name: session-desktop + # this has to be the first shared parent on the upload artefact task for Desktop + path: "${{ github.workspace }}/desktop" + - name: Create Desktop Pull Request uses: peter-evans/create-pull-request@v6 with: path: 'desktop' token: ${{ secrets.CROWDIN_PR_TOKEN }} - title: "[Automated] Update translations from Crowdin" - body: | - [Automated] - This PR includes the latest translations from Crowdin - - Session uses the community-driven translation platform Crowdin for localization, anyone can contribute at https://getsession.org/translate + title: "{{env.PR_TITLE}}" + body: "{{env.PR_DESCRIPTION}}" branch: feature/update-crowdin-translations commit-message: "[Automated] Update translations from Crowdin" delete-branch: true + + + make_ios_pr: + needs: [make_prs] + runs-on: ubuntu-latest + steps: + - name: Checkout Repo Content + uses: actions/checkout@v4 + with: + path: 'scripts' + # don't provide a branch (ref) so it uses the default for that event + - name: Checkout iOS + uses: ./scripts/actions/checkout_ios + + - uses: actions/download-artifact@v4 + with: + name: session-ios + # this has to be the first shared parent on the upload artefact task for iOS + path: "${{ github.workspace }}/ios" + - name: Create iOS Pull Request uses: peter-evans/create-pull-request@v6 with: path: 'ios' token: ${{ secrets.CROWDIN_PR_TOKEN }} - title: "[Automated] Update translations from Crowdin" - body: | - [Automated] - This PR includes the latest translations from Crowdin - - Session uses the community-driven translation platform Crowdin for localization, anyone can contribute at https://getsession.org/translate + title: "{{env.PR_TITLE}}" + body: "{{env.PR_DESCRIPTION}}" branch: feature/update-crowdin-translations commit-message: "[Automated] Update translations from Crowdin" delete-branch: true + diff --git a/actions/checkout_android/action.yml b/actions/checkout_android/action.yml new file mode 100644 index 0000000..1aa4dce --- /dev/null +++ b/actions/checkout_android/action.yml @@ -0,0 +1,16 @@ +name: 'Setup for all' +description: "Setup shared for all jobs" +runs: + using: 'composite' + steps: + - name: Checkout Android + uses: actions/checkout@v4 + with: + repository: 'oxen-io/session-android' + path: 'android' + submodules: recursive + ref: 'release/1.20.0' + - name: Remove existing strings + shell: bash + run: | + rm -rf ${{ github.workspace }}/android/libsession/src/main/res/values*/strings.xml diff --git a/actions/checkout_desktop/action.yml b/actions/checkout_desktop/action.yml new file mode 100644 index 0000000..4b15fd2 --- /dev/null +++ b/actions/checkout_desktop/action.yml @@ -0,0 +1,15 @@ +name: 'Setup for all' +description: "Setup shared for all jobs" +runs: + using: 'composite' + steps: + - name: Checkout Desktop + uses: actions/checkout@v4 + with: + repository: 'oxen-io/session-desktop' + path: 'desktop' + ref: 'standardised_strings_qa_2' + - name: Remove existing strings + shell: bash + run: | + rm -rf ${{ github.workspace }}/desktop/_locales/* \ No newline at end of file diff --git a/actions/checkout_ios/action.yml b/actions/checkout_ios/action.yml new file mode 100644 index 0000000..b498b48 --- /dev/null +++ b/actions/checkout_ios/action.yml @@ -0,0 +1,11 @@ +name: 'Setup for all' +description: "Setup shared for all jobs" +runs: + using: 'composite' + steps: + - name: Checkout iOS + uses: actions/checkout@v4 + with: + repository: 'oxen-io/session-ios' + path: 'ios' + ref: 'dev' \ No newline at end of file diff --git a/actions/setup_shared/action.yml b/actions/setup_shared/action.yml new file mode 100644 index 0000000..f8ce360 --- /dev/null +++ b/actions/setup_shared/action.yml @@ -0,0 +1,23 @@ +name: 'Setup for all' +description: "Setup shared for all jobs" +runs: + using: 'composite' + steps: + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: 3.8 + cache: 'pip' # caching pip dependencies + - name: Install Dependencies + shell: bash + run: | + pip install -r ${{ github.workspace }}/scripts/crowdin/requirements.txt + + - uses: actions/download-artifact@v4 + with: + name: session-download + path: "${{ github.workspace }}/raw_translations" + + - name: Display structure of downloaded files + shell: bash + run: ls ${{ github.workspace }}/raw_translations \ No newline at end of file diff --git a/crowdin/generate_android_strings.py b/crowdin/generate_android_strings.py index ee8d532..02958c7 100644 --- a/crowdin/generate_android_strings.py +++ b/crowdin/generate_android_strings.py @@ -10,6 +10,14 @@ # Variables that should be treated as numeric (using %d) NUMERIC_VARIABLES = ['count', 'found_count', 'total_count'] +# Customizable mapping for output folder hierarchy +# Add entries here to customize the output path for specific locales +# Format: 'input_locale': 'output_path' +LOCALE_PATH_MAPPING = { + 'es-419': 'b+es+419', + # Add more mappings as needed +} + # Parse command-line arguments parser = argparse.ArgumentParser(description='Convert a XLIFF translation files to Android XML.') parser.add_argument('raw_translations_directory', help='Directory which contains the raw translation files') @@ -26,7 +34,7 @@ def parse_xliff(file_path): root = tree.getroot() namespace = {'ns': 'urn:oasis:names:tc:xliff:document:1.2'} translations = {} - + # Handle plural groups for group in root.findall('.//ns:group[@restype="x-gettext-plurals"]', namespaces=namespace): plural_forms = {} @@ -42,7 +50,7 @@ def parse_xliff(file_path): plural_forms[form] = target.text if resname and plural_forms: translations[resname] = plural_forms - + # Handle non-plural translations for trans_unit in root.findall('.//ns:trans-unit', namespaces=namespace): resname = trans_unit.get('resname') @@ -50,14 +58,14 @@ def parse_xliff(file_path): target = trans_unit.find('ns:target', namespaces=namespace) if target is not None and target.text: translations[resname] = target.text - + return translations def convert_placeholders(text): def repl(match): var_name = match.group(1) index = len(set(re.findall(r'\{([^}]+)\}', text[:match.start()]))) + 1 - + if var_name in NUMERIC_VARIABLES: return f"%{index}$d" else: @@ -105,7 +113,7 @@ def generate_android_xml(translations, app_name): return result -def convert_xliff_to_android_xml(input_file, output_dir, source_locale, locale, app_name): +def convert_xliff_to_android_xml(input_file, output_dir, source_locale, locale, locale_two_letter_code, app_name): if not os.path.exists(input_file): raise FileNotFoundError(f"Could not find '{input_file}' in raw translations directory") @@ -115,25 +123,20 @@ def convert_xliff_to_android_xml(input_file, output_dir, source_locale, locale, output_data = generate_android_xml(translations, app_name if is_source_language else None) # Generate output files - language_code = locale.split('-')[0] - region_code = locale.split('-')[1] if '-' in locale else None + output_locale = LOCALE_PATH_MAPPING.get(locale, LOCALE_PATH_MAPPING.get(locale_two_letter_code, locale_two_letter_code)) + if is_source_language: language_output_dir = os.path.join(output_dir, 'values') else: - language_output_dir = os.path.join(output_dir, f'values-{language_code}') + language_output_dir = os.path.join(output_dir, f'values-{output_locale}') os.makedirs(language_output_dir, exist_ok=True) language_output_file = os.path.join(language_output_dir, 'strings.xml') with open(language_output_file, 'w', encoding='utf-8') as file: file.write(output_data) - if region_code: - region_output_dir = os.path.join(output_dir, f'values-{language_code}-r{region_code}') - os.makedirs(region_output_dir, exist_ok=True) - region_output_file = os.path.join(region_output_dir, 'strings.xml') - with open(region_output_file, 'w', encoding='utf-8') as file: - file.write(output_data) + def convert_non_translatable_strings_to_kotlin(input_file, output_path): if not os.path.exists(input_file): @@ -179,7 +182,7 @@ def convert_all_files(input_directory): project_details = {} with open(project_info_file, 'r') as file: project_details = json.load(file) - + # Extract the language info and sort the target languages alphabetically by locale source_language = project_details['data']['sourceLanguage'] target_languages = project_details['data']['targetLanguages'] @@ -198,9 +201,10 @@ def convert_all_files(input_directory): source_locale = source_language['locale'] for language in [source_language] + target_languages: lang_locale = language['locale'] + lang_two_letter_code = language['twoLettersCode'] print(f"\033[2K{Fore.WHITE}⏳ Converting translations for {lang_locale} to target format...{Style.RESET_ALL}", end='\r') input_file = os.path.join(input_directory, f"{lang_locale}.xliff") - convert_xliff_to_android_xml(input_file, TRANSLATIONS_OUTPUT_DIRECTORY, source_locale, lang_locale, app_name) + convert_xliff_to_android_xml(input_file, TRANSLATIONS_OUTPUT_DIRECTORY, source_locale, lang_locale, lang_two_letter_code, app_name) print(f"\033[2K{Fore.GREEN}✅ All conversions complete{Style.RESET_ALL}") if __name__ == "__main__":