From 2eb4e55cbafb0ceb5e883f13beba2774983d1aca Mon Sep 17 00:00:00 2001 From: Audric Ackermann Date: Thu, 19 Sep 2024 16:26:05 +1000 Subject: [PATCH] chore: test breaking workflow into separate jobs --- .../workflows/check_for_crowdin_updates.yml | 234 +++++++++++------- actions/setup_shared/action.yml | 23 ++ crowdin/generate_android_strings.py | 18 +- 3 files changed, 180 insertions(+), 95 deletions(-) create mode 100644 actions/setup_shared/action.yml diff --git a/.github/workflows/check_for_crowdin_updates.yml b/.github/workflows/check_for_crowdin_updates.yml index 25a8877..45c24e7 100644 --- a/.github/workflows/check_for_crowdin_updates.yml +++ b/.github/workflows/check_for_crowdin_updates.yml @@ -5,8 +5,21 @@ on: - cron: '0 0 * * 1' # Every Monday at 12:00 AM UTC, which is 10:00 AM Melbourne time (AEST) workflow_dispatch: +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +env: + PR_TITLE: "[Automated] Update translations from Crowdin" + PR_DESCRIPTION: | + [Automated] + This PR includes the latest translations from Crowdin + + Session uses the community-driven translation platform Crowdin for localization, anyone can contribute at https://getsession.org/translate + jobs: - fetch_convert_and_diff_translations: + + fetch_translations: runs-on: ubuntu-latest steps: - name: Checkout Repo Content @@ -14,41 +27,13 @@ jobs: with: path: 'scripts' # don't provide a branch (ref) so it uses the default for that event - - name: Checkout Android - uses: actions/checkout@v4 - with: - repository: 'oxen-io/session-android' - path: 'android' - submodules: recursive - ref: 'release/1.20.0' - - name: Checkout Desktop - uses: actions/checkout@v4 - with: - repository: 'oxen-io/session-desktop' - path: 'desktop' - ref: 'standardised_strings_qa_2' - - name: Checkout iOS - uses: actions/checkout@v4 - with: - repository: 'oxen-io/session-ios' - path: 'ios' - ref: 'dev' - name: Setup Python uses: actions/setup-python@v5 with: python-version: 3.8 cache: 'pip' # caching pip dependencies - - name: Setup Java - uses: actions/setup-java@v4 - with: - distribution: 'temurin' - java-version: 17 - cache: gradle - - - name: Setup Gradle - uses: gradle/actions/setup-gradle@v4 - - name: Install Dependencies + shell: bash run: | pip install -r ${{ github.workspace }}/scripts/crowdin/requirements.txt - name: Download Translations @@ -62,53 +47,82 @@ jobs: --glossary_id 407522 \ --concept_id 36 \ --skip-untranslated-strings - - - name: Upload xliff artefacts + - name: Upload download artefacts uses: actions/upload-artifact@v4 with: - name: session-xliff-artefact - path: "{{ github.workspace }}/raw_translations/*" + name: session-download-artefact + path: | + ${{ github.workspace }}/raw_translations/*.xliff + ${{ github.workspace }}/raw_translations/_non_translatable_strings.json + ${{ github.workspace }}/raw_translations/_project_info.json overwrite: true if-no-files-found: warn retention-days: 7 - - name: Prepare Android Strings - run: | - python "${{ github.workspace }}/scripts/crowdin/generate_android_strings.py" \ - "${{ github.workspace }}/raw_translations" \ - "${{ github.workspace }}/android/libsession/src/main/res" \ - "${{ github.workspace }}/android/libsession/src/main/java/org/session/libsession/utilities/NonTranslatableStringConstants.kt" - - + build_ios: + runs-on: ubuntu-latest + needs: [fetch_translations] + steps: + - name: Checkout Repo Content + uses: actions/checkout@v4 + with: + path: 'scripts' + # don't provide a branch (ref) so it uses the default for that event + - name: Checkout iOS + uses: actions/checkout@v4 + with: + repository: 'oxen-io/session-ios' + path: 'ios' + ref: 'dev' - - name: Print Android Strings - run: | - ls -l "${{ github.workspace }}/android/libsession/src/main/res/" + - name: Setup shared + uses: ./scripts/actions/setup_shared - - name: Prepare Desktop Strings - run: | - rm -rf ${{ github.workspace }}/desktop/_locales/* - python "${{ github.workspace }}/scripts/crowdin/generate_desktop_strings.py" \ - "${{ github.workspace }}/raw_translations" \ - "${{ github.workspace }}/desktop/_locales" \ - "${{ github.workspace }}/desktop/ts/localization/constants.ts" - name: Prepare iOS Strings run: | python "${{ github.workspace }}/scripts/crowdin/generate_ios_strings.py" \ "${{ github.workspace }}/raw_translations" \ "${{ github.workspace }}/ios/Session/Meta" \ "${{ github.workspace }}/ios/SessionUtilitiesKit/General/Constants.swift" - - - name: Upload Android artefacts + - name: Upload iOS artefacts uses: actions/upload-artifact@v4 with: - name: session-android-artefact + name: session-ios-artifact path: | - ${{ github.workspace }}/android/libsession/src/main/res/values*/strings.xml - ${{ github.workspace }}/android/libsession/src/main/java/org/session/libsession/utilities/NonTranslatableStringConstants.kt + ${{ github.workspace }}/ios/Session/Meta/Localizable.xcstrings + ${{ github.workspace }}/ios/SessionUtilitiesKit/General/Constants.swift overwrite: true if-no-files-found: warn retention-days: 7 + + + + build_desktop: + needs: [fetch_translations] + runs-on: ubuntu-latest + steps: + - name: Checkout Repo Content + uses: actions/checkout@v4 + with: + path: 'scripts' + # don't provide a branch (ref) so it uses the default for that event + - name: Setup shared + uses: ./scripts/actions/setup_shared + + - name: Checkout Desktop + uses: actions/checkout@v4 + with: + repository: 'oxen-io/session-desktop' + path: 'desktop' + ref: 'standardised_strings_qa_2' + + - name: Prepare Desktop Strings + run: | + rm -rf ${{ github.workspace }}/desktop/_locales/* + python "${{ github.workspace }}/scripts/crowdin/generate_desktop_strings.py" \ + "${{ github.workspace }}/raw_translations" \ + "${{ github.workspace }}/desktop/_locales" \ + "${{ github.workspace }}/desktop/ts/localization/constants.ts" - name: Upload Desktop artifacts uses: actions/upload-artifact@v4 with: @@ -119,20 +133,6 @@ jobs: overwrite: true if-no-files-found: warn retention-days: 7 - - name: Upload iOS artefacts - uses: actions/upload-artifact@v4 - with: - name: session-ios-artifact - path: | - ${{ github.workspace }}/ios/Session/Meta/Localizable.xcstrings - ${{ github.workspace }}/ios/SessionUtilitiesKit/General/Constants.swift - overwrite: true - if-no-files-found: warn - retention-days: 7 - - - name: Validate strings for Android - run: ${{ github.workspace }}/android/gradlew :libsession:packageDebugResources - # It's easier to find what went wrong with some strings if we can get the files from the artefact upload step above. # The job will still be marked as failed and no Pull Requests will be made. - name: Prepare QA strings @@ -151,10 +151,66 @@ jobs: if-no-files-found: warn retention-days: 7 + build_android: + runs-on: ubuntu-latest + needs: [fetch_translations] + + steps: + - name: Checkout Repo Content + uses: actions/checkout@v4 + with: + path: 'scripts' + # don't provide a branch (ref) so it uses the default for that event + - name: Checkout Android + uses: actions/checkout@v4 + with: + repository: 'oxen-io/session-android' + path: 'android' + submodules: recursive + ref: 'release/1.20.0' + + - name: Setup shared + uses: ./scripts/actions/setup_shared + - name: Setup Java + uses: actions/setup-java@v4 + with: + distribution: 'temurin' + java-version: 17 + cache: gradle + + - name: Setup Gradle + uses: gradle/actions/setup-gradle@v4 + + - name: Prepare Android Strings + run: | + python "${{ github.workspace }}/scripts/crowdin/generate_android_strings.py" \ + "${{ github.workspace }}/raw_translations" \ + "${{ github.workspace }}/android/libsession/src/main/res" \ + "${{ github.workspace }}/android/libsession/src/main/java/org/session/libsession/utilities/NonTranslatableStringConstants.kt" + - name: Upload Android artefacts + uses: actions/upload-artifact@v4 + with: + name: session-android-artefact + path: | + ${{ github.workspace }}/android/libsession/src/main/res/values*/strings.xml + ${{ github.workspace }}/android/libsession/src/main/java/org/session/libsession/utilities/NonTranslatableStringConstants.kt + overwrite: true + if-no-files-found: warn + retention-days: 7 + - name: Validate strings for Android + run: cd ${{ github.workspace }}/android && ${{ github.workspace }}/android/gradlew app:mergePlayDebugResources + make_prs: + needs: [build_android, build_ios, build_desktop] + runs-on: ubuntu-latest + + make_android_pr: + needs: [make_prs] + runs-on: ubuntu-latest + steps: # Not sure why yet, but uploading artefacts after creating the pull requests # seems to only include a part of what should be in. # As a dirty fix we upload the artefacts first, and then make the pull request @@ -163,40 +219,40 @@ jobs: with: path: 'android' token: ${{ secrets.CROWDIN_PR_TOKEN }} - title: "[Automated] Update translations from Crowdin" - body: | - [Automated] - This PR includes the latest translations from Crowdin - - Session uses the community-driven translation platform Crowdin for localization, anyone can contribute at https://getsession.org/translate + title: "{{env.PR_TITLE}}" + body: "{{env.PR_DESCRIPTION}}" branch: feature/update-crowdin-translations commit-message: "[Automated] Update translations from Crowdin" delete-branch: true + + make_desktop_pr: + needs: [make_prs] + runs-on: ubuntu-latest + steps: - name: Create Desktop Pull Request uses: peter-evans/create-pull-request@v6 with: path: 'desktop' token: ${{ secrets.CROWDIN_PR_TOKEN }} - title: "[Automated] Update translations from Crowdin" - body: | - [Automated] - This PR includes the latest translations from Crowdin - - Session uses the community-driven translation platform Crowdin for localization, anyone can contribute at https://getsession.org/translate + title: "{{env.PR_TITLE}}" + body: "{{env.PR_DESCRIPTION}}" branch: feature/update-crowdin-translations commit-message: "[Automated] Update translations from Crowdin" delete-branch: true + + + make_ios_pr: + needs: [make_prs] + runs-on: ubuntu-latest + steps: - name: Create iOS Pull Request uses: peter-evans/create-pull-request@v6 with: path: 'ios' token: ${{ secrets.CROWDIN_PR_TOKEN }} - title: "[Automated] Update translations from Crowdin" - body: | - [Automated] - This PR includes the latest translations from Crowdin - - Session uses the community-driven translation platform Crowdin for localization, anyone can contribute at https://getsession.org/translate + title: "{{env.PR_TITLE}}" + body: "{{env.PR_DESCRIPTION}}" branch: feature/update-crowdin-translations commit-message: "[Automated] Update translations from Crowdin" delete-branch: true + diff --git a/actions/setup_shared/action.yml b/actions/setup_shared/action.yml new file mode 100644 index 0000000..d6223ea --- /dev/null +++ b/actions/setup_shared/action.yml @@ -0,0 +1,23 @@ +name: 'Setup for all' +description: "Setup shared for all jobs" +runs: + using: 'composite' + steps: + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: 3.8 + cache: 'pip' # caching pip dependencies + - name: Install Dependencies + shell: bash + run: | + pip install -r ${{ github.workspace }}/scripts/crowdin/requirements.txt + + - uses: actions/download-artifact@v4 + with: + name: session-download-artefact + path: "${{ github.workspace }}/raw_translations" + + - name: Display structure of downloaded files + shell: bash + run: ls ${{ github.workspace }}/raw_translations \ No newline at end of file diff --git a/crowdin/generate_android_strings.py b/crowdin/generate_android_strings.py index ee8d532..2dc9783 100644 --- a/crowdin/generate_android_strings.py +++ b/crowdin/generate_android_strings.py @@ -26,7 +26,7 @@ def parse_xliff(file_path): root = tree.getroot() namespace = {'ns': 'urn:oasis:names:tc:xliff:document:1.2'} translations = {} - + # Handle plural groups for group in root.findall('.//ns:group[@restype="x-gettext-plurals"]', namespaces=namespace): plural_forms = {} @@ -42,7 +42,7 @@ def parse_xliff(file_path): plural_forms[form] = target.text if resname and plural_forms: translations[resname] = plural_forms - + # Handle non-plural translations for trans_unit in root.findall('.//ns:trans-unit', namespaces=namespace): resname = trans_unit.get('resname') @@ -50,14 +50,14 @@ def parse_xliff(file_path): target = trans_unit.find('ns:target', namespaces=namespace) if target is not None and target.text: translations[resname] = target.text - + return translations def convert_placeholders(text): def repl(match): var_name = match.group(1) index = len(set(re.findall(r'\{([^}]+)\}', text[:match.start()]))) + 1 - + if var_name in NUMERIC_VARIABLES: return f"%{index}$d" else: @@ -114,10 +114,15 @@ def convert_xliff_to_android_xml(input_file, output_dir, source_locale, locale, translations = parse_xliff(input_file) output_data = generate_android_xml(translations, app_name if is_source_language else None) + + # Generate output files language_code = locale.split('-')[0] region_code = locale.split('-')[1] if '-' in locale else None + print("language_code",language_code) + print("region_code",region_code) + if is_source_language: language_output_dir = os.path.join(output_dir, 'values') else: @@ -129,7 +134,8 @@ def convert_xliff_to_android_xml(input_file, output_dir, source_locale, locale, file.write(output_data) if region_code: - region_output_dir = os.path.join(output_dir, f'values-{language_code}-r{region_code}') + region_output_file = f'values-{language_code}-r{region_code}' if region_code != "419" else "values-b+es+419" + region_output_dir = os.path.join(output_dir,region_output_file ) os.makedirs(region_output_dir, exist_ok=True) region_output_file = os.path.join(region_output_dir, 'strings.xml') with open(region_output_file, 'w', encoding='utf-8') as file: @@ -179,7 +185,7 @@ def convert_all_files(input_directory): project_details = {} with open(project_info_file, 'r') as file: project_details = json.load(file) - + # Extract the language info and sort the target languages alphabetically by locale source_language = project_details['data']['sourceLanguage'] target_languages = project_details['data']['targetLanguages']