From 39b3a5d58477e9ca10da45071437194788c1aac7 Mon Sep 17 00:00:00 2001 From: Gabriel Omar Cotelli Date: Mon, 26 Aug 2024 16:53:29 -0300 Subject: [PATCH 1/5] Add NaturalLanguageTranslationScanner --- ...NaturalLanguageTranslationScanner.class.st | 97 +++++++++++++++++++ 1 file changed, 97 insertions(+) create mode 100644 source/Buoy-Development-Tools-Pharo-12/NaturalLanguageTranslationScanner.class.st diff --git a/source/Buoy-Development-Tools-Pharo-12/NaturalLanguageTranslationScanner.class.st b/source/Buoy-Development-Tools-Pharo-12/NaturalLanguageTranslationScanner.class.st new file mode 100644 index 0000000..f6f002b --- /dev/null +++ b/source/Buoy-Development-Tools-Pharo-12/NaturalLanguageTranslationScanner.class.st @@ -0,0 +1,97 @@ +" +I'm a tool scanning the source code of a project to detect senders of the localization messages to string literals, +so they can be exported and later translated. + +Use me with: +```smalltalk +(NaturalLanguageTranslationScanner forProjectNamed: 'ProjectName') export +``` +" +Class { + #name : 'NaturalLanguageTranslationScanner', + #superclass : 'Object', + #instVars : [ + 'targetRepository', + 'base64Codec', + 'environment' + ], + #category : 'Buoy-Development-Tools-Pharo-12', + #package : 'Buoy-Development-Tools-Pharo-12' +} + +{ #category : 'instance creation' } +NaturalLanguageTranslationScanner class >> forProjectNamed: aProjectName [ + + ^ self new initializeForProjectNamed: aProjectName +] + +{ #category : 'exporting' } +NaturalLanguageTranslationScanner >> export [ + + ^ self exportTo: targetRepository repositoryDirectory / 'locales' / 'en' + / ( '<1s>.json' expandMacrosWith: self projectName asLowercase ) +] + +{ #category : 'exporting' } +NaturalLanguageTranslationScanner >> exportOn: writeStream [ + + | stringsToTranslate translator translations | + stringsToTranslate := SortedCollection new. + self scanProjectMethodsCollectingTranslationsIn: stringsToTranslate. + translator := MonoglotNaturalLanguageTranslator for: 'en' asLanguageRange. + translations := OrderedDictionary new. + stringsToTranslate do: [ :string | + translations at: ( base64Codec encode: ( translator hashCodeFor: string ) ) put: string ]. + + ( NeoJSONWriter on: writeStream ) + prettyPrint: true; + nextPut: ( OrderedDictionary new + at: self projectName put: translations; + yourself ). + writeStream lf +] + +{ #category : 'exporting' } +NaturalLanguageTranslationScanner >> exportTo: targetFileReference [ + + targetFileReference + ensureCreateFile; + writeStreamDo: [ :stream | self exportOn: stream ] +] + +{ #category : 'initialization' } +NaturalLanguageTranslationScanner >> initializeForProjectNamed: aProjectName [ + + targetRepository := IceRepository registry + detect: [ :repository | repository name = aProjectName ] + ifNone: [ + ObjectNotFound signal: + ( 'Missing repository for project named <1s>' expandMacrosWith: + aProjectName ) + ]. + base64Codec := ZnBase64Encoder new + beForURLEncoding; + noPadding. + environment := RBBrowserEnvironment new forPackageNames: + ( targetRepository project packageNames select: [ :name | + PackageOrganizer default hasPackage: name ] ) +] + +{ #category : 'accessing' } +NaturalLanguageTranslationScanner >> projectName [ + + ^ targetRepository name +] + +{ #category : 'private' } +NaturalLanguageTranslationScanner >> scanProjectMethodsCollectingTranslationsIn: stringsToTranslate [ + + | searchRule | + searchRule := RBParseTreeLintRule new. + searchRule matcher + matches: '`#string localized' + do: [ :node :answer | stringsToTranslate add: node receiver value ]; + matches: '`#string localizedWithAll: `@values' + do: [ :node :answer | stringsToTranslate add: node receiver value ]. + searchRule runOnEnvironment: environment +] From 083f6450f301d9f5aca039cbd5545323b5a7edc6 Mon Sep 17 00:00:00 2001 From: Gabriel Omar Cotelli Date: Tue, 27 Aug 2024 11:35:04 -0300 Subject: [PATCH 2/5] Add loadJSONTranslationFilesIn: to PolyglotNaturalLanguageTranslator --- ...glotNaturalLanguageTranslatorTest.class.st | 50 +++++++++++++++++++ ...PolyglotNaturalLanguageTranslator.class.st | 31 ++++++++++++ 2 files changed, 81 insertions(+) diff --git a/source/Buoy-Localization-Tests/PolyglotNaturalLanguageTranslatorTest.class.st b/source/Buoy-Localization-Tests/PolyglotNaturalLanguageTranslatorTest.class.st index 7713e97..7ef1141 100644 --- a/source/Buoy-Localization-Tests/PolyglotNaturalLanguageTranslatorTest.class.st +++ b/source/Buoy-Localization-Tests/PolyglotNaturalLanguageTranslatorTest.class.st @@ -11,6 +11,20 @@ Class { #package : 'Buoy-Localization-Tests' } +{ #category : 'private' } +PolyglotNaturalLanguageTranslatorTest >> exportSpanishTranslationsIn: location [ + + | spanishLocation translationsToExport | + spanishLocation := location / 'locales' / 'es'. + spanishLocation ensureCreateDirectory. + + translationsToExport := self + translationDictionaryFor: #( 'Argentina' 'Brazil' 'USA' ) + and: #( 'Argentina' 'Brasil' 'EEUU' ). + spanishLocation / 'tests.json' writeStreamDo: [ :stream | + ( NeoJSONWriter on: stream ) nextPut: translationsToExport ] +] + { #category : 'running' } PolyglotNaturalLanguageTranslatorTest >> setUp [ @@ -18,6 +32,23 @@ PolyglotNaturalLanguageTranslatorTest >> setUp [ translator := PolyglotNaturalLanguageTranslator new ] +{ #category : 'tests' } +PolyglotNaturalLanguageTranslatorTest >> testLoadJSONTranslationFilesIn [ + + | tempLocation | + tempLocation := FileLocator temp / 'Buoy' / self selector asString. + tempLocation ensureCreateDirectory. + [ + self exportSpanishTranslationsIn: tempLocation. + + self assert: ( translator localize: 'Brazil' to: 'es' ) equals: 'Brazil'. + + translator loadJSONTranslationFilesIn: tempLocation / 'locales'. + self assert: ( translator localize: 'Brazil' to: 'es' ) equals: 'Brasil'. + self assert: ( translator localize: 'USA' to: 'es' ) equals: 'EEUU' + ] ensure: tempLocation ensureDeleteAll +] + { #category : 'tests' } PolyglotNaturalLanguageTranslatorTest >> testLocalizedToLanguageWithTranslation [ @@ -59,3 +90,22 @@ PolyglotNaturalLanguageTranslatorTest >> testLocalizedToLanguageWithoutTranslati assert: ( translator localize: 'Happy {1} year' withAll: { 2024 } to: 'es-AR' ) equals: 'Happy 2024 year' ] + +{ #category : 'private' } +PolyglotNaturalLanguageTranslatorTest >> translationDictionaryFor: original and: translated [ + + | englishTranslator base64Codec translations | + englishTranslator := MonoglotNaturalLanguageTranslator for: 'en' asLanguageRange. + base64Codec := ZnBase64Encoder new + beForURLEncoding; + noPadding. + translations := OrderedDictionary new. + original with: translated do: [ :string :translatedString | + translations + at: ( base64Codec encode: ( englishTranslator hashCodeFor: string ) ) + put: translatedString + ]. + ^ OrderedDictionary new + at: 'Tests' put: translations; + yourself +] diff --git a/source/Buoy-Localization/PolyglotNaturalLanguageTranslator.class.st b/source/Buoy-Localization/PolyglotNaturalLanguageTranslator.class.st index c0962ea..b8587ea 100644 --- a/source/Buoy-Localization/PolyglotNaturalLanguageTranslator.class.st +++ b/source/Buoy-Localization/PolyglotNaturalLanguageTranslator.class.st @@ -22,6 +22,37 @@ PolyglotNaturalLanguageTranslator >> initialize [ translators := SortedCollection sortBlock: [ :a :b | a specificity > b specificity ] ] +{ #category : 'configuring' } +PolyglotNaturalLanguageTranslator >> loadJSONTranslationFilesIn: aLocaleFileReference [ + "Loads in the language translator the translation files in the location. + It expectes a directory for each language tag containing any number of JSON + files. " + + aLocaleFileReference directories do: [ :localeDirectory | + self loadJSONTranslationsFor: localeDirectory basename asLanguageRange in: localeDirectory ] +] + +{ #category : 'private' } +PolyglotNaturalLanguageTranslator >> loadJSONTranslationsFor: languageRange in: localeDirectory [ + "Each file has the following structure: a first key level with a group, + and a second level where the key is the translation key and the value is the + translated string." + + | codec monoglotTranslator | + codec := ZnBase64Encoder new. + codec beForURLEncoding. + monoglotTranslator := self translatorFor: languageRange. + + ^ localeDirectory files select: [ :file | file extension = 'json' ] thenDo: [ :file | + | groupedTranslations | + groupedTranslations := NeoJSONObject fromString: file contents. + groupedTranslations keysAndValuesDo: [ :group :translations | + translations keysAndValuesDo: [ :translationKey :translation | + monoglotTranslator translationAt: ( codec decode: translationKey ) put: translation ] + ] + ] +] + { #category : 'localization' } PolyglotNaturalLanguageTranslator >> localize: string to: stringOrLanguageTag [ From 50a31fdcc7f508f4b9a310db00dba06e317f66c2 Mon Sep 17 00:00:00 2001 From: Gabriel Omar Cotelli Date: Tue, 27 Aug 2024 11:47:02 -0300 Subject: [PATCH 3/5] Fix test to not depend on NeoJSON --- ...glotNaturalLanguageTranslatorTest.class.st | 30 ++++--------------- 1 file changed, 5 insertions(+), 25 deletions(-) diff --git a/source/Buoy-Localization-Tests/PolyglotNaturalLanguageTranslatorTest.class.st b/source/Buoy-Localization-Tests/PolyglotNaturalLanguageTranslatorTest.class.st index 7ef1141..f3e1a53 100644 --- a/source/Buoy-Localization-Tests/PolyglotNaturalLanguageTranslatorTest.class.st +++ b/source/Buoy-Localization-Tests/PolyglotNaturalLanguageTranslatorTest.class.st @@ -14,15 +14,14 @@ Class { { #category : 'private' } PolyglotNaturalLanguageTranslatorTest >> exportSpanishTranslationsIn: location [ - | spanishLocation translationsToExport | + | spanishLocation | spanishLocation := location / 'locales' / 'es'. spanishLocation ensureCreateDirectory. - translationsToExport := self - translationDictionaryFor: #( 'Argentina' 'Brazil' 'USA' ) - and: #( 'Argentina' 'Brasil' 'EEUU' ). spanishLocation / 'tests.json' writeStreamDo: [ :stream | - ( NeoJSONWriter on: stream ) nextPut: translationsToExport ] + stream nextPutAll: + '{"Tests":{"6guTfqMXEB7iwmsDpIQ6Gc7O2KK5Zzw89AmnJsorD9g":"Argentina","B_YrAhdx089n4uH68YdpzF5cEZrX1NGEehHhHW1afss":"Brasil","qlqzWpF0wgYrf3aXsz-v5c5ATPX-z2v7vw3Ja6DZAEY":"EEUU"}}' + ] ] { #category : 'running' } @@ -46,7 +45,7 @@ PolyglotNaturalLanguageTranslatorTest >> testLoadJSONTranslationFilesIn [ translator loadJSONTranslationFilesIn: tempLocation / 'locales'. self assert: ( translator localize: 'Brazil' to: 'es' ) equals: 'Brasil'. self assert: ( translator localize: 'USA' to: 'es' ) equals: 'EEUU' - ] ensure: tempLocation ensureDeleteAll + ] ensure: [tempLocation ensureDeleteAll] ] { #category : 'tests' } @@ -90,22 +89,3 @@ PolyglotNaturalLanguageTranslatorTest >> testLocalizedToLanguageWithoutTranslati assert: ( translator localize: 'Happy {1} year' withAll: { 2024 } to: 'es-AR' ) equals: 'Happy 2024 year' ] - -{ #category : 'private' } -PolyglotNaturalLanguageTranslatorTest >> translationDictionaryFor: original and: translated [ - - | englishTranslator base64Codec translations | - englishTranslator := MonoglotNaturalLanguageTranslator for: 'en' asLanguageRange. - base64Codec := ZnBase64Encoder new - beForURLEncoding; - noPadding. - translations := OrderedDictionary new. - original with: translated do: [ :string :translatedString | - translations - at: ( base64Codec encode: ( englishTranslator hashCodeFor: string ) ) - put: translatedString - ]. - ^ OrderedDictionary new - at: 'Tests' put: translations; - yourself -] From 5a780e23d2bee38cc484f7bb34e53e191c35d188 Mon Sep 17 00:00:00 2001 From: Gabriel Omar Cotelli Date: Tue, 27 Aug 2024 11:51:12 -0300 Subject: [PATCH 4/5] Avoid NeoJSON dependency --- .../PolyglotNaturalLanguageTranslator.class.st | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/Buoy-Localization/PolyglotNaturalLanguageTranslator.class.st b/source/Buoy-Localization/PolyglotNaturalLanguageTranslator.class.st index b8587ea..a852a4b 100644 --- a/source/Buoy-Localization/PolyglotNaturalLanguageTranslator.class.st +++ b/source/Buoy-Localization/PolyglotNaturalLanguageTranslator.class.st @@ -45,7 +45,7 @@ PolyglotNaturalLanguageTranslator >> loadJSONTranslationsFor: languageRange in: ^ localeDirectory files select: [ :file | file extension = 'json' ] thenDo: [ :file | | groupedTranslations | - groupedTranslations := NeoJSONObject fromString: file contents. + groupedTranslations := STON fromString: file contents. groupedTranslations keysAndValuesDo: [ :group :translations | translations keysAndValuesDo: [ :translationKey :translation | monoglotTranslator translationAt: ( codec decode: translationKey ) put: translation ] From 65d8c7a70282d67693c6af0b15fff5f34a6d6a2f Mon Sep 17 00:00:00 2001 From: Gabriel Omar Cotelli Date: Tue, 27 Aug 2024 15:49:54 -0300 Subject: [PATCH 5/5] Avoid Zinc dependency --- .../NaturalLanguageTranslationScanner.class.st | 6 +----- .../PolyglotNaturalLanguageTranslatorTest.class.st | 2 +- .../PolyglotNaturalLanguageTranslator.class.st | 8 ++++---- 3 files changed, 6 insertions(+), 10 deletions(-) diff --git a/source/Buoy-Development-Tools-Pharo-12/NaturalLanguageTranslationScanner.class.st b/source/Buoy-Development-Tools-Pharo-12/NaturalLanguageTranslationScanner.class.st index f6f002b..a27c63a 100644 --- a/source/Buoy-Development-Tools-Pharo-12/NaturalLanguageTranslationScanner.class.st +++ b/source/Buoy-Development-Tools-Pharo-12/NaturalLanguageTranslationScanner.class.st @@ -12,7 +12,6 @@ Class { #superclass : 'Object', #instVars : [ 'targetRepository', - 'base64Codec', 'environment' ], #category : 'Buoy-Development-Tools-Pharo-12', @@ -41,7 +40,7 @@ NaturalLanguageTranslationScanner >> exportOn: writeStream [ translator := MonoglotNaturalLanguageTranslator for: 'en' asLanguageRange. translations := OrderedDictionary new. stringsToTranslate do: [ :string | - translations at: ( base64Codec encode: ( translator hashCodeFor: string ) ) put: string ]. + translations at: ( translator hashCodeFor: string ) hex put: string ]. ( NeoJSONWriter on: writeStream ) prettyPrint: true; @@ -69,9 +68,6 @@ NaturalLanguageTranslationScanner >> initializeForProjectNamed: aProjectName [ ( 'Missing repository for project named <1s>' expandMacrosWith: aProjectName ) ]. - base64Codec := ZnBase64Encoder new - beForURLEncoding; - noPadding. environment := RBBrowserEnvironment new forPackageNames: ( targetRepository project packageNames select: [ :name | PackageOrganizer default hasPackage: name ] ) diff --git a/source/Buoy-Localization-Tests/PolyglotNaturalLanguageTranslatorTest.class.st b/source/Buoy-Localization-Tests/PolyglotNaturalLanguageTranslatorTest.class.st index f3e1a53..277ecff 100644 --- a/source/Buoy-Localization-Tests/PolyglotNaturalLanguageTranslatorTest.class.st +++ b/source/Buoy-Localization-Tests/PolyglotNaturalLanguageTranslatorTest.class.st @@ -20,7 +20,7 @@ PolyglotNaturalLanguageTranslatorTest >> exportSpanishTranslationsIn: location [ spanishLocation / 'tests.json' writeStreamDo: [ :stream | stream nextPutAll: - '{"Tests":{"6guTfqMXEB7iwmsDpIQ6Gc7O2KK5Zzw89AmnJsorD9g":"Argentina","B_YrAhdx089n4uH68YdpzF5cEZrX1NGEehHhHW1afss":"Brasil","qlqzWpF0wgYrf3aXsz-v5c5ATPX-z2v7vw3Ja6DZAEY":"EEUU"}}' + '{"Tests":{"ea0b937ea317101ee2c26b03a4843a19ceced8a2b9673c3cf409a726ca2b0fd8":"Argentina","07f62b021771d3cf67e2e1faf18769cc5e5c119ad7d4d1847a11e11d6d5a7ecb":"Brasil","aa5ab35a9174c2062b7f7697b33fafe5ce404cf5fecf6bfbbf0dc96ba0d90046":"EEUU"}}' ] ] diff --git a/source/Buoy-Localization/PolyglotNaturalLanguageTranslator.class.st b/source/Buoy-Localization/PolyglotNaturalLanguageTranslator.class.st index a852a4b..3cc35ac 100644 --- a/source/Buoy-Localization/PolyglotNaturalLanguageTranslator.class.st +++ b/source/Buoy-Localization/PolyglotNaturalLanguageTranslator.class.st @@ -38,9 +38,7 @@ PolyglotNaturalLanguageTranslator >> loadJSONTranslationsFor: languageRange in: and a second level where the key is the translation key and the value is the translated string." - | codec monoglotTranslator | - codec := ZnBase64Encoder new. - codec beForURLEncoding. + | monoglotTranslator | monoglotTranslator := self translatorFor: languageRange. ^ localeDirectory files select: [ :file | file extension = 'json' ] thenDo: [ :file | @@ -48,7 +46,9 @@ PolyglotNaturalLanguageTranslator >> loadJSONTranslationsFor: languageRange in: groupedTranslations := STON fromString: file contents. groupedTranslations keysAndValuesDo: [ :group :translations | translations keysAndValuesDo: [ :translationKey :translation | - monoglotTranslator translationAt: ( codec decode: translationKey ) put: translation ] + monoglotTranslator + translationAt: ( ByteArray readHexFrom: translationKey ) + put: translation ] ] ] ]