Skip to content

Commit

Permalink
Added push ai translations support to json sync
Browse files Browse the repository at this point in the history
  • Loading branch information
@maallen
maallen committed Oct 21, 2024
1 parent 5ef5679 commit 1cd24c2
Show file tree
Hide file tree
Showing 4 changed files with 213 additions and 50 deletions.
18 changes: 15 additions & 3 deletions webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyTMSSmartling.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -420,7 +420,6 @@ public void push(
SmartlingOptions options = SmartlingOptions.parseList(optionList);

if (options.isJsonSync()) {
// TODO(mallen) Need to update json sync push to handle AI translations
thirdPartyTMSSmartlingWithJson.push(
repository,
projectId,
Expand Down Expand Up @@ -761,8 +760,13 @@ public void pushAITranslations(
.tag("repository", repository.getName())) {
SmartlingOptions options = SmartlingOptions.parseList(optionList);
if (options.isJsonSync()) {
// TODO: Add json sync implementation
throw new RuntimeException("Not yet implemented");
thirdPartyTMSSmartlingWithJson.pushAiTranslations(
repository,
projectId,
localeMapping,
skipTextUnitsWithPattern,
skipAssetsWithPathPattern,
includeTextUnitsWithPattern);
}

AndroidStringDocumentMapper mapper = new AndroidStringDocumentMapper(pluralSeparator, null);
Expand Down Expand Up @@ -862,6 +866,14 @@ private SmartlingFile uploadAiTranslationBatch(
filterTmTextUnitIds);
aiTranslationService.updateVariantStatusToMTReview(
batch.stream().map(TextUnitDTO::getTmTextUnitCurrentVariantId).toList());
meterRegistry
.counter(
"SmartlingSync.uploadAiTranslationsBatch",
"repository",
repository.getName(),
"jsonSync",
"false")
.increment(batch.size());
return file;
}

Expand Down
179 changes: 143 additions & 36 deletions .../src/main/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyTMSSmartlingWithJson.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
import com.box.l10n.mojito.entity.RepositoryLocale;
import com.box.l10n.mojito.iterators.PageFetcherOffsetAndLimitSplitIterator;
import com.box.l10n.mojito.iterators.Spliterators;
import com.box.l10n.mojito.service.ai.translation.AITranslationConfiguration;
import com.box.l10n.mojito.service.thirdparty.smartling.SmartlingJsonConverter;
import com.box.l10n.mojito.service.thirdparty.smartling.SmartlingOptions;
import com.box.l10n.mojito.service.tm.importer.TextUnitBatchImporterService;
Expand All @@ -32,6 +33,7 @@
import java.util.regex.Pattern;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import java.util.stream.StreamSupport;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
Expand Down Expand Up @@ -61,6 +63,8 @@ public class ThirdPartyTMSSmartlingWithJson {

ThirdPartyFileChecksumRepository thirdPartyFileChecksumRepository;

AITranslationConfiguration aiTranslationConfiguration;

int batchSize = 5000;

public ThirdPartyTMSSmartlingWithJson(
Expand All @@ -70,14 +74,16 @@ public ThirdPartyTMSSmartlingWithJson(
TextUnitBatchImporterService textUnitBatchImporterService,
SmartlingJsonKeys smartlingJsonKeys,
MeterRegistry meterRegistry,
ThirdPartyFileChecksumRepository thirdPartyFileChecksumRepository) {
ThirdPartyFileChecksumRepository thirdPartyFileChecksumRepository,
AITranslationConfiguration aiTranslationConfiguration) {
this.smartlingClient = smartlingClient;
this.smartlingJsonConverter = smartlingJsonConverter;
this.textUnitSearcher = textUnitSearcher;
this.textUnitBatchImporterService = textUnitBatchImporterService;
this.smartlingJsonKeys = smartlingJsonKeys;
this.meterRegistry = meterRegistry;
this.thirdPartyFileChecksumRepository = thirdPartyFileChecksumRepository;
this.aiTranslationConfiguration = aiTranslationConfiguration;
}

void push(
Expand Down Expand Up @@ -255,6 +261,101 @@ && isFileEqualToPreviousRun(
});
}

public void pushAiTranslations(
Repository repository,
String projectId,
Map<String, String> localeMapping,
String skipTextUnitsWithPattern,
String skipAssetsWithPathPattern,
String includeTextUnitsWithPattern) {

logger.info(
"Push AI translations from repository: {} into project: {}",
repository.getName(),
projectId);

try (var timer =
Timer.resource(meterRegistry, "ThirdPartyTMSSmartlingWithJson.pushAiTranslations")
.tag("repository", repository.getName())) {
getRepositoryLocaleWithoutRootStream(repository)
.forEach(
repositoryLocale -> {
Map<String, List<TextUnitDTO>> textUnitsByUploadedFileUri =
StreamSupport.stream(
getTargetTextUnitIterator(
repository,
repositoryLocale.getLocale().getId(),
skipTextUnitsWithPattern,
skipAssetsWithPathPattern,
includeTextUnitsWithPattern,
StatusFilter.MT_TRANSLATED),
false)
.collect(Collectors.groupingBy(TextUnitDTO::getUploadedFileUri));

textUnitsByUploadedFileUri.forEach(
(uploadedFileUri, textUnitDTOS) -> {
try (var timer2 =
Timer.resource(
meterRegistry,
"ThirdPartyTMSSmartlingWithJson.pushAiTranslations.batch")
.tag("repository", repository.getName())) {

uploadLocalizedFile(
projectId,
localeMapping,
repositoryLocale,
uploadedFileUri,
textUnitDTOS);
meterRegistry
.counter(
"SmartlingSync.uploadAiTranslationsBatch",
"repository",
repository.getName(),
"jsonSync",
"true")
.increment(textUnitDTOS.size());
}
});
});
}
}

private void uploadLocalizedFile(
String projectId,
Map<String, String> localeMapping,
RepositoryLocale repositoryLocale,
String uploadedFileUri,
List<TextUnitDTO> textUnitDTOS) {
String fileContent =
smartlingJsonConverter.textUnitDTOsToJsonString(textUnitDTOS, TextUnitDTO::getTarget);
String smartlingLocale = getSmartlingLocale(localeMapping, repositoryLocale);

Mono.fromCallable(
() ->
smartlingClient.uploadLocalizedFile(
projectId, uploadedFileUri, "json", smartlingLocale, fileContent, null, null))
.retryWhen(
smartlingClient
.getRetryConfiguration()
.doBeforeRetry(
e ->
logger.info(
String.format(
"Retrying after failure to upload localized file %s in project %s",
uploadedFileUri, projectId),
e.failure())))
.doOnError(
e -> {
String msg =
String.format(
"Error uploading localized file %s in project %s",
uploadedFileUri, projectId);
logger.error(msg, e);
throw new SmartlingClientException(msg, e);
})
.block();
}

public void pushTranslations(
Repository repository,
String projectId,
Expand Down Expand Up @@ -291,41 +392,12 @@ public void pushTranslations(
.tag("repository", repository.getName())) {

String fileName = getSourceFileName(repository.getName(), index);
String fileContent =
smartlingJsonConverter.textUnitDTOsToJsonString(
textUnitDTOS, TextUnitDTO::getTarget);
String smartlingLocale =
getSmartlingLocale(localeMapping, repositoryLocale);
Mono.fromCallable(
() ->
smartlingClient.uploadLocalizedFile(
projectId,
fileName,
"json",
smartlingLocale,
fileContent,
null,
null))
.retryWhen(
smartlingClient
.getRetryConfiguration()
.doBeforeRetry(
e ->
logger.info(
String.format(
"Retrying after failure to upload localized file %s in project %s",
fileName, projectId),
e.failure())))
.doOnError(
e -> {
String msg =
String.format(
"Error uploading localized file %s in project %s",
fileName, projectId);
logger.error(msg, e);
throw new SmartlingClientException(msg, e);
})
.block();
uploadLocalizedFile(
projectId,
localeMapping,
repositoryLocale,
fileName,
textUnitDTOS);
return index;
}
})
Expand Down Expand Up @@ -353,6 +425,41 @@ PageFetcherOffsetAndLimitSplitIterator<TextUnitDTO> getSourceTextUnitIterator(
parameters.setLimit(limit);
parameters.setPluralFormsFiltered(true);
parameters.setOrderByTextUnitID(true);
parameters.setExcludeUnexpiredPendingMT(aiTranslationConfiguration.isEnabled());
parameters.setAiTranslationExpiryDuration(
aiTranslationConfiguration.getExpiryDuration());
List<TextUnitDTO> search = textUnitSearcher.search(parameters);
return search;
},
batchSize);

return textUnitDTOPageFetcherOffsetAndLimitSplitIterator;
}

PageFetcherOffsetAndLimitSplitIterator<TextUnitDTO> getTargetTextUnitIterator(
Repository repository,
Long localeId,
String skipTextUnitsWithPattern,
String skipAssetsWithPathPattern,
String includeTextUnitsWithPattern,
StatusFilter statusFilter) {

PageFetcherOffsetAndLimitSplitIterator<TextUnitDTO>
textUnitDTOPageFetcherOffsetAndLimitSplitIterator =
new PageFetcherOffsetAndLimitSplitIterator<>(
(offset, limit) -> {
TextUnitSearcherParameters parameters = new TextUnitSearcherParameters();
parameters.setRepositoryIds(repository.getId());
parameters.setLocaleId(localeId);
parameters.setDoNotTranslateFilter(false);
parameters.setStatusFilter(statusFilter);
parameters.setUsedFilter(UsedFilter.USED);
parameters.setSkipTextUnitWithPattern(skipTextUnitsWithPattern);
parameters.setSkipAssetPathWithPattern(skipAssetsWithPathPattern);
parameters.setIncludeTextUnitsWithPattern(includeTextUnitsWithPattern);
parameters.setOffset(offset);
parameters.setLimit(limit);
parameters.setPluralFormsFiltered(true);
List<TextUnitDTO> search = textUnitSearcher.search(parameters);
return search;
},
Expand Down
54 changes: 44 additions & 10 deletions webapp/src/test/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyTMSSmartlingTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -990,7 +990,8 @@ public void testPushTranslationsSingularsNoBatches()
eq(locale.getBcp47Tag()),
startsWith("<?xml version="),
eq(null),
eq(null));
eq(null),
eq("PUBLISHED"));
});
}

Expand All @@ -1000,7 +1001,14 @@ public void testRetriesExhaustedPushTranslations()
doThrow(new SmartlingClientException(new HttpServerErrorException(HttpStatus.GATEWAY_TIMEOUT)))
.when(smartlingClient)
.uploadLocalizedFile(
anyString(), anyString(), anyString(), anyString(), anyString(), any(), any());
anyString(),
anyString(),
anyString(),
anyString(),
anyString(),
any(),
any(),
anyString());
List<SmartlingFile> result;
Repository repository =
repositoryService.createRepository(testIdWatcher.getEntityName("batchRepo"));
Expand Down Expand Up @@ -1050,15 +1058,29 @@ public void testRetriesExhaustedPushTranslations()

verify(smartlingClient, times(11))
.uploadLocalizedFile(
anyString(), anyString(), anyString(), anyString(), anyString(), any(), any());
anyString(),
anyString(),
anyString(),
anyString(),
anyString(),
any(),
any(),
eq("PUBLISHED"));
}

@Test
public void testRetryPushTranslations()
throws RepositoryNameAlreadyUsedException, RepositoryLocaleCreationException {
// First request results in timeout, retry then successful
when(smartlingClient.uploadLocalizedFile(
anyString(), anyString(), anyString(), anyString(), anyString(), any(), any()))
anyString(),
anyString(),
anyString(),
anyString(),
anyString(),
any(),
any(),
anyString()))
.thenThrow(
new SmartlingClientException(new HttpServerErrorException(HttpStatus.GATEWAY_TIMEOUT)))
.thenReturn(null);
Expand Down Expand Up @@ -1131,7 +1153,14 @@ public void testRetryPushTranslations()
// Verify upload localized file called three times due to retry on first request
verify(smartlingClient, times(3))
.uploadLocalizedFile(
anyString(), anyString(), anyString(), anyString(), anyString(), eq(null), eq(null));
anyString(),
anyString(),
anyString(),
anyString(),
anyString(),
eq(null),
eq(null),
eq("PUBLISHED"));
}

@Test
Expand Down Expand Up @@ -1334,7 +1363,8 @@ public void testPushTranslationsNoBatches()
eq(locale.getBcp47Tag()),
startsWith("<?xml version="),
eq(null),
eq(null));
eq(null),
eq("PUBLISHED"));

verify(smartlingClient, times(1))
.uploadLocalizedFile(
Expand All @@ -1344,7 +1374,8 @@ public void testPushTranslationsNoBatches()
eq(locale.getBcp47Tag()),
startsWith("<?xml version="),
eq(null),
eq(null));
eq(null),
eq("PUBLISHED"));
});
}

Expand Down Expand Up @@ -1463,7 +1494,8 @@ public void testPushTranslationsPluralFix()
eq(locale.getBcp47Tag()),
startsWith("<?xml version="),
eq(null),
eq(null)));
eq(null),
eq("PUBLISHED")));
}

@Test
Expand Down Expand Up @@ -1609,7 +1641,8 @@ public void testPushTranslationsInBatches()
eq(locale.getBcp47Tag()),
startsWith("<?xml version="),
eq(null),
eq(null)));
eq(null),
eq("PUBLISHED")));

IntStream.range(0, pluralBatches)
.forEachOrdered(
Expand All @@ -1622,7 +1655,8 @@ public void testPushTranslationsInBatches()
eq(locale.getBcp47Tag()),
startsWith("<?xml version="),
eq(null),
eq(null)));
eq(null),
eq("PUBLISHED")));
});
}

Expand Down
Loading

0 comments on commit 1cd24c2

Please sign in to comment.