diff --git a/webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyTMSSmartling.java b/webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyTMSSmartling.java index 926a346891..dda767b8c8 100644 --- a/webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyTMSSmartling.java +++ b/webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyTMSSmartling.java @@ -1,7 +1,9 @@ package com.box.l10n.mojito.service.thirdparty; import static com.box.l10n.mojito.android.strings.AndroidPluralQuantity.MANY; -import static com.box.l10n.mojito.service.thirdparty.smartling.SmartlingFileUtils.*; +import static com.box.l10n.mojito.service.thirdparty.smartling.SmartlingFileUtils.getOutputSourceFile; +import static com.box.l10n.mojito.service.thirdparty.smartling.SmartlingFileUtils.getOutputTargetFile; +import static com.box.l10n.mojito.service.thirdparty.smartling.SmartlingFileUtils.isPluralFile; import static com.google.common.collect.Streams.mapWithIndex; import com.box.l10n.mojito.android.strings.AndroidStringDocumentMapper; @@ -10,9 +12,18 @@ import com.box.l10n.mojito.entity.Repository; import com.box.l10n.mojito.entity.RepositoryLocale; import com.box.l10n.mojito.service.assetExtraction.AssetTextUnitToTMTextUnitRepository; -import com.box.l10n.mojito.service.thirdparty.smartling.*; +import com.box.l10n.mojito.service.thirdparty.smartling.SmartlingFile; +import com.box.l10n.mojito.service.thirdparty.smartling.SmartlingFileUtils; +import com.box.l10n.mojito.service.thirdparty.smartling.SmartlingOptions; +import com.box.l10n.mojito.service.thirdparty.smartling.SmartlingPluralFix; +import com.box.l10n.mojito.service.thirdparty.smartling.SmartlingResultProcessor; import com.box.l10n.mojito.service.tm.importer.TextUnitBatchImporterService; -import com.box.l10n.mojito.service.tm.search.*; +import com.box.l10n.mojito.service.tm.search.SearchType; +import com.box.l10n.mojito.service.tm.search.StatusFilter; +import com.box.l10n.mojito.service.tm.search.TextUnitDTO; +import com.box.l10n.mojito.service.tm.search.TextUnitSearcher; +import com.box.l10n.mojito.service.tm.search.TextUnitSearcherParameters; +import com.box.l10n.mojito.service.tm.search.UsedFilter; import com.box.l10n.mojito.smartling.AssetPathAndTextUnitNameKeys; import com.box.l10n.mojito.smartling.SmartlingClient; import com.box.l10n.mojito.smartling.SmartlingClientException; @@ -20,7 +31,6 @@ import com.box.l10n.mojito.smartling.request.Bindings; import com.box.l10n.mojito.smartling.response.File; import com.box.l10n.mojito.smartling.response.StringInfo; -import com.google.common.base.Stopwatch; import com.google.common.base.Strings; import com.google.common.collect.ImmutableList; import com.google.common.collect.Iterables; @@ -28,6 +38,7 @@ import com.google.common.io.Files; import io.micrometer.core.annotation.Timed; import io.micrometer.core.instrument.MeterRegistry; +import io.micrometer.core.instrument.Tags; import java.io.IOException; import java.util.List; import java.util.Map; @@ -372,7 +383,6 @@ public void createImageToTextUnitMappings( } @Override - @Timed("SmartlingSync.push") public void push( Repository repository, String projectId, @@ -381,45 +391,58 @@ public void push( String skipAssetsWithPathPattern, List optionList) { - SmartlingOptions options = SmartlingOptions.parseList(optionList); - - if (options.isJsonSync()) { - thirdPartyTMSSmartlingWithJson.push( - repository, - projectId, - pluralSeparator, - skipTextUnitsWithPattern, - skipAssetsWithPathPattern, - options); - return; - } - - Stopwatch totalStopwatch = Stopwatch.createStarted(); - - AndroidStringDocumentMapper mapper = new AndroidStringDocumentMapper(pluralSeparator, null); - - Stream singularFiles = - mapWithIndex( - partitionSingulars( - repository.getId(), LOCALE_EN, skipTextUnitsWithPattern, skipAssetsWithPathPattern), - (batch, index) -> - processPushBatch( - batch, index, mapper, repository, projectId, options, Prefix.SINGULAR)); - - Stream pluralFiles = - mapWithIndex( - partitionPlurals( - repository.getId(), LOCALE_EN, skipTextUnitsWithPattern, skipAssetsWithPathPattern), - (batch, index) -> - processPushBatch( - batch, index, mapper, repository, projectId, options, Prefix.PLURAL)); - - List result = - Stream.concat(singularFiles, pluralFiles).collect(Collectors.toList()); - resultProcessor.processPush(result, options); meterRegistry - .timer("ThirdPartyTMSSmartling.push", "repository", repository.getName()) - .record(totalStopwatch.stop().elapsed()); + .timer("SmartlingSync.push", Tags.of("repository", repository.getName())) + .record( + () -> { + SmartlingOptions options = SmartlingOptions.parseList(optionList); + + if (options.isJsonSync()) { + thirdPartyTMSSmartlingWithJson.push( + repository, + projectId, + pluralSeparator, + skipTextUnitsWithPattern, + skipAssetsWithPathPattern, + options); + return; + } + + AndroidStringDocumentMapper mapper = + new AndroidStringDocumentMapper(pluralSeparator, null); + + Stream singularFiles = + mapWithIndex( + partitionSingulars( + repository.getId(), + LOCALE_EN, + skipTextUnitsWithPattern, + skipAssetsWithPathPattern), + (batch, index) -> + processPushBatch( + batch, + index, + mapper, + repository, + projectId, + options, + Prefix.SINGULAR)); + + Stream pluralFiles = + mapWithIndex( + partitionPlurals( + repository.getId(), + LOCALE_EN, + skipTextUnitsWithPattern, + skipAssetsWithPathPattern), + (batch, index) -> + processPushBatch( + batch, index, mapper, repository, projectId, options, Prefix.PLURAL)); + + List result = + Stream.concat(singularFiles, pluralFiles).collect(Collectors.toList()); + resultProcessor.processPush(result, options); + }); } private SmartlingFile processPushBatch( @@ -431,7 +454,22 @@ private SmartlingFile processPushBatch( SmartlingOptions options, Prefix filePrefix) { - Stopwatch stopwatch = Stopwatch.createStarted(); + return meterRegistry + .timer("SmartlingSync.processPushBatch", Tags.of("repository", repository.getName())) + .record( + () -> + uploadTextUnitsToSmartling( + result, batchNumber, mapper, repository, projectId, options, filePrefix)); + } + + private SmartlingFile uploadTextUnitsToSmartling( + List result, + long batchNumber, + AndroidStringDocumentMapper mapper, + Repository repository, + String projectId, + SmartlingOptions options, + Prefix filePrefix) { logger.debug("Convert text units to AndroidString for asset number: {}", batchNumber); SmartlingFile file = new SmartlingFile(); file.setFileName(getOutputSourceFile(batchNumber, repository.getName(), filePrefix.getType())); @@ -478,14 +516,10 @@ private SmartlingFile processPushBatch( }) .block(); } - meterRegistry - .timer("ThirdPartyTMSSmartling.processPushBatch", "repository", repository.getName()) - .record(stopwatch.stop().elapsed()); return file; } @Override - @Timed("SmartlingSync.pull") public void pull( Repository repository, String projectId, @@ -495,51 +529,58 @@ public void pull( String skipAssetsWithPathPattern, List optionList) { - Stopwatch stopwatch = Stopwatch.createStarted(); - SmartlingOptions options = SmartlingOptions.parseList(optionList); - - if (options.isJsonSync()) { - thirdPartyTMSSmartlingWithJson.pull(repository, projectId, localeMapping); - return; - } - - if (options.isGlossarySync()) { - thirdPartyTMSSmartlingGlossary.pull(repository, projectId, localeMapping); - return; - } - - Long singulars = - singularCount( - repository.getId(), LOCALE_EN, skipTextUnitsWithPattern, skipAssetsWithPathPattern); - LongStream.range(0, batchesFor(singulars)) - .forEach( - num -> - processPullBatch( - num, - pluralSeparator, - repository, - projectId, - options, - localeMapping, - Prefix.SINGULAR)); - - Long plurals = - pluralCount( - repository.getId(), LOCALE_EN, skipTextUnitsWithPattern, skipAssetsWithPathPattern); - LongStream.range(0, batchesFor(plurals)) - .forEach( - num -> - processPullBatch( - num, - pluralSeparator, - repository, - projectId, - options, - localeMapping, - Prefix.PLURAL)); meterRegistry - .timer("ThirdPartyTMSSmartling.pull", "repository", repository.getName()) - .record(stopwatch.stop().elapsed()); + .timer("SmartlingSync.pull", Tags.of("repository", repository.getName())) + .record( + () -> { + SmartlingOptions options = SmartlingOptions.parseList(optionList); + + if (options.isJsonSync()) { + thirdPartyTMSSmartlingWithJson.pull(repository, projectId, localeMapping); + return; + } + + if (options.isGlossarySync()) { + thirdPartyTMSSmartlingGlossary.pull(repository, projectId, localeMapping); + return; + } + + Long singulars = + singularCount( + repository.getId(), + LOCALE_EN, + skipTextUnitsWithPattern, + skipAssetsWithPathPattern); + LongStream.range(0, batchesFor(singulars)) + .forEach( + num -> + processPullBatch( + num, + pluralSeparator, + repository, + projectId, + options, + localeMapping, + Prefix.SINGULAR)); + + Long plurals = + pluralCount( + repository.getId(), + LOCALE_EN, + skipTextUnitsWithPattern, + skipAssetsWithPathPattern); + LongStream.range(0, batchesFor(plurals)) + .forEach( + num -> + processPullBatch( + num, + pluralSeparator, + repository, + projectId, + options, + localeMapping, + Prefix.PLURAL)); + }); } private void processPullBatch( @@ -551,90 +592,88 @@ private void processPullBatch( Map localeMapping, Prefix filePrefix) { - AndroidStringDocumentMapper mapper; - for (RepositoryLocale locale : repository.getRepositoryLocales()) { - Stopwatch stopwatch = Stopwatch.createStarted(); - if (locale.getParentLocale() == null) { continue; } - String localeTag = locale.getLocale().getBcp47Tag(); - String smartlingLocale = getSmartlingLocale(localeMapping, localeTag); - String fileName = - getOutputSourceFile(batchNumber, repository.getName(), filePrefix.getType()); - mapper = - new AndroidStringDocumentMapper(pluralSeparator, null, localeTag, repository.getName()); - - logger.debug( - "Download localized file from Smartling for file: {}, Mojito locale: {} and Smartling locale: {}", - fileName, - localeTag, - smartlingLocale); - - String fileContent = - Mono.fromCallable( - () -> - smartlingClient.downloadPublishedFile( - projectId, smartlingLocale, fileName, false)) - .retryWhen( - smartlingClient - .getRetryConfiguration() - .doBeforeRetry( - e -> - logger.info( - String.format( - "Retrying after failure to download file %s", fileName), - e.failure()))) - .doOnError( - e -> { - String msg = - String.format("Error downloading file %s: %s", fileName, e.getMessage()); - logger.error(msg, e); - throw new SmartlingClientException(msg, e); - }) - .blockOptional() - .orElseThrow( - () -> - new SmartlingClientException( - "Error with download from Smartling, file content string is not present.")); - - List textUnits; - - try { - textUnits = mapper.mapToTextUnits(AndroidStringDocumentReader.fromText(fileContent)); - } catch (ParserConfigurationException | IOException | SAXException e) { - String msg = "An error ocurred when processing a pull batch"; - logger.error(msg, e); - throw new RuntimeException(msg, e); - } - - if (!textUnits.isEmpty() - && filePrefix.isPlural() - && options.getPluralFixForLocales().contains(localeTag)) { - textUnits = SmartlingPluralFix.fixTextUnits(textUnits); - } - - if (!options.isDryRun()) { - logger.debug("Importing text units for locale: {}", smartlingLocale); - textUnitBatchImporterService.importTextUnits(textUnits, false, true); - } - meterRegistry .timer( - "ThirdPartyTMSSmartling.processPullBatch", - "repository", - repository.getName(), - "locale", - localeTag) - .record(stopwatch.stop().elapsed()); + "SmartlingSync.processPullBatch", + Tags.of( + "repository", repository.getName(), "locale", locale.getLocale().getBcp47Tag())) + .record( + () -> { + String localeTag = locale.getLocale().getBcp47Tag(); + String smartlingLocale = getSmartlingLocale(localeMapping, localeTag); + String fileName = + getOutputSourceFile(batchNumber, repository.getName(), filePrefix.getType()); + AndroidStringDocumentMapper mapper = + new AndroidStringDocumentMapper( + pluralSeparator, null, localeTag, repository.getName()); + + logger.debug( + "Download localized file from Smartling for file: {}, Mojito locale: {} and Smartling locale: {}", + fileName, + localeTag, + smartlingLocale); + + String fileContent = + Mono.fromCallable( + () -> + smartlingClient.downloadPublishedFile( + projectId, smartlingLocale, fileName, false)) + .retryWhen( + smartlingClient + .getRetryConfiguration() + .doBeforeRetry( + e -> + logger.info( + String.format( + "Retrying after failure to download file %s", + fileName), + e.failure()))) + .doOnError( + e -> { + String msg = + String.format( + "Error downloading file %s: %s", fileName, e.getMessage()); + logger.error(msg, e); + throw new SmartlingClientException(msg, e); + }) + .blockOptional() + .orElseThrow( + () -> + new SmartlingClientException( + "Error with download from Smartling, file content string is not present.")); + + List textUnits; + + try { + textUnits = + mapper.mapToTextUnits(AndroidStringDocumentReader.fromText(fileContent)); + } catch (ParserConfigurationException | IOException | SAXException e) { + String msg = "An error ocurred when processing a pull batch"; + logger.error(msg, e); + throw new RuntimeException(msg, e); + } + + if (!textUnits.isEmpty() + && filePrefix.isPlural() + && options.getPluralFixForLocales().contains(localeTag)) { + textUnits = SmartlingPluralFix.fixTextUnits(textUnits); + } + + if (!options.isDryRun()) { + logger.debug("Importing text units for locale: {}", smartlingLocale); + textUnitBatchImporterService.importTextUnits(textUnits, false, true); + } + }); } } @Override - @Timed("SmartlingSync.pushTranslations") public void pushTranslations( Repository repository, String projectId, @@ -645,82 +684,84 @@ public void pushTranslations( String includeTextUnitsWithPattern, List optionList) { - Stopwatch stopwatch = Stopwatch.createStarted(); - - SmartlingOptions options = SmartlingOptions.parseList(optionList); - if (options.isJsonSync()) { - thirdPartyTMSSmartlingWithJson.pushTranslations( - repository, - projectId, - pluralSeparator, - localeMapping, - skipTextUnitsWithPattern, - skipAssetsWithPathPattern, - includeTextUnitsWithPattern, - options); - return; - } - - List result; - - AndroidStringDocumentMapper mapper = new AndroidStringDocumentMapper(pluralSeparator, null); - - Set filterTmTextUnitIds = getFilterTmTextUnitIdsForPushTranslation(options); - - result = - repository.getRepositoryLocales().stream() - .map(l -> l.getLocale().getBcp47Tag()) - .filter( - localeTag -> - !localeTag.equalsIgnoreCase(repository.getSourceLocale().getBcp47Tag())) - .flatMap( - localeTag -> - Stream.concat( - mapWithIndex( - partitionSingulars( - repository.getId(), - localeTag, - skipTextUnitsWithPattern, - skipAssetsWithPathPattern, - includeTextUnitsWithPattern), - (list, batch) -> - processTranslationBatch( - list, - batch, - localeTag, - mapper, - repository, - projectId, - options, - localeMapping, - Prefix.SINGULAR, - filterTmTextUnitIds)), - mapWithIndex( - partitionPlurals( - repository.getId(), - localeTag, - skipTextUnitsWithPattern, - skipAssetsWithPathPattern, - options.getPluralFixForLocales(), - includeTextUnitsWithPattern), - (list, batch) -> - processTranslationBatch( - list, - batch, - localeTag, - mapper, - repository, - projectId, - options, - localeMapping, - Prefix.PLURAL, - filterTmTextUnitIds)))) - .collect(Collectors.toList()); - - resultProcessor.processPushTranslations(result, options); meterRegistry - .timer("ThirdPartyTMSSmartling.pushTranslations", "repository", repository.getName()) - .record(stopwatch.stop().elapsed()); + .timer("SmartlingSync.pushTranslations", Tags.of("repository", repository.getName())) + .record( + () -> { + SmartlingOptions options = SmartlingOptions.parseList(optionList); + if (options.isJsonSync()) { + thirdPartyTMSSmartlingWithJson.pushTranslations( + repository, + projectId, + pluralSeparator, + localeMapping, + skipTextUnitsWithPattern, + skipAssetsWithPathPattern, + includeTextUnitsWithPattern, + options); + return; + } + + List result; + + AndroidStringDocumentMapper mapper = + new AndroidStringDocumentMapper(pluralSeparator, null); + + Set filterTmTextUnitIds = getFilterTmTextUnitIdsForPushTranslation(options); + + result = + repository.getRepositoryLocales().stream() + .map(l -> l.getLocale().getBcp47Tag()) + .filter( + localeTag -> + !localeTag.equalsIgnoreCase( + repository.getSourceLocale().getBcp47Tag())) + .flatMap( + localeTag -> + Stream.concat( + mapWithIndex( + partitionSingulars( + repository.getId(), + localeTag, + skipTextUnitsWithPattern, + skipAssetsWithPathPattern, + includeTextUnitsWithPattern), + (list, batch) -> + processTranslationBatch( + list, + batch, + localeTag, + mapper, + repository, + projectId, + options, + localeMapping, + Prefix.SINGULAR, + filterTmTextUnitIds)), + mapWithIndex( + partitionPlurals( + repository.getId(), + localeTag, + skipTextUnitsWithPattern, + skipAssetsWithPathPattern, + options.getPluralFixForLocales(), + includeTextUnitsWithPattern), + (list, batch) -> + processTranslationBatch( + list, + batch, + localeTag, + mapper, + repository, + projectId, + options, + localeMapping, + Prefix.PLURAL, + filterTmTextUnitIds)))) + .collect(Collectors.toList()); + + resultProcessor.processPushTranslations(result, options); + }); } private Set getFilterTmTextUnitIdsForPushTranslation(SmartlingOptions options) { @@ -735,7 +776,6 @@ private Set getFilterTmTextUnitIdsForPushTranslation(SmartlingOptions opti } @Override - @Timed("SmartlingSync.pullSource") public void pullSource( Repository repository, String thirdPartyProjectId, @@ -743,12 +783,13 @@ public void pullSource( Map localeMapping) { SmartlingOptions options = SmartlingOptions.parseList(optionList); if (options.isGlossarySync()) { - Stopwatch stopwatch = Stopwatch.createStarted(); - thirdPartyTMSSmartlingGlossary.pullSourceTextUnits( - repository, thirdPartyProjectId, localeMapping); meterRegistry - .timer("ThirdPartyTMSSmartling.pullSource", "repository", repository.getName()) - .record(stopwatch.stop().elapsed()); + .timer("SmartlingSync.pullSource", "repository", repository.getName()) + .record( + () -> + thirdPartyTMSSmartlingGlossary.pullSourceTextUnits( + repository, thirdPartyProjectId, localeMapping)); + } else { throw new UnsupportedOperationException( "Pull source is only supported with glossary sync enabled."); @@ -767,80 +808,84 @@ private SmartlingFile processTranslationBatch( Prefix filePrefix, Set filterTmTextUnitIds) { - logger.debug("Process {} batch: {}", localeTag, batchNumber); - logger.debug("Convert text units to AndroidString for asset number: {}", batchNumber); - Stopwatch stopwatch = Stopwatch.createStarted(); - String sourceFilename = - getOutputSourceFile(batchNumber, repository.getName(), filePrefix.getType()); - String targetFilename = - getOutputTargetFile(batchNumber, repository.getName(), filePrefix.getType(), localeTag); - SmartlingFile file = new SmartlingFile(); - file.setFileName(targetFilename); - - try { - - logger.debug("Save target file to: {}", file.getFileName()); - - if (filterTmTextUnitIds != null) { - batch = - batch.stream() - .filter(textUnitDTO -> filterTmTextUnitIds.contains(textUnitDTO.getTmTextUnitId())) - .collect(Collectors.toList()); - } - - AndroidStringDocumentWriter writer = - new AndroidStringDocumentWriter(mapper.readFromTargetTextUnits(batch)); - file.setFileContent(writer.toText()); - - } catch (ParserConfigurationException | TransformerException e) { - logger.error("An error ocurred when processing a push_translations batch", e); - throw new RuntimeException(e); - } - - if (!options.isDryRun()) { - logger.debug( - "Push Android file to Smartling project: {} and locale: {}", projectId, localeTag); - Mono.fromCallable( - () -> - smartlingClient.uploadLocalizedFile( - projectId, - sourceFilename, - "android", - getSmartlingLocale(localeMapping, localeTag), - file.getFileContent(), - options.getPlaceholderFormat(), - options.getCustomPlaceholderFormat())) - .retryWhen( - smartlingClient - .getRetryConfiguration() - .doBeforeRetry( - e -> - logger.info( - String.format( - "Retrying after failure to upload localized file: %s, project id: %s", - sourceFilename, projectId), - e.failure()))) - .doOnError( - e -> { - String msg = - String.format( - "Error uploading localized file to Smartling for file %s in project %s", - sourceFilename, projectId); - logger.error(msg, e); - throw new SmartlingClientException(msg, e); - }) - .block(); - } - - meterRegistry + return meterRegistry .timer( - "ThirdPartyTMSSmartlingWithJson.processTranslationBatch", - "repository", - repository.getName(), - "locale", - localeTag) - .record(stopwatch.stop().elapsed()); - return file; + "SmartlingSync.processTranslationBatch", + Tags.of("repository", repository.getName(), "locale", localeTag)) + .record( + () -> { + logger.debug("Process {} batch: {}", localeTag, batchNumber); + logger.debug("Convert text units to AndroidString for asset number: {}", batchNumber); + List fileBatch = batch; + String sourceFilename = + getOutputSourceFile(batchNumber, repository.getName(), filePrefix.getType()); + String targetFilename = + getOutputTargetFile( + batchNumber, repository.getName(), filePrefix.getType(), localeTag); + SmartlingFile file = new SmartlingFile(); + file.setFileName(targetFilename); + + try { + + logger.debug("Save target file to: {}", file.getFileName()); + + if (filterTmTextUnitIds != null) { + fileBatch = + fileBatch.stream() + .filter( + textUnitDTO -> + filterTmTextUnitIds.contains(textUnitDTO.getTmTextUnitId())) + .collect(Collectors.toList()); + } + + AndroidStringDocumentWriter writer = + new AndroidStringDocumentWriter(mapper.readFromTargetTextUnits(batch)); + file.setFileContent(writer.toText()); + + } catch (ParserConfigurationException | TransformerException e) { + logger.error("An error ocurred when processing a push_translations batch", e); + throw new RuntimeException(e); + } + + if (!options.isDryRun()) { + logger.debug( + "Push Android file to Smartling project: {} and locale: {}", + projectId, + localeTag); + Mono.fromCallable( + () -> + smartlingClient.uploadLocalizedFile( + projectId, + sourceFilename, + "android", + getSmartlingLocale(localeMapping, localeTag), + file.getFileContent(), + options.getPlaceholderFormat(), + options.getCustomPlaceholderFormat())) + .retryWhen( + smartlingClient + .getRetryConfiguration() + .doBeforeRetry( + e -> + logger.info( + String.format( + "Retrying after failure to upload localized file: %s, project id: %s", + sourceFilename, projectId), + e.failure()))) + .doOnError( + e -> { + String msg = + String.format( + "Error uploading localized file to Smartling for file %s in project %s", + sourceFilename, projectId); + logger.error(msg, e); + throw new SmartlingClientException(msg, e); + }) + .block(); + } + + return file; + }); } private Stream> partitionSingulars( diff --git a/webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyTMSSmartlingGlossary.java b/webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyTMSSmartlingGlossary.java index c8dfc7f851..2e372a11f7 100644 --- a/webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyTMSSmartlingGlossary.java +++ b/webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyTMSSmartlingGlossary.java @@ -21,7 +21,6 @@ import com.box.l10n.mojito.smartling.response.GlossaryTargetTerm; import com.box.l10n.mojito.smartling.response.GlossaryTermTranslation; import com.google.common.base.Strings; -import io.micrometer.core.instrument.MeterRegistry; import java.io.ByteArrayInputStream; import java.nio.charset.StandardCharsets; import java.util.ArrayList; @@ -59,8 +58,6 @@ public class ThirdPartyTMSSmartlingGlossary { @Autowired AssetTextUnitRepository assetTextUnitRepository; - @Autowired MeterRegistry meterRegistry; - @Value("${l10n.smartling.accountId:}") String accountId; diff --git a/webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyTMSSmartlingWithJson.java b/webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyTMSSmartlingWithJson.java index a9b0d6b942..9c10a5ff37 100644 --- a/webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyTMSSmartlingWithJson.java +++ b/webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyTMSSmartlingWithJson.java @@ -21,9 +21,9 @@ import com.box.l10n.mojito.smartling.response.Items; import com.box.l10n.mojito.smartling.response.StringInfo; import com.google.common.base.Preconditions; -import com.google.common.base.Stopwatch; import com.google.common.collect.ImmutableList; import io.micrometer.core.instrument.MeterRegistry; +import io.micrometer.core.instrument.Tags; import java.util.List; import java.util.Map; import java.util.function.Function; @@ -88,63 +88,66 @@ void push( repository.getName(), projectId); - Stopwatch totalStopwatch = Stopwatch.createStarted(); - - long partitionCount = - Spliterators.partitionStreamWithIndex( - getSourceTextUnitIterator( - repository, skipTextUnitsWithPattern, skipAssetsWithPathPattern), - batchSize, - (textUnitDTOS, index) -> { - Stopwatch batchStopWatch = Stopwatch.createStarted(); - String fileName = getSourceFileName(repository.getName(), index); - String fileContent = - smartlingJsonConverter.textUnitDTOsToJsonString( - textUnitDTOS, TextUnitDTO::getSource); - Mono.fromCallable( - () -> - smartlingClient.uploadFile( - projectId, - fileName, - "json", - fileContent, - smartlingOptions.getPlaceholderFormat(), - smartlingOptions.getCustomPlaceholderFormat(), - smartlingOptions.getStringFormat())) - .retryWhen( - smartlingClient - .getRetryConfiguration() - .doBeforeRetry( - e -> - logger.info( - String.format( - "Retrying failed upload file request for file %s in project %s", - fileName, projectId), - e.failure()))) - .doOnError( - e -> { - String msg = - String.format( - "Upload file request failed for file %s in project %s", - fileName, projectId); - logger.error(msg, e); - throw new SmartlingClientException(msg, e); - }) - .block(); - meterRegistry - .timer( - "ThirdPartyTMSSmartlingWithJson.push.batch", - "repository", - repository.getName()) - .record(batchStopWatch.stop().elapsed()); - return index; - }) - .count(); - - removeFileForBatchNumberGreaterOrEquals(repository.getName(), projectId, partitionCount); meterRegistry - .timer("ThirdPartyTMSSmartlingWithJson.push") - .record(totalStopwatch.stop().elapsed()); + .timer("ThirdPartyTMSSmartlingWithJson.push", Tags.of("repository", repository.getName())) + .record( + () -> { + long partitionCount = + Spliterators.partitionStreamWithIndex( + getSourceTextUnitIterator( + repository, skipTextUnitsWithPattern, skipAssetsWithPathPattern), + batchSize, + (textUnitDTOS, index) -> + meterRegistry + .timer( + "ThirdPartyTMSSmartlingWithJson.push.batch", + Tags.of("repository", repository.getName())) + .record( + () -> { + String fileName = + getSourceFileName(repository.getName(), index); + String fileContent = + smartlingJsonConverter.textUnitDTOsToJsonString( + textUnitDTOS, TextUnitDTO::getSource); + Mono.fromCallable( + () -> + smartlingClient.uploadFile( + projectId, + fileName, + "json", + fileContent, + smartlingOptions.getPlaceholderFormat(), + smartlingOptions + .getCustomPlaceholderFormat(), + smartlingOptions.getStringFormat())) + .retryWhen( + smartlingClient + .getRetryConfiguration() + .doBeforeRetry( + e -> + logger.info( + String.format( + "Retrying failed upload file request for file %s in project %s", + fileName, projectId), + e.failure()))) + .doOnError( + e -> { + String msg = + String.format( + "Upload file request failed for file %s in project %s", + fileName, projectId); + logger.error(msg, e); + throw new SmartlingClientException(msg, e); + }) + .block(); + + return index; + })) + .count(); + + removeFileForBatchNumberGreaterOrEquals( + repository.getName(), projectId, partitionCount); + }); } void removeFileForBatchNumberGreaterOrEquals( @@ -244,73 +247,79 @@ public void pushTranslations( logger.info( "Push translation from repository: {} into project: {}", repository.getName(), projectId); - Stopwatch stopwatch = Stopwatch.createStarted(); - - getRepositoryLocaleWithoutRootStream(repository) - .forEach( - repositoryLocale -> { - long partitionCount = - Spliterators.partitionStreamWithIndex( - getTargetTextUnitIterator( - repository, - repositoryLocale.getLocale().getId(), - skipTextUnitsWithPattern, - skipAssetsWithPathPattern, - includeTextUnitsWithPattern), - batchSize, - (textUnitDTOS, index) -> { - Stopwatch batchStopwatch = Stopwatch.createStarted(); - String fileName = getSourceFileName(repository.getName(), index); - String fileContent = - smartlingJsonConverter.textUnitDTOsToJsonString( - textUnitDTOS, TextUnitDTO::getTarget); - String smartlingLocale = - getSmartlingLocale(localeMapping, repositoryLocale); - Mono.fromCallable( - () -> - smartlingClient.uploadLocalizedFile( - projectId, - fileName, - "json", - smartlingLocale, - fileContent, - null, - null)) - .retryWhen( - smartlingClient - .getRetryConfiguration() - .doBeforeRetry( - e -> - logger.info( - String.format( - "Retrying after failure to upload localized file %s in project %s", - fileName, projectId), - e.failure()))) - .doOnError( - e -> { - String msg = - String.format( - "Error uploading localized file %s in project %s", - fileName, projectId); - logger.error(msg, e); - throw new SmartlingClientException(msg, e); - }) - .block(); - meterRegistry - .timer( - "ThirdPartyTMSSmartlingWithJson.pushTranslations.batch", - "repository", - repository.getName()) - .record(batchStopwatch.stop().elapsed()); - return index; - }) - .count(); - logger.debug("Processed {} partitions", partitionCount); - }); meterRegistry .timer( - "ThirdPartyTMSSmartlingWithJson.pushTranslations", "repository", repository.getName()) - .record(stopwatch.stop().elapsed()); + "ThirdPartyTMSSmartlingWithJson.pushTranslations", + Tags.of("repository", repository.getName())) + .record( + () -> { + getRepositoryLocaleWithoutRootStream(repository) + .forEach( + repositoryLocale -> { + long partitionCount = + Spliterators.partitionStreamWithIndex( + getTargetTextUnitIterator( + repository, + repositoryLocale.getLocale().getId(), + skipTextUnitsWithPattern, + skipAssetsWithPathPattern, + includeTextUnitsWithPattern), + batchSize, + (textUnitDTOS, index) -> + meterRegistry + .timer( + "ThirdPartyTMSSmartlingWithJson.pushTranslations.batch", + "repository", + repository.getName()) + .record( + () -> { + String fileName = + getSourceFileName( + repository.getName(), index); + String fileContent = + smartlingJsonConverter + .textUnitDTOsToJsonString( + textUnitDTOS, TextUnitDTO::getTarget); + String smartlingLocale = + getSmartlingLocale( + localeMapping, repositoryLocale); + Mono.fromCallable( + () -> + smartlingClient.uploadLocalizedFile( + projectId, + fileName, + "json", + smartlingLocale, + fileContent, + null, + null)) + .retryWhen( + smartlingClient + .getRetryConfiguration() + .doBeforeRetry( + e -> + logger.info( + String.format( + "Retrying after failure to upload localized file %s in project %s", + fileName, projectId), + e.failure()))) + .doOnError( + e -> { + String msg = + String.format( + "Error uploading localized file %s in project %s", + fileName, projectId); + logger.error(msg, e); + throw new SmartlingClientException( + msg, e); + }) + .block(); + return index; + })) + .count(); + logger.debug("Processed {} partitions", partitionCount); + }); + }); } PageFetcherOffsetAndLimitSplitIterator getSourceTextUnitIterator( diff --git a/webapp/src/test/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyTMSSmartlingTest.java b/webapp/src/test/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyTMSSmartlingTest.java index 75117929a1..1829cdec5f 100644 --- a/webapp/src/test/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyTMSSmartlingTest.java +++ b/webapp/src/test/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyTMSSmartlingTest.java @@ -9,7 +9,6 @@ import static org.mockito.ArgumentMatchers.anyBoolean; import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.ArgumentMatchers.eq; -import static org.mockito.ArgumentMatchers.isA; import static org.mockito.ArgumentMatchers.isNull; import static org.mockito.ArgumentMatchers.matches; import static org.mockito.ArgumentMatchers.startsWith; @@ -61,7 +60,6 @@ import com.google.common.collect.Iterables; import com.google.common.primitives.Ints; import io.micrometer.core.instrument.MeterRegistry; -import io.micrometer.core.instrument.Timer; import java.io.IOException; import java.time.Duration; import java.util.ArrayList; @@ -102,10 +100,6 @@ public class ThirdPartyTMSSmartlingTest extends ServiceTestBase { @Mock SmartlingClient smartlingClient; - @Mock MeterRegistry meterRegistryMock; - - @Mock Timer timerMock; - @Mock ThirdPartyTMSSmartlingWithJson mockThirdPartyTMSSmartlingWithJson; @Mock ThirdPartyTMSSmartlingGlossary mockThirdPartyTMSSmartlingGlossary; @@ -138,6 +132,8 @@ public class ThirdPartyTMSSmartlingTest extends ServiceTestBase { @Autowired LocaleMappingHelper localeMappingHelper; + @Autowired MeterRegistry meterRegistry; + StubSmartlingResultProcessor resultProcessor; @Mock TextUnitBatchImporterService mockTextUnitBatchImporterService; @@ -176,10 +172,6 @@ public void setUp() { doReturn(null) .when(mockTextUnitBatchImporterService) .importTextUnits(any(), eq(false), eq(true)); - doReturn(timerMock) - .when(meterRegistryMock) - .timer(anyString(), anyString(), anyString(), anyString(), anyString()); - doReturn(timerMock).when(meterRegistryMock).timer(anyString(), anyString(), anyString()); resultProcessor = new StubSmartlingResultProcessor(); tmsSmartling = @@ -192,7 +184,7 @@ public void setUp() { mockThirdPartyTMSSmartlingWithJson, mockThirdPartyTMSSmartlingGlossary, assetTextUnitToTMTextUnitRepository, - meterRegistryMock); + meterRegistry); mapper = new AndroidStringDocumentMapper(pluralSep, null); RetryBackoffSpec retryConfiguration = @@ -243,7 +235,7 @@ public void testPushInBatchesWithSingularsAndNoPlurals() mockThirdPartyTMSSmartlingGlossary, assetTextUnitToTMTextUnitRepository, batchSize, - meterRegistryMock); + meterRegistry); TM tm = repository.getTm(); Asset asset = @@ -277,7 +269,6 @@ public void testPushInBatchesWithSingularsAndNoPlurals() .satisfies( file -> assertThat(readTextUnits(file, pluralSep)).hasSize(singularTextUnits % batchSize)); - verify(timerMock, atLeastOnce()).record(isA((Duration.class))); } @Test @@ -297,7 +288,7 @@ public void testRetryDuringPush() throws RepositoryNameAlreadyUsedException { mockThirdPartyTMSSmartlingGlossary, assetTextUnitToTMTextUnitRepository, batchSize, - meterRegistryMock); + meterRegistry); // throw timeout exception for first request, following request should be successful when(smartlingClient.uploadFile(any(), any(), any(), any(), any(), any(), any())) .thenThrow( @@ -362,7 +353,7 @@ public void testRetriesExhaustedDuringPush() throws RepositoryNameAlreadyUsedExc mockThirdPartyTMSSmartlingGlossary, assetTextUnitToTMTextUnitRepository, batchSize, - meterRegistryMock); + meterRegistry); TM tm = repository.getTm(); Asset asset = @@ -412,7 +403,7 @@ public void testPushInBatchesWithNoSingularsAndPlurals() mockThirdPartyTMSSmartlingGlossary, assetTextUnitToTMTextUnitRepository, batchSize, - meterRegistryMock); + meterRegistry); TM tm = repository.getTm(); Asset asset = @@ -450,7 +441,6 @@ public void testPushInBatchesWithNoSingularsAndPlurals() .satisfies( file -> assertThat(readTextUnits(file, pluralSep)).hasSize(pluralTextUnits % batchSize)); - verify(timerMock, atLeastOnce()).record(isA((Duration.class))); } @Test @@ -471,7 +461,7 @@ public void testPushInBatchesWithSingularsAndPlurals() throws RepositoryNameAlre mockThirdPartyTMSSmartlingGlossary, assetTextUnitToTMTextUnitRepository, batchSize, - meterRegistryMock); + meterRegistry); TM tm = repository.getTm(); Asset asset = @@ -538,8 +528,6 @@ public void testPushInBatchesWithSingularsAndPlurals() throws RepositoryNameAlre .satisfies( file -> assertThat(readTextUnits(file, pluralSep)).hasSize(pluralTextUnits % batchSize)); - - verify(timerMock, atLeastOnce()).record(isA((Duration.class))); } @Test @@ -602,8 +590,6 @@ public void testPushWithUploadNoBatches() { eq("NONE"), eq("^some(.*)pattern$"), isNull()); - - verify(timerMock, atLeastOnce()).record(isA((Duration.class))); } @Test @@ -689,7 +675,7 @@ public void testPullNoBatches() throws RepositoryLocaleCreationException { mockThirdPartyTMSSmartlingWithJson, mockThirdPartyTMSSmartlingGlossary, assetTextUnitToTMTextUnitRepository, - meterRegistryMock); + meterRegistry); tmsSmartling.pull( repository, "projectId", pluralSep, localeMapping, null, null, Collections.emptyList()); @@ -803,7 +789,7 @@ public void testRetrySuccessDuringPull() mockThirdPartyTMSSmartlingWithJson, mockThirdPartyTMSSmartlingGlossary, assetTextUnitToTMTextUnitRepository, - meterRegistryMock); + meterRegistry); tmsSmartling.pull( repository, "projectId", pluralSep, localeMapping, null, null, Collections.emptyList()); @@ -897,7 +883,7 @@ public void testRetriesExhaustedDuringPull() mockThirdPartyTMSSmartlingWithJson, mockThirdPartyTMSSmartlingGlossary, assetTextUnitToTMTextUnitRepository, - meterRegistryMock); + meterRegistry); RuntimeException exception = assertThrows( SmartlingClientException.class, @@ -957,7 +943,7 @@ public void testPullNoBatchesLocaleMapping() throws RepositoryLocaleCreationExce mockThirdPartyTMSSmartlingWithJson, mockThirdPartyTMSSmartlingGlossary, assetTextUnitToTMTextUnitRepository, - meterRegistryMock); + meterRegistry); tmsSmartling.pull( repository, "projectId", pluralSep, localeMapping, null, null, Collections.emptyList()); @@ -1020,8 +1006,6 @@ public void testPullNoBatchesLocaleMapping() throws RepositoryLocaleCreationExce assetPath, "fr-CA", repository.getName())); - - verify(timerMock, atLeastOnce()).record(isA((Duration.class))); } @Test @@ -1068,7 +1052,7 @@ public void testPullNoBatchesPluralFix() throws RepositoryLocaleCreationExceptio mockThirdPartyTMSSmartlingWithJson, mockThirdPartyTMSSmartlingGlossary, assetTextUnitToTMTextUnitRepository, - meterRegistryMock); + meterRegistry); tmsSmartling.pull( repository, "projectId", @@ -1201,7 +1185,7 @@ public void testPullDryRunNoBatches() throws RepositoryLocaleCreationException { mockThirdPartyTMSSmartlingWithJson, mockThirdPartyTMSSmartlingGlossary, assetTextUnitToTMTextUnitRepository, - meterRegistryMock); + meterRegistry); tmsSmartling.pull( repository, "projectId", @@ -1234,7 +1218,7 @@ public void testPullInBatchesWithSingularsAndPlurals() mockThirdPartyTMSSmartlingGlossary, assetTextUnitToTMTextUnitRepository, batchSize, - meterRegistryMock); + meterRegistry); Repository repository = repositoryService.createRepository(testIdWatcher.getEntityName("batchRepo")); Locale frCA = localeService.findByBcp47Tag("fr-CA"); @@ -1329,8 +1313,6 @@ public void testPullInBatchesWithSingularsAndPlurals() assertThat(captured.subList(10, 14)).allSatisfy(list -> assertThat(list).hasSize(batchSize)); assertThat(captured.subList(14, 16)) .allSatisfy(list -> assertThat(list).hasSize(pluralTextUnits % batchSize)); - - verify(timerMock, atLeastOnce()).record(isA((Duration.class))); } @Test @@ -1412,8 +1394,6 @@ public void testPushTranslationsSingularsNoBatches() eq(null), eq(null)); }); - - verify(timerMock, atLeastOnce()).record(isA((Duration.class))); } @Test @@ -1906,7 +1886,7 @@ public void testPushTranslationsInBatches() mockThirdPartyTMSSmartlingGlossary, assetTextUnitToTMTextUnitRepository, batchSize, - meterRegistryMock); + meterRegistry); Repository repository = repositoryService.createRepository(testIdWatcher.getEntityName("batchRepo")); Locale frCA = localeService.findByBcp47Tag("fr-CA"); @@ -2059,7 +2039,7 @@ public void testBatchesFor() { mockThirdPartyTMSSmartlingGlossary, assetTextUnitToTMTextUnitRepository, 3, - meterRegistryMock); + meterRegistry); assertThat(tmsSmartling.batchesFor(0)).isEqualTo(0); assertThat(tmsSmartling.batchesFor(1)).isEqualTo(1); @@ -2081,7 +2061,7 @@ public void testBatchesFor() { mockThirdPartyTMSSmartlingGlossary, assetTextUnitToTMTextUnitRepository, 35, - meterRegistryMock); + meterRegistry); assertThat(tmsSmartling.batchesFor(0)).isEqualTo(0); assertThat(tmsSmartling.batchesFor(1)).isEqualTo(1); @@ -2101,7 +2081,7 @@ public void testBatchesFor() { mockThirdPartyTMSSmartlingGlossary, assetTextUnitToTMTextUnitRepository, 4231, - meterRegistryMock); + meterRegistry); assertThat(tmsSmartling.batchesFor(0)).isEqualTo(0); assertThat(tmsSmartling.batchesFor(1)).isEqualTo(1);