Skip to content

Commit

Permalink
Added metrics to Smartling TMS sync
Browse files Browse the repository at this point in the history
  • Loading branch information
@maallen
maallen committed Jul 24, 2023
1 parent d7db1a4 commit 56353d2
Show file tree
Hide file tree
Showing 6 changed files with 171 additions and 23 deletions.
57 changes: 53 additions & 4 deletions webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyTMSSmartling.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,12 +20,14 @@
import com.box.l10n.mojito.smartling.request.Bindings;
import com.box.l10n.mojito.smartling.response.File;
import com.box.l10n.mojito.smartling.response.StringInfo;
import com.google.common.base.Stopwatch;
import com.google.common.base.Strings;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Iterables;
import com.google.common.collect.Sets;
import com.google.common.io.Files;
import io.micrometer.core.annotation.Timed;
import io.micrometer.core.instrument.MeterRegistry;
import java.io.IOException;
import java.util.List;
import java.util.Map;
Expand Down Expand Up @@ -83,6 +85,8 @@ public class ThirdPartyTMSSmartling implements ThirdPartyTMS {
private final ThirdPartyTMSSmartlingGlossary thirdPartyTMSSmartlingGlossary;
private final AssetTextUnitToTMTextUnitRepository assetTextUnitToTMTextUnitRepository;

private final MeterRegistry meterRegistry;

private final Set<String> supportedImageExtensions =
Sets.newHashSet("png", "jpg", "jpeg", "gif", "tiff");

Expand All @@ -99,7 +103,8 @@ public ThirdPartyTMSSmartling(
SmartlingResultProcessor resultProcessor,
ThirdPartyTMSSmartlingWithJson thirdPartyTMSSmartlingWithJson,
ThirdPartyTMSSmartlingGlossary thirdPartyTMSSmartlingGlossary,
AssetTextUnitToTMTextUnitRepository assetTextUnitToTMTextUnitRepository) {
AssetTextUnitToTMTextUnitRepository assetTextUnitToTMTextUnitRepository,
MeterRegistry meterRegistry) {
this(
smartlingClient,
textUnitSearcher,
Expand All @@ -109,7 +114,8 @@ public ThirdPartyTMSSmartling(
thirdPartyTMSSmartlingWithJson,
thirdPartyTMSSmartlingGlossary,
assetTextUnitToTMTextUnitRepository,
DEFAULT_BATCH_SIZE);
DEFAULT_BATCH_SIZE,
meterRegistry);
}

public ThirdPartyTMSSmartling(
Expand All @@ -121,7 +127,8 @@ public ThirdPartyTMSSmartling(
ThirdPartyTMSSmartlingWithJson thirdPartyTMSSmartlingWithJson,
ThirdPartyTMSSmartlingGlossary thirdPartyTMSSmartlingGlossary,
AssetTextUnitToTMTextUnitRepository assetTextUnitToTMTextUnitRepository,
int batchSize) {
int batchSize,
MeterRegistry meterRegistry) {
this.smartlingClient = smartlingClient;
this.assetPathAndTextUnitNameKeys = assetPathAndTextUnitNameKeys;
this.textUnitBatchImporterService = textUnitBatchImporterService;
Expand All @@ -131,6 +138,7 @@ public ThirdPartyTMSSmartling(
this.thirdPartyTMSSmartlingWithJson = thirdPartyTMSSmartlingWithJson;
this.thirdPartyTMSSmartlingGlossary = thirdPartyTMSSmartlingGlossary;
this.assetTextUnitToTMTextUnitRepository = assetTextUnitToTMTextUnitRepository;
this.meterRegistry = meterRegistry;
}

@Override
Expand Down Expand Up @@ -386,6 +394,8 @@ public void push(
return;
}

Stopwatch totalStopwatch = Stopwatch.createStarted();

AndroidStringDocumentMapper mapper = new AndroidStringDocumentMapper(pluralSeparator, null);

Stream<SmartlingFile> singularFiles =
Expand All @@ -407,6 +417,9 @@ public void push(
List<SmartlingFile> result =
Stream.concat(singularFiles, pluralFiles).collect(Collectors.toList());
resultProcessor.processPush(result, options);
meterRegistry
.timer("ThirdPartyTMSSmartling.push", "repository", repository.getName())
.record(totalStopwatch.stop().elapsed());
}

private SmartlingFile processPushBatch(
Expand All @@ -418,6 +431,7 @@ private SmartlingFile processPushBatch(
SmartlingOptions options,
Prefix filePrefix) {

Stopwatch stopwatch = Stopwatch.createStarted();
logger.debug("Convert text units to AndroidString for asset number: {}", batchNumber);
SmartlingFile file = new SmartlingFile();
file.setFileName(getOutputSourceFile(batchNumber, repository.getName(), filePrefix.getType()));
Expand Down Expand Up @@ -464,7 +478,9 @@ private SmartlingFile processPushBatch(
})
.block();
}

meterRegistry
.timer("ThirdPartyTMSSmartling.processPushBatch", "repository", repository.getName())
.record(stopwatch.stop().elapsed());
return file;
}

Expand All @@ -479,6 +495,7 @@ public void pull(
String skipAssetsWithPathPattern,
List<String> optionList) {

Stopwatch stopwatch = Stopwatch.createStarted();
SmartlingOptions options = SmartlingOptions.parseList(optionList);

if (options.isJsonSync()) {
Expand Down Expand Up @@ -520,6 +537,9 @@ public void pull(
options,
localeMapping,
Prefix.PLURAL));
meterRegistry
.timer("ThirdPartyTMSSmartling.pull", "repository", repository.getName())
.record(stopwatch.stop().elapsed());
}

private void processPullBatch(
Expand All @@ -535,6 +555,8 @@ private void processPullBatch(

for (RepositoryLocale locale : repository.getRepositoryLocales()) {

Stopwatch stopwatch = Stopwatch.createStarted();

if (locale.getParentLocale() == null) {
continue;
}
Expand Down Expand Up @@ -599,6 +621,15 @@ private void processPullBatch(
logger.debug("Importing text units for locale: {}", smartlingLocale);
textUnitBatchImporterService.importTextUnits(textUnits, false, true);
}

meterRegistry
.timer(
"ThirdPartyTMSSmartling.processPullBatch",
"repository",
repository.getName(),
"locale",
localeTag)
.record(stopwatch.stop().elapsed());
}
}

Expand All @@ -614,6 +645,8 @@ public void pushTranslations(
String includeTextUnitsWithPattern,
List<String> optionList) {

Stopwatch stopwatch = Stopwatch.createStarted();

SmartlingOptions options = SmartlingOptions.parseList(optionList);
if (options.isJsonSync()) {
thirdPartyTMSSmartlingWithJson.pushTranslations(
Expand Down Expand Up @@ -685,6 +718,9 @@ public void pushTranslations(
.collect(Collectors.toList());

resultProcessor.processPushTranslations(result, options);
meterRegistry
.timer("ThirdPartyTMSSmartling.pushTranslations", "repository", repository.getName())
.record(stopwatch.stop().elapsed());
}

private Set<Long> getFilterTmTextUnitIdsForPushTranslation(SmartlingOptions options) {
Expand All @@ -707,8 +743,12 @@ public void pullSource(
Map<String, String> localeMapping) {
SmartlingOptions options = SmartlingOptions.parseList(optionList);
if (options.isGlossarySync()) {
Stopwatch stopwatch = Stopwatch.createStarted();
thirdPartyTMSSmartlingGlossary.pullSourceTextUnits(
repository, thirdPartyProjectId, localeMapping);
meterRegistry
.timer("ThirdPartyTMSSmartling.pullSource", "repository", repository.getName())
.record(stopwatch.stop().elapsed());
} else {
throw new UnsupportedOperationException(
"Pull source is only supported with glossary sync enabled.");
Expand All @@ -729,6 +769,7 @@ private SmartlingFile processTranslationBatch(

logger.debug("Process {} batch: {}", localeTag, batchNumber);
logger.debug("Convert text units to AndroidString for asset number: {}", batchNumber);
Stopwatch stopwatch = Stopwatch.createStarted();
String sourceFilename =
getOutputSourceFile(batchNumber, repository.getName(), filePrefix.getType());
String targetFilename =
Expand Down Expand Up @@ -791,6 +832,14 @@ private SmartlingFile processTranslationBatch(
.block();
}

meterRegistry
.timer(
"ThirdPartyTMSSmartlingWithJson.processTranslationBatch",
"repository",
repository.getName(),
"locale",
localeTag)
.record(stopwatch.stop().elapsed());
return file;
}

Expand Down
3 changes: 3 additions & 0 deletions .../src/main/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyTMSSmartlingGlossary.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@
import com.box.l10n.mojito.smartling.response.GlossaryTargetTerm;
import com.box.l10n.mojito.smartling.response.GlossaryTermTranslation;
import com.google.common.base.Strings;
import io.micrometer.core.instrument.MeterRegistry;
import java.io.ByteArrayInputStream;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
Expand Down Expand Up @@ -58,6 +59,8 @@ public class ThirdPartyTMSSmartlingGlossary {

@Autowired AssetTextUnitRepository assetTextUnitRepository;

@Autowired MeterRegistry meterRegistry;

@Value("${l10n.smartling.accountId:}")
String accountId;

Expand Down
33 changes: 32 additions & 1 deletion .../src/main/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyTMSSmartlingWithJson.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,9 @@
import com.box.l10n.mojito.smartling.response.Items;
import com.box.l10n.mojito.smartling.response.StringInfo;
import com.google.common.base.Preconditions;
import com.google.common.base.Stopwatch;
import com.google.common.collect.ImmutableList;
import io.micrometer.core.instrument.MeterRegistry;
import java.util.List;
import java.util.Map;
import java.util.function.Function;
Expand Down Expand Up @@ -54,19 +56,23 @@ public class ThirdPartyTMSSmartlingWithJson {
TextUnitBatchImporterService textUnitBatchImporterService;
SmartlingJsonKeys smartlingJsonKeys;

MeterRegistry meterRegistry;

int batchSize = 5000;

public ThirdPartyTMSSmartlingWithJson(
SmartlingClient smartlingClient,
SmartlingJsonConverter smartlingJsonConverter,
TextUnitSearcher textUnitSearcher,
TextUnitBatchImporterService textUnitBatchImporterService,
SmartlingJsonKeys smartlingJsonKeys) {
SmartlingJsonKeys smartlingJsonKeys,
MeterRegistry meterRegistry) {
this.smartlingClient = smartlingClient;
this.smartlingJsonConverter = smartlingJsonConverter;
this.textUnitSearcher = textUnitSearcher;
this.textUnitBatchImporterService = textUnitBatchImporterService;
this.smartlingJsonKeys = smartlingJsonKeys;
this.meterRegistry = meterRegistry;
}

void push(
Expand All @@ -82,12 +88,15 @@ void push(
repository.getName(),
projectId);

Stopwatch totalStopwatch = Stopwatch.createStarted();

long partitionCount =
Spliterators.partitionStreamWithIndex(
getSourceTextUnitIterator(
repository, skipTextUnitsWithPattern, skipAssetsWithPathPattern),
batchSize,
(textUnitDTOS, index) -> {
Stopwatch batchStopWatch = Stopwatch.createStarted();
String fileName = getSourceFileName(repository.getName(), index);
String fileContent =
smartlingJsonConverter.textUnitDTOsToJsonString(
Expand Down Expand Up @@ -122,11 +131,20 @@ void push(
throw new SmartlingClientException(msg, e);
})
.block();
meterRegistry
.timer(
"ThirdPartyTMSSmartlingWithJson.push.batch",
"repository",
repository.getName())
.record(batchStopWatch.stop().elapsed());
return index;
})
.count();

removeFileForBatchNumberGreaterOrEquals(repository.getName(), projectId, partitionCount);
meterRegistry
.timer("ThirdPartyTMSSmartlingWithJson.push")
.record(totalStopwatch.stop().elapsed());
}

void removeFileForBatchNumberGreaterOrEquals(
Expand Down Expand Up @@ -226,6 +244,8 @@ public void pushTranslations(
logger.info(
"Push translation from repository: {} into project: {}", repository.getName(), projectId);

Stopwatch stopwatch = Stopwatch.createStarted();

getRepositoryLocaleWithoutRootStream(repository)
.forEach(
repositoryLocale -> {
Expand All @@ -239,6 +259,7 @@ public void pushTranslations(
includeTextUnitsWithPattern),
batchSize,
(textUnitDTOS, index) -> {
Stopwatch batchStopwatch = Stopwatch.createStarted();
String fileName = getSourceFileName(repository.getName(), index);
String fileContent =
smartlingJsonConverter.textUnitDTOsToJsonString(
Expand Down Expand Up @@ -275,11 +296,21 @@ public void pushTranslations(
throw new SmartlingClientException(msg, e);
})
.block();
meterRegistry
.timer(
"ThirdPartyTMSSmartlingWithJson.pushTranslations.batch",
"repository",
repository.getName())
.record(batchStopwatch.stop().elapsed());
return index;
})
.count();
logger.debug("Processed {} partitions", partitionCount);
});
meterRegistry
.timer(
"ThirdPartyTMSSmartlingWithJson.pushTranslations", "repository", repository.getName())
.record(stopwatch.stop().elapsed());
}

PageFetcherOffsetAndLimitSplitIterator<TextUnitDTO> getSourceTextUnitIterator(
Expand Down
21 changes: 21 additions & 0 deletions ...p/src/main/java/com/box/l10n/mojito/service/tm/importer/TextUnitBatchImporterService.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,9 +34,11 @@
import com.box.l10n.mojito.service.tm.search.TextUnitSearcher;
import com.box.l10n.mojito.service.tm.textunitdtocache.TextUnitDTOsCacheService;
import com.box.l10n.mojito.service.tm.textunitdtocache.UpdateType;
import com.google.common.base.Stopwatch;
import com.google.common.cache.CacheBuilder;
import com.google.common.cache.CacheLoader;
import com.google.common.cache.LoadingCache;
import io.micrometer.core.instrument.MeterRegistry;
import java.util.AbstractMap.SimpleEntry;
import java.util.List;
import java.util.Map;
Expand Down Expand Up @@ -83,6 +85,8 @@ public class TextUnitBatchImporterService {

@Autowired TMTextUnitVariantCommentService tmMTextUnitVariantCommentService;

@Autowired MeterRegistry meterRegistry;

/**
* Imports a batch of text units.
*
Expand Down Expand Up @@ -120,6 +124,8 @@ public PollableFuture<Void> importTextUnits(
boolean integrityCheckSkipped,
boolean integrityCheckKeepStatusIfFailedAndSameTarget) {

Stopwatch stopwatchTotal = Stopwatch.createStarted();

logger.debug("Import {} text units", textUnitDTOs.size());
List<TextUnitForBatchMatcherImport> textUnitForBatchImports =
skipInvalidAndConvertToTextUnitForBatchImport(textUnitDTOs);
Expand All @@ -138,15 +144,30 @@ public PollableFuture<Void> importTextUnits(
(asset, textUnitsForBatchImport) -> {
mapTextUnitsToImportWithExistingTextUnits(locale, asset, textUnitsForBatchImport);
if (!integrityCheckSkipped) {
Stopwatch integrityCheckStopwatch = Stopwatch.createStarted();
applyIntegrityChecks(
asset,
textUnitsForBatchImport,
integrityCheckKeepStatusIfFailedAndSameTarget);
meterRegistry
.timer(
"TextUnitBatchImporterService.importTextUnits.integrityChecks",
"repository",
asset.getRepository().getName(),
"asset",
asset.getPath(),
"locale",
locale.getBcp47Tag())
.record(integrityCheckStopwatch.stop().elapsed());
}
importTextUnitsOfLocaleAndAsset(locale, asset, textUnitsForBatchImport);
});
});

meterRegistry
.timer("TextUnitBatchImporterService.importTextUnits")
.record(stopwatchTotal.stop().elapsed());

return new PollableFutureTaskResult<>();
}

Expand Down
Loading

0 comments on commit 56353d2

Please sign in to comment.