From 53370ce3173421a1d76c360c0137606062c61755 Mon Sep 17 00:00:00 2001 From: Mark Allen <3417310+maallen@users.noreply.github.com> Date: Fri, 4 Aug 2023 17:43:13 +0100 Subject: [PATCH] Added file checksum handling to ThirdPartyTMSSmartlingWithJSON --- .../thirdparty/ThirdPartyTMSSmartling.java | 32 ++++------------- .../ThirdPartyTMSSmartlingWithJson.java | 20 ++++++++++- .../thirdparty/ThirdPartyTMSUtils.java | 35 +++++++++++++++++++ .../ThirdPartyTMSSmartlingWithJsonTest.java | 5 ++- 4 files changed, 64 insertions(+), 28 deletions(-) create mode 100644 webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyTMSUtils.java diff --git a/webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyTMSSmartling.java b/webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyTMSSmartling.java index be1efd26c2..3759079dbe 100644 --- a/webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyTMSSmartling.java +++ b/webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyTMSSmartling.java @@ -1,6 +1,7 @@ package com.box.l10n.mojito.service.thirdparty; import static com.box.l10n.mojito.android.strings.AndroidPluralQuantity.MANY; +import static com.box.l10n.mojito.service.thirdparty.ThirdPartyTMSUtils.isFileEqualToPreviousRun; import static com.box.l10n.mojito.service.thirdparty.smartling.SmartlingFileUtils.getOutputSourceFile; import static com.box.l10n.mojito.service.thirdparty.smartling.SmartlingFileUtils.getOutputTargetFile; import static com.box.l10n.mojito.service.thirdparty.smartling.SmartlingFileUtils.isPluralFile; @@ -9,11 +10,8 @@ import com.box.l10n.mojito.android.strings.AndroidStringDocumentMapper; import com.box.l10n.mojito.android.strings.AndroidStringDocumentReader; import com.box.l10n.mojito.android.strings.AndroidStringDocumentWriter; -import com.box.l10n.mojito.entity.Locale; import com.box.l10n.mojito.entity.Repository; import com.box.l10n.mojito.entity.RepositoryLocale; -import com.box.l10n.mojito.entity.ThirdPartyFileChecksum; -import com.box.l10n.mojito.entity.ThirdPartyFileChecksumId; import com.box.l10n.mojito.service.assetExtraction.AssetTextUnitToTMTextUnitRepository; import com.box.l10n.mojito.service.thirdparty.smartling.SmartlingFile; import com.box.l10n.mojito.service.thirdparty.smartling.SmartlingFileUtils; @@ -45,7 +43,6 @@ import java.io.IOException; import java.util.List; import java.util.Map; -import java.util.Optional; import java.util.Set; import java.util.function.Function; import java.util.regex.Pattern; @@ -55,7 +52,6 @@ import java.util.stream.StreamSupport; import javax.xml.parsers.ParserConfigurationException; import javax.xml.transform.TransformerException; -import org.apache.commons.codec.digest.DigestUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; @@ -660,7 +656,11 @@ private void processPullBatch( "Error with download from Smartling, file content string is not present.")); if (isFileEqualToPreviousRun( - repository, locale.getLocale(), fileName, fileContent)) { + thirdPartyFileChecksumRepository, + repository, + locale.getLocale(), + fileName, + fileContent)) { logger.info("Checksum match for " + fileName + ", skipping text unit import."); return; } @@ -690,26 +690,6 @@ private void processPullBatch( } } - private boolean isFileEqualToPreviousRun( - Repository repository, Locale locale, String fileName, String fileContent) { - - String currentChecksum = DigestUtils.md5Hex(fileContent); - ThirdPartyFileChecksumId thirdPartyFileChecksumId = - new ThirdPartyFileChecksumId(repository, locale, fileName); - - Optional thirdPartyFileChecksum = - thirdPartyFileChecksumRepository.findByThirdPartyFileChecksumId(thirdPartyFileChecksumId); - if (thirdPartyFileChecksum.isPresent() - && thirdPartyFileChecksum.get().getChecksum().equals(currentChecksum)) { - return true; - } else { - thirdPartyFileChecksumRepository.save( - new ThirdPartyFileChecksum(thirdPartyFileChecksumId, currentChecksum)); - } - - return false; - } - @Override public void pushTranslations( Repository repository, diff --git a/webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyTMSSmartlingWithJson.java b/webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyTMSSmartlingWithJson.java index 9c10a5ff37..d043299b80 100644 --- a/webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyTMSSmartlingWithJson.java +++ b/webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyTMSSmartlingWithJson.java @@ -1,5 +1,6 @@ package com.box.l10n.mojito.service.thirdparty; +import static com.box.l10n.mojito.service.thirdparty.ThirdPartyTMSUtils.isFileEqualToPreviousRun; import static com.box.l10n.mojito.service.thirdparty.smartling.SmartlingFileUtils.isPluralFile; import com.box.l10n.mojito.entity.Repository; @@ -58,6 +59,8 @@ public class ThirdPartyTMSSmartlingWithJson { MeterRegistry meterRegistry; + ThirdPartyFileChecksumRepository thirdPartyFileChecksumRepository; + int batchSize = 5000; public ThirdPartyTMSSmartlingWithJson( @@ -66,13 +69,15 @@ public ThirdPartyTMSSmartlingWithJson( TextUnitSearcher textUnitSearcher, TextUnitBatchImporterService textUnitBatchImporterService, SmartlingJsonKeys smartlingJsonKeys, - MeterRegistry meterRegistry) { + MeterRegistry meterRegistry, + ThirdPartyFileChecksumRepository thirdPartyFileChecksumRepository) { this.smartlingClient = smartlingClient; this.smartlingJsonConverter = smartlingJsonConverter; this.textUnitSearcher = textUnitSearcher; this.textUnitBatchImporterService = textUnitBatchImporterService; this.smartlingJsonKeys = smartlingJsonKeys; this.meterRegistry = meterRegistry; + this.thirdPartyFileChecksumRepository = thirdPartyFileChecksumRepository; } void push( @@ -201,6 +206,19 @@ void pull(Repository repository, String projectId, Map localeMap String localizedFileContent = getLocalizedFileContent(projectId, file, smartlingLocale, false); + if (isFileEqualToPreviousRun( + thirdPartyFileChecksumRepository, + repository, + repositoryLocale.getLocale(), + file.getFileUri(), + localizedFileContent)) { + logger.info( + "Checksum match for " + + file.getFileUri() + + ", skipping text unit import."); + return; + } + ImmutableList textUnitDTOS = smartlingJsonConverter.jsonStringToTextUnitDTOs( localizedFileContent, TextUnitDTO::setTarget); diff --git a/webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyTMSUtils.java b/webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyTMSUtils.java new file mode 100644 index 0000000000..e06d603419 --- /dev/null +++ b/webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyTMSUtils.java @@ -0,0 +1,35 @@ +package com.box.l10n.mojito.service.thirdparty; + +import com.box.l10n.mojito.entity.Locale; +import com.box.l10n.mojito.entity.Repository; +import com.box.l10n.mojito.entity.ThirdPartyFileChecksum; +import com.box.l10n.mojito.entity.ThirdPartyFileChecksumId; +import java.util.Optional; +import org.apache.commons.codec.digest.DigestUtils; + +public class ThirdPartyTMSUtils { + + public static boolean isFileEqualToPreviousRun( + ThirdPartyFileChecksumRepository thirdPartyFileChecksumRepository, + Repository repository, + Locale locale, + String fileName, + String fileContent) { + + String currentChecksum = DigestUtils.md5Hex(fileContent); + ThirdPartyFileChecksumId thirdPartyFileChecksumId = + new ThirdPartyFileChecksumId(repository, locale, fileName); + + Optional thirdPartyFileChecksum = + thirdPartyFileChecksumRepository.findByThirdPartyFileChecksumId(thirdPartyFileChecksumId); + if (thirdPartyFileChecksum.isPresent() + && thirdPartyFileChecksum.get().getChecksum().equals(currentChecksum)) { + return true; + } else { + thirdPartyFileChecksumRepository.save( + new ThirdPartyFileChecksum(thirdPartyFileChecksumId, currentChecksum)); + } + + return false; + } +} diff --git a/webapp/src/test/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyTMSSmartlingWithJsonTest.java b/webapp/src/test/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyTMSSmartlingWithJsonTest.java index c2da4e4850..64ba30d6fa 100644 --- a/webapp/src/test/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyTMSSmartlingWithJsonTest.java +++ b/webapp/src/test/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyTMSSmartlingWithJsonTest.java @@ -80,6 +80,8 @@ public class ThirdPartyTMSSmartlingWithJsonTest extends ServiceTestBase { @Autowired TextUnitSearcher textUnitSearcher; + @Mock ThirdPartyFileChecksumRepository thirdPartyFileChecksumRepository; + SmartlingJsonConverter smartlingJsonConverter = new SmartlingJsonConverter(ObjectMapper.withIndentedOutput(), new SmartlingJsonKeys()); @@ -250,7 +252,8 @@ public void testGetTranslatedUnits() { ImmutableList.of(translatedTextUnitDto, untranslatedTextUnitDtoWithOriginalString); ThirdPartyTMSSmartlingWithJson thirdPartyTMSSmartlingWithJson = - new ThirdPartyTMSSmartlingWithJson(null, null, null, null, null, meterRegistryMock); + new ThirdPartyTMSSmartlingWithJson( + null, null, null, null, null, meterRegistryMock, thirdPartyFileChecksumRepository); ImmutableList result = thirdPartyTMSSmartlingWithJson.getTranslatedUnits(