From 9e0acd46e6b68e3f2e360199332471f5f0ad5d96 Mon Sep 17 00:00:00 2001 From: Mark Allen <3417310+maallen@users.noreply.github.com> Date: Thu, 17 Oct 2024 10:01:52 +0100 Subject: [PATCH] AI translation updates to TMTextUnitSearcher and store uploaded file uri in mapping step (#162) * Updated TextUnitSearcher to exclude AI translations based on duration or null id, updated map step to record uploaded file uri in DB * Added unit tests for text unit searcher * Unit test update to include file uri * Remove unnecessary code --- .../mojito/entity/ThirdPartyTextUnit.java | 11 ++++ .../service/thirdparty/ThirdPartyService.java | 2 + .../thirdparty/ThirdPartyTMSSmartling.java | 16 ++++- .../thirdparty/ThirdPartyTextUnit.java | 10 +++ .../service/tm/search/TextUnitSearcher.java | 18 ++++++ .../tm/search/TextUnitSearcherParameters.java | 19 ++++++ .../thirdparty/ThirdPartyServiceTest.java | 34 +++++++--- .../ThirdPartyTMSSmartlingTest.java | 42 ++++++++---- .../l10n/mojito/service/tm/TMTestData.java | 47 +++++++++++++- .../tm/search/TextUnitSearcherTest.java | 64 +++++++++++++++++++ 10 files changed, 237 insertions(+), 26 deletions(-) diff --git a/webapp/src/main/java/com/box/l10n/mojito/entity/ThirdPartyTextUnit.java b/webapp/src/main/java/com/box/l10n/mojito/entity/ThirdPartyTextUnit.java index eb58456101..1ec1b341cb 100644 --- a/webapp/src/main/java/com/box/l10n/mojito/entity/ThirdPartyTextUnit.java +++ b/webapp/src/main/java/com/box/l10n/mojito/entity/ThirdPartyTextUnit.java @@ -41,6 +41,9 @@ public class ThirdPartyTextUnit extends AuditableEntity { foreignKey = @ForeignKey(name = "FK__THIRD_PARTY_TEXT_UNIT__TM_TEXT_UNIT__ID")) TMTextUnit tmTextUnit; + @Column(name = "uploaded_file_uri") + String uploadedFileUri; + public String getThirdPartyId() { return thirdPartyId; } @@ -64,4 +67,12 @@ public Asset getAsset() { public void setAsset(Asset asset) { this.asset = asset; } + + public String getUploadedFileUri() { + return uploadedFileUri; + } + + public void setUploadedFileUri(String uploadedFileUri) { + this.uploadedFileUri = uploadedFileUri; + } } diff --git a/webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyService.java b/webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyService.java index 0524677f9c..cecd5686a3 100644 --- a/webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyService.java +++ b/webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyService.java @@ -361,6 +361,8 @@ void saveMojitoToThirdPartyTextUnitMapping( thirdPartyTextUnit.setThirdPartyId( thirdPartyTextUnitForMapping.getId()); thirdPartyTextUnit.setAsset(asset); + thirdPartyTextUnit.setUploadedFileUri( + thirdPartyTextUnitForMapping.getUploadedFileUri()); TMTextUnit tmTextUnit = tmTextUnitRepository.getOne(tmTextUnitId); if (tmTextUnitAlreadySaved.containsKey(tmTextUnitId)) { diff --git a/webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyTMSSmartling.java b/webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyTMSSmartling.java index 763e976119..3270556c28 100644 --- a/webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyTMSSmartling.java +++ b/webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyTMSSmartling.java @@ -12,6 +12,7 @@ import com.box.l10n.mojito.entity.Repository; import com.box.l10n.mojito.quartz.QuartzJobInfo; import com.box.l10n.mojito.quartz.QuartzPollableTaskScheduler; +import com.box.l10n.mojito.service.ai.translation.AITranslationConfiguration; import com.box.l10n.mojito.service.assetExtraction.AssetTextUnitToTMTextUnitRepository; import com.box.l10n.mojito.service.pollableTask.PollableFuture; import com.box.l10n.mojito.service.thirdparty.smartling.SmartlingFile; @@ -100,6 +101,8 @@ public class ThirdPartyTMSSmartling implements ThirdPartyTMS { private final QuartzPollableTaskScheduler quartzPollableTaskScheduler; + private final AITranslationConfiguration aiTranslationConfiguration; + private final Set supportedImageExtensions = Sets.newHashSet("png", "jpg", "jpeg", "gif", "tiff"); @@ -118,7 +121,8 @@ public ThirdPartyTMSSmartling( ThirdPartyTMSSmartlingGlossary thirdPartyTMSSmartlingGlossary, AssetTextUnitToTMTextUnitRepository assetTextUnitToTMTextUnitRepository, MeterRegistry meterRegistry, - QuartzPollableTaskScheduler quartzPollableTaskScheduler) { + QuartzPollableTaskScheduler quartzPollableTaskScheduler, + AITranslationConfiguration aiTranslationConfiguration) { this( smartlingClient, textUnitSearcher, @@ -130,7 +134,8 @@ public ThirdPartyTMSSmartling( assetTextUnitToTMTextUnitRepository, DEFAULT_BATCH_SIZE, meterRegistry, - quartzPollableTaskScheduler); + quartzPollableTaskScheduler, + aiTranslationConfiguration); } public ThirdPartyTMSSmartling( @@ -144,7 +149,8 @@ public ThirdPartyTMSSmartling( AssetTextUnitToTMTextUnitRepository assetTextUnitToTMTextUnitRepository, int batchSize, MeterRegistry meterRegistry, - QuartzPollableTaskScheduler quartzPollableTaskScheduler) { + QuartzPollableTaskScheduler quartzPollableTaskScheduler, + AITranslationConfiguration aiTranslationConfiguration) { this.smartlingClient = smartlingClient; this.assetPathAndTextUnitNameKeys = assetPathAndTextUnitNameKeys; this.textUnitBatchImporterService = textUnitBatchImporterService; @@ -156,6 +162,7 @@ public ThirdPartyTMSSmartling( this.assetTextUnitToTMTextUnitRepository = assetTextUnitToTMTextUnitRepository; this.meterRegistry = meterRegistry; this.quartzPollableTaskScheduler = quartzPollableTaskScheduler; + this.aiTranslationConfiguration = aiTranslationConfiguration; } @Override @@ -337,6 +344,7 @@ public List getThirdPartyTextUnits( thirdPartyTextUnit.setAssetPath(key.getAssetPath()); thirdPartyTextUnit.setName(key.getTextUnitName()); thirdPartyTextUnit.setNamePluralPrefix(isPluralFile(file.getFileUri())); + thirdPartyTextUnit.setUploadedFileUri(file.getFileUri()); return thirdPartyTextUnit; }); @@ -968,6 +976,8 @@ private TextUnitSearcherParameters baseParams( result.setPluralFormsExcluded(pluralFormsExcluded); result.setSkipTextUnitWithPattern(skipTextUnitsWithPattern); result.setSkipAssetPathWithPattern(skipAssetsWithPathPattern); + result.setExcludeUnexpiredPendingMT(aiTranslationConfiguration.isEnabled()); + result.setAiTranslationExpiryDuration(aiTranslationConfiguration.getExpiryDuration()); if (!Strings.isNullOrEmpty(pluralFormOther)) { result.setPluralFormOther(pluralFormOther); } diff --git a/webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyTextUnit.java b/webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyTextUnit.java index 3f3fda32ef..8e37f3ccc9 100644 --- a/webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyTextUnit.java +++ b/webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyTextUnit.java @@ -30,6 +30,8 @@ public class ThirdPartyTextUnit implements TextUnitForBatchMatcher { */ boolean namePluralPrefix; + String uploadedFileUri; + public String getId() { return id; } @@ -80,4 +82,12 @@ public boolean isNamePluralPrefix() { public void setNamePluralPrefix(boolean namePluralPrefix) { this.namePluralPrefix = namePluralPrefix; } + + public String getUploadedFileUri() { + return uploadedFileUri; + } + + public void setUploadedFileUri(String uploadedFileUri) { + this.uploadedFileUri = uploadedFileUri; + } } diff --git a/webapp/src/main/java/com/box/l10n/mojito/service/tm/search/TextUnitSearcher.java b/webapp/src/main/java/com/box/l10n/mojito/service/tm/search/TextUnitSearcher.java index 846fbd2e44..728da69b9b 100644 --- a/webapp/src/main/java/com/box/l10n/mojito/service/tm/search/TextUnitSearcher.java +++ b/webapp/src/main/java/com/box/l10n/mojito/service/tm/search/TextUnitSearcher.java @@ -24,6 +24,7 @@ import com.github.pnowy.nc.core.expressions.NativeProjection; import com.github.pnowy.nc.core.mappers.CriteriaResultTransformer; import com.google.common.base.Preconditions; +import java.time.ZonedDateTime; import java.util.Arrays; import java.util.List; import org.apache.commons.codec.digest.DigestUtils; @@ -172,6 +173,11 @@ NativeCriteria getCriteriaForSearch(TextUnitSearcherParameters searchParameters) c.addJoin(NativeExps.crossJoin("locale", "l")); c.addJoin(NativeExps.innerJoin("asset", "a", "a.id", "tu.asset_id")); c.addJoin(NativeExps.innerJoin("repository", "r", "r.id", "a.repository_id")); + if (searchParameters.isExcludeUnexpiredPendingMT()) { + c.addJoin( + NativeExps.leftJoin( + "tm_text_unit_pending_mt", "tmtupmt", "tmtupmt.tm_text_unit_id", "tu.id")); + } NativeJunctionExp onClauseRepositoryLocale = NativeExps.conjunction(); onClauseRepositoryLocale.add(new NativeColumnEqExp("rl.locale_id", "l.id")); @@ -378,6 +384,18 @@ NativeCriteria getCriteriaForSearch(TextUnitSearcherParameters searchParameters) new NativeNotILikeExp("a.path", searchParameters.getSkipAssetPathWithPattern())); } + if (searchParameters.isExcludeUnexpiredPendingMT()) { + ZonedDateTime durationThreshold = + ZonedDateTime.now().minus(searchParameters.getAiTranslationExpiryDuration()); + + // Include rows where the id in the left table is null (i.e. no match) or the created_date is + // outside the configured expiry duration + NativeJunctionExp mtExclusionFilter = NativeExps.disjunction(); + mtExclusionFilter.add(NativeExps.isNull("tmtupmt.id")); + mtExclusionFilter.add(new NativeDateLteExp("tmtupmt.created_date", durationThreshold)); + conjunction.add(mtExclusionFilter); + } + StatusFilter statusFilter = searchParameters.getStatusFilter(); if (statusFilter != null) { diff --git a/webapp/src/main/java/com/box/l10n/mojito/service/tm/search/TextUnitSearcherParameters.java b/webapp/src/main/java/com/box/l10n/mojito/service/tm/search/TextUnitSearcherParameters.java index 6c1b95dd76..ae3354a915 100644 --- a/webapp/src/main/java/com/box/l10n/mojito/service/tm/search/TextUnitSearcherParameters.java +++ b/webapp/src/main/java/com/box/l10n/mojito/service/tm/search/TextUnitSearcherParameters.java @@ -1,6 +1,7 @@ package com.box.l10n.mojito.service.tm.search; import com.box.l10n.mojito.service.NormalizationUtils; +import java.time.Duration; import java.time.ZonedDateTime; import java.util.Arrays; import java.util.List; @@ -50,6 +51,8 @@ public class TextUnitSearcherParameters { String skipTextUnitWithPattern; String includeTextUnitsWithPattern; String skipAssetPathWithPattern; + boolean isExcludeUnexpiredPendingMT = false; + Duration aiTranslationExpiryDuration; public String getName() { return name; @@ -327,4 +330,20 @@ public boolean isOrderedByTextUnitID() { public void setOrderByTextUnitID(boolean ordered) { isOrderedByTextUnitID = ordered; } + + public boolean isExcludeUnexpiredPendingMT() { + return isExcludeUnexpiredPendingMT; + } + + public void setExcludeUnexpiredPendingMT(boolean excludeUnexpiredPendingMT) { + isExcludeUnexpiredPendingMT = excludeUnexpiredPendingMT; + } + + public Duration getAiTranslationExpiryDuration() { + return aiTranslationExpiryDuration; + } + + public void setAiTranslationExpiryDuration(Duration aiTranslationExpiryDuration) { + this.aiTranslationExpiryDuration = aiTranslationExpiryDuration; + } } diff --git a/webapp/src/test/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyServiceTest.java b/webapp/src/test/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyServiceTest.java index 0d3acf7263..17b2c26c35 100644 --- a/webapp/src/test/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyServiceTest.java +++ b/webapp/src/test/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyServiceTest.java @@ -197,34 +197,49 @@ public void mapMojitoAndThirdPartyTextUnits() throws InterruptedException, Execu .extracting( t -> t.getAsset().getId(), com.box.l10n.mojito.entity.ThirdPartyTextUnit::getThirdPartyId, - t -> t.getTmTextUnit().getId()) + t -> t.getTmTextUnit().getId(), + com.box.l10n.mojito.entity.ThirdPartyTextUnit::getUploadedFileUri) .containsExactly( - tuple(asset.getId(), "3rd-hello", thirdPartyServiceTestData.tmTextUnitHello.getId()), - tuple(asset.getId(), "3rd-bye", thirdPartyServiceTestData.tmTextUnitBye.getId()), + tuple( + asset.getId(), + "3rd-hello", + thirdPartyServiceTestData.tmTextUnitHello.getId(), + "testFileUri"), + tuple( + asset.getId(), + "3rd-bye", + thirdPartyServiceTestData.tmTextUnitBye.getId(), + "testFileUri"), tuple( asset.getId(), "3rd-plural_things", - thirdPartyServiceTestData.tmTextUnitPluralThingsZero.getId()), + thirdPartyServiceTestData.tmTextUnitPluralThingsZero.getId(), + "testFileUri"), tuple( asset.getId(), "3rd-plural_things", - thirdPartyServiceTestData.tmTextUnitPluralThingsOne.getId()), + thirdPartyServiceTestData.tmTextUnitPluralThingsOne.getId(), + "testFileUri"), tuple( asset.getId(), "3rd-plural_things", - thirdPartyServiceTestData.tmTextUnitPluralThingsTwo.getId()), + thirdPartyServiceTestData.tmTextUnitPluralThingsTwo.getId(), + "testFileUri"), tuple( asset.getId(), "3rd-plural_things", - thirdPartyServiceTestData.tmTextUnitPluralThingsFew.getId()), + thirdPartyServiceTestData.tmTextUnitPluralThingsFew.getId(), + "testFileUri"), tuple( asset.getId(), "3rd-plural_things", - thirdPartyServiceTestData.tmTextUnitPluralThingsMany.getId()), + thirdPartyServiceTestData.tmTextUnitPluralThingsMany.getId(), + "testFileUri"), tuple( asset.getId(), "3rd-plural_things", - thirdPartyServiceTestData.tmTextUnitPluralThingsOther.getId())); + thirdPartyServiceTestData.tmTextUnitPluralThingsOther.getId(), + "testFileUri")); logger.debug("Verify behavior"); verify(thirdPartyTMSMock, times(3)) @@ -741,6 +756,7 @@ ThirdPartyTextUnit createThirdPartyTextUnit( thirdPartyTextUnit.setId(id); thirdPartyTextUnit.setName(name); thirdPartyTextUnit.setNamePluralPrefix(isNamePluralPrefix); + thirdPartyTextUnit.setUploadedFileUri("testFileUri"); return thirdPartyTextUnit; } } diff --git a/webapp/src/test/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyTMSSmartlingTest.java b/webapp/src/test/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyTMSSmartlingTest.java index e79e185914..f819dd220c 100644 --- a/webapp/src/test/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyTMSSmartlingTest.java +++ b/webapp/src/test/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyTMSSmartlingTest.java @@ -34,6 +34,7 @@ import com.box.l10n.mojito.quartz.QuartzJobInfo; import com.box.l10n.mojito.quartz.QuartzPollableTaskScheduler; import com.box.l10n.mojito.quartz.QuartzSchedulerManager; +import com.box.l10n.mojito.service.ai.translation.AITranslationConfiguration; import com.box.l10n.mojito.service.asset.AssetService; import com.box.l10n.mojito.service.assetExtraction.AssetExtractionRepository; import com.box.l10n.mojito.service.assetExtraction.AssetExtractionService; @@ -161,6 +162,8 @@ public class ThirdPartyTMSSmartlingTest extends ServiceTestBase { @Mock TextUnitBatchImporterService mockTextUnitBatchImporterService; + @Mock AITranslationConfiguration aiTranslationConfiguration; + @Captor ArgumentCaptor> textUnitListCaptor; @Captor @@ -216,7 +219,8 @@ public void setUp() throws SchedulerException { mockThirdPartyTMSSmartlingGlossary, assetTextUnitToTMTextUnitRepository, meterRegistry, - mockQuartzPollableTaskScheduler); + mockQuartzPollableTaskScheduler, + aiTranslationConfiguration); mapper = new AndroidStringDocumentMapper(pluralSep, null); RetryBackoffSpec retryConfiguration = @@ -292,7 +296,8 @@ public void testPushInBatchesWithSingularsAndNoPlurals() assetTextUnitToTMTextUnitRepository, batchSize, meterRegistry, - mockQuartzPollableTaskScheduler); + mockQuartzPollableTaskScheduler, + aiTranslationConfiguration); TM tm = repository.getTm(); Asset asset = @@ -346,7 +351,8 @@ public void testRetryDuringPush() throws RepositoryNameAlreadyUsedException { assetTextUnitToTMTextUnitRepository, batchSize, meterRegistry, - mockQuartzPollableTaskScheduler); + mockQuartzPollableTaskScheduler, + aiTranslationConfiguration); // throw timeout exception for first request, following request should be successful when(smartlingClient.uploadFile(any(), any(), any(), any(), any(), any(), any())) .thenThrow( @@ -412,7 +418,8 @@ public void testRetriesExhaustedDuringPush() throws RepositoryNameAlreadyUsedExc assetTextUnitToTMTextUnitRepository, batchSize, meterRegistry, - mockQuartzPollableTaskScheduler); + mockQuartzPollableTaskScheduler, + aiTranslationConfiguration); TM tm = repository.getTm(); Asset asset = @@ -463,7 +470,8 @@ public void testPushInBatchesWithNoSingularsAndPlurals() assetTextUnitToTMTextUnitRepository, batchSize, meterRegistry, - mockQuartzPollableTaskScheduler); + mockQuartzPollableTaskScheduler, + aiTranslationConfiguration); TM tm = repository.getTm(); Asset asset = @@ -522,7 +530,8 @@ public void testPushInBatchesWithSingularsAndPlurals() throws RepositoryNameAlre assetTextUnitToTMTextUnitRepository, batchSize, meterRegistry, - mockQuartzPollableTaskScheduler); + mockQuartzPollableTaskScheduler, + aiTranslationConfiguration); TM tm = repository.getTm(); Asset asset = @@ -737,7 +746,8 @@ public void testPullNoBatches() throws RepositoryLocaleCreationException, Interr mockThirdPartyTMSSmartlingGlossary, assetTextUnitToTMTextUnitRepository, meterRegistry, - mockQuartzPollableTaskScheduler); + mockQuartzPollableTaskScheduler, + aiTranslationConfiguration); tmsSmartling.pull( repository, "projectId", @@ -809,7 +819,8 @@ public void testPullNoBatchesPluralFix() throws RepositoryLocaleCreationExceptio mockThirdPartyTMSSmartlingGlossary, assetTextUnitToTMTextUnitRepository, meterRegistry, - mockQuartzPollableTaskScheduler); + mockQuartzPollableTaskScheduler, + aiTranslationConfiguration); tmsSmartling.pull( repository, "projectId", @@ -872,7 +883,8 @@ public void testPullDryRunNoBatches() throws RepositoryLocaleCreationException { mockThirdPartyTMSSmartlingGlossary, assetTextUnitToTMTextUnitRepository, meterRegistry, - mockQuartzPollableTaskScheduler); + mockQuartzPollableTaskScheduler, + aiTranslationConfiguration); tmsSmartling.pull( repository, "projectId", @@ -1461,7 +1473,8 @@ public void testPushTranslationsInBatches() assetTextUnitToTMTextUnitRepository, batchSize, meterRegistry, - mockQuartzPollableTaskScheduler); + mockQuartzPollableTaskScheduler, + aiTranslationConfiguration); Repository repository = repositoryService.createRepository(testIdWatcher.getEntityName("batchRepo")); Locale frCA = localeService.findByBcp47Tag("fr-CA"); @@ -1615,7 +1628,8 @@ public void testBatchesFor() { assetTextUnitToTMTextUnitRepository, 3, meterRegistry, - mockQuartzPollableTaskScheduler); + mockQuartzPollableTaskScheduler, + aiTranslationConfiguration); assertThat(tmsSmartling.batchesFor(0)).isEqualTo(0); assertThat(tmsSmartling.batchesFor(1)).isEqualTo(1); @@ -1638,7 +1652,8 @@ public void testBatchesFor() { assetTextUnitToTMTextUnitRepository, 35, meterRegistry, - mockQuartzPollableTaskScheduler); + mockQuartzPollableTaskScheduler, + aiTranslationConfiguration); assertThat(tmsSmartling.batchesFor(0)).isEqualTo(0); assertThat(tmsSmartling.batchesFor(1)).isEqualTo(1); @@ -1659,7 +1674,8 @@ public void testBatchesFor() { assetTextUnitToTMTextUnitRepository, 4231, meterRegistry, - mockQuartzPollableTaskScheduler); + mockQuartzPollableTaskScheduler, + aiTranslationConfiguration); assertThat(tmsSmartling.batchesFor(0)).isEqualTo(0); assertThat(tmsSmartling.batchesFor(1)).isEqualTo(1); diff --git a/webapp/src/test/java/com/box/l10n/mojito/service/tm/TMTestData.java b/webapp/src/test/java/com/box/l10n/mojito/service/tm/TMTestData.java index 5b4e110d62..e538d2a698 100644 --- a/webapp/src/test/java/com/box/l10n/mojito/service/tm/TMTestData.java +++ b/webapp/src/test/java/com/box/l10n/mojito/service/tm/TMTestData.java @@ -12,6 +12,8 @@ import com.box.l10n.mojito.entity.TM; import com.box.l10n.mojito.entity.TMTextUnit; import com.box.l10n.mojito.entity.TMTextUnitVariant; +import com.box.l10n.mojito.entity.TmTextUnitPendingMT; +import com.box.l10n.mojito.service.ai.translation.TmTextUnitPendingMTRepository; import com.box.l10n.mojito.service.asset.AssetService; import com.box.l10n.mojito.service.assetExtraction.AssetExtractionRepository; import com.box.l10n.mojito.service.assetExtraction.AssetExtractionService; @@ -62,6 +64,8 @@ public class TMTestData { @Autowired PluralFormService pluralFormService; + @Autowired TmTextUnitPendingMTRepository tmTextUnitPendingMTRepository; + public Repository repository; public TM tm; public TMTextUnit addTMTextUnit1; @@ -88,8 +92,15 @@ public class TMTestData { private boolean addUsage; + private boolean addPendingMTs; + + private ZonedDateTime pendingMTCreatedDate; + + private String repositoryName; + public TMTestData(TestIdWatcher testIdWatcher) { this.testIdWatcher = testIdWatcher; + this.repositoryName = testIdWatcher.getEntityName("repository"); } public TMTestData(TestIdWatcher testIdWatcher, boolean addUsage) { @@ -97,6 +108,32 @@ public TMTestData(TestIdWatcher testIdWatcher, boolean addUsage) { this.addUsage = addUsage; } + public TMTestData(TestIdWatcher testIdWatcher, boolean addUsage, String repositoryName) { + this(testIdWatcher, addUsage); + this.repositoryName = testIdWatcher.getEntityName(repositoryName); + } + + public TMTestData( + TestIdWatcher testIdWatcher, boolean addUsage, String repositoryName, boolean addPendingMTs) { + this( + testIdWatcher, + addUsage, + repositoryName, + addPendingMTs, + JSR310Migration.newDateTimeEmptyCtor()); + } + + public TMTestData( + TestIdWatcher testIdWatcher, + boolean addUsage, + String repositoryName, + boolean addPendingMTs, + ZonedDateTime pendingMTCreatedDate) { + this(testIdWatcher, addUsage, repositoryName); + this.addPendingMTs = addPendingMTs; + this.pendingMTCreatedDate = pendingMTCreatedDate; + } + @PostConstruct @Transactional private void createData() throws Exception { @@ -127,7 +164,8 @@ private void createData() throws Exception { assetExtractionOther, "TEST2", "Content2", "Comment2"); // This is the actual data that should be proccessed - repository = repositoryService.createRepository(testIdWatcher.getEntityName("repository")); + repository = repositoryService.createRepository(repositoryName); + repoLocaleKoKR = repositoryService.addRepositoryLocale(repository, koKR.getBcp47Tag()); repoLocaleFrFR = repositoryService.addRepositoryLocale(repository, frFR.getBcp47Tag()); repositoryService.addRepositoryLocale(repository, frCA.getBcp47Tag()); @@ -202,6 +240,13 @@ private void createData() throws Exception { addCurrentTMTextUnitVariant3FrCA = tmService.addCurrentTMTextUnitVariant( addTMTextUnit3.getId(), frCA.getId(), "Content3 fr-CA"); + + if (addPendingMTs) { + TmTextUnitPendingMT tmTextUnitPendingMT = new TmTextUnitPendingMT(); + tmTextUnitPendingMT.setTmTextUnitId(addTMTextUnit1.getId()); + tmTextUnitPendingMT.setCreatedDate(pendingMTCreatedDate); + tmTextUnitPendingMTRepository.save(tmTextUnitPendingMT); + } } public ImmutableMap addPluralString(String basename) { diff --git a/webapp/src/test/java/com/box/l10n/mojito/service/tm/search/TextUnitSearcherTest.java b/webapp/src/test/java/com/box/l10n/mojito/service/tm/search/TextUnitSearcherTest.java index be5ecd358f..f76f9105f9 100644 --- a/webapp/src/test/java/com/box/l10n/mojito/service/tm/search/TextUnitSearcherTest.java +++ b/webapp/src/test/java/com/box/l10n/mojito/service/tm/search/TextUnitSearcherTest.java @@ -23,6 +23,7 @@ import com.box.l10n.mojito.test.TestIdWatcher; import jakarta.persistence.EntityManager; import jakarta.persistence.TypedQuery; +import java.time.Duration; import java.time.ZonedDateTime; import java.util.ArrayList; import java.util.Arrays; @@ -33,6 +34,7 @@ import java.util.Optional; import org.apache.commons.lang3.builder.ToStringBuilder; import org.apache.commons.lang3.builder.ToStringStyle; +import org.assertj.core.api.Condition; import org.junit.Assert; import org.junit.Rule; import org.junit.Test; @@ -841,6 +843,68 @@ public void testILikeLocationUsageByUsages() { testSearchText("usage", "%USAGE_%TEST", SearchType.ILIKE, List.of("usage_test"), true); } + @Test + @Transactional + public void testExclusionOfPendingMTTextUnits() { + + TMTestData tmTestData = new TMTestData(testIdWatcher, false, "pendingMT", true); + TextUnitSearcherParameters textUnitSearcherParameters = new TextUnitSearcherParameters(); + textUnitSearcherParameters.setRepositoryIds(tmTestData.repository.getId()); + textUnitSearcherParameters.setUsedFilter(UsedFilter.USED); + textUnitSearcherParameters.setExcludeUnexpiredPendingMT(true); + textUnitSearcherParameters.setAiTranslationExpiryDuration(Duration.ofMinutes(10)); + + List results = textUnitSearcher.search(textUnitSearcherParameters); + assertEquals(4, results.size()); + assertThat(results) + .extracting(TextUnitDTO::getTmTextUnitId) + .doesNotContain(tmTestData.addTMTextUnit1.getId()); + + textUnitSearcherParameters.setExcludeUnexpiredPendingMT(false); + results = textUnitSearcher.search(textUnitSearcherParameters); + assertEquals(8, results.size()); + assertThat(results) + .extracting(TextUnitDTO::getTmTextUnitId) + .has( + new Condition<>( + ids -> ids.contains(tmTestData.addTMTextUnit1.getId()), + "TmTextUnit id is present")); + } + + @Test + @Transactional + public void testExpiredPendingMTIsIncludedInResults() { + + TMTestData tmTestData = + new TMTestData( + testIdWatcher, + false, + "expiredPendingMT", + true, + ZonedDateTime.now().minus(Duration.ofMinutes(30))); + TextUnitSearcherParameters textUnitSearcherParameters = new TextUnitSearcherParameters(); + textUnitSearcherParameters.setRepositoryIds(tmTestData.repository.getId()); + textUnitSearcherParameters.setUsedFilter(UsedFilter.USED); + textUnitSearcherParameters.setExcludeUnexpiredPendingMT(true); + textUnitSearcherParameters.setAiTranslationExpiryDuration(Duration.ofMinutes(45)); + + List results = textUnitSearcher.search(textUnitSearcherParameters); + assertEquals(4, results.size()); + assertThat(results) + .extracting(TextUnitDTO::getTmTextUnitId) + .doesNotContain(tmTestData.addTMTextUnit1.getId()); + + textUnitSearcherParameters.setAiTranslationExpiryDuration(Duration.ofMinutes(10)); + results = textUnitSearcher.search(textUnitSearcherParameters); + assertEquals(8, results.size()); + assertThat(results) + .extracting(TextUnitDTO::getTmTextUnitId) + .has( + new Condition<>( + ids -> ids.contains(tmTestData.addTMTextUnit1.getId()), + "TmTextUnit id is present")); + } + private List getAssetTextUnitToTMTextUnit( AssetExtraction assetExtraction) { TypedQuery query =