AI translation updates to TMTextUnitSearcher and store uploaded file …

…uri in mapping step (#162)

* Updated TextUnitSearcher to exclude AI translations based on duration or null id, updated map step to record uploaded file uri in DB

* Added unit tests for text unit searcher

* Unit test update to include file uri

* Remove unnecessary code

webapp/src/main/java/com/box/l10n/mojito/entity/ThirdPartyTextUnit.java

@@ -41,6 +41,9 @@ public class ThirdPartyTextUnit extends AuditableEntity {
       foreignKey = @ForeignKey(name = "FK__THIRD_PARTY_TEXT_UNIT__TM_TEXT_UNIT__ID"))
   TMTextUnit tmTextUnit;
 
+  @Column(name = "uploaded_file_uri")
+  String uploadedFileUri;
+
   public String getThirdPartyId() {
     return thirdPartyId;
   }
@@ -64,4 +67,12 @@ public Asset getAsset() {
   public void setAsset(Asset asset) {
     this.asset = asset;
   }
+
+  public String getUploadedFileUri() {
+    return uploadedFileUri;
+  }
+
+  public void setUploadedFileUri(String uploadedFileUri) {
+    this.uploadedFileUri = uploadedFileUri;
+  }
 }

webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyService.java

@@ -361,6 +361,8 @@ void saveMojitoToThirdPartyTextUnitMapping(
                             thirdPartyTextUnit.setThirdPartyId(
                                 thirdPartyTextUnitForMapping.getId());
                             thirdPartyTextUnit.setAsset(asset);
+                            thirdPartyTextUnit.setUploadedFileUri(
+                                thirdPartyTextUnitForMapping.getUploadedFileUri());
                             TMTextUnit tmTextUnit = tmTextUnitRepository.getOne(tmTextUnitId);
 
                             if (tmTextUnitAlreadySaved.containsKey(tmTextUnitId)) {

webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyTMSSmartling.java

@@ -12,6 +12,7 @@
 import com.box.l10n.mojito.entity.Repository;
 import com.box.l10n.mojito.quartz.QuartzJobInfo;
 import com.box.l10n.mojito.quartz.QuartzPollableTaskScheduler;
+import com.box.l10n.mojito.service.ai.translation.AITranslationConfiguration;
 import com.box.l10n.mojito.service.assetExtraction.AssetTextUnitToTMTextUnitRepository;
 import com.box.l10n.mojito.service.pollableTask.PollableFuture;
 import com.box.l10n.mojito.service.thirdparty.smartling.SmartlingFile;
@@ -100,6 +101,8 @@ public class ThirdPartyTMSSmartling implements ThirdPartyTMS {
 
   private final QuartzPollableTaskScheduler quartzPollableTaskScheduler;
 
+  private final AITranslationConfiguration aiTranslationConfiguration;
+
   private final Set<String> supportedImageExtensions =
       Sets.newHashSet("png", "jpg", "jpeg", "gif", "tiff");
 
@@ -118,7 +121,8 @@ public ThirdPartyTMSSmartling(
       ThirdPartyTMSSmartlingGlossary thirdPartyTMSSmartlingGlossary,
       AssetTextUnitToTMTextUnitRepository assetTextUnitToTMTextUnitRepository,
       MeterRegistry meterRegistry,
-      QuartzPollableTaskScheduler quartzPollableTaskScheduler) {
+      QuartzPollableTaskScheduler quartzPollableTaskScheduler,
+      AITranslationConfiguration aiTranslationConfiguration) {
     this(
         smartlingClient,
         textUnitSearcher,
@@ -130,7 +134,8 @@ public ThirdPartyTMSSmartling(
         assetTextUnitToTMTextUnitRepository,
         DEFAULT_BATCH_SIZE,
         meterRegistry,
-        quartzPollableTaskScheduler);
+        quartzPollableTaskScheduler,
+        aiTranslationConfiguration);
   }
 
   public ThirdPartyTMSSmartling(
@@ -144,7 +149,8 @@ public ThirdPartyTMSSmartling(
       AssetTextUnitToTMTextUnitRepository assetTextUnitToTMTextUnitRepository,
       int batchSize,
       MeterRegistry meterRegistry,
-      QuartzPollableTaskScheduler quartzPollableTaskScheduler) {
+      QuartzPollableTaskScheduler quartzPollableTaskScheduler,
+      AITranslationConfiguration aiTranslationConfiguration) {
     this.smartlingClient = smartlingClient;
     this.assetPathAndTextUnitNameKeys = assetPathAndTextUnitNameKeys;
     this.textUnitBatchImporterService = textUnitBatchImporterService;
@@ -156,6 +162,7 @@ public ThirdPartyTMSSmartling(
     this.assetTextUnitToTMTextUnitRepository = assetTextUnitToTMTextUnitRepository;
     this.meterRegistry = meterRegistry;
     this.quartzPollableTaskScheduler = quartzPollableTaskScheduler;
+    this.aiTranslationConfiguration = aiTranslationConfiguration;
   }
 
   @Override
@@ -337,6 +344,7 @@ public List<ThirdPartyTextUnit> getThirdPartyTextUnits(
                         thirdPartyTextUnit.setAssetPath(key.getAssetPath());
                         thirdPartyTextUnit.setName(key.getTextUnitName());
                         thirdPartyTextUnit.setNamePluralPrefix(isPluralFile(file.getFileUri()));
+                        thirdPartyTextUnit.setUploadedFileUri(file.getFileUri());
 
                         return thirdPartyTextUnit;
                       });
@@ -968,6 +976,8 @@ private TextUnitSearcherParameters baseParams(
     result.setPluralFormsExcluded(pluralFormsExcluded);
     result.setSkipTextUnitWithPattern(skipTextUnitsWithPattern);
     result.setSkipAssetPathWithPattern(skipAssetsWithPathPattern);
+    result.setExcludeUnexpiredPendingMT(aiTranslationConfiguration.isEnabled());
+    result.setAiTranslationExpiryDuration(aiTranslationConfiguration.getExpiryDuration());
     if (!Strings.isNullOrEmpty(pluralFormOther)) {
       result.setPluralFormOther(pluralFormOther);
     }

webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyTextUnit.java

@@ -30,6 +30,8 @@ public class ThirdPartyTextUnit implements TextUnitForBatchMatcher {
    */
   boolean namePluralPrefix;
 
+  String uploadedFileUri;
+
   public String getId() {
     return id;
   }
@@ -80,4 +82,12 @@ public boolean isNamePluralPrefix() {
   public void setNamePluralPrefix(boolean namePluralPrefix) {
     this.namePluralPrefix = namePluralPrefix;
   }
+
+  public String getUploadedFileUri() {
+    return uploadedFileUri;
+  }
+
+  public void setUploadedFileUri(String uploadedFileUri) {
+    this.uploadedFileUri = uploadedFileUri;
+  }
 }

webapp/src/main/java/com/box/l10n/mojito/service/tm/search/TextUnitSearcher.java

@@ -24,6 +24,7 @@
 import com.github.pnowy.nc.core.expressions.NativeProjection;
 import com.github.pnowy.nc.core.mappers.CriteriaResultTransformer;
 import com.google.common.base.Preconditions;
+import java.time.ZonedDateTime;
 import java.util.Arrays;
 import java.util.List;
 import org.apache.commons.codec.digest.DigestUtils;
@@ -172,6 +173,11 @@ NativeCriteria getCriteriaForSearch(TextUnitSearcherParameters searchParameters)
     c.addJoin(NativeExps.crossJoin("locale", "l"));
     c.addJoin(NativeExps.innerJoin("asset", "a", "a.id", "tu.asset_id"));
     c.addJoin(NativeExps.innerJoin("repository", "r", "r.id", "a.repository_id"));
+    if (searchParameters.isExcludeUnexpiredPendingMT()) {
+      c.addJoin(
+          NativeExps.leftJoin(
+              "tm_text_unit_pending_mt", "tmtupmt", "tmtupmt.tm_text_unit_id", "tu.id"));
+    }
 
     NativeJunctionExp onClauseRepositoryLocale = NativeExps.conjunction();
     onClauseRepositoryLocale.add(new NativeColumnEqExp("rl.locale_id", "l.id"));
@@ -378,6 +384,18 @@ NativeCriteria getCriteriaForSearch(TextUnitSearcherParameters searchParameters)
           new NativeNotILikeExp("a.path", searchParameters.getSkipAssetPathWithPattern()));
     }
 
+    if (searchParameters.isExcludeUnexpiredPendingMT()) {
+      ZonedDateTime durationThreshold =
+          ZonedDateTime.now().minus(searchParameters.getAiTranslationExpiryDuration());
+
+      // Include rows where the id in the left table is null (i.e. no match) or the created_date is
+      // outside the configured expiry duration
+      NativeJunctionExp mtExclusionFilter = NativeExps.disjunction();
+      mtExclusionFilter.add(NativeExps.isNull("tmtupmt.id"));
+      mtExclusionFilter.add(new NativeDateLteExp("tmtupmt.created_date", durationThreshold));
+      conjunction.add(mtExclusionFilter);
+    }
+
     StatusFilter statusFilter = searchParameters.getStatusFilter();
 
     if (statusFilter != null) {

webapp/src/main/java/com/box/l10n/mojito/service/tm/search/TextUnitSearcherParameters.java

@@ -1,6 +1,7 @@
 package com.box.l10n.mojito.service.tm.search;
 
 import com.box.l10n.mojito.service.NormalizationUtils;
+import java.time.Duration;
 import java.time.ZonedDateTime;
 import java.util.Arrays;
 import java.util.List;
@@ -50,6 +51,8 @@ public class TextUnitSearcherParameters {
   String skipTextUnitWithPattern;
   String includeTextUnitsWithPattern;
   String skipAssetPathWithPattern;
+  boolean isExcludeUnexpiredPendingMT = false;
+  Duration aiTranslationExpiryDuration;
 
   public String getName() {
     return name;
@@ -327,4 +330,20 @@ public boolean isOrderedByTextUnitID() {
   public void setOrderByTextUnitID(boolean ordered) {
     isOrderedByTextUnitID = ordered;
   }
+
+  public boolean isExcludeUnexpiredPendingMT() {
+    return isExcludeUnexpiredPendingMT;
+  }
+
+  public void setExcludeUnexpiredPendingMT(boolean excludeUnexpiredPendingMT) {
+    isExcludeUnexpiredPendingMT = excludeUnexpiredPendingMT;
+  }
+
+  public Duration getAiTranslationExpiryDuration() {
+    return aiTranslationExpiryDuration;
+  }
+
+  public void setAiTranslationExpiryDuration(Duration aiTranslationExpiryDuration) {
+    this.aiTranslationExpiryDuration = aiTranslationExpiryDuration;
+  }
 }

webapp/src/test/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyServiceTest.java

@@ -197,34 +197,49 @@ public void mapMojitoAndThirdPartyTextUnits() throws InterruptedException, Execu
         .extracting(
             t -> t.getAsset().getId(),
             com.box.l10n.mojito.entity.ThirdPartyTextUnit::getThirdPartyId,
-            t -> t.getTmTextUnit().getId())
+            t -> t.getTmTextUnit().getId(),
+            com.box.l10n.mojito.entity.ThirdPartyTextUnit::getUploadedFileUri)
         .containsExactly(
-            tuple(asset.getId(), "3rd-hello", thirdPartyServiceTestData.tmTextUnitHello.getId()),
-            tuple(asset.getId(), "3rd-bye", thirdPartyServiceTestData.tmTextUnitBye.getId()),
+            tuple(
+                asset.getId(),
+                "3rd-hello",
+                thirdPartyServiceTestData.tmTextUnitHello.getId(),
+                "testFileUri"),
+            tuple(
+                asset.getId(),
+                "3rd-bye",
+                thirdPartyServiceTestData.tmTextUnitBye.getId(),
+                "testFileUri"),
             tuple(
                 asset.getId(),
                 "3rd-plural_things",
-                thirdPartyServiceTestData.tmTextUnitPluralThingsZero.getId()),
+                thirdPartyServiceTestData.tmTextUnitPluralThingsZero.getId(),
+                "testFileUri"),
             tuple(
                 asset.getId(),
                 "3rd-plural_things",
-                thirdPartyServiceTestData.tmTextUnitPluralThingsOne.getId()),
+                thirdPartyServiceTestData.tmTextUnitPluralThingsOne.getId(),
+                "testFileUri"),
             tuple(
                 asset.getId(),
                 "3rd-plural_things",
-                thirdPartyServiceTestData.tmTextUnitPluralThingsTwo.getId()),
+                thirdPartyServiceTestData.tmTextUnitPluralThingsTwo.getId(),
+                "testFileUri"),
             tuple(
                 asset.getId(),
                 "3rd-plural_things",
-                thirdPartyServiceTestData.tmTextUnitPluralThingsFew.getId()),
+                thirdPartyServiceTestData.tmTextUnitPluralThingsFew.getId(),
+                "testFileUri"),
             tuple(
                 asset.getId(),
                 "3rd-plural_things",
-                thirdPartyServiceTestData.tmTextUnitPluralThingsMany.getId()),
+                thirdPartyServiceTestData.tmTextUnitPluralThingsMany.getId(),
+                "testFileUri"),
             tuple(
                 asset.getId(),
                 "3rd-plural_things",
-                thirdPartyServiceTestData.tmTextUnitPluralThingsOther.getId()));
+                thirdPartyServiceTestData.tmTextUnitPluralThingsOther.getId(),
+                "testFileUri"));
 
     logger.debug("Verify behavior");
     verify(thirdPartyTMSMock, times(3))
@@ -741,6 +756,7 @@ ThirdPartyTextUnit createThirdPartyTextUnit(
     thirdPartyTextUnit.setId(id);
     thirdPartyTextUnit.setName(name);
     thirdPartyTextUnit.setNamePluralPrefix(isNamePluralPrefix);
+    thirdPartyTextUnit.setUploadedFileUri("testFileUri");
     return thirdPartyTextUnit;
   }
 }