Update

deepjavalibrary · Sep 12, 2024 · bcafd27 · bcafd27
1 parent 2c14fde
commit bcafd27
Showing 1 changed file with 8 additions and 0 deletions.
diff --git a/extensions/tokenizers/src/main/java/ai/djl/huggingface/tokenizers/Encoding.java b/extensions/tokenizers/src/main/java/ai/djl/huggingface/tokenizers/Encoding.java
@@ -16,11 +16,16 @@
 import ai.djl.ndarray.NDList;
 import ai.djl.ndarray.NDManager;
 
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
 import java.util.Arrays;
 
 /** A class holds token encoding information. */
 public class Encoding {
 
+    private static final Logger logger = LoggerFactory.getLogger(Encoding.class);
+
     private long[] ids;
     private long[] typeIds;
     private String[] tokens;
@@ -63,6 +68,9 @@ public NDList toNDList(NDManager manager, boolean withTokenType) {
         NDList list = new NDList(withTokenType ? 3 : 2);
         int[] ids2 = Arrays.stream(ids).mapToInt(i -> (int) i).toArray();
         int[] attentionMask2 = Arrays.stream(attentionMask).mapToInt(i -> (int) i).toArray();
+        logger.info(
+                "Converting encoding to NDList, attentionMask: "
+                        + manager.create(attentionMask2).toDebugString());
         list.add(manager.create(ids2));
         list.add(manager.create(attentionMask2));
         if (withTokenType) {