diff --git a/extensions/tokenizers/src/main/java/ai/djl/huggingface/tokenizers/Encoding.java b/extensions/tokenizers/src/main/java/ai/djl/huggingface/tokenizers/Encoding.java index 920f13d43a6..fe0346abc7e 100644 --- a/extensions/tokenizers/src/main/java/ai/djl/huggingface/tokenizers/Encoding.java +++ b/extensions/tokenizers/src/main/java/ai/djl/huggingface/tokenizers/Encoding.java @@ -16,11 +16,16 @@ import ai.djl.ndarray.NDList; import ai.djl.ndarray.NDManager; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + import java.util.Arrays; /** A class holds token encoding information. */ public class Encoding { + private static final Logger logger = LoggerFactory.getLogger(Encoding.class); + private long[] ids; private long[] typeIds; private String[] tokens; @@ -63,6 +68,9 @@ public NDList toNDList(NDManager manager, boolean withTokenType) { NDList list = new NDList(withTokenType ? 3 : 2); int[] ids2 = Arrays.stream(ids).mapToInt(i -> (int) i).toArray(); int[] attentionMask2 = Arrays.stream(attentionMask).mapToInt(i -> (int) i).toArray(); + logger.info( + "Converting encoding to NDList, attentionMask: " + + manager.create(attentionMask2).toDebugString()); list.add(manager.create(ids2)); list.add(manager.create(attentionMask2)); if (withTokenType) {