diff --git a/pylate/models/colbert.py b/pylate/models/colbert.py
index 16a088a..0de7cfc 100644
--- a/pylate/models/colbert.py
+++ b/pylate/models/colbert.py
@@ -261,6 +261,9 @@ def __init__(
                         use_auth_token,
                     )
                 )
+                # Setting the prefixes from stanford-nlp models
+                self.query_prefix = "[unused0]"
+                self.document_prefix = "[unused1]"
                 logger.warning("Loaded the ColBERT model from Stanford NLP.")
             else:
                 # Add a linear projection layer to the model in order to project the embeddings to the desired size