Add the configured prefix (e.g. ##) when decoding suffix tokens at the start

* Fixes an issue where suffix tokens appearing at the start of a decoded string lost their prefix.

* Added initializing of ContinuingSubwordPrefix from tokenizer.json

* Test DecodeStartingFromSuffix changed beacuse of new behaviour.

* Revert "Added initializing of ContinuingSubwordPrefix from tokenizer.json"

This reverts commit d312979.

* Revert "Fixes an issue where suffix tokens appearing at the start of a decoded string lost their prefix."

This reverts commit f43f344.

* Added decoder suffix and its initialization from tokenizer.json.

* Set _decoderPrefix for vocab.txt

---------

Co-authored-by: Georg Jung <[email protected]>

Fixes #102

Author: kudima03

src/FastBertTokenizer.Tests/Decode.cs

@@ -49,7 +49,7 @@ public void DecodeStartingFromSuffix()
 
         long[] startsWithSuffix = loremIpsum.AsSpan(2).ToArray();
         decoded = _uut.Decode(startsWithSuffix);
-        decoded.ShouldStartWith("m ipsum");
+        decoded.ShouldStartWith("##m ipsum");
     }
 
     [Fact]

src/FastBertTokenizer/BertTokenizer.Decode.cs

@@ -31,6 +31,7 @@ public string Decode(ReadOnlySpan<long> tokenIds, bool cleanupTokenizationSpaces
         else
         {
             // Our decoded text does not start with a word start but in the middle of a word.
+            sb.Append(_decoderPrefix);
             sb.Append(_decodeSuffixes[tokenIds[0]]);
         }
 

src/FastBertTokenizer/BertTokenizer.LoadTokenizerJson.cs

@@ -174,6 +174,7 @@ void HandleLine(string line, int tokenId)
         // https://huggingface.co/docs/tokenizers/python/latest/api/reference.html#module-tokenizers.normalizers
         // strip_accents (bool, optional) – Whether to strip all accents. If this option is not specified (ie == None), then it will be determined by the value for lowercase (as in the original Bert).
         _stripAccents = tok.Normalizer.StripAccents ?? _lowercaseInput;
+        _decoderPrefix = tok.Decoder?.Prefix ?? "##";
         _normalization = normalization;
         _unk = (unkId ?? throw new InvalidOperationException($"Vocabulary does not contain unknown token {unkToken}."), unkToken);
         _cls = (clsId ?? throw new InvalidOperationException($"Vocabulary does not contain cls token {clsToken}."), clsToken);

src/FastBertTokenizer/BertTokenizer.LoadVocab.cs

@@ -11,6 +11,8 @@ namespace FastBertTokenizer;
 
 public partial class BertTokenizer
 {
+    private const string VocabTxtDefaultContinuingSubwordPrefix = "##";
+
     /// <summary>
     /// Load a vocab.txt file that assigns an id to each token based on the line number.
     /// </summary>
@@ -75,7 +77,7 @@ void HandleLine(string line)
         {
             if (!string.IsNullOrEmpty(line))
             {
-                if (line.StartsWith("##", StringComparison.Ordinal))
+                if (line.StartsWith(VocabTxtDefaultContinuingSubwordPrefix, StringComparison.Ordinal))
                 {
                     suffixes[new StringSpanOrdinalKey(line[2..])] = i;
                 }
@@ -149,6 +151,7 @@ void Finish()
             _suffixes = suffixes;
 #endif
             _lowercaseInput = convertInputToLowercase;
+            _decoderPrefix = VocabTxtDefaultContinuingSubwordPrefix;
             _normalization = normalization;
             _addedTokens = new([(unknownToken, false), (clsToken, false), (sepToken, false), (padToken, false)]);
         }

src/FastBertTokenizer/BertTokenizer.cs

@@ -35,6 +35,7 @@ public partial class BertTokenizer
     private bool _stripAccents = true;
     private NormalizationForm _normalization;
     private AddedTokens _addedTokens = default!;
+    private string _decoderPrefix = default!;
 
     // These will just be used if the consumer calls an API that _returns_ ReadOnlyMemory.
     // They will be reused for subsequent calls to avoid allocations.

src/FastBertTokenizer/TokenizerJson.cs

@@ -25,6 +25,8 @@ internal record TokenizerJson
 
     public required ModelSection Model { get; init; }
 
+    public DecoderSection? Decoder { get; init; }
+
     // https://huggingface.co/docs/tokenizers/python/latest/api/reference.html#tokenizers.AddedToken
     internal record AddedToken
     {
@@ -91,4 +93,13 @@ internal record ModelSection
 
         public required Dictionary<string, int> Vocab { get; set; }
     }
+
+    internal record DecoderSection
+    {
+        public string? Type { get; init; }
+
+        public string? Prefix { get; init; }
+
+        public bool? Cleanup { get; init; }
+    }
 }