Optimize performance: EmitNoSpaceBefore, AddedTokens, and Decode dictionaries

Co-authored-by: georg-jung <[email protected]>

Author: Copilot

src/FastBertTokenizer/AddedTokens.cs

@@ -13,15 +13,22 @@ public AddedTokens(IEnumerable<(string Content, bool Normalize)> addedTokens)
 
         // This logic might not be perfect. Are there chars that are equal to others in an invariant case insesitive comparison
         // but are neither the upper nor the lower variant of the original?
-        var firstLettersToSearch = addedTokens
-            .SelectMany(x => x.Normalize
-                ? (IEnumerable<char>)[x.Content[0], char.ToLowerInvariant(x.Content[0]), char.ToUpperInvariant(x.Content[0])]
-                : [x.Content[0]])
-            .Distinct();
+        var firstLettersSet = new HashSet<char>();
+        foreach (var (content, normalize) in addedTokens)
+        {
+            var firstChar = content[0];
+            firstLettersSet.Add(firstChar);
+            if (normalize)
+            {
+                firstLettersSet.Add(char.ToLowerInvariant(firstChar));
+                firstLettersSet.Add(char.ToUpperInvariant(firstChar));
+            }
+        }
+
 #if NET8_0_OR_GREATER
-        FirstLetters = SearchValues.Create([.. firstLettersToSearch]);
+        FirstLetters = SearchValues.Create([.. firstLettersSet]);
 #else
-        FirstLetters = [.. firstLettersToSearch];
+        FirstLetters = [.. firstLettersSet];
 #endif
     }
 

src/FastBertTokenizer/BertTokenizer.Decode.cs

@@ -15,8 +15,10 @@ public string Decode(ReadOnlySpan<long> tokenIds)
         _ = _prefixes ?? throw new InvalidOperationException("Vocabulary not loaded.");
         _ = _suffixes ?? throw new InvalidOperationException("Vocabulary not loaded.");
 
-        _decodeSuffixes ??= _suffixes.ToDictionary(x => x.Value, x => x.Key.ToString());
-        _decodePrefixes ??= _prefixes.ToDictionary(x => x.Value, x => x.Key.ToString());
+        if (_decodeSuffixes is null || _decodePrefixes is null)
+        {
+            InitializeDecodeDictionaries();
+        }
 
         if (tokenIds.Length == 0)
         {
@@ -72,9 +74,24 @@ public string Decode(ReadOnlySpan<long> tokenIds)
     // See https://github.com/huggingface/tokenizers/blob/daf361676bdfd14088f7e0bc087effc6a9cfdf3e/tokenizers/src/decoders/wordpiece.rs#L31
     private bool EmitNoSpaceBefore(string prefix)
     {
-        return ".".Equals(prefix, StringComparison.Ordinal)
-            || "?".Equals(prefix, StringComparison.Ordinal)
-            || "!".Equals(prefix, StringComparison.Ordinal)
-            || ",".Equals(prefix, StringComparison.Ordinal);
+        return prefix.Length == 1 && (prefix[0] == '.' || prefix[0] == '?' || prefix[0] == '!' || prefix[0] == ',');
+    }
+
+    private void InitializeDecodeDictionaries()
+    {
+        var decodeSuffixes = new Dictionary<long, string>(_suffixes!.Count);
+        foreach (var kvp in _suffixes!)
+        {
+            decodeSuffixes[kvp.Value] = kvp.Key.ToString();
+        }
+
+        var decodePrefixes = new Dictionary<long, string>(_prefixes!.Count);
+        foreach (var kvp in _prefixes!)
+        {
+            decodePrefixes[kvp.Value] = kvp.Key.ToString();
+        }
+
+        _decodeSuffixes = decodeSuffixes;
+        _decodePrefixes = decodePrefixes;
     }
 }