Tokenizers and Transformers are ready to use.

Author: joaopauloschuler

neural/neuralnetwork.pas

@@ -1738,7 +1738,6 @@   TNNet = class(TMObject)
       function AddAvgMaxPool(pPoolSize: integer; pMaxPoolDropout: TNeuralFloat = 0; pKeepDepth:boolean = false; pAfterLayer: TNNetLayer = nil): TNNetLayer;
       function AddMinMaxChannel(pAfterLayer: TNNetLayer = nil): TNNetLayer;
       function AddAvgMaxChannel(pMaxPoolDropout: TNeuralFloat = 0; pKeepDepth:boolean = false; pAfterLayer: TNNetLayer = nil): TNNetLayer;
-      // Transformers, AddSingleHeadSelfAttention and AddSingleHeadTransformerBlock are under construction - do not use it
       procedure AddSingleHeadSelfAttention(out Attended, W: TNNetLayer; NoForward:boolean = false);
       function AddSelfAttention(Heads: integer; NoForward:boolean = false; HasNorm: boolean = false): TNNetLayer;
       function AddSelfAttentionCAI(Heads: integer; NoForward:boolean = false; HasNorm: boolean = false; CellMul: boolean = false): TNNetLayer;

neural/neuraltokenizer.pas

@@ -539,7 +539,6 @@ procedure TestNeuralTokenizer();
   X: TNeuralTokenizer;
 begin
   X := TNeuralTokenizer.Create;
-  //X.Test;
   X.FitOnFile('datasets/tinystories-10k.txt', 3000, true);
   X.SaveToFile('datasets/experiment-tinystories-vocab-3k-cai.txt');
   //X.LoadVocabularyFromFile('datasets/tinystories-vocab-3k-cai.txt');