Other: Multi-game based AI training. (WIP)

Author: DeanEdisGG

G33kShell.Desktop/Console/Screensavers/AI/AiGameCanvasBase.cs

@@ -23,16 +23,18 @@ namespace G33kShell.Desktop.Console.Screensavers.AI;
 /// </summary>
 public abstract class AiGameCanvasBase : ScreensaverBase
 {
-    private int m_generation;
-    private double m_savedRating;
     private const int InitialPopSize = 300;
     private const int MinPopSize = 150;
+    private const int MaxGoatBrains = 5;
+    
+    private readonly List<(double Rating, AiBrainBase Brain)> m_goatBrains = new List<(double Rating, AiBrainBase Brain)>(MaxGoatBrains);
+    private int m_generation;
+    private double m_savedRating;
     private int m_generationsSinceImprovement;
     private int m_currentPopSize = InitialPopSize;
     private Task m_trainingTask;
     private bool m_stopTraining;
-
-    protected AiGameBase[] m_games;
+    private List<AiBrainBase> m_nextGenBrains;
 
     protected int ArenaWidth { get; }
     protected int ArenaHeight { get; }
@@ -83,35 +85,70 @@ public override void StopScreensaver()
 
     private void TrainAiImpl(Action<byte[]> saveBrainBytes)
     {
-        m_games ??= Enumerable.Range(0, m_currentPopSize).Select(_ => CreateGameWithSeed(m_generation)).ToArray();
+        m_nextGenBrains ??= Enumerable.Range(0, InitialPopSize).Select(_ => CreateGame().Brain).ToList();
 
-        Parallel.For(0, m_games.Length, i =>
+        var games = new (double AverageRating, AiGameBase Game, AiBrainBase Brain)[m_nextGenBrains.Count];
+        Parallel.For(0, games.Length, i =>
         {
-            var game = m_games[i];
-            while (!game.IsGameOver)
-                game.Tick();
+            // Play the base game.
+            var baseGame = CreateGameWithSeed(m_generation);
+            while (!baseGame.IsGameOver && !m_stopTraining)
+                baseGame.Tick();
+
+            var totalRating = baseGame.Rating;
+            if (baseGame.Rating > 0.0)
+            {
+                // Play several more games.
+                var gameCount = 1;
+                for (var trial = 0; trial < 4 && !m_stopTraining; trial++, gameCount++)
+                {
+                    var game = CreateGameWithSeed(Random.Shared.Next());
+                    game.Brain = baseGame.Brain;
+                    while (!game.IsGameOver)
+                        game.Tick();
+                    totalRating += game.Rating;
+                    if (game.Rating <= 0.0001)
+                        break; // No score, no point in continuing.
+                }
+
+                totalRating /= gameCount;
+            }
+
+            games[i] = (totalRating, baseGame, baseGame.Brain);
         });
 
         // Select the breeders.
-        var orderedGames = m_games.OrderByDescending(o => o.Rating).ToArray();
-        
+        var orderedGames = games.OrderByDescending(o => o.AverageRating).ToArray();
+        var theBest = orderedGames[0];
+
         // Report summary of results.
         m_generation++;
-        var veryBest = orderedGames[0];
-        var stats = $"Gen {m_generation}|Pop {m_currentPopSize}|Rating {veryBest.Rating:F1}|GOAT {m_savedRating:F1}";
-        var extraStats = veryBest.ExtraGameStats().Select(o => $" {o.Name}: {o.Value}").ToArray().ToCsv().Trim();
+        var stats = $"Gen {m_generation}|Pop {m_currentPopSize}|Rating {theBest.AverageRating:F1}|GOAT {m_savedRating:F1}";
+        var extraStats = theBest.Game.ExtraGameStats().Select(o => $" {o.Name}: {o.Value}").ToArray().ToCsv().Trim();
         if (!string.IsNullOrEmpty(extraStats))
             stats += $"|{extraStats}";
         System.Console.WriteLine(stats);
+        
+        // Remember the GOAT brains.
+        var worstGoatRating = m_goatBrains.Count > 0 ? m_goatBrains.FastFindMin(o => o.Rating).Rating : 0.0;
+        for (var i = 0; i < orderedGames.Length; i++)
+        {
+            if (orderedGames[i].AverageRating > worstGoatRating)
+                m_goatBrains.Add((orderedGames[i].AverageRating, orderedGames[i].Brain));
+        }
+
+        while (m_goatBrains.Count > MaxGoatBrains)
+        {
+            var toRemove = m_goatBrains.FastFindMin(o => o.Rating);
+            m_goatBrains.Remove(toRemove);
+        }
 
         // Persist brain improvements.
-        AiBrainBase goatBrain = null;
-        if (veryBest.Rating > m_savedRating)
+        if (theBest.AverageRating > m_savedRating)
         {
-            m_savedRating = veryBest.Rating;
+            m_savedRating = theBest.AverageRating;
             System.Console.WriteLine("Saved.");
-            saveBrainBytes(veryBest.Brain.Save());
-            goatBrain = veryBest.Brain.Clone();
+            saveBrainBytes(theBest.Brain.Save());
 
             m_generationsSinceImprovement = 0;
         }
@@ -124,39 +161,30 @@ private void TrainAiImpl(Action<byte[]> saveBrainBytes)
             {
                 m_generationsSinceImprovement = 0;
                 m_currentPopSize = InitialPopSize;
-                System.Console.WriteLine("Stagnation detected — Increasing population size.");
+                System.Console.WriteLine("Stagnation detected - Increasing population size.");
             }
         }
 
         // Build the brains for the next generation.
-        var nextBrains = new List<AiBrainBase>(m_games.Length);
-        
-        // The GOAT lives on.
-        if (goatBrain != null)
-        {
-            nextBrains.Add(goatBrain.Clone());
-            nextBrains.AddRange(Enumerable.Range(0, (int)(m_currentPopSize * 0.05)).Select(_ => goatBrain.Clone().Mutate(0.03)));
-        }
+        var nextBrains = new List<AiBrainBase>(games.Length);
+        nextBrains.AddRange(m_goatBrains.Select(o => o.Brain.Clone()));
 
         // Spawn 5% pure randoms.
-        nextBrains.AddRange(Enumerable.Range(0, (int)(m_currentPopSize * 0.05)).Select(_ => veryBest.Brain.Clone().Randomize()));
+        nextBrains.AddRange(Enumerable.Range(0, (int)(m_currentPopSize * 0.05)).Select(_ => CreateGameWithSeed(0).Brain.Clone().Randomize()));
 
         // Elite get to be parents.
+        var breeders = orderedGames.Select(o => (o.AverageRating, o.Brain)).ToList();
+        breeders.AddRange(m_goatBrains);
         while (nextBrains.Count < m_currentPopSize)
         {
-            var mumBrain = Random.Shared.RouletteSelection(m_games, o => o.Rating).Brain;
-            var dadBrain = Random.Shared.RouletteSelection(m_games, o => o.Rating).Brain;
+            var mumBrain = Random.Shared.RouletteSelection(breeders, o => o.AverageRating).Brain;
+            var dadBrain = Random.Shared.RouletteSelection(breeders, o => o.AverageRating).Brain;
             var childBrain = mumBrain.Clone().CrossWith(dadBrain, 0.5).Mutate(0.05);
             nextBrains.Add(childBrain);
         }
 
         // Make the next generation of games.
-        m_games = nextBrains.Select(o =>
-        {
-            var newGame = CreateGameWithSeed(m_generation);
-            newGame.Brain = o;
-            return newGame;
-        }).ToArray();
+        m_nextGenBrains = nextBrains;
     }
 
     private AiGameBase CreateGameWithSeed(int seed)

G33kShell.Desktop/Console/Screensavers/Asteroids/AsteroidsCanvas.cs

@@ -25,6 +25,8 @@ namespace G33kShell.Desktop.Console.Screensavers.Asteroids;
 [UsedImplicitly]
 public class AsteroidsCanvas : AiGameCanvasBase
 {
+    private Game m_game;
+
     public AsteroidsCanvas(int screenWidth, int screenHeight) : base(screenWidth, screenHeight, 60)
     {
         Name = "asciiroids";
@@ -43,17 +45,17 @@ public override void UpdateFrame(ScreenData screen)
     [UsedImplicitly]
     private void PlayGame(ScreenData screen)
     {
-        if (m_games == null)
+        if (m_game == null)
         {
-            m_games = [CreateGame().ResetGame()];
-            m_games[0].LoadBrainData(Settings.Instance.AsteroidsBrain);
+            m_game = (Game)CreateGame().ResetGame();
+            m_game.LoadBrainData(Settings.Instance.AsteroidsBrain);
         }
 
-        m_games[0].Tick();
-        DrawGame(screen, m_games[0]);
+        m_game.Tick();
+        DrawGame(screen, m_game);
 
-        if (((Game)m_games[0]).IsGameOver)
-            m_games[0].ResetGame();
+        if (m_game.IsGameOver)
+            m_game.ResetGame();
     }
 
     private void DrawGame(ScreenData screen, AiGameBase aiGame)

G33kShell.Desktop/Console/Screensavers/Asteroids/Game.cs

@@ -17,16 +17,16 @@
 
 namespace G33kShell.Desktop.Console.Screensavers.Asteroids;
 
-[DebuggerDisplay("Rating = {Rating}, Score = {Score}, Lives = {m_lives}")]
+[DebuggerDisplay("Rating = {Rating}, Score = {Score}")]
 public class Game : AiGameBase
 {
     private int m_bulletsFired;
     private int m_gameTicks;
     private int m_leftTurns;
     private int m_rightTurns;
     private GameState m_gameState;
-    private int m_lives;
     private int m_thrustTicks;
+    private int m_ticksSinceScore;
 
     /// <summary>
     /// Score used for public display.
@@ -42,7 +42,8 @@ public override double Rating
             if (m_thrustTicks == 0)
                 return 0.0; // Penalize non-thrusters.
 
-            return Score * HitRatio * HitRatio * m_gameTicks * 0.001;
+            var rating = 1.0 + Score * HitRatio * 0.001;
+            return rating * rating;
         }
     }
 
@@ -66,7 +67,7 @@ private double TurnEquality
         }
     }
 
-    public override bool IsGameOver => m_lives == 0 || m_gameTicks > 200_000;
+    public override bool IsGameOver => m_ticksSinceScore >= 200_000 || Ship.Shield <= 0.02;
     public Ship Ship { get; private set; }
     public List<Asteroid> Asteroids { get; } = [];
     public List<Bullet> Bullets { get; } = [];
@@ -90,7 +91,7 @@ public override Game ResetGame()
         m_leftTurns = 0;
         m_rightTurns = 0;
         m_thrustTicks = 0;
-        m_lives = 3;
+        m_ticksSinceScore = 0;
 
         EnsureMinimumAsteroidCount();
 
@@ -115,6 +116,7 @@ public override void Tick()
             return;
 
         m_gameTicks++;
+        m_ticksSinceScore++;
 
         // Spawn asteroids.
         EnsureMinimumAsteroidCount();
@@ -148,32 +150,38 @@ public override void Tick()
         Bullets.RemoveAll(o => o.IsExpired);
 
         // Check for bullet/asteroid collisions.
-        var bulletsToRemove = new List<Bullet>();
-        for (var index = 0; index < Bullets.Count; index++)
+        var hitDetected = false;
+        if (Bullets.Count > 0)
         {
-            var bullet = Bullets[index];
-            Asteroid hitAsteroid = null;
-            // ReSharper disable once ForCanBeConvertedToForeach
-            // ReSharper disable once LoopCanBeConvertedToQuery
-            for (var i = 0; i < Asteroids.Count; i++)
+            var bulletsToRemove = new List<Bullet>();
+            for (var index = 0; index < Bullets.Count; index++)
             {
-                if (Asteroids[i].IsInvulnerable || !Asteroids[i].Contains(bullet.Position))
-                    continue;
-                hitAsteroid = Asteroids[i];
-                break;
-            }
+                var bullet = Bullets[index];
+                Asteroid hitAsteroid = null;
+                // ReSharper disable once ForCanBeConvertedToForeach
+                // ReSharper disable once LoopCanBeConvertedToQuery
+                for (var i = 0; i < Asteroids.Count; i++)
+                {
+                    if (Asteroids[i].IsInvulnerable || !Asteroids[i].Contains(bullet.Position))
+                        continue;
+                    hitAsteroid = Asteroids[i];
+                    break;
+                }
 
-            if (hitAsteroid == null)
-                continue; // Bullet not hitting anything.
+                if (hitAsteroid == null)
+                    continue; // Bullet not hitting anything.
 
-            // Bullet hit an asteroid.
-            bulletsToRemove.Add(bullet);
-            hitAsteroid.Explode(Asteroids);
-            Score++;
-        }
+                // Bullet hit an asteroid.
+                bulletsToRemove.Add(bullet);
+                hitAsteroid.Explode(Asteroids);
+                Score++;
+                m_ticksSinceScore = 0;
+                hitDetected = true;
+            }
 
-        for (var i = 0; i < bulletsToRemove.Count; i++)
-            Bullets.Remove(bulletsToRemove[i]);
+            for (var i = 0; i < bulletsToRemove.Count; i++)
+                Bullets.Remove(bulletsToRemove[i]);
+        }
 
         // Check for ship/asteroid collisions.
         const float shipRadius = 4.5f;
@@ -182,18 +190,12 @@ public override void Tick()
             var distance = Asteroids[i].DistanceTo(Ship.Position);
             if (distance < Asteroids[i].Radius + shipRadius)
             {
-                var newShield = Ship.Shield - 0.08;
-                if (newShield < 0.0)
+                Ship.Shield = Math.Max(0.0, Ship.Shield - 0.08);
+                if (Ship.Shield <= 0.001)
                 {
-                    // Ship is dead - Reset asteroids and bullets.
-                    m_lives--;
-                    Asteroids.Clear();
-                    Bullets.Clear();
-                    Ship.Reset();
+                    // Ship is dead.
                     return;
                 }
-                
-                Ship.Shield = newShield;
                 break;
             }
         }

G33kShell.Desktop/Console/Screensavers/Pong/PongCanvas.cs

@@ -27,6 +27,7 @@ namespace G33kShell.Desktop.Console.Screensavers.Pong;
 public class PongCanvas : AiGameCanvasBase
 {
     private readonly FIGletFont m_font;
+    private Game m_game;
 
     public PongCanvas(int screenWidth, int screenHeight) : base(screenWidth, screenHeight, 60)
     {
@@ -49,17 +50,17 @@ public override void UpdateFrame(ScreenData screen)
     [UsedImplicitly]
     private void PlayGame(ScreenData screen)
     {
-        if (m_games == null)
+        if (m_game == null)
         {
-            m_games = [CreateGame().ResetGame()];
-            m_games[0].LoadBrainData(Settings.Instance.PongBrain);
+            m_game = (Game)CreateGame().ResetGame();
+            m_game.LoadBrainData(Settings.Instance.PongBrain);
         }
 
-        DrawGame(screen, m_games[0]);
+        DrawGame(screen, m_game);
 
-        m_games[0].Tick();
-        if (((Game)m_games[0]).IsGameOver)
-            m_games[0].ResetGame();
+        m_game.Tick();
+        if (m_game.IsGameOver)
+            m_game.ResetGame();
     }
 
     private void DrawGame(ScreenData screen, AiGameBase aiGame)

G33kShell.Desktop/Console/Screensavers/Snake/SnakeCanvas.cs

@@ -22,6 +22,8 @@ namespace G33kShell.Desktop.Console.Screensavers.Snake;
 [UsedImplicitly]
 public class SnakeCanvas : AiGameCanvasBase
 {
+    private Game m_game;
+
     public SnakeCanvas(int screenWidth, int screenHeight) : base(screenWidth, screenHeight, 60)
     {
         Name = "snake";
@@ -40,15 +42,15 @@ public override void UpdateFrame(ScreenData screen)
     [UsedImplicitly]
     private void PlayGame(ScreenData screen)
     {
-        if (m_games == null)
+        if (m_game == null)
         {
-            m_games = [new Game(ArenaWidth, ArenaHeight).ResetGame()];
-            m_games[0].LoadBrainData(Settings.Instance.SnakeBrain);
+            m_game = (Game)new Game(ArenaWidth, ArenaHeight).ResetGame();
+            m_game.LoadBrainData(Settings.Instance.SnakeBrain);
         }
 
-        DrawGame(screen, m_games[0]);
+        DrawGame(screen, m_game);
 
-        m_games[0].Tick();
+        m_game.Tick();
     }
 
     private static void DrawGame(ScreenData screen, AiGameBase aiGame)