enhance logging and script management for agents and binaries

Author: naderzare

.gitignore

@@ -4,3 +4,4 @@ scripts/rcssserver
 logs/
 __pycache__/
 utils/__pycache__/
+scripts/binary/

scripts/create_binary.sh

@@ -0,0 +1,45 @@
+#!/bin/bash
+
+
+# show error if binary directory already exists
+
+if [ -d "binary" ]; then
+  echo "binary directory already exists. Please remove it before running this script."
+  exit 1
+fi
+
+# create binary directory
+
+mkdir binary
+
+# copy scripts/proxy to binary directory
+
+mkdir binary/scripts
+cp -r proxy binary/scripts/proxy
+
+# copy formations directory to binary directory
+
+mkdir binary/src
+cp -r ../src/formations binary/src/formations
+
+# create binary
+
+nuitka --standalone --onefile --output-dir=binary ../start_agent.py
+
+# remove build directory
+
+rm -rf binary/start_agent.build
+rm -rf binary/start_agent.dist
+rm -rf binary/start_agent.onefile-build
+
+# copy start_agents.sh to binary directory
+
+cp ../start_agents.sh binary/start_agents.sh
+
+# change all `python3 start_agent.py`` to ./start_agent.bin in binary/start_agents.sh
+
+sed -i 's/run_bin=false/run_bin=true/g' binary/start_agents.sh
+
+# copy start to binary directory
+
+cp start binary/start

scripts/start

@@ -0,0 +1,23 @@
+#!/bin/sh
+
+HOST=$1
+BASEDIR=$2
+NUM=$3
+
+teamname="CLSF"
+
+cd $BASEDIR
+
+options="--team_name $teamname --server-host $HOST --use-random-port --close-server --disable_log_file"
+
+case $NUM in
+    1)
+        ./start_agent.bin $options --goalie
+        ;;
+    12)
+        ./start_agent.bin $options --coach
+        ;;
+    *)
+        ./start_agent.bin $options
+        ;;
+esac

scripts/start_agents.sh

@@ -0,0 +1,66 @@
+#!/bin/bash
+
+# active .venv
+source .venv/bin/activate
+
+server_port=6000
+log_to_file=false
+
+while [ $# -gt 0 ]
+do
+  case $1 in
+    --server-port)
+      server_port=$2
+      shift
+      ;;
+    --log-to-file)
+      log_to_file=true
+      ;;
+    *)
+      echo 1>&2
+      echo "invalid option \"${1}\"." 1>&2
+      echo 1>&2
+      usage
+      exit 1
+      ;;
+  esac
+    shift 1
+done
+
+# Function to handle termination
+cleanup() {
+  echo "Terminating all start_agent.py processes..."
+  for pid in "${pids[@]}"; do
+    kill $pid
+  done
+  exit 0
+}
+
+# Trap SIGTERM and SIGINT signals
+trap cleanup SIGTERM SIGINT
+
+# Init log directory
+log_dir="$(pwd)/logs/$(date +'%Y-%m-%d_%H-%M-%S')_$((RANDOM % 900000 + 100000))"
+mkdir -p $log_dir
+
+# Initialize an empty array to store PIDs
+pids=()
+
+# Run start_agent.py 11 times and store each PID in the array
+python3 start_agent.py --server-port $server_port --use-random-port --close-server --goalie --log-dir $log_dir &
+pids+=($!)
+
+sleep 2
+
+for i in {2..11}; do
+  python3 start_agent.py --server-port $server_port --use-random-port --close-server --log-dir $log_dir &
+  pids+=($!)
+done
+
+python3 start_agent.py --server-port $server_port --use-random-port --close-server --coach --log-dir $log_dir &
+pids+=($!)
+
+# Wait for all background processes to finish
+for pid in "${pids[@]}"; do
+  wait $pid
+done

server.py

@@ -16,9 +16,9 @@
 
 
 console_logging_level = logging.INFO
-file_logging_level = logging.DEBUG
+file_logging_level = logging.INFO
 player_console_logging_level = logging.INFO
-player_file_logging_level = logging.DEBUG
+player_file_logging_level = logging.INFO
 
 main_logger = None
 log_dir = None
@@ -31,12 +31,11 @@ def __init__(self, agent_type, uniform_number, logger) -> None:
         self.agent: IAgent = None
         self.logger: logging.Logger = logger
         if self.agent_type == pb2.AgentType.PlayerT:
-            self.agent = SamplePlayerAgent()
+            self.agent = SamplePlayerAgent(self.logger)
         elif self.agent_type == pb2.AgentType.CoachT:
-            self.agent = SampleCoachAgent()
+            self.agent = SampleCoachAgent(self.logger)
         elif self.agent_type == pb2.AgentType.TrainerT:
-            self.agent = SampleTrainerAgent()
-        self.agent.set_logger(self.logger)
+            self.agent = SampleTrainerAgent(self.logger)
         self.debug_mode: bool = False
 
 
@@ -189,13 +188,20 @@ def serve(port, shared_lock, shared_number_of_connections):
 
 
 def main():
-    global main_logger, log_dir
+    global main_logger, log_dir, file_logging_level, player_file_logging_level
     parser = argparse.ArgumentParser(description='Run play maker server')
     parser.add_argument('-p', '--rpc-port', required=False, help='The port of the server', default=50051)
     parser.add_argument('-l', '--log-dir', required=False, help='The directory of the log file', 
                         default=f'logs/{datetime.datetime.now().strftime("%Y-%m-%d_%H-%M-%S")}')
+    parser.add_argument('--disable-log-file', required=False, help='Disable logging to a file', default=False, action='store_true')
+    
     args = parser.parse_args()
+    
     log_dir = args.log_dir
+    if args.disable_log_file:
+        file_logging_level = None
+        player_file_logging_level = None
+        
     main_logger = setup_logger("pmservice", log_dir, console_level=console_logging_level, file_level=file_logging_level)
     main_logger.info("Starting server")
     manager = Manager()

src/interfaces/IAgent.py

@@ -6,15 +6,15 @@
 
 
 class IAgent(ABC):
-    def __init__(self) -> None:
+    def __init__(self, logger) -> None:
         super().__init__()
         self.wm: Union[WorldModel, None] = None
         self.actions: list[Union[PlayerAction, CoachAction, TrainerAction]] = []
         self.server_params: Union[ServerParam, None] = None
         self.player_params: Union[PlayerParam, None] = None
         self.player_types: dict[PlayerType] = {}
         self.debug_mode: bool = False
-        self.logger: logging.Logger= None
+        self.logger: logging.Logger = logger
 
     def set_server_params(self, server_param: ServerParam):
         self.server_params = server_param
@@ -87,8 +87,4 @@ def add_action(self, action: Union[PlayerAction, CoachAction, TrainerAction]):
         self.actions.append(action)
 
     def get_actions(self) -> list[Union[PlayerAction, CoachAction, TrainerAction]]:
-        return self.actions
-
-    def set_logger(self, logger):
-        self.logger = logger
-        self.logger.debug(f"IAgent.set_logger: {logger}")
+        return self.actions

src/sample_coach_agent.py

@@ -4,8 +4,9 @@
 
 
 class SampleCoachAgent(IAgent, ABC):
-    def __init__(self):
-        super().__init__()
+    def __init__(self, logger) -> None:
+        super().__init__(logger)
+        self.logger.info('SampleCoachAgent created')
         self.wm: WorldModel = None
         self.first_substitution = True
 

src/sample_player_agent.py

@@ -6,8 +6,9 @@
 
 
 class SamplePlayerAgent(IAgent, ABC):
-    def __init__(self):
-        super().__init__()
+    def __init__(self, logger) -> None:
+        super().__init__(logger)
+        self.logger.info('SamplePlayerAgent created')
         self.decision_maker = DecisionMaker()
         self.strategy = FormationStrategy()
         self.wm: WorldModel = None

src/sample_trainer_agent.py

@@ -4,8 +4,9 @@
 
 
 class SampleTrainerAgent(IAgent, ABC):
-    def __init__(self):
-        super().__init__()
+    def __init__(self, logger) -> None:
+        super().__init__(logger)
+        self.logger.info('SampleTrainerAgent created')
         self.wm: WorldModel = None
         self.first_substitution = True
 

start_agent.py

@@ -21,7 +21,7 @@ def run_server_script(args):
     # Define a wrapper function to pass the arguments to the main function
     def server_main():
         import sys
-        sys.argv = ['server.py', '--rpc-port', str(args.rpc_port), '--log-dir', log_dir]
+        sys.argv = ['server.py', '--rpc-port', str(args.rpc_port), '--log-dir', log_dir, '--disable-log-file' if args.disable_log_file else '']
         main()
 
     # Start the main function as a new process
@@ -32,7 +32,10 @@ def server_main():
 def run_start_script(args):
     # Start the start.sh script in its own directory as a new process group
     process = subprocess.Popen(
-        ['bash', 'start-agent.sh' if not args.debug else 'start-debug.sh', '-t', args.team_name, '--rpc-port', args.rpc_port, '--rpc-type', 'grpc', '-p', args.server_port, 
+        ['bash', 'start-agent.sh' if not args.debug else 'start-debug.sh', 
+         '-t', args.team_name, 
+         '--rpc-port', args.rpc_port, '--rpc-type', 'grpc', 
+         '-p', args.server_port, '-h', args.server_host,
          '--coach' if args.coach else '--goalie' if args.goalie else '--player'],
         cwd='scripts/proxy',  # Corrected directory to where start.sh is located
         preexec_fn=os.setsid,  # Create a new session and set the process group ID
@@ -44,16 +47,19 @@ def run_start_script(args):
 stop_thread = threading.Event()
 
 
-def stream_output_to_file(process, name):
+def stream_output_to_file(process, name, args):
     # Stream output from the process and log it to a file with the given name
-    with open(f"{log_dir}/{name}.log", "a") as f:
-        while not stop_thread.is_set():
-            output = process.stdout.readline()
-            if output:
+    f = None if args.disable_log_file else open(f"{log_dir}/{name}.log", "a")
+    while not stop_thread.is_set():
+        output = process.stdout.readline()
+        if output:
+            if f:
                 f.write(output.decode())
                 f.flush()
             else:
-                break
+                print(output.decode())
+        else:
+            break
     process.stdout.close()
 
 def kill_process_group(process):
@@ -79,10 +85,12 @@ def kill_rpc_server_process(process):
     parser.add_argument('-d', '--debug', required=False, help='Enable debug mode', default=False, action='store_true')
     parser.add_argument('--use-random-port', required=False, help='Use a random port for the server', default=False, action='store_true')
     parser.add_argument('--use-random-name', required=False, help='Use a random team name', default=False, action='store_true')
+    parser.add_argument('--server-host', required=False, help='The host of the server', default='localhost')
     parser.add_argument('--server-port', required=False, help='The port of the server', default='6000')
     parser.add_argument('--close-server', required=False, help='Close the server', default=False, action='store_true')
     parser.add_argument('--coach', required=False, help='Use coach instead of proxy', default=False, action='store_true')
     parser.add_argument('--goalie', required=False, help='Use goalie instead of proxy', default=False, action='store_true')
+    parser.add_argument('--disable-log-file', required=False, help='Disable logging to a file', default=False, action='store_true')
     parser.add_argument('--log-dir', required=False, help='The directory to store logs', default=None)
     args = parser.parse_args()
 
@@ -91,7 +99,8 @@ def kill_rpc_server_process(process):
         log_dir = os.path.join(os.getcwd(), 'logs', f"{datetime.datetime.now().strftime('%Y-%m-%d_%H-%M-%S')}_{random.randint(100000, 999999)}")
     else:
         log_dir = args.log_dir
-    start_team_logger = setup_logger('start-team', log_dir, console_level=logging.DEBUG, file_level=logging.DEBUG, console_format_str='%(message)s')
+    start_team_logger = setup_logger('start-team', log_dir, console_level=logging.DEBUG, file_level=logging.DEBUG if not args.disable_log_file else None,
+                                     console_format_str='%(message)s')
 
     start_team_logger.debug(f"Arguments: {args=}")
 
@@ -119,7 +128,7 @@ def kill_rpc_server_process(process):
         # Monitor both processes and log their outputs
         start_team_logger.debug("Monitoring processes...")
 
-        start_thread = threading.Thread(target=stream_output_to_file, args=(start_process, 'proxy'))
+        start_thread = threading.Thread(target=stream_output_to_file, args=(start_process, 'proxy', args))
         start_thread.start()
 
         def signal_handler(sig, frame):